diff options
526 files changed, 21826 insertions, 14522 deletions
diff --git a/bootstrap/bin/start.boot b/bootstrap/bin/start.boot Binary files differindex c4244d91e4..25c092961c 100644 --- a/bootstrap/bin/start.boot +++ b/bootstrap/bin/start.boot diff --git a/bootstrap/bin/start_clean.boot b/bootstrap/bin/start_clean.boot Binary files differindex c4244d91e4..25c092961c 100644 --- a/bootstrap/bin/start_clean.boot +++ b/bootstrap/bin/start_clean.boot diff --git a/bootstrap/lib/compiler/ebin/beam_asm.beam b/bootstrap/lib/compiler/ebin/beam_asm.beam Binary files differindex 4e31355235..b25c33084f 100644 --- a/bootstrap/lib/compiler/ebin/beam_asm.beam +++ b/bootstrap/lib/compiler/ebin/beam_asm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_bsm.beam b/bootstrap/lib/compiler/ebin/beam_bsm.beam Binary files differindex 6fc683bf76..a6757dfcfb 100644 --- a/bootstrap/lib/compiler/ebin/beam_bsm.beam +++ b/bootstrap/lib/compiler/ebin/beam_bsm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_dead.beam b/bootstrap/lib/compiler/ebin/beam_dead.beam Binary files differindex 0bd9767208..6ae6de1d8b 100644 --- a/bootstrap/lib/compiler/ebin/beam_dead.beam +++ b/bootstrap/lib/compiler/ebin/beam_dead.beam diff --git a/bootstrap/lib/compiler/ebin/beam_dict.beam b/bootstrap/lib/compiler/ebin/beam_dict.beam Binary files differindex ccab57cc81..17b3c40eda 100644 --- a/bootstrap/lib/compiler/ebin/beam_dict.beam +++ b/bootstrap/lib/compiler/ebin/beam_dict.beam diff --git a/bootstrap/lib/compiler/ebin/beam_jump.beam b/bootstrap/lib/compiler/ebin/beam_jump.beam Binary files differindex 136036ae39..8dd6375403 100644 --- a/bootstrap/lib/compiler/ebin/beam_jump.beam +++ b/bootstrap/lib/compiler/ebin/beam_jump.beam diff --git a/bootstrap/lib/compiler/ebin/beam_type.beam b/bootstrap/lib/compiler/ebin/beam_type.beam Binary files differindex 4094cec9e6..cb3bc1a8bd 100644 --- a/bootstrap/lib/compiler/ebin/beam_type.beam +++ b/bootstrap/lib/compiler/ebin/beam_type.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_sets.beam b/bootstrap/lib/compiler/ebin/cerl_sets.beam Binary files differnew file mode 100644 index 0000000000..d16543cdbe --- /dev/null +++ b/bootstrap/lib/compiler/ebin/cerl_sets.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_trees.beam b/bootstrap/lib/compiler/ebin/cerl_trees.beam Binary files differindex 45f76f5c89..fc1a7e04f8 100644 --- a/bootstrap/lib/compiler/ebin/cerl_trees.beam +++ b/bootstrap/lib/compiler/ebin/cerl_trees.beam diff --git a/bootstrap/lib/compiler/ebin/compile.beam b/bootstrap/lib/compiler/ebin/compile.beam Binary files differindex e648a70b2f..111f85acd2 100644 --- a/bootstrap/lib/compiler/ebin/compile.beam +++ b/bootstrap/lib/compiler/ebin/compile.beam diff --git a/bootstrap/lib/compiler/ebin/compiler.app b/bootstrap/lib/compiler/ebin/compiler.app index 39635bd447..814c2f0df5 100644 --- a/bootstrap/lib/compiler/ebin/compiler.app +++ b/bootstrap/lib/compiler/ebin/compiler.app @@ -18,7 +18,7 @@ {application, compiler, [{description, "ERTS CXC 138 10"}, - {vsn, "5.0.4"}, + {vsn, "6.0"}, {modules, [ beam_a, beam_asm, @@ -45,6 +45,7 @@ cerl, cerl_clauses, cerl_inline, + cerl_sets, cerl_trees, compile, core_scan, @@ -69,5 +70,5 @@ {registered, []}, {applications, [kernel, stdlib]}, {env, []}, - {runtime_dependencies, ["stdlib-2.0","kernel-3.0","hipe-3.10.3","erts-7.0", - "crypto-3.3"]}]}. + {runtime_dependencies, ["stdlib-2.5","kernel-4.0","hipe-3.12","erts-7.0", + "crypto-3.6"]}]}. diff --git a/bootstrap/lib/compiler/ebin/sys_core_fold.beam b/bootstrap/lib/compiler/ebin/sys_core_fold.beam Binary files differindex 2ed29957b3..1b5467a54b 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_fold.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_fold.beam diff --git a/bootstrap/lib/compiler/ebin/v3_codegen.beam b/bootstrap/lib/compiler/ebin/v3_codegen.beam Binary files differindex 7b33f9cbff..e53f0fcd12 100644 --- a/bootstrap/lib/compiler/ebin/v3_codegen.beam +++ b/bootstrap/lib/compiler/ebin/v3_codegen.beam diff --git a/bootstrap/lib/compiler/ebin/v3_kernel.beam b/bootstrap/lib/compiler/ebin/v3_kernel.beam Binary files differindex 5817c6ae58..f5cdbb6e40 100644 --- a/bootstrap/lib/compiler/ebin/v3_kernel.beam +++ b/bootstrap/lib/compiler/ebin/v3_kernel.beam diff --git a/bootstrap/lib/kernel/ebin/application.beam b/bootstrap/lib/kernel/ebin/application.beam Binary files differindex 6f68b83ad5..d8e98c021b 100644 --- a/bootstrap/lib/kernel/ebin/application.beam +++ b/bootstrap/lib/kernel/ebin/application.beam diff --git a/bootstrap/lib/kernel/ebin/application_controller.beam b/bootstrap/lib/kernel/ebin/application_controller.beam Binary files differindex 9a7907cb38..c4fa46e33e 100644 --- a/bootstrap/lib/kernel/ebin/application_controller.beam +++ b/bootstrap/lib/kernel/ebin/application_controller.beam diff --git a/bootstrap/lib/kernel/ebin/application_master.beam b/bootstrap/lib/kernel/ebin/application_master.beam Binary files differindex d8018f0a40..b81dfa81d3 100644 --- a/bootstrap/lib/kernel/ebin/application_master.beam +++ b/bootstrap/lib/kernel/ebin/application_master.beam diff --git a/bootstrap/lib/kernel/ebin/code.beam b/bootstrap/lib/kernel/ebin/code.beam Binary files differindex 9a644b6b48..55d123c6a2 100644 --- a/bootstrap/lib/kernel/ebin/code.beam +++ b/bootstrap/lib/kernel/ebin/code.beam diff --git a/bootstrap/lib/kernel/ebin/code_server.beam b/bootstrap/lib/kernel/ebin/code_server.beam Binary files differindex 7b0517c500..aa75ae9bd1 100644 --- a/bootstrap/lib/kernel/ebin/code_server.beam +++ b/bootstrap/lib/kernel/ebin/code_server.beam diff --git a/bootstrap/lib/kernel/ebin/erts_debug.beam b/bootstrap/lib/kernel/ebin/erts_debug.beam Binary files differindex f162729d6a..bd73ff6c5c 100644 --- a/bootstrap/lib/kernel/ebin/erts_debug.beam +++ b/bootstrap/lib/kernel/ebin/erts_debug.beam diff --git a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam Binary files differindex 46e3a567b5..fc12b6b194 100644 --- a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam +++ b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam diff --git a/bootstrap/lib/kernel/ebin/inet.beam b/bootstrap/lib/kernel/ebin/inet.beam Binary files differindex 36869dd323..a76806293f 100644 --- a/bootstrap/lib/kernel/ebin/inet.beam +++ b/bootstrap/lib/kernel/ebin/inet.beam diff --git a/bootstrap/lib/kernel/ebin/inet_db.beam b/bootstrap/lib/kernel/ebin/inet_db.beam Binary files differindex bb2a6cb5e0..8c7c6ba218 100644 --- a/bootstrap/lib/kernel/ebin/inet_db.beam +++ b/bootstrap/lib/kernel/ebin/inet_db.beam diff --git a/bootstrap/lib/kernel/ebin/inet_dns.beam b/bootstrap/lib/kernel/ebin/inet_dns.beam Binary files differindex 6d04559d9a..0c5b6c73e1 100644 --- a/bootstrap/lib/kernel/ebin/inet_dns.beam +++ b/bootstrap/lib/kernel/ebin/inet_dns.beam diff --git a/bootstrap/lib/kernel/ebin/inet_parse.beam b/bootstrap/lib/kernel/ebin/inet_parse.beam Binary files differindex 6d28aa3c95..294afcea30 100644 --- a/bootstrap/lib/kernel/ebin/inet_parse.beam +++ b/bootstrap/lib/kernel/ebin/inet_parse.beam diff --git a/bootstrap/lib/kernel/ebin/inet_sctp.beam b/bootstrap/lib/kernel/ebin/inet_sctp.beam Binary files differindex 503cbc11e2..19506c14e9 100644 --- a/bootstrap/lib/kernel/ebin/inet_sctp.beam +++ b/bootstrap/lib/kernel/ebin/inet_sctp.beam diff --git a/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam b/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam Binary files differindex c322eb9fdc..a3635e5dde 100644 --- a/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam +++ b/bootstrap/lib/kernel/ebin/inet_tcp_dist.beam diff --git a/bootstrap/lib/kernel/ebin/kernel.app b/bootstrap/lib/kernel/ebin/kernel.app index e2cc4bac63..54a61e6ef6 100644 --- a/bootstrap/lib/kernel/ebin/kernel.app +++ b/bootstrap/lib/kernel/ebin/kernel.app @@ -21,7 +21,7 @@ {application, kernel, [ {description, "ERTS CXC 138 10"}, - {vsn, "3.2"}, + {vsn, "4.0"}, {modules, [application, application_controller, application_master, @@ -115,6 +115,6 @@ {applications, []}, {env, [{error_logger, tty}]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-7.0", "stdlib-2.0", "sasl-2.4"]} + {runtime_dependencies, ["erts-7.0", "stdlib-2.5", "sasl-2.4"]} ] }. diff --git a/bootstrap/lib/kernel/ebin/kernel.appup b/bootstrap/lib/kernel/ebin/kernel.appup index 4cdfb47c24..a0c5e763c1 100644 --- a/bootstrap/lib/kernel/ebin/kernel.appup +++ b/bootstrap/lib/kernel/ebin/kernel.appup @@ -1,7 +1,7 @@ %% -*- erlang -*- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2014. All Rights Reserved. +%% Copyright Ericsson AB 1999-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -15,9 +15,9 @@ %% under the License. %% %% %CopyrightEnd% -{"3.2", +{"4.0", %% Up from - max one major revision back - [{<<"3\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-17 + [{<<"3\\.[0-2](\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-17 %% Down to - max one major revision back - [{<<"3\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-17 + [{<<"3\\.[0-2](\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-17 }. diff --git a/bootstrap/lib/kernel/ebin/pg2.beam b/bootstrap/lib/kernel/ebin/pg2.beam Binary files differindex 63bedc72d7..944f6a27a4 100644 --- a/bootstrap/lib/kernel/ebin/pg2.beam +++ b/bootstrap/lib/kernel/ebin/pg2.beam diff --git a/bootstrap/lib/kernel/ebin/user_drv.beam b/bootstrap/lib/kernel/ebin/user_drv.beam Binary files differindex 54cf78d6ce..726a130b55 100644 --- a/bootstrap/lib/kernel/ebin/user_drv.beam +++ b/bootstrap/lib/kernel/ebin/user_drv.beam diff --git a/bootstrap/lib/stdlib/ebin/c.beam b/bootstrap/lib/stdlib/ebin/c.beam Binary files differindex a08df44eb7..0e07dc1531 100644 --- a/bootstrap/lib/stdlib/ebin/c.beam +++ b/bootstrap/lib/stdlib/ebin/c.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_anno.beam b/bootstrap/lib/stdlib/ebin/erl_anno.beam Binary files differindex f30442bc06..4807dac5f9 100644 --- a/bootstrap/lib/stdlib/ebin/erl_anno.beam +++ b/bootstrap/lib/stdlib/ebin/erl_anno.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_lint.beam b/bootstrap/lib/stdlib/ebin/erl_lint.beam Binary files differindex 7c80cbe624..16cacefb7c 100644 --- a/bootstrap/lib/stdlib/ebin/erl_lint.beam +++ b/bootstrap/lib/stdlib/ebin/erl_lint.beam diff --git a/bootstrap/lib/stdlib/ebin/erl_parse.beam b/bootstrap/lib/stdlib/ebin/erl_parse.beam Binary files differindex adf1cfb43e..0522f5c05e 100644 --- a/bootstrap/lib/stdlib/ebin/erl_parse.beam +++ b/bootstrap/lib/stdlib/ebin/erl_parse.beam diff --git a/bootstrap/lib/stdlib/ebin/ets.beam b/bootstrap/lib/stdlib/ebin/ets.beam Binary files differindex 6f2085cf22..7d30fc9fc1 100644 --- a/bootstrap/lib/stdlib/ebin/ets.beam +++ b/bootstrap/lib/stdlib/ebin/ets.beam diff --git a/bootstrap/lib/stdlib/ebin/gb_sets.beam b/bootstrap/lib/stdlib/ebin/gb_sets.beam Binary files differindex cf763f51a0..71da2376ba 100644 --- a/bootstrap/lib/stdlib/ebin/gb_sets.beam +++ b/bootstrap/lib/stdlib/ebin/gb_sets.beam diff --git a/bootstrap/lib/stdlib/ebin/gb_trees.beam b/bootstrap/lib/stdlib/ebin/gb_trees.beam Binary files differindex 6660dcb787..db59d5af19 100644 --- a/bootstrap/lib/stdlib/ebin/gb_trees.beam +++ b/bootstrap/lib/stdlib/ebin/gb_trees.beam diff --git a/bootstrap/lib/stdlib/ebin/maps.beam b/bootstrap/lib/stdlib/ebin/maps.beam Binary files differindex 5209c7cfd8..d1aa8bb9dd 100644 --- a/bootstrap/lib/stdlib/ebin/maps.beam +++ b/bootstrap/lib/stdlib/ebin/maps.beam diff --git a/bootstrap/lib/stdlib/ebin/shell_default.beam b/bootstrap/lib/stdlib/ebin/shell_default.beam Binary files differindex fa64e33080..f295c9f116 100644 --- a/bootstrap/lib/stdlib/ebin/shell_default.beam +++ b/bootstrap/lib/stdlib/ebin/shell_default.beam diff --git a/bootstrap/lib/stdlib/ebin/stdlib.app b/bootstrap/lib/stdlib/ebin/stdlib.app index 50eb39d712..6e830917ad 100644 --- a/bootstrap/lib/stdlib/ebin/stdlib.app +++ b/bootstrap/lib/stdlib/ebin/stdlib.app @@ -19,7 +19,7 @@ %% {application, stdlib, [{description, "ERTS CXC 138 10"}, - {vsn, "2.4"}, + {vsn, "2.5"}, {modules, [array, base64, beam_lib, @@ -104,7 +104,7 @@ dets]}, {applications, [kernel]}, {env, []}, - {runtime_dependencies, ["sasl-2.4","kernel-3.0.2","erts-7.0","crypto-3.3", + {runtime_dependencies, ["sasl-2.4","kernel-4.0","erts-7.0","crypto-3.3", "compiler-5.0"]} ]}. diff --git a/bootstrap/lib/stdlib/ebin/stdlib.appup b/bootstrap/lib/stdlib/ebin/stdlib.appup index 2457f9b4ed..a1038a43b6 100644 --- a/bootstrap/lib/stdlib/ebin/stdlib.appup +++ b/bootstrap/lib/stdlib/ebin/stdlib.appup @@ -1,7 +1,7 @@ %% -*- erlang -*- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2014. All Rights Reserved. +%% Copyright Ericsson AB 1999-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -15,11 +15,9 @@ %% under the License. %% %% %CopyrightEnd% -{"2.4", +{"2.5", %% Up from - max one major revision back - [{<<"2\\.[1-3](\\.[0-9]+)*">>,[restart_new_emulator]}, %% 17.1-17.3 - {<<"2\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}], %% 17.0 + [{<<"2\\.[0-4](\\.[0-9]+)*">>,[restart_new_emulator]}], %% 17.0-17.5 %% Down to - max one major revision back - [{<<"2\\.[1-3](\\.[0-9]+)*">>,[restart_new_emulator]}, %% 17.1-17.3 - {<<"2\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}] %% 17.0 + [{<<"2\\.[0-4](\\.[0-9]+)*">>,[restart_new_emulator]}] %% 17.0-17.5 }. diff --git a/bootstrap/lib/stdlib/ebin/supervisor.beam b/bootstrap/lib/stdlib/ebin/supervisor.beam Binary files differindex ec0c45c134..6dd01bf004 100644 --- a/bootstrap/lib/stdlib/ebin/supervisor.beam +++ b/bootstrap/lib/stdlib/ebin/supervisor.beam diff --git a/erts/configure.in b/erts/configure.in index 62515fe081..ce0cef871f 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -649,6 +649,7 @@ case $chk_arch_ in powerpc) ARCH=ppc;; ppc) ARCH=ppc;; ppc64) ARCH=ppc64;; + ppc64le) ARCH=ppc64;; "Power Macintosh") ARCH=ppc;; armv5b) ARCH=arm;; armv5teb) ARCH=arm;; @@ -4709,6 +4710,8 @@ case $host_os in use_cpu_sup=yes ;; linux*) use_cpu_sup=yes ;; + freebsd*) + use_cpu_sup=yes ;; esac if test "$use_cpu_sup" = "yes"; then diff --git a/erts/doc/src/absform.xml b/erts/doc/src/absform.xml index 835a4fc692..e1a8c2e517 100644 --- a/erts/doc/src/absform.xml +++ b/erts/doc/src/absform.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2001</year><year>2013</year> + <year>2001</year><year>2015</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -80,6 +80,28 @@ <item>If F is a record declaration <c><![CDATA[-record(Name,{V_1, ..., V_k})]]></c>, then Rep(F) = <c><![CDATA[{attribute,LINE,record,{Name,[Rep(V_1), ..., Rep(V_k)]}}]]></c>. For Rep(V), see below.</item> + <item>If F is a type attribute (i.e. <c><![CDATA[opaque]]></c> or + <c><![CDATA[type]]></c>) + <c><![CDATA[-Attr Name(A_1, ..., A_k) :: T]]></c> where each + <c><![CDATA[A_i]]></c> is a variable, then Rep(F) = + <c><![CDATA[{attribute,LINE,Attr,{Name,Rep(T),[Rep(A_1), ..., Rep(A_k)]}}]]></c>. + For Rep(T), see below.</item> + <item>If F is a type spec (i.e. <c><![CDATA[callback]]></c> or + <c><![CDATA[spec]]></c>) + <c><![CDATA[-Attr F Tc_1; ...; Tc_k]]></c>, + where each <c><![CDATA[Tc_i]]></c> is a fun type clause with an + argument sequence of the same length <c><![CDATA[Arity]]></c>, then + Rep(F) = + <c><![CDATA[{Attr,LINE,{{F,Arity},[Rep(Tc_1), ..., Rep(Tc_k)]}}]]></c>. + For Rep(Tc_i), see below.</item> + <item>If F is a type spec (i.e. <c><![CDATA[callback]]></c> or + <c><![CDATA[spec]]></c>) + <c><![CDATA[-Attr Mod:F Tc_1; ...; Tc_k]]></c>, + where each <c><![CDATA[Tc_i]]></c> is a fun type clause with an + argument sequence of the same length <c><![CDATA[Arity]]></c>, then + Rep(F) = + <c><![CDATA[{Attr,LINE,{{Mod,F,Arity},[Rep(Tc_1), ..., Rep(Tc_k)]}}]]></c>. + For Rep(Tc_i), see below.</item> <item>If F is a wild attribute <c><![CDATA[-A(T)]]></c>, then Rep(F) = <c><![CDATA[{attribute,LINE,A,T}]]></c>. <br></br></item> @@ -90,6 +112,127 @@ </list> <section> + <title>Type clauses</title> + <list type="bulleted"> + <item>If T is a fun type clause + <c><![CDATA[(A_1, ..., A_n) -> Ret]]></c>, where each + <c><![CDATA[A_i]]></c> and <c><![CDATA[Ret]]></c> are types, then + Rep(T) = + <c><![CDATA[{type,LINE,'fun',[{type,LINE,product,[Rep(A_1), ..., Rep(A_n)]},Rep(Ret)]}]]></c>. + </item> + <item>If T is a bounded fun type clause <c><![CDATA[Tc when Tg]]></c>, + where <c><![CDATA[Tc]]></c> is an unbounded fun type clause and + <c><![CDATA[Tg]]></c> is a type guard sequence, then Rep(T) = + <c><![CDATA[{type,LINE,bounded_fun,[Rep(Tc),Rep(Tg)]}]]></c>.</item> + </list> + </section> + + <section> + <title>Type guards</title> + <list type="bulleted"> + <item>If G is a constraint <c><![CDATA[F(A_1, ..., A_k)]]></c>, where + <c><![CDATA[F]]></c> is an atom and each <c><![CDATA[A_i]]></c> is a + type, then Rep(G) = + <c><![CDATA[{type,LINE,constraint,[Rep(F),[Rep(A_1), ..., Rep(A_k)]]}]]></c>. + </item> + <item>If G is a type definition <c><![CDATA[Name :: Type]]></c>, + where <c><![CDATA[Name]]></c> is a variable and + <c><![CDATA[Type]]></c> is a type, then Rep(G) = + <c><![CDATA[{type,LINE,constraint,[{atom,LINE,is_subtype},[Rep(Name),Rep(Type)]]}]]></c>.</item> + </list> + </section> + + <section> + <title>Types</title> + <list type="bulleted"> + <item>If T is a type definition <c><![CDATA[Name :: Type]]></c>, + where <c><![CDATA[Name]]></c> is a variable and + <c><![CDATA[Type]]></c> is a type, then Rep(T) = + <c><![CDATA[{ann_type,LINE,[Rep(Name),Rep(Type)]}]]></c>.</item> + <item>If T is a type union <c><![CDATA[A_1 | ... | A_k]]></c>, + where each <c><![CDATA[A_i]]></c> is a type, then Rep(T) = + <c><![CDATA[{type,LINE,union,[Rep(A_1), ..., Rep(A_k)]}]]></c>.</item> + <item>If T is a type range <c><![CDATA[L .. R]]></c>, + where <c><![CDATA[L]]></c> and <c><![CDATA[R]]></c> are types, then + Rep(T) = <c><![CDATA[{type,LINE,range,[Rep(L), Rep(R)]}]]></c>.</item> + <item>If T is a binary operation <c><![CDATA[L Op R]]></c>, + where <c><![CDATA[Op]]></c> is an arithmetic or bitwise binary operator + and <c><![CDATA[L]]></c> and <c><![CDATA[R]]></c> are types, then + Rep(T) = <c><![CDATA[{op,LINE,Op,Rep(L),Rep(R)}]]></c>.</item> + <item>If T is <c><![CDATA[Op A]]></c>, where <c><![CDATA[Op]]></c> is an + arithmetic or bitwise unary operator and <c><![CDATA[A]]></c> is a + type, then Rep(T) = <c><![CDATA[{op,LINE,Op,Rep(A)}]]></c>.</item> + <item>If T is a fun type <c><![CDATA[fun()]]></c>, then Rep(T) = + <c><![CDATA[{type,LINE,'fun',[]}]]></c>.</item> + <item>If T is a variable <c><![CDATA[V]]></c>, then Rep(T) = + <c><![CDATA[{var,LINE,A}]]></c>, where <c><![CDATA[A]]></c> is an atom + with a printname consisting of the same characters as + <c><![CDATA[V]]></c>.</item> + <item>If T is an atomic literal L and L is not a string literal, then + Rep(T) = Rep(L).</item> + <item>If T is a tuple or map type <c><![CDATA[F()]]></c> (i.e. + <c><![CDATA[tuple]]></c> or <c><![CDATA[map]]></c>), then Rep(T) = + <c><![CDATA[{type,LINE,F,any}]]></c>.</item> + <item>If T is a type <c><![CDATA[F(A_1, ..., A_k)]]></c>, where each + <c><![CDATA[A_i]]></c> is a type, then Rep(T) = + <c><![CDATA[{user_type,LINE,F,[Rep(A_1), ..., Rep(A_k)]}]]></c>.</item> + <item>If T is a remote type <c><![CDATA[M:F(A_1, ..., A_k)]]></c>, where + each <c><![CDATA[A_i]]></c> is a type and <c><![CDATA[M]]></c> and + <c><![CDATA[F]]></c>, then Rep(T) = + <c><![CDATA[{remote_type,LINE,[Rep(M),Rep(F),[Rep(A_1), ..., Rep(A_k)]]}]]></c>. + </item> + <item>If T is the nil type <c><![CDATA[[]]]></c>, then Rep(T) = + <c><![CDATA[{type,LINE,nil,[]}]]></c>.</item> + <item>If T is a list type <c><![CDATA[[A]]]></c>, where + <c><![CDATA[A]]></c> is a type, then Rep(T) = + <c><![CDATA[{type,LINE,list,[Rep(A)]}]]></c>.</item> + <item>If T is a non-empty list type <c><![CDATA[[A, ...]]]></c>, where + <c><![CDATA[A]]></c> is a type, then Rep(T) = + <c><![CDATA[{type,LINE,nonempty_list,[Rep(A)]}]]></c>.</item> + <item>If T is a map type <c><![CDATA[#{P_1, ..., P_k}]]></c>, where each + <c><![CDATA[P_i]]></c> is a map pair type, then Rep(T) = + <c><![CDATA[{type,LINE,map,[Rep(P_1), ..., Rep(P_k)]}]]></c>.</item> + <item>If T is a map pair type <c><![CDATA[K => V]]></c>, where + <c><![CDATA[K]]></c> and <c><![CDATA[V]]></c> are types, + then Rep(T) = + <c><![CDATA[{type,LINE,map_field_assoc,[Rep(K),Rep(V)]}]]></c>.</item> + <item>If T is a tuple type <c><![CDATA[{A_1, ..., A_k}]]></c>, where + each <c><![CDATA[A_i]]></c> is a type, then Rep(T) = + <c><![CDATA[{type,LINE,tuple,[Rep(A_1), ..., Rep(A_k)]}]]></c>.</item> + <item>If T is a record type <c><![CDATA[#Name{}]]></c>, where + <c><![CDATA[Name]]></c> is an atom, then Rep(T) = + <c><![CDATA[{type,LINE,record,[Rep(Name)]}]]></c>.</item> + <item>If T is a record type <c><![CDATA[#Name{F_1, ..., F_k}]]></c>, + where <c><![CDATA[Name]]></c> is an atom, then Rep(T) = + <c><![CDATA[{type,LINE,record,[Rep(Name),[Rep(F_1), ..., Rep(F_k)]]}]]></c>. + </item> + <item>If T is a record field type <c><![CDATA[Name :: Type]]></c>, + where <c><![CDATA[Name]]></c> is an atom, then Rep(T) = + <c><![CDATA[{type,LINE,field_type,[Rep(Name),Rep(Type)]}]]></c>.</item> + <item>If T is a record field type <c><![CDATA[<<>>]]></c>, then Rep(T) = + <c><![CDATA[{type,LINE,binary,[{integer,LINE,0},{integer,LINE,0}]}]]></c>. + </item> + <item>If T is a binary type <c><![CDATA[<< _ : B >>]]></c>, where + <c><![CDATA[B]]></c> is a type, then Rep(T) = + <c><![CDATA[{type,LINE,binary,[Rep(B),{integer,LINE,0}]}]]></c>.</item> + <item>If T is a binary type <c><![CDATA[<< _ : _ * U >>]]></c>, + where <c><![CDATA[U]]></c> is a type, then Rep(T) = + <c><![CDATA[{type,LINE,binary,[{integer,LINE,0},Rep(U)]}]]></c>.</item> + <item>If T is a binary type <c><![CDATA[<< _ : B , _ : _ * U >>]]></c>, + where <c><![CDATA[B]]></c> and <c><![CDATA[U]]></c> is a type, then + Rep(T) = + <c><![CDATA[{type,LINE,binary,[Rep(B),Rep(U)]}]]></c>.</item> + + <item>If T is a fun type <c><![CDATA[fun((...) -> Ret)]]></c>, then + Rep(T) = <c><![CDATA[{type,LINE,'fun',[{type,LINE,product,[]},Rep(Ret)]}]]></c>. + </item> + <item>If T is a fun type <c><![CDATA[fun(Tc)]]></c>, where + <c><![CDATA[Tc]]></c> is an unbounded fun type clause, + then Rep(T) = <c><![CDATA[Rep(Tc)]]></c>.</item> + </list> + </section> + + <section> <title>Record fields</title> <p>Each field in a record declaration may have an optional explicit default initializer expression</p> @@ -98,6 +241,21 @@ Rep(V) = <c><![CDATA[{record_field,LINE,Rep(A)}]]></c>.</item> <item>If V is <c><![CDATA[A = E]]></c>, then Rep(V) = <c><![CDATA[{record_field,LINE,Rep(A),Rep(E)}]]></c>.</item> + <item>If V is <c><![CDATA[A :: T]]></c>, where <c><![CDATA[A]]></c> is + an atom and <c><![CDATA[T]]></c> is a type and it does not contain + <c><![CDATA[undefined]]></c> syntactically, then Rep(V) = + <c><![CDATA[{typed_record_field,{record_field,LINE,Rep(A)},Rep(undefined | T)}]]></c>. + Note that if <![CDATA[T]]> is an annotated type, it will be wrapped in + parentheses.</item> + <item>If V is <c><![CDATA[A :: T]]></c>, where <c><![CDATA[A]]></c> is + an atom and <c><![CDATA[T]]></c> is a type, then Rep(V) = + <c><![CDATA[{typed_record_field,{record_field,LINE,Rep(A)},Rep(T)}]]></c>. + </item> + <item>If V is <c><![CDATA[A = E :: T]]></c>, where <c><![CDATA[A]]></c> + is an atom, <c><![CDATA[E]]></c> is an expression and + <c><![CDATA[T]]></c> is a type, then Rep(V) = + <c><![CDATA[{typed_record_field,{record_field,LINE,Rep(A),Rep(E)},Rep(T)}]]></c>. + </item> </list> </section> diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index 00da503469..f41b6e6149 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -1322,13 +1322,14 @@ <item> <p>Verbose.</p> </item> - <tag><c><![CDATA[+W w | i]]></c></tag> + <tag><c><![CDATA[+W w | i | e]]></c></tag> <item> <p>Sets the mapping of warning messages for <c><![CDATA[error_logger]]></c>. Messages sent to the error logger using one of the warning - routines can be mapped either to errors (default), warnings - (<c><![CDATA[+W w]]></c>), or info reports (<c><![CDATA[+W i]]></c>). The current - mapping can be retrieved using + routines can be mapped either to errors (<c><![CDATA[+W e]]></c>), + warnings (<c><![CDATA[+W w]]></c>), or info reports + (<c><![CDATA[+W i]]></c>). The default is warnings. + The current mapping can be retrieved using <c><![CDATA[error_logger:warning_map/0]]></c>. See <seealso marker="kernel:error_logger#warning_map/0">error_logger(3)</seealso> for further information.</p> diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index 4bad8b253c..f64381c99d 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -461,8 +461,9 @@ ok independent environment with all its terms is valid until you explicitly invalidates it with <seealso marker="#enif_free_env">enif_free_env</seealso> or <c>enif_send</c>.</p> - <p>All elements of a list/tuple must belong to the same environment as the - list/tuple itself. Terms can be copied between environments with + <p>All contained terms of a list/tuple/map must belong to the same + environment as the list/tuple/map itself. Terms can be copied between + environments with <seealso marker="#enif_make_copy">enif_make_copy</seealso>.</p> </item> <tag><marker id="ErlNifFunc"/>ErlNifFunc</tag> @@ -564,11 +565,11 @@ typedef enum { <funcs> <func><name><ret>void *</ret><nametext>enif_alloc(size_t size)</nametext></name> - <fsummary>Allocate dynamic memory.</fsummary> + <fsummary>Allocate dynamic memory</fsummary> <desc><p>Allocate memory of <c>size</c> bytes. Return NULL if allocation failed.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_alloc_binary(size_t size, ErlNifBinary* bin)</nametext></name> - <fsummary>Create a new binary.</fsummary> + <fsummary>Create a new binary</fsummary> <desc><p>Allocate a new binary of size <c>size</c> bytes. Initialize the structure pointed to by <c>bin</c> to refer to the allocated binary. The binary must either be released by @@ -595,7 +596,7 @@ typedef enum { <desc><p>Allocate a memory managed resource object of type <c>type</c> and size <c>size</c> bytes.</p></desc> </func> <func><name><ret>void</ret><nametext>enif_clear_env(ErlNifEnv* env)</nametext></name> - <fsummary>Clear an environment for reuse.</fsummary> + <fsummary>Clear an environment for reuse</fsummary> <desc><p>Free all terms in an environment and clear it for reuse. The environment must have been allocated with <seealso marker="#enif_alloc_env">enif_alloc_env</seealso>. </p></desc> @@ -683,14 +684,14 @@ typedef enum { <c>size-1</c>.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_get_atom_length(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len, ErlNifCharEncoding encode)</nametext></name> - <fsummary>Get the length of atom <c>term</c>.</fsummary> + <fsummary>Get the length of atom <c>term</c></fsummary> <desc><p>Set <c>*len</c> to the length (number of bytes excluding terminating null character) of the atom <c>term</c> with encoding <c>encode</c>. Return true on success or false if <c>term</c> is not an atom.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_get_double(ErlNifEnv* env, ERL_NIF_TERM term, double* dp)</nametext></name> - <fsummary>Read a floating-point number term.</fsummary> + <fsummary>Read a floating-point number term</fsummary> <desc><p>Set <c>*dp</c> to the floating point value of <c>term</c>. Return true on success or false if <c>term</c> is not a float.</p></desc> </func> @@ -719,17 +720,28 @@ typedef enum { non-empty list.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_get_list_length(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len)</nametext></name> - <fsummary>Get the length of list <c>term</c>.</fsummary> + <fsummary>Get the length of list <c>term</c></fsummary> <desc><p>Set <c>*len</c> to the length of list <c>term</c> and return true, or return false if <c>term</c> is not a list.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_get_long(ErlNifEnv* env, ERL_NIF_TERM term, long int* ip)</nametext></name> - <fsummary>Read an long integer term.</fsummary> + <fsummary>Read an long integer term</fsummary> <desc><p>Set <c>*ip</c> to the long integer value of <c>term</c> and return true, or return false if <c>term</c> is not an integer or is outside the bounds of type <c>long int</c>.</p></desc> </func> - <func><name><ret>int</ret><nametext>enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp)</nametext></name> + <func><name><ret>int</ret><nametext>enif_get_map_size(ErlNifEnv* env, ERL_NIF_TERM term, size_t *size)</nametext></name> + <fsummary>Read the size of a map term</fsummary> + <desc><p>Set <c>*size</c> to the number of key-value pairs in the map <c>term</c> and + return true, or return false if <c>term</c> is not a map.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_get_map_value(ErlNifEnv* env, ERL_NIF_TERM map, ERL_NIF_TERM key, ERL_NIF_TERM* value)</nametext></name> + <fsummary>Get the value of a key in a map</fsummary> + <desc><p>Set <c>*value</c> to the value associated with <c>key</c> in the + map <c>map</c> and return true. Return false if <c>map</c> is not a map + or if <c>map</c> does not contain <c>key</c>.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp)</nametext></name> <fsummary>Get the pointer to a resource object</fsummary> <desc><p>Set <c>*objp</c> to point to the resource object referred to by <c>term</c>.</p> <p>Return true on success or false if <c>term</c> is not a handle to a resource object @@ -738,7 +750,7 @@ typedef enum { <func><name><ret>int</ret><nametext>enif_get_string(ErlNifEnv* env, ERL_NIF_TERM list, char* buf, unsigned size, ErlNifCharEncoding encode)</nametext></name> - <fsummary>Get a C-string from a list.</fsummary> + <fsummary>Get a C-string from a list</fsummary> <desc><p>Write a null-terminated string, in the buffer pointed to by <c>buf</c> with size <c>size</c>, consisting of the characters in the string <c>list</c>. The characters are written using encoding @@ -751,7 +763,7 @@ typedef enum { <c>size</c> is less than 1.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_get_tuple(ErlNifEnv* env, ERL_NIF_TERM term, int* arity, const ERL_NIF_TERM** array)</nametext></name> - <fsummary>Inspect the elements of a tuple.</fsummary> + <fsummary>Inspect the elements of a tuple</fsummary> <desc><p>If <c>term</c> is a tuple, set <c>*array</c> to point to an array containing the elements of the tuple and set <c>*arity</c> to the number of elements. Note that the array @@ -761,28 +773,35 @@ typedef enum { tuple.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_get_uint(ErlNifEnv* env, ERL_NIF_TERM term, unsigned int* ip)</nametext></name> - <fsummary>Read an unsigned integer term.</fsummary> + <fsummary>Read an unsigned integer term</fsummary> <desc><p>Set <c>*ip</c> to the unsigned integer value of <c>term</c> and return true, or return false if <c>term</c> is not an unsigned integer or is outside the bounds of type <c>unsigned int</c>.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_get_uint64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifUInt64* ip)</nametext></name> - <fsummary>Read an unsigned 64-bit integer term.</fsummary> + <fsummary>Read an unsigned 64-bit integer term</fsummary> <desc><p>Set <c>*ip</c> to the unsigned integer value of <c>term</c> and return true, or return false if <c>term</c> is not an unsigned integer or is outside the bounds of an unsigned 64-bit integer.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_get_ulong(ErlNifEnv* env, ERL_NIF_TERM term, unsigned long* ip)</nametext></name> - <fsummary>Read an unsigned integer term.</fsummary> + <fsummary>Read an unsigned integer term</fsummary> <desc><p>Set <c>*ip</c> to the unsigned long integer value of <c>term</c> and return true, or return false if <c>term</c> is not an unsigned integer or is outside the bounds of type <c>unsigned long</c>.</p></desc> </func> - <func><name><ret>int</ret><nametext>enif_has_pending_exception(ErlNifEnv* env)</nametext></name> - <fsummary>Check if an exception has been raised.</fsummary> + <func><name><ret>int</ret><nametext>enif_has_pending_exception(ErlNifEnv* env, ERL_NIF_TERM* reason)</nametext></name> + <fsummary>Check if an exception has been raised</fsummary> <desc><p>Return true if a pending exception is associated - with the environment <c>env</c>. The only possible exception is currently - <c>badarg</c> (see <seealso marker="#enif_make_badarg">enif_make_badarg</seealso>).</p></desc> + with the environment <c>env</c>. If <c>reason</c> is a null pointer, ignore it. + Otherwise, if there's a pending exception associated with <c>env</c>, set the ERL_NIF_TERM + to which <c>reason</c> points to the value of the exception's term. For example, if + <seealso marker="#enif_make_badarg">enif_make_badarg</seealso> is called to set a + pending <c>badarg</c> exception, a subsequent call to <c>enif_has_pending_exception(env, &reason)</c> + will set <c>reason</c> to the atom <c>badarg</c>, then return true.</p> + <p>See also: <seealso marker="#enif_make_badarg">enif_make_badarg</seealso> + and <seealso marker="#enif_raise_exception">enif_raise_exception</seealso>.</p> + </desc> </func> <func><name><ret>int</ret><nametext>enif_inspect_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, ErlNifBinary* bin)</nametext></name> <fsummary>Inspect the content of a binary</fsummary> @@ -817,6 +836,10 @@ typedef enum { <fsummary>Determine if a term is an exception</fsummary> <desc><p>Return true if <c>term</c> is an exception.</p></desc> </func> + <func><name><ret>int</ret><nametext>enif_is_map(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name> + <fsummary>Determine if a term is a map</fsummary> + <desc><p>Return true if <c>term</c> is a map, false otherwise.</p></desc> + </func> <func><name><ret>int</ret><nametext>enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name> <fsummary>Determine if a term is a number (integer or float)</fsummary> <desc><p>Return true if <c>term</c> is a number.</p></desc> @@ -890,18 +913,19 @@ typedef enum { </p></desc> </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_badarg(ErlNifEnv* env)</nametext></name> - <fsummary>Make a badarg exception.</fsummary> + <fsummary>Make a badarg exception</fsummary> <desc><p>Make a badarg exception to be returned from a NIF, and associate it with the environment <c>env</c>. Once a NIF or any function it calls invokes <c>enif_make_badarg</c>, the runtime ensures that a <c>badarg</c> exception is raised when the NIF returns, even if the NIF attempts to return a non-exception term instead. - The return value from <c>enif_make_badarg</c> may only be used as - return value from the NIF that invoked it (direct or indirectly) + The return value from <c>enif_make_badarg</c> may be used only as the + return value from the NIF that invoked it (directly or indirectly) or be passed to <seealso marker="#enif_is_exception">enif_is_exception</seealso>, but not to any other NIF API function.</p> - <p>See also: <seealso marker="#enif_has_pending_exception">enif_has_pending_exception</seealso>. + <p>See also: <seealso marker="#enif_has_pending_exception">enif_has_pending_exception</seealso> + and <seealso marker="#enif_raise_exception">enif_raise_exception</seealso> </p> <note><p>In earlier versions (older than erts-7.0, OTP 18) the return value from <c>enif_make_badarg</c> had to be returned from the NIF. This @@ -909,14 +933,14 @@ typedef enum { if <c>enif_make_badarg</c> has been invoked.</p></note></desc> </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin)</nametext></name> - <fsummary>Make a binary term.</fsummary> + <fsummary>Make a binary term</fsummary> <desc><p>Make a binary term from <c>bin</c>. Any ownership of the binary data will be transferred to the created term and <c>bin</c> should be considered read-only for the rest of the NIF call and then as released.</p></desc> </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)</nametext></name> - <fsummary>Make a copy of a term.</fsummary> + <fsummary>Make a copy of a term</fsummary> <desc><p>Make a copy of term <c>src_term</c>. The copy will be created in environment <c>dst_env</c>. The source term may be located in any environment.</p></desc> @@ -957,7 +981,7 @@ typedef enum { <desc><p>Create an integer term from a signed 64-bit integer.</p></desc> </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_list(ErlNifEnv* env, unsigned cnt, ...)</nametext></name> - <fsummary>Create a list term.</fsummary> + <fsummary>Create a list term</fsummary> <desc><p>Create an ordinary list term of length <c>cnt</c>. Expects <c>cnt</c> number of arguments (after <c>cnt</c>) of type ERL_NIF_TERM as the elements of the list. An empty list is returned if <c>cnt</c> is 0.</p></desc> @@ -971,28 +995,21 @@ typedef enum { <name><ret>ERL_NIF_TERM</ret><nametext>enif_make_list7(ErlNifEnv* env, ERL_NIF_TERM e1, ..., ERL_NIF_TERM e7)</nametext></name> <name><ret>ERL_NIF_TERM</ret><nametext>enif_make_list8(ErlNifEnv* env, ERL_NIF_TERM e1, ..., ERL_NIF_TERM e8)</nametext></name> <name><ret>ERL_NIF_TERM</ret><nametext>enif_make_list9(ErlNifEnv* env, ERL_NIF_TERM e1, ..., ERL_NIF_TERM e9)</nametext></name> - <fsummary>Create a list term.</fsummary> + <fsummary>Create a list term</fsummary> <desc><p>Create an ordinary list term with length indicated by the function name. Prefer these functions (macros) over the variadic <c>enif_make_list</c> to get a compile time error if the number of arguments does not match.</p></desc> </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_list_cell(ErlNifEnv* env, ERL_NIF_TERM head, ERL_NIF_TERM tail)</nametext></name> - <fsummary>Create a list cell.</fsummary> + <fsummary>Create a list cell</fsummary> <desc><p>Create a list cell <c>[head | tail]</c>.</p></desc> </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_list_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt)</nametext></name> - <fsummary>Create a list term from an array.</fsummary> + <fsummary>Create a list term from an array</fsummary> <desc><p>Create an ordinary list containing the elements of array <c>arr</c> of length <c>cnt</c>. An empty list is returned if <c>cnt</c> is 0.</p></desc> </func> - <func><name><ret>int</ret><nametext>enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list)</nametext></name> - <fsummary>Create the reverse list of the list <c>term</c>.</fsummary> - <desc><p>Set <c>*list</c> to the reverse list of the list <c>term</c> and return true, - or return false if <c>term</c> is not a list. This function should only be used on - short lists as a copy will be created of the list which will not be released until after the - nif returns.</p></desc> - </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_long(ErlNifEnv* env, long int i)</nametext></name> <fsummary>Create an integer term from a long int</fsummary> <desc><p>Create an integer term from a <c>long int</c>.</p></desc> @@ -1007,12 +1024,42 @@ typedef enum { reallocated.</p><p>Return a pointer to the raw binary data and set <c>*termp</c> to the binary term.</p></desc> </func> + <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_new_map(ErlNifEnv* env)</nametext></name> + <fsummary>Make an empty map term</fsummary> + <desc><p>Make an empty map term.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_make_map_put(ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM value, ERL_NIF_TERM* map_out)</nametext></name> + <fsummary>Insert key-value pair in map</fsummary> + <desc><p>Make a copy of map <c>map_in</c> and insert <c>key</c> with + <c>value</c>. If <c>key</c> already exists in <c>map_in</c>, the old + associated value is replaced by <c>value</c>. If successful set + <c>*map_out</c> to the new map and return true. Return false if + <c>map_in</c> is not a map.</p> + <p>The <c>map_in</c> term must belong to the environment <c>env</c>.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_make_map_update(ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM new_value, ERL_NIF_TERM* map_out)</nametext></name> + <fsummary>Replace value for key in map</fsummary> + <desc><p>Make a copy of map <c>map_in</c> and replace the old associated + value for <c>key</c> with <c>new_value</c>. If successful set + <c>*map_out</c> to the new map and return true. Return false if + <c>map_in</c> is not a map or if it does no contain <c>key</c>.</p> + <p>The <c>map_in</c> term must belong to the environment <c>env</c>.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_make_map_remove(ErlNifEnv* env, ERL_NIF_TERM map_in, ERL_NIF_TERM key, ERL_NIF_TERM* map_out)</nametext></name> + <fsummary>Remove key from map</fsummary> + <desc><p>If map <c>map_in</c> contains <c>key</c>, make a copy of + <c>map_in</c> in <c>*map_out</c> and remove <c>key</c> and associated + value. If map <c>map_in</c> does not contain <c>key</c>, set + <c>*map_out</c> to <c>map_in</c>. Return true for success or false if + <c>map_in</c> is not a map.</p> + <p>The <c>map_in</c> term must belong to the environment <c>env</c>.</p></desc> + </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_pid(ErlNifEnv* env, const ErlNifPid* pid)</nametext></name> <fsummary>Make a pid term</fsummary> <desc><p>Make a pid term from <c>*pid</c>.</p></desc> </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_ref(ErlNifEnv* env)</nametext></name> - <fsummary>Create a reference.</fsummary> + <fsummary>Create a reference</fsummary> <desc><p>Create a reference like <seealso marker="erlang#make_ref-0">erlang:make_ref/0</seealso>.</p></desc> </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_resource(ErlNifEnv* env, void* obj)</nametext></name> @@ -1050,20 +1097,28 @@ typedef enum { <seealso marker="#enif_release_resource">enif_release_resource</seealso>.</p> </desc> </func> + <func><name><ret>int</ret><nametext>enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM list_in, ERL_NIF_TERM *list_out)</nametext></name> + <fsummary>Create the reverse of a list</fsummary> + <desc><p>Set <c>*list_out</c> to the reverse list of the list <c>list_in</c> and return true, + or return false if <c>list_in</c> is not a list. This function should only be used on + short lists as a copy will be created of the list which will not be released until after the + nif returns.</p> + <p>The <c>list_in</c> term must belong to the environment <c>env</c>.</p></desc> + </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_string(ErlNifEnv* env, const char* string, ErlNifCharEncoding encoding)</nametext></name> - <fsummary>Create a string.</fsummary> + <fsummary>Create a string</fsummary> <desc><p>Create a list containing the characters of the null-terminated string <c>string</c> with encoding <seealso marker="#ErlNifCharEncoding">encoding</seealso>.</p></desc> </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_string_len(ErlNifEnv* env, const char* string, size_t len, ErlNifCharEncoding encoding)</nametext></name> - <fsummary>Create a string.</fsummary> + <fsummary>Create a string</fsummary> <desc><p>Create a list containing the characters of the string <c>string</c> with length <c>len</c> and encoding <seealso marker="#ErlNifCharEncoding">encoding</seealso>. Null-characters are treated as any other characters.</p></desc> </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, size_t pos, size_t size)</nametext></name> - <fsummary>Make a subbinary term.</fsummary> + <fsummary>Make a subbinary term</fsummary> <desc><p>Make a subbinary of binary <c>bin_term</c>, starting at zero-based position <c>pos</c> with a length of <c>size</c> bytes. <c>bin_term</c> must be a binary or bitstring and @@ -1071,7 +1126,7 @@ typedef enum { bytes in <c>bin_term</c>.</p></desc> </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...)</nametext></name> - <fsummary>Create a tuple term.</fsummary> + <fsummary>Create a tuple term</fsummary> <desc><p>Create a tuple term of arity <c>cnt</c>. Expects <c>cnt</c> number of arguments (after <c>cnt</c>) of type ERL_NIF_TERM as the elements of the tuple.</p></desc> @@ -1085,14 +1140,14 @@ typedef enum { <name><ret>ERL_NIF_TERM</ret><nametext>enif_make_tuple7(ErlNifEnv* env, ERL_NIF_TERM e1, ..., ERL_NIF_TERM e7)</nametext></name> <name><ret>ERL_NIF_TERM</ret><nametext>enif_make_tuple8(ErlNifEnv* env, ERL_NIF_TERM e1, ..., ERL_NIF_TERM e8)</nametext></name> <name><ret>ERL_NIF_TERM</ret><nametext>enif_make_tuple9(ErlNifEnv* env, ERL_NIF_TERM e1, ..., ERL_NIF_TERM e9)</nametext></name> - <fsummary>Create a tuple term.</fsummary> + <fsummary>Create a tuple term</fsummary> <desc><p>Create a tuple term with length indicated by the function name. Prefer these functions (macros) over the variadic <c>enif_make_tuple</c> to get a compile time error if the number of arguments does not match.</p></desc> </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_tuple_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt)</nametext></name> - <fsummary>Create a tuple term from an array.</fsummary> + <fsummary>Create a tuple term from an array</fsummary> <desc><p>Create a tuple containing the elements of array <c>arr</c> of length <c>cnt</c>.</p></desc> </func> @@ -1108,6 +1163,72 @@ typedef enum { <fsummary>Create an integer term from an unsigned long int</fsummary> <desc><p>Create an integer term from an <c>unsigned long int</c>.</p></desc> </func> + <func><name><ret>int</ret><nametext>enif_map_iterator_create(ErlNifEnv *env, ERL_NIF_TERM map, ErlNifMapIterator *iter, ErlNifMapIteratorEntry entry)</nametext></name> + <fsummary>Create a map iterator</fsummary> + <desc><p>Create an iterator for the map <c>map</c> by initializing the + structure pointed to by <c>iter</c>. The <c>entry</c> argument determines + the start position of the iterator: <c>ERL_NIF_MAP_ITERATOR_FIRST</c> or + <c>ERL_NIF_MAP_ITERATOR_LAST</c>. Return true on success or false if + <c>map</c> is not a map.</p> + <p>A map iterator is only useful during the lifetime of the environment + <c>env</c> that the <c>map</c> belongs to. The iterator must be destroyed by + calling <seealso marker="#enif_map_iterator_destroy"> + enif_map_iterator_destroy</seealso>.</p> + <code type="none"> +ERL_NIF_TERM key, value; +ErlNifMapIterator iter; +enif_map_iterator_create(env, my_map, ERL_NIF_MAP_ITERATOR_FIRST); + +while (enif_map_iterator_get_pair(env, &iter, &key, &value)) { + do_something(key,value); + enif_map_iterator_next(env, &iter); +} +enif_map_iterator_destroy(env, &iter); + </code> + <note><p>The key-value pairs of a map have no defined iteration + order. The only guarantee is that the iteration order of a single map + instance is preserved during the lifetime of the environment that the map + belongs to.</p> + </note> + </desc> + </func> + <func><name><ret>void</ret><nametext>enif_map_iterator_destroy(ErlNifEnv *env, ErlNifMapIterator *iter)</nametext></name> + <fsummary>Destroy a map iterator</fsummary> + <desc><p>Destroy a map iterator created by + <seealso marker="#enif_map_iterator_create">enif_map_iterator_create</seealso>. + </p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_map_iterator_get_pair(ErlNifEnv *env, ErlNifMapIterator *iter, ERL_NIF_TERM *key, ERL_NIF_TERM *value)</nametext></name> + <fsummary>Get key and value at current map iterator position</fsummary> + <desc><p>Get key and value terms at current map iterator position. + On success set <c>*key</c> and <c>*value</c> and return true. + Return false if the iterator is positioned at head (before first entry) + or tail (beyond last entry).</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_map_iterator_is_head(ErlNifEnv *env, ErlNifMapIterator *iter)</nametext></name> + <fsummary>Check if map iterator is positioned before first</fsummary> + <desc><p>Return true if map iterator <c>iter</c> is positioned + before first entry.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_map_iterator_is_tail(ErlNifEnv *env, ErlNifMapIterator *iter)</nametext></name> + <fsummary>Check if map iterator is positioned after last</fsummary> + <desc><p>Return true if map iterator <c>iter</c> is positioned + after last entry.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_map_iterator_next(ErlNifEnv *env, ErlNifMapIterator *iter)</nametext></name> + <fsummary>Increment map iterator to point to next entry</fsummary> + <desc><p>Increment map iterator to point to next key-value entry. + Return true if the iterator is now positioned at a valid key-value entry, + or false if the iterator is positioned at the tail (beyond the last + entry).</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_map_iterator_prev(ErlNifEnv *env, ErlNifMapIterator *iter)</nametext></name> + <fsummary>Decrement map iterator to point to previous entry</fsummary> + <desc><p>Decrement map iterator to point to previous key-value entry. + Return true if the iterator is now positioned at a valid key-value entry, + or false if the iterator is positioned at the head (before the first + entry).</p></desc> + </func> <func><name><ret>ErlNifMutex *</ret><nametext>enif_mutex_create(char *name)</nametext></name> <fsummary></fsummary> <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_create">erl_drv_mutex_create</seealso>. @@ -1168,19 +1289,32 @@ typedef enum { <c>reload</c> or <c>upgrade</c>.</p> <p>Was previously named <c>enif_get_data</c>.</p></desc> </func> + <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_raise_exception(ErlNifEnv* env, ERL_NIF_TERM reason)</nametext></name> + <fsummary>Raise a NIF error exception</fsummary> + <desc><p>Create an error exception with the term <c>reason</c> to be returned from a NIF, + and associate it with the environment <c>env</c>. Once a NIF or any function it calls + invokes <c>enif_raise_exception</c>, the runtime ensures that the exception it creates + is raised when the NIF returns, even if the NIF attempts to return a non-exception + term instead. The return value from <c>enif_raise_exception</c> may be used only as + the return value from the NIF that invoked it (directly or indirectly) or be passed + to <seealso marker="#enif_is_exception">enif_is_exception</seealso>, but + not to any other NIF API function.</p> + <p>See also: <seealso marker="#enif_has_pending_exception">enif_has_pending_exception</seealso> + and <seealso marker="#enif_make_badarg">enif_make_badarg</seealso>.</p></desc> + </func> <func><name><ret>int</ret><nametext>enif_realloc_binary(ErlNifBinary* bin, size_t size)</nametext></name> - <fsummary>Change the size of a binary.</fsummary> + <fsummary>Change the size of a binary</fsummary> <desc><p>Change the size of a binary <c>bin</c>. The source binary may be read-only, in which case it will be left untouched and a mutable copy is allocated and assigned to <c>*bin</c>. Return true on success, false if memory allocation failed.</p></desc> </func> <func><name><ret>void</ret><nametext>enif_release_binary(ErlNifBinary* bin)</nametext></name> - <fsummary>Release a binary.</fsummary> + <fsummary>Release a binary</fsummary> <desc><p>Release a binary obtained from <c>enif_alloc_binary</c>.</p></desc> </func> <func><name><ret>void</ret><nametext>enif_release_resource(void* obj)</nametext></name> - <fsummary>Release a resource object.</fsummary> + <fsummary>Release a resource object</fsummary> <desc><p>Remove a reference to resource object <c>obj</c>obtained from <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>. The resource object will be destructed when the last reference is removed. @@ -1256,12 +1390,12 @@ typedef enum { </desc> </func> <func><name><ret>ErlNifPid *</ret><nametext>enif_self(ErlNifEnv* caller_env, ErlNifPid* pid)</nametext></name> - <fsummary>Get the pid of the calling process.</fsummary> + <fsummary>Get the pid of the calling process</fsummary> <desc><p>Initialize the pid variable <c>*pid</c> to represent the calling process. Return <c>pid</c>.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_send(ErlNifEnv* env, ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg)</nametext></name> - <fsummary>Send a message to a process.</fsummary> + <fsummary>Send a message to a process</fsummary> <desc><p>Send a message to a process.</p> <taglist> <tag><c>env</c></tag> diff --git a/erts/doc/src/init.xml b/erts/doc/src/init.xml index 09b5493341..c5a1a92b92 100644 --- a/erts/doc/src/init.xml +++ b/erts/doc/src/init.xml @@ -248,7 +248,7 @@ evaluation), Erlang stops with an error message. Here is an example that seeds the random number generator:</p> <pre> -% <input>erl -eval '{X,Y,Z}' = now(), random:seed(X,Y,Z).'</input></pre> +% <input>erl -eval '{X,Y,Z} = now(), random:seed(X,Y,Z).'</input></pre> <p>This example uses Erlang as a hexadecimal calculator:</p> <pre> % <input>erl -noshell -eval 'R = 16#1F+16#A0, io:format("~.16B~n", [R])' \\</input> diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index c85cbe543d..35e6e55e72 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -30,508 +30,6 @@ </header> <p>This document describes the changes made to the ERTS application.</p> -<section><title>Erts 7.0</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Fix issuing with spaces and quoting in the arguments when - using erlang:open_port spawn_executable on windows. The - behavior now mimics how unix works. This change implies a - backwards incompatibility for how spawn_executable works - on windows.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-11905</p> - </item> - <item> - <p> - Fix global call trace when hipe compiled code call beam - compiled functions. Tracing of beam functions should now - alway work regardless who the caller is.</p> - <p> - Own Id: OTP-11939</p> - </item> - <item> - <p> - Correct cache alignment for ETS <c>write_concurrency</c> - locks to improve performance by reduced false sharing. - May increase memory footprint for tables with - <c>write_concurrency</c>.</p> - <p> - Own Id: OTP-11974</p> - </item> - <item> - <p> - All possibly blocking operations in the fd/spawn and - terminal driver have been converted to non-blocking - operations. Before this fix it was possible for the VM to - be blocked for a long time if the entity consuming - stdout/stderr did not consume it fast enough.</p> - <p> - Own Id: OTP-12239</p> - </item> - <item> - <p> - Add missing overhead for offheap binaries created from - external format. This fix can improve the garbage - collection of large binaries originating from - <c>binary_to_term</c> or messages from remote nodes.</p> - <p> - Own Id: OTP-12554</p> - </item> - <item> - <p> - Ensure hashing of zero is consistent</p> - <p> Erlang treats positive and negative zero as - equal:</p> - <p> - <c>true = 0.0 =:= 0.0/-1</c></p> - <p>However, Erlangs hash functions: hash, phash and - phash2 did not reflect this behaviour. The hash values - produced by the different hash functions would not be - identical for positive and negative zero.</p> <p>This - change ensures that hash value of positive zero is always - produced regardless of the signedness of the zero float, - i.e.,</p> - <p> - <c>true = erlang:phash2(0.0) =:= - erlang:phash2(0.0/-1)</c></p> - <p> - Own Id: OTP-12641</p> - </item> - <item> - <p> - Ensure NIF term creation disallows illegal floating point - values and too long atoms. Such values will cause a NIF - to throw badarg exception when it returns.</p> - <p> - Own Id: OTP-12655</p> - </item> - <item> - <p> - Fixed building of Map results from match_specs</p> - <p> - A faulty "box-value" entered into the heap which could - cause a segmentation fault in the garbage collector if it - was written on a heap fragment.</p> - <p> - Own Id: OTP-12656</p> - </item> - <item> - <p> - Fix hipe bug when matching a "writable" binary. The bug - has been seen to sometimes cause a failed binary matching - of a correct utf8 character, but other symptoms are also - possible.</p> - <p> - Own Id: OTP-12667</p> - </item> - <item> - <p> - Keep dirty schedulers from waking other schedulers.</p> - <p> - Own Id: OTP-12685</p> - </item> - <item> - <p> - Disable floating point exceptions if the VM is compiled - by clang/llvm. This is a known long-standing problem in - clang/llvm.</p> - <p> - Own Id: OTP-12717</p> - </item> - <item> - <p> - Fix bug in <c>file:sendfile</c> for FreeBSD causing not - the entire file to be sent.</p> - <p> - Own Id: OTP-12720</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Add <c>md5</c> and <c>module</c> entries to - <c>?MODULE:module_info/0/1</c> and remove obsolete entry - 'import'.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-11940</p> - </item> - <item> - <p> - Debug function <c>erlang:display/1</c> shows content of - binaries and bitstrings, not only the length.</p> - <p> - Own Id: OTP-11941</p> - </item> - <item> - <p>The time functionality of Erlang has been extended. - This both includes a <seealso - marker="time_correction#The_New_Time_API">new - API</seealso> for time, as well as <seealso - marker="time_correction#Time_Warp_Modes">time warp - modes</seealso> which alters the behavior of the system - when system time changes. <em>You are strongly encouraged - to use the new API</em> instead of the old API based on - <seealso - marker="erlang#now/0"><c>erlang:now/0</c></seealso>. - <c>erlang:now/0</c> has been deprecated since it is and - forever will be a scalability bottleneck. For more - information see the <seealso - marker="time_correction">Time and Time - Correction</seealso> chapter of the ERTS User's - Guide.</p> - <p>Besides the API changes and time warp modes a lot of - scalability and performance improvements regarding time - management has been made internally in the runtime - system. Examples of such improvements are scheduler - specific timer wheels, scheduler specific BIF timer - management, parallel retrieval of monotonic time and - system time on systems with primitives that are not - buggy.</p> - <p> - Own Id: OTP-11997</p> - </item> - <item> - <p><c>erlang:function_exported(M, F, A)</c> will now - return <c>true</c> if <c>M:F/A</c> refers to a BIF.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12099</p> - </item> - <item> - <p> - New BIF: <c>erlang:get_keys/0</c>, lists all keys - associated with the process dictionary. Note: - <c>erlang:get_keys/0</c> is auto-imported.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12151 Aux Id: seq12521 </p> - </item> - <item> - <p> - Make distributed send of large messages yield to improve - real-time characteristics.</p> - <p> - Own Id: OTP-12232</p> - </item> - <item> - <p> - Use high accuracy poll timeouts</p> - <p> - Where available, use poll/select API's that can handle - time resolutions less than 1ms. In the cases where such - API's are not available the timeout is rounded up to the - nearest ms.</p> - <p> - Own Id: OTP-12236</p> - </item> - <item> - <p> - The internal group to user_drv protocol has been changed - to be synchronous in order to guarantee that output sent - to a process implementing the user_drv protocol is - printed before replying. This protocol is used by the - standard_output device and the ssh application when - acting as a client. </p> - <p> - This change changes the previous unlimited buffer when - printing to standard_io and other devices that end up in - user_drv to 1KB.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12240</p> - </item> - <item> - <p>The previously introduced "eager check I/O" feature is - now enabled by default.</p> - <p>Eager check I/O can be disabled using the <c>erl</c> - command line argument: <seealso - marker="erl#+secio"><c>+secio false</c></seealso></p> - <p>Characteristics impact compared to previous - default:</p> <list> <item>Lower latency and smoother - management of externally triggered I/O operations.</item> - <item>A slightly reduced priority of externally triggered - I/O operations.</item> </list> - <p> - Own Id: OTP-12254 Aux Id: OTP-12117 </p> - </item> - <item> - <p> - Properly support maps in match_specs</p> - <p> - Own Id: OTP-12270</p> - </item> - <item> - <p> - The notice that a crashdump has been written has been - moved to be printed before the crashdump is generated - instead of afterwords. The wording of the notice has also - been changed.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12292</p> - </item> - <item> - <p> - New function <c>ets:take/2</c>. Works the same as - <c>ets:delete/2</c> but also returns the deleted - object(s).</p> - <p> - Own Id: OTP-12309</p> - </item> - <item> - <p> - Tracing with cpu_timestamp option has been enabled on - Linux.</p> - <p> - Own Id: OTP-12366</p> - </item> - <item> - <p> - ets:info/1,2 now contains information about whether - write_concurrency or read_concurrency is enabled.</p> - <p> - Own Id: OTP-12376</p> - </item> - <item> - <p> - Improved usage of <c>gcc</c>'s builtins for atomic memory - access. These are used when no other implementation of - atomic memory operations is available. For example, when - compiling for ARM when <c>libatomic_ops</c> is not - available.</p> - <p> - The largest improvement will be seen when compiling with - a <c>gcc</c> with support for the <c>__atomic_*</c> - builtins (using a <c>gcc</c> of at least version 4.7), - but also when only the legacy <c>__sync_*</c> builtins - are available (using a <c>gcc</c> of at least version - 4.1) an improvement can be seen.</p> - <p> - For more information see the "<seealso - marker="doc/installation_guide:INSTALL#Advanced-configuration-and-build-of-ErlangOTP_Configuring_Atomic-Memory-Operations-and-the-VM">Atomic - Memory Operations and the VM</seealso>" section of - <c>$ERL_TOP/HOWTO/INSTALL.md</c>.</p> - <p> - Own Id: OTP-12383</p> - </item> - <item> - <p> - Introduce <c>math:log2/1</c> function to math module.</p> - <p> - Own Id: OTP-12411</p> - </item> - <item> - <p> - Remove perfctr support</p> - <p> - Development of perfctr in the linux kernel ceased in - 2010. The perfctr support code in the Erlang VM is thus - effectively dead code and therefor removed.</p> - <p> - Own Id: OTP-12508</p> - </item> - <item> - <p><c>zlib:inflateChunk/2</c> has been added. It works - like <c>zlib:inflate/2</c>, but decompresses no more data - than will fit in the buffer configured by - <c>zlib:setBufSize/2</c>.</p> - <p> - Own Id: OTP-12548</p> - </item> - <item> - <p> - Use linear search for small select_val arrays</p> - <p> - Own Id: OTP-12555</p> - </item> - <item> - <p> - New BIF ets:update_counter/4 with a default object as - argument, which will be inserted in the table if the key - was not found.</p> - <p> - Own Id: OTP-12563</p> - </item> - <item> - <p> - Export missing types from zlib module</p> - <p> - Own Id: OTP-12584</p> - </item> - <item> - <p> - Use persistent hashmaps for large Maps <p>Maps will use a - persistent hashmap implementation when the number of - pairs in a Map becomes sufficiently large. The change - will occur when a Map reaches 33 pairs in size but this - limit might change in the future.</p></p> - <p>The most significant impact for the user by this - change is speed, and to a lesser degree memory - consumption and introspection of Maps. Memory consumption - size is probalistic but lesser than <c>gb_trees</c> or - <c>dict</c> for instance. Any other impacts will be - transparent for the user except for the following - changes.</p> - <p>Semantics of Maps have changed in two incompatible - ways compared to the experimental implementation in OTP - 17:</p> <list> <item>Hashing of maps is done different by - <c>erlang:phash2/1,2</c>, <c>erlang:phash/1</c> and - <c>erlang:hash/2</c>.</item> <item>Comparing two maps - with ==, /=, =<, <, >= and >, is done - different if the keys contain floating point - numbers.</item> </list> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12585</p> - </item> - <item> - <p> - Scalability improvement for <seealso - marker="erlang#make_ref/0">erlang:make_ref/0</seealso>, - and other functionality that create references. Each - scheduler now manage its own set of references. By this - no communication at all is needed when creating - references.</p> - <p> - Previous implementation generated a strictly - monotonically increasing sequence of references - corresponding to creation time on the runtime system - instance. This is <em>not</em> the case with current - implementation. You can only expect reference to be - unique. The Erlang/OTP documentation has never mentioned - anything else but the uniqueness property, so this change - <em>is</em> fully compatible. The only reason we've - marked this as a potential incompatibility is since an - early draft for an Erlang specification mentions strict - monotonicity as a property.</p> - <p> - If you need to create data with a strict monotonicity - property use <seealso - marker="erlang#unique_integer/1">erlang:unique_integer([monotonic])</seealso>. - Do <em>not</em> use the deprecated <seealso - marker="erlang:now/0">erlang:now()</seealso>.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12610</p> - </item> - <item> - <p> - Enable different abort signal from heart</p> - <p>By using environment variable HEART_KILL_SIGNAL, heart - can now use a different signal to kill the old running - Erlang.</p> - <p>By default the signal is SIGKILL but SIGABRT may also - be used by setting environment variable: - HEART_KILL_SIGNAL=SIGABRT</p> - <p> - Own Id: OTP-12613 Aux Id: seq12826 </p> - </item> - <item> - <p> - Update autconf to latest version 2015-03-04</p> - <p> - Own Id: OTP-12646</p> - </item> - <item> - <p> - Optimization of timers internally in the VM. This include - process timers (<c>receive ... after</c>), port timers - (<c>driver_set_timer()</c>) as well as BIF timers - (<c>erlang:send_after()</c>/<c>erlang:start_timer()</c>).</p> - <p> - Each scheduler thread now has its own lock-free timer - service instead of one locked central service. This - dramatically improves performance of timer management on - systems with a large amount of schedulers and timers.</p> - <p> - The timer service internal data structure has also been - optimized to be able to handle more timers than before. - That is, each timer service is by its self able to handle - more timers without dramatic performance loss than the - old centralized timer service.</p> - <p> - The API of BIF timers has also been extended. Timeout - values are for example no longer limited to 32-bit - integers. For more information see the documentation of - <seealso - marker="erlang#start_timer/4"><c>erlang:start_timer/4</c></seealso>, - <seealso - marker="erlang#send_after/4"><c>erlang:send_after/4</c></seealso>, - <seealso - marker="erlang#cancel_timer/2"><c>erlang:cancel_timer/2</c></seealso>, - and <seealso - marker="erlang#read_timer/2"><c>erlang:read_timer/2</c></seealso>.</p> - <p> - Own Id: OTP-12650 Aux Id: OTP-11997 </p> - </item> - <item> - <p> - Specialize instructions from common assembler patterns</p> - <p>Specialize common instructions of <c>rem</c>, - <c>band</c>, <c>minus</c> and <c>plus</c> in the beam - loader. This will reduce the number of fetches and thus - lessen the instruction dispatch pressure during runtime - and speed up those operations in some common cases.</p> - <p>Specialize move patterns from x-registers to the stack - with a new <c>move_window</c> instruction. This change - will reduce instruction dispatch pressure.</p> - <p> - Own Id: OTP-12690</p> - </item> - <item> - <p> - Fix cross compilation for Android.</p> - <p> - Own Id: OTP-12693</p> - </item> - <item> - <p> - Fix incorrect use of autoconf macro AC_EGREP_CPP, which - could cause faulty configuration if run from a path - containing the string 'yes'.</p> - <p> - Own Id: OTP-12706</p> - </item> - <item> - <p> - Minimal Java version is now 1.6</p> - <p> - Own Id: OTP-12718</p> - </item> - <item> - <p> - Send format and args on process exit to error_logger</p> - <p> - Previously, the emulator would generate a whole string - with values and call the error_logger passing - <c>"~s~n"</c>. This changes it to a format string - containing <c>~p</c> with the respective values as - arguments.</p> - <p> - Own Id: OTP-12735</p> - </item> - </list> - </section> - -</section> - <section><title>Erts 6.4.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 5ec1409adf..74b42c647e 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -350,6 +350,7 @@ atom message atom message_binary atom message_queue_len atom messages +atom merge_trap atom meta atom meta_match_spec atom micro_seconds diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 500a98195b..c769428266 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -33,6 +33,7 @@ #include "beam_catches.h" #include "erl_binary.h" #include "erl_nif.h" +#include "erl_bits.h" #include "erl_thr_progress.h" static void set_default_trace_pattern(Eterm module); @@ -937,7 +938,15 @@ any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) break; case TAG_PRIMARY_HEADER: if (!header_is_transparent(val)) { - Eterm* new_p = p + thing_arityval(val); + Eterm* new_p; + if (header_is_bin_matchstate(val)) { + ErlBinMatchState *ms = (ErlBinMatchState*) p; + ErlBinMatchBuffer *mb = &(ms->mb); + if (in_area(EXPAND_POINTER(mb->orig), mod_start, mod_size)) { + return 1; + } + } + new_p = p + thing_arityval(val); ASSERT(start <= new_p && new_p < end); p = new_p; } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index a21622f424..2d4bf4240c 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -6653,8 +6653,9 @@ new_map(Process* p, Eterm* reg, BeamInstr* I) p->htop = mhp; - factory.p = p; + erts_factory_proc_init(&factory, p); res = erts_hashmap_from_array(&factory, thp, n/2, 0); + erts_factory_close(&factory); if (p->mbuf) { Uint live = Arg(2); reg[live] = res; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 0d40201934..006ba5d0db 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -205,10 +205,7 @@ typedef struct { typedef struct { Eterm term; /* The tagged term (in the heap). */ - Uint heap_size; /* (Exact) size on the heap. */ - SWord offset; /* Offset from temporary location to final. */ - ErlOffHeap off_heap; /* Start of linked list of ProcBins. */ - Eterm* heap; /* Heap for term. */ + ErlHeapFragment* heap_frags; } Literal; /* @@ -477,6 +474,8 @@ typedef struct LoaderState { static void free_loader_state(Binary* magic); +static ErlHeapFragment* new_literal_fragment(Uint size); +static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); static Eterm insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, @@ -525,13 +524,16 @@ static void new_literal_patch(LoaderState* stp, int pos); static void new_string_patch(LoaderState* stp, int pos); static Uint new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size); static int genopargcompare(GenOpArg* a, GenOpArg* b); -static Eterm exported_from_module(Process* p, Eterm mod); -static Eterm functions_in_module(Process* p, Eterm mod); -static Eterm attributes_for_module(Process* p, Eterm mod); -static Eterm compilation_info_for_module(Process* p, Eterm mod); -static Eterm md5_of_module(Process* p, Eterm mod); -static Eterm has_native(Process* p, Eterm mod); -static Eterm native_addresses(Process* p, Eterm mod); +static Eterm get_module_info(Process* p, ErtsCodeIndex code_ix, + BeamInstr* code, Eterm module, Eterm what); +static Eterm exported_from_module(Process* p, ErtsCodeIndex code_ix, + Eterm mod); +static Eterm functions_in_module(Process* p, BeamInstr* code); +static Eterm attributes_for_module(Process* p, BeamInstr* code); +static Eterm compilation_info_for_module(Process* p, BeamInstr* code); +static Eterm md5_of_module(Process* p, BeamInstr* code); +static Eterm has_native(BeamInstr* code); +static Eterm native_addresses(Process* p, BeamInstr* code); int patch_funentries(Eterm Patchlist); int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); @@ -883,6 +885,28 @@ free_loader_state(Binary* magic) } } +static ErlHeapFragment* new_literal_fragment(Uint size) +{ + ErlHeapFragment* bp; + bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(ERTS_ALC_T_PREPARED_CODE, + ERTS_HEAP_FRAG_SIZE(size)); + ERTS_INIT_HEAP_FRAG(bp, size); + return bp; +} + +static void free_literal_fragment(ErlHeapFragment* bp) +{ + ASSERT(bp != NULL); + do { + ErlHeapFragment* next_bp = bp->next; + + erts_cleanup_offheap(&bp->off_heap); + ERTS_HEAP_FREE(ERTS_ALC_T_PREPARED_CODE, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->size)); + bp = next_bp; + }while (bp != NULL); +} + /* * This destructor function can safely be called multiple times. */ @@ -922,10 +946,9 @@ loader_state_dtor(Binary* magic) if (stp->literals != 0) { int i; for (i = 0; i < stp->num_literals; i++) { - if (stp->literals[i].heap != 0) { - erts_free(ERTS_ALC_T_PREPARED_CODE, - (void *) stp->literals[i].heap); - stp->literals[i].heap = 0; + if (stp->literals[i].heap_frags != 0) { + free_literal_fragment(stp->literals[i].heap_frags); + stp->literals[i].heap_frags = 0; } } erts_free(ERTS_ALC_T_PREPARED_CODE, (void *) stp->literals); @@ -1450,6 +1473,7 @@ read_lambda_table(LoaderState* stp) return 0; } + static int read_literal_table(LoaderState* stp) { @@ -1471,7 +1495,7 @@ read_literal_table(LoaderState* stp) stp->allocated_literals = stp->num_literals; for (i = 0; i < stp->num_literals; i++) { - stp->literals[i].heap = 0; + stp->literals[i].heap_frags = 0; } for (i = 0; i < stp->num_literals; i++) { @@ -1479,28 +1503,38 @@ read_literal_table(LoaderState* stp) Sint heap_size; byte* p; Eterm val; - Eterm* hp; + ErtsHeapFactory factory; GetInt(stp, 4, sz); /* Size of external term format. */ GetString(stp, p, sz); if ((heap_size = erts_decode_ext_size(p, sz)) < 0) { LoadError1(stp, "literal %d: bad external format", i); } - hp = stp->literals[i].heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, - heap_size*sizeof(Eterm)); - stp->literals[i].off_heap.first = 0; - stp->literals[i].off_heap.overhead = 0; - val = erts_decode_ext(&hp, &stp->literals[i].off_heap, &p); - stp->literals[i].heap_size = hp - stp->literals[i].heap; - if (stp->literals[i].heap_size > heap_size) { - erl_exit(1, "overrun by %d word(s) for literal heap, term %d", - stp->literals[i].heap_size - heap_size, i); - } - if (is_non_value(val)) { - LoadError1(stp, "literal %d: bad external format", i); - } - stp->literals[i].term = val; - stp->total_literal_size += stp->literals[i].heap_size; + + if (heap_size > 0) { + erts_factory_message_init(&factory, NULL, NULL, + new_literal_fragment(heap_size)); + factory.alloc_type = ERTS_ALC_T_PREPARED_CODE; + val = erts_decode_ext(&factory, &p); + + if (is_non_value(val)) { + LoadError1(stp, "literal %d: bad external format", i); + } + erts_factory_close(&factory); + stp->literals[i].heap_frags = factory.heap_frags; + stp->total_literal_size += erts_used_frag_sz(factory.heap_frags); + } + else { + erts_factory_dummy_init(&factory); + val = erts_decode_ext(&factory, &p); + if (is_non_value(val)) { + LoadError1(stp, "literal %d: bad external format", i); + } + ASSERT(is_immed(val)); + stp->literals[i].heap_frags = NULL; + } + stp->literals[i].term = val; + } erts_free(ERTS_ALC_T_TMP, uncompressed); return 1; @@ -4370,8 +4404,9 @@ freeze_code(LoaderState* stp) Uint* low; Uint* high; LiteralPatch* lp; - struct erl_off_heap_header* off_heap = 0; - struct erl_off_heap_header** off_heap_last = &off_heap; + ErlOffHeap code_off_heap; + + ERTS_INIT_OFF_HEAP(&code_off_heap); low = (Uint *) (code+stp->ci); high = low + stp->total_literal_size; @@ -4379,73 +4414,21 @@ freeze_code(LoaderState* stp) code[MI_LITERALS_END] = (BeamInstr) high; ptr = low; for (i = 0; i < stp->num_literals; i++) { - SWord offset; - struct erl_off_heap_header* t_off_heap; - - sys_memcpy(ptr, stp->literals[i].heap, - stp->literals[i].heap_size*sizeof(Eterm)); - offset = ptr - stp->literals[i].heap; - stp->literals[i].offset = offset; - high = ptr + stp->literals[i].heap_size; - while (ptr < high) { - Eterm val = *ptr; - switch (primary_tag(val)) { - case TAG_PRIMARY_LIST: - case TAG_PRIMARY_BOXED: - *ptr++ = offset_ptr(val, offset); - break; - case TAG_PRIMARY_HEADER: - if (header_is_transparent(val)) { - ptr++; - } else { - if (thing_subtag(val) == REFC_BINARY_SUBTAG) { - struct erl_off_heap_header* oh; - - oh = (struct erl_off_heap_header*) ptr; - if (oh->next) { - Eterm** uptr = (Eterm **) (void *) &oh->next; - *uptr += offset; - } - } - ptr += 1 + thing_arityval(val); - } - break; - default: - ptr++; - break; - } - } - ASSERT(ptr == high); - - /* - * Re-link the off_heap list for this term onto the - * off_heap list for the entire module. - */ - t_off_heap = stp->literals[i].off_heap.first; - if (t_off_heap) { - t_off_heap = (struct erl_off_heap_header *) - offset_ptr((UWord) t_off_heap, offset); - while (t_off_heap) { - *off_heap_last = t_off_heap; - off_heap_last = &t_off_heap->next; - t_off_heap = t_off_heap->next; - } - } + if (stp->literals[i].heap_frags) { + move_multi_frags(&ptr, &code_off_heap, stp->literals[i].heap_frags, + &stp->literals[i].term, 1); + } + else ASSERT(is_immed(stp->literals[i].term)); } - code[MI_LITERALS_OFF_HEAP] = (BeamInstr) off_heap; + code[MI_LITERALS_OFF_HEAP] = (BeamInstr) code_off_heap.first; lp = stp->literal_patches; while (lp != 0) { BeamInstr* op_ptr; - Uint literal; Literal* lit; op_ptr = code + lp->pos; lit = &stp->literals[op_ptr[0]]; - literal = lit->term; - if (is_boxed(literal) || is_list(literal)) { - literal = offset_ptr(literal, lit->offset); - } - op_ptr[0] = literal; + op_ptr[0] = lit->term; lp = lp->next; } literal_end += stp->total_literal_size; @@ -5376,19 +5359,18 @@ new_literal(LoaderState* stp, Eterm** hpp, Uint heap_size) stp->total_literal_size += heap_size; lit = stp->literals + stp->num_literals; - lit->offset = 0; - lit->heap_size = heap_size; - lit->heap = erts_alloc(ERTS_ALC_T_PREPARED_CODE, heap_size*sizeof(Eterm)); - lit->term = make_boxed(lit->heap); - lit->off_heap.first = 0; - lit->off_heap.overhead = 0; - *hpp = lit->heap; + lit->heap_frags = new_literal_fragment(heap_size); + lit->term = make_boxed(lit->heap_frags->mem); + *hpp = lit->heap_frags->mem; return stp->num_literals++; } Eterm erts_module_info_0(Process* p, Eterm module) { + Module* modp; + ErtsCodeIndex code_ix = erts_active_code_ix(); + BeamInstr* code; Eterm *hp; Eterm list = NIL; Eterm tup; @@ -5397,12 +5379,18 @@ erts_module_info_0(Process* p, Eterm module) return THE_NON_VALUE; } - if (erts_get_module(module, erts_active_code_ix()) == NULL) { + modp = erts_get_module(module, code_ix); + if (modp == NULL) { return THE_NON_VALUE; } + code = modp->curr.code; + if (code == NULL) { + return THE_NON_VALUE; + } + #define BUILD_INFO(What) \ - tup = erts_module_info_1(p, module, What); \ + tup = get_module_info(p, code_ix, code, module, What); \ hp = HAlloc(p, 5); \ tup = TUPLE2(hp, What, tup); \ hp += 3; \ @@ -5423,22 +5411,47 @@ erts_module_info_0(Process* p, Eterm module) Eterm erts_module_info_1(Process* p, Eterm module, Eterm what) { + Module* modp; + ErtsCodeIndex code_ix = erts_active_code_ix(); + BeamInstr* code; + + if (is_not_atom(module)) { + return THE_NON_VALUE; + } + + modp = erts_get_module(module, code_ix); + if (modp == NULL) { + return THE_NON_VALUE; + } + + code = modp->curr.code; + if (code == NULL) { + return THE_NON_VALUE; + } + + return get_module_info(p, code_ix, code, module, what); +} + +static Eterm +get_module_info(Process* p, ErtsCodeIndex code_ix, BeamInstr* code, + Eterm module, Eterm what) +{ if (what == am_module) { return module; } else if (what == am_md5) { - return md5_of_module(p, module); + return md5_of_module(p, code); } else if (what == am_exports) { - return exported_from_module(p, module); + return exported_from_module(p, code_ix, module); } else if (what == am_functions) { - return functions_in_module(p, module); + return functions_in_module(p, code); } else if (what == am_attributes) { - return attributes_for_module(p, module); + return attributes_for_module(p, code); } else if (what == am_compile) { - return compilation_info_for_module(p, module); + return compilation_info_for_module(p, code); } else if (what == am_native_addresses) { - return native_addresses(p, module); + return native_addresses(p, code); } else if (what == am_native) { - return has_native(p, module); + return has_native(code); } return THE_NON_VALUE; } @@ -5446,16 +5459,12 @@ erts_module_info_1(Process* p, Eterm module, Eterm what) /* * Builds a list of all functions in the given module: * [{Name, Arity},...] - * - * Returns a tagged term, or 0 on error. */ Eterm functions_in_module(Process* p, /* Process whose heap to use. */ - Eterm mod) /* Tagged atom for module. */ + BeamInstr* code) { - Module* modp; - BeamInstr* code; int i; Uint num_functions; Uint need; @@ -5463,15 +5472,6 @@ functions_in_module(Process* p, /* Process whose heap to use. */ Eterm* hp_end; Eterm result = NIL; - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - code = modp->curr.code; num_functions = code[MI_NUM_FUNCTIONS]; need = 5*num_functions; hp = HAlloc(p, need); @@ -5503,23 +5503,12 @@ functions_in_module(Process* p, /* Process whose heap to use. */ */ static Eterm -has_native(Process* p, Eterm mod) +has_native(BeamInstr *code) { Eterm result = am_false; #ifdef HIPE - Module* modp; - - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - - if (erts_is_module_native(modp->curr.code)) { - result = am_true; + if (erts_is_module_native(code)) { + result = am_true; } #endif return result; @@ -5548,15 +5537,11 @@ erts_is_module_native(BeamInstr* code) /* * Builds a list of all functions including native addresses. * [{Name,Arity,NativeAddress},...] - * - * Returns a tagged term, or 0 on error. */ static Eterm -native_addresses(Process* p, Eterm mod) +native_addresses(Process* p, BeamInstr* code) { - Module* modp; - BeamInstr* code; int i; Eterm* hp; Uint num_functions; @@ -5564,16 +5549,6 @@ native_addresses(Process* p, Eterm mod) Eterm* hp_end; Eterm result = NIL; - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - - code = modp->curr.code; num_functions = code[MI_NUM_FUNCTIONS]; need = (6+BIG_UINT_HEAP_SIZE)*num_functions; hp = HAlloc(p, need); @@ -5600,25 +5575,18 @@ native_addresses(Process* p, Eterm mod) /* * Builds a list of all exported functions in the given module: * [{Name, Arity},...] - * - * Returns a tagged term, or 0 on error. */ Eterm exported_from_module(Process* p, /* Process whose heap to use. */ + ErtsCodeIndex code_ix, Eterm mod) /* Tagged atom for module. */ { int i; Eterm* hp = NULL; Eterm* hend = NULL; Eterm result = NIL; - ErtsCodeIndex code_ix; - - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - code_ix = erts_active_code_ix(); for (i = 0; i < export_list_size(code_ix); i++) { Export* ep = export_list(i,code_ix); @@ -5648,112 +5616,59 @@ exported_from_module(Process* p, /* Process whose heap to use. */ /* * Returns a list of all attributes for the module. - * - * Returns a tagged term, or 0 on error. */ Eterm attributes_for_module(Process* p, /* Process whose heap to use. */ - Eterm mod) /* Tagged atom for module. */ - + BeamInstr* code) { - Module* modp; - BeamInstr* code; - Eterm* hp; byte* ext; Eterm result = NIL; - Eterm* end; - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - code = modp->curr.code; ext = (byte *) code[MI_ATTR_PTR]; if (ext != NULL) { - hp = HAlloc(p, code[MI_ATTR_SIZE_ON_HEAP]); - end = hp + code[MI_ATTR_SIZE_ON_HEAP]; - result = erts_decode_ext(&hp, &MSO(p), &ext); + ErtsHeapFactory factory; + erts_factory_proc_prealloc_init(&factory, p, code[MI_ATTR_SIZE_ON_HEAP]); + result = erts_decode_ext(&factory, &ext); if (is_value(result)) { - ASSERT(hp <= end); + erts_factory_close(&factory); } - HRelease(p,end,hp); } return result; } /* * Returns a list containing compilation information. - * - * Returns a tagged term, or 0 on error. */ Eterm compilation_info_for_module(Process* p, /* Process whose heap to use. */ - Eterm mod) /* Tagged atom for module. */ + BeamInstr* code) { - Module* modp; - BeamInstr* code; - Eterm* hp; byte* ext; Eterm result = NIL; - Eterm* end; - - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - code = modp->curr.code; ext = (byte *) code[MI_COMPILE_PTR]; if (ext != NULL) { - hp = HAlloc(p, code[MI_COMPILE_SIZE_ON_HEAP]); - end = hp + code[MI_COMPILE_SIZE_ON_HEAP]; - result = erts_decode_ext(&hp, &MSO(p), &ext); + ErtsHeapFactory factory; + erts_factory_proc_prealloc_init(&factory, p, code[MI_COMPILE_SIZE_ON_HEAP]); + result = erts_decode_ext(&factory, &ext); if (is_value(result)) { - ASSERT(hp <= end); + erts_factory_close(&factory); } - HRelease(p,end,hp); } return result; } /* * Returns the MD5 checksum for a module - * - * Returns a tagged term, or 0 on error. */ Eterm md5_of_module(Process* p, /* Process whose heap to use. */ - Eterm mod) /* Tagged atom for module. */ + BeamInstr* code) { - Module* modp; - BeamInstr* code; - Eterm res = NIL; - - if (is_not_atom(mod)) { - return THE_NON_VALUE; - } - - modp = erts_get_module(mod, erts_active_code_ix()); - if (modp == NULL) { - return THE_NON_VALUE; - } - code = modp->curr.code; - if (code[MI_MD5_PTR] != 0) { - res = new_binary(p, (byte *) code[MI_MD5_PTR], MD5_SIZE); - } else { - res = am_undefined; - } - return res; + return new_binary(p, (byte *) code[MI_MD5_PTR], MD5_SIZE); } /* diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 4e4611de16..5b5550da43 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -85,6 +85,7 @@ typedef Uint dsize_t; /* Vector size type */ /* The heap size needed for a bignum */ #define BIG_NEED_SIZE(x) ((x) + 1) +#define BIG_NEED_FOR_BITS(bits) BIG_NEED_SIZE(((bits)-1)/D_EXP + 1) #define BIG_UINT_HEAP_SIZE (1 + 1) /* always, since sizeof(Uint) <= sizeof(Eterm) */ diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 850606dd86..cddadb346c 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -34,7 +34,7 @@ #include "erl_bits.h" #include "dtrace-wrapper.h" -static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap*); +static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*); /* * Copy object "obj" to process p. @@ -661,8 +661,7 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, unsigned i; for (bp=first; bp!=NULL; bp=bp->next) { - move_one_frag(hpp, bp->mem, bp->used_size, off_heap); - OH_OVERHEAD(off_heap, bp->off_heap.overhead); + move_one_frag(hpp, bp, off_heap); } hp_end = *hpp; for (hp=hp_start; hp<hp_end; ++hp) { @@ -698,10 +697,10 @@ void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, } static void -move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap) +move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap) { - Eterm* ptr = src; - Eterm* end = ptr + src_sz; + Eterm* ptr = frag->mem; + Eterm* end = ptr + frag->used_size; Eterm dummy_ref; Eterm* hp = *hpp; @@ -732,5 +731,7 @@ move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap) } } *hpp = hp; + OH_OVERHEAD(off_heap, frag->off_heap.overhead); + frag->off_heap.first = NULL; } diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index cfdede793c..ae46174a14 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1149,6 +1149,7 @@ int erts_net_message(Port *prt, DeclareTmpHeapNoproc(ctl_default,DIST_CTL_DEFAULT_SIZE); Eterm* ctl = ctl_default; ErlOffHeap off_heap; + ErtsHeapFactory factory; Eterm* hp; Sint type; Eterm token; @@ -1225,7 +1226,8 @@ int erts_net_message(Port *prt, } hp = ctl; - arg = erts_decode_dist_ext(&hp, &off_heap, &ede); + erts_factory_static_init(&factory, ctl, ctl_len, &off_heap); + arg = erts_decode_dist_ext(&factory, &ede); if (is_non_value(arg)) { #ifdef ERTS_DIST_MSG_DBG erts_fprintf(stderr, "DIST MSG DEBUG: erts_dist_ext_size(CTL) failed:\n"); diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 23fc4f915e..397c68e199 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1106,19 +1106,18 @@ process_info_aux(Process *BIF_P, heap_need += mq[i].copy_struct_size; } else { - mq[i].copy_struct_size = 0; - if (mp->data.attached) - heap_need += erts_msg_attached_data_size(mp); + mq[i].copy_struct_size = mp->data.attached ? + erts_msg_attached_data_size(mp) : 0; } i++; } - hp = HAlloc(BIF_P, heap_need); - hp_end = hp + heap_need; - ASSERT(i == n); - for (i--; i >= 0; i--) { - Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp); - if (rp != BIF_P) { + if (rp != BIF_P) { + hp = HAlloc(BIF_P, heap_need); + hp_end = hp + heap_need; + ASSERT(i == n); + for (i--; i >= 0; i--) { + Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp); if (is_value(msg)) { if (mq[i].copy_struct_size) msg = copy_struct(msg, @@ -1152,9 +1151,9 @@ process_info_aux(Process *BIF_P, } else { /* Make our copy of the message */ - ASSERT(size_object(msg) == hfp->used_size); + ASSERT(size_object(msg) == erts_used_frag_sz(hfp)); msg = copy_struct(msg, - hfp->used_size, + erts_used_frag_sz(hfp), &hp, &MSO(BIF_P)); } @@ -1164,27 +1163,38 @@ process_info_aux(Process *BIF_P, remove_bad_messages = 1; continue; } + res = CONS(hp, msg, res); + hp += 2; } - else { + HRelease(BIF_P, hp_end, hp+3); + } + else { + for (i--; i >= 0; i--) { + ErtsHeapFactory factory; + Eterm msg = ERL_MESSAGE_TERM(mq[i].msgp); + + erts_factory_proc_prealloc_init(&factory, BIF_P, + mq[i].copy_struct_size+2); if (mq[i].msgp->data.attached) { /* Decode it on the heap */ - erts_move_msg_attached_data_to_heap(&hp, - &MSO(BIF_P), + erts_move_msg_attached_data_to_heap(&factory, mq[i].msgp); msg = ERL_MESSAGE_TERM(mq[i].msgp); ASSERT(!mq[i].msgp->data.attached); - if (is_non_value(msg)) { - /* Bad distribution message; ignore */ - remove_bad_messages = 1; - continue; - } - } + } + if (is_value(msg)) { + hp = erts_produce_heap(&factory, 2, 0); + res = CONS(hp, msg, res); + } + else { + /* Bad distribution message; ignore */ + remove_bad_messages = 1; + continue; + } + erts_factory_close(&factory); } - - res = CONS(hp, msg, res); - hp += 2; + hp = HAlloc(BIF_P, 3); } - HRelease(BIF_P, hp_end, hp+3); erts_free(ERTS_ALC_T_TMP, mq); if (remove_bad_messages) { ErlMessage **mpp; diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 8d264d166e..6b96787d40 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -194,6 +194,9 @@ ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc_fnf(Binary *bp, Uint size); ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size); ERTS_GLB_INLINE void erts_bin_free(Binary *bp); +ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size, + void (*destructor)(Binary *), + int unaligned); ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size, void (*destructor)(Binary *)); @@ -332,21 +335,30 @@ erts_bin_free(Binary *bp) } ERTS_GLB_INLINE Binary * -erts_create_magic_binary(Uint size, void (*destructor)(Binary *)) +erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *), + int unaligned) { - Uint bsize = ERTS_MAGIC_BIN_SIZE(size); + Uint bsize = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_SIZE(size) + : ERTS_MAGIC_BIN_SIZE(size); Binary* bptr = erts_alloc_fnf(ERTS_ALC_T_BINARY, bsize); ASSERT(bsize > size); if (!bptr) erts_alloc_n_enomem(ERTS_ALC_T2N(ERTS_ALC_T_BINARY), bsize); ERTS_CHK_BIN_ALIGNMENT(bptr); bptr->flags = BIN_FLAG_MAGIC; - bptr->orig_size = ERTS_MAGIC_BIN_ORIG_SIZE(size); + bptr->orig_size = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(size) + : ERTS_MAGIC_BIN_ORIG_SIZE(size); erts_refc_init(&bptr->refc, 0); ERTS_MAGIC_BIN_DESTRUCTOR(bptr) = destructor; return bptr; } +ERTS_GLB_INLINE Binary * +erts_create_magic_binary(Uint size, void (*destructor)(Binary *)) +{ + return erts_create_magic_binary_x(size, destructor, 0); +} + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* !__ERL_BINARY_H */ diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index b8ae93fa58..2e29bf8895 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -107,6 +107,14 @@ erts_bits_destroy_state(ERL_BITS_PROTO_0) void erts_init_bits(void) { + ERTS_CT_ASSERT(offsetof(Binary,orig_bytes) % 8 == 0); + ERTS_CT_ASSERT(offsetof(ErtsMagicBinary,u.aligned.data) % 8 == 0); + ERTS_CT_ASSERT(ERTS_MAGIC_BIN_BYTES_TO_ALIGN == + (offsetof(ErtsMagicBinary,u.aligned.data) + - offsetof(ErtsMagicBinary,u.unaligned.data))); + ERTS_CT_ASSERT(offsetof(ErtsBinary,driver.binary.orig_bytes) + == offsetof(Binary,orig_bytes)); + erts_smp_atomic_init_nob(&bits_bufs_size, 0); #if defined(ERTS_SMP) /* erl_process.c calls erts_bits_init_state() on all state instances */ diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index f6ea57e513..dce0a3d621 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1051,7 +1051,6 @@ static int db_get_element_hash(Process *p, DbTable *tbl, Eterm copy = db_copy_element_from_ets(&tb->common, p, &b->dbterm, ndex, &hp, 2); elem_list = CONS(hp, copy, elem_list); - hp += 2; } b = b->next; } diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index c6c3c55a7e..d47ff03a30 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -2175,11 +2175,12 @@ restart: { ErtsHeapFactory factory; Uint ix; - factory.p = build_proc; for (ix = 0; ix < 2*n; ix++){ ehp[ix] = esp[ix]; } + erts_factory_proc_init(&factory, build_proc); t = erts_hashmap_from_array(&factory, ehp, n, 0); + erts_factory_close(&factory); } *esp++ = t; break; @@ -3192,6 +3193,7 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, { Eterm* hp = *hpp; int i, arity = arityval(bp->tpl[0]); + ErtsHeapFactory factory; hp[0] = bp->tpl[0]; *hpp += arity + 1; @@ -3199,17 +3201,23 @@ Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos], size_object_rel(bp->tpl[tb->keypos], bp->tpl), hpp, off_heap, bp->tpl, NULL); + + erts_factory_static_init(&factory, *hpp, bp->size - (arity+1), off_heap); + for (i=arity; i>0; i--) { if (i != tb->keypos) { if (is_immed(bp->tpl[i])) { hp[i] = bp->tpl[i]; } else { - hp[i] = erts_decode_ext_ets(hpp, off_heap, + hp[i] = erts_decode_ext_ets(&factory, elem2ext(bp->tpl, i)); } } } + *hpp = factory.hp; + erts_factory_close(&factory); + ASSERT((*hpp - hp) <= bp->size); #ifdef DEBUG_CLONE ASSERT(eq_rel(make_tuple(hp),NULL,make_tuple(bp->debug_clone),bp->debug_clone)); @@ -3228,12 +3236,13 @@ Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, if (tb->compress && pos != tb->keypos) { byte* ext = elem2ext(obj->tpl, pos); Sint sz = erts_decode_ext_size_ets(ext, db_alloced_size_comp(obj)) + extra; - Eterm* hp = HAlloc(p, sz); - Eterm* endp = hp + sz; - Eterm copy = erts_decode_ext_ets(&hp, &MSO(p), ext); - *hpp = hp; - hp += extra; - HRelease(p, endp, hp); + Eterm copy; + ErtsHeapFactory factory; + + erts_factory_proc_prealloc_init(&factory, p, sz); + copy = erts_decode_ext_ets(&factory, ext); + *hpp = erts_produce_heap(&factory, extra, 0); + erts_factory_close(&factory); #ifdef DEBUG_CLONE ASSERT(eq_rel(copy, NULL, obj->debug_clone[pos], obj->debug_clone)); #endif diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 1785fc27be..71ca2713b2 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -108,8 +108,7 @@ static Eterm* sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint src_size); static Eterm* collect_heap_frags(Process* p, Eterm* heap, Eterm* htop, Eterm* objv, int nobj); -static Uint adjust_after_fullsweep(Process *p, Uint size_before, - int need, Eterm *objv, int nobj); +static void adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); static void sweep_off_heap(Process *p, int fullsweep); @@ -677,7 +676,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint area_size; Eterm* old_htop; Uint n; - struct erl_off_heap_header** prev; + struct erl_off_heap_header** prev = NULL; if (p->flags & F_DISABLE_GC) return; @@ -786,10 +785,10 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, */ if (oh) { - prev = &MSO(p).first; - while (*prev) { - prev = &(*prev)->next; - } + prev = &MSO(p).first; + while (*prev) { + prev = &(*prev)->next; + } } /* @@ -818,6 +817,10 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, oh = oh->next; } + if (prev) { + *prev = NULL; + } + /* * We no longer need this temporary area. */ @@ -868,29 +871,37 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) ErlMessage *msgp; Uint size_after; Uint need_after; - Uint stack_size = STACK_SZ_ON_HEAP(p); - Uint fragments = MBUF_SIZE(p) + combined_message_size(p); - Uint size_before = fragments + (HEAP_TOP(p) - HEAP_START(p)); - Uint new_sz = next_heap_size(p, HEAP_SIZE(p) + fragments, 0); + const Uint stack_size = STACK_SZ_ON_HEAP(p); + const Uint size_before = MBUF_SIZE(p) + (HEAP_TOP(p) - HEAP_START(p)); + Uint new_sz = HEAP_SIZE(p) + MBUF_SIZE(p) + combined_message_size(p); + new_sz = next_heap_size(p, new_sz, 0); do_minor(p, new_sz, objv, nobj); - /* + size_after = HEAP_TOP(p) - HEAP_START(p); + *recl += (size_before - size_after); + + /* * Copy newly received message onto the end of the new heap. */ - ErtsGcQuickSanityCheck(p); - for (msgp = p->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached) { - erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp); - ErtsGcQuickSanityCheck(p); - } - } + ErtsGcQuickSanityCheck(p); + for (msgp = p->msg.first; msgp; msgp = msgp->next) { + if (msgp->data.attached) { + ErtsHeapFactory factory; + erts_factory_proc_prealloc_init(&factory, p, + erts_msg_attached_data_size(msgp)); + erts_move_msg_attached_data_to_heap(&factory, msgp); + erts_factory_close(&factory); + ErtsGcQuickSanityCheck(p); + } + } ErtsGcQuickSanityCheck(p); GEN_GCS(p)++; - size_after = HEAP_TOP(p) - HEAP_START(p); - need_after = size_after + need + stack_size; - *recl += (size_before - size_after); + need_after = ((HEAP_TOP(p) - HEAP_START(p)) + + erts_used_frag_sz(MBUF(p)) + + need + + stack_size); /* * Excessively large heaps should be shrunk, but @@ -925,6 +936,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) } ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); + ASSERT(MBUF(p) == NULL); return 1; /* We are done. */ } @@ -933,6 +945,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * The heap size turned out to be just right. We are done. */ ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); + ASSERT(MBUF(p) == NULL); return 1; } } @@ -1212,7 +1225,9 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) { Rootset rootset; Roots* roots; - Uint size_before; + const Uint size_before = ((HEAP_TOP(p) - HEAP_START(p)) + + (OLD_HTOP(p) - OLD_HEAP(p)) + + MBUF_SIZE(p)); Eterm* n_heap; Eterm* n_htop; char* src = (char *) HEAP_START(p); @@ -1221,24 +1236,15 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) Uint oh_size = (char *) OLD_HTOP(p) - oh; Uint n; Uint new_sz; - Uint fragments = MBUF_SIZE(p) + combined_message_size(p); - - size_before = fragments + (HEAP_TOP(p) - HEAP_START(p)); /* * Do a fullsweep GC. First figure out the size of the heap * to receive all live data. */ - new_sz = HEAP_SIZE(p) + fragments + (OLD_HTOP(p) - OLD_HEAP(p)); - /* - * We used to do - * - * new_sz += STACK_SZ_ON_HEAP(p); - * - * here for no obvious reason. (The stack size is already counted once - * in HEAP_SIZE(p).) - */ + new_sz = (HEAP_SIZE(p) + MBUF_SIZE(p) + + combined_message_size(p) + + (OLD_HTOP(p) - OLD_HEAP(p))); new_sz = next_heap_size(p, new_sz, 0); /* @@ -1431,20 +1437,27 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) ErtsGcQuickSanityCheck(p); + *recl += size_before - (HEAP_TOP(p) - HEAP_START(p)); + { ErlMessage *msgp; + /* * Copy newly received message onto the end of the new heap. */ - for (msgp = p->msg.first; msgp; msgp = msgp->next) { + for (msgp = p->msg.first; msgp; msgp = msgp->next) { if (msgp->data.attached) { - erts_move_msg_attached_data_to_heap(&p->htop, &p->off_heap, msgp); + ErtsHeapFactory factory; + erts_factory_proc_prealloc_init(&factory, p, + erts_msg_attached_data_size(msgp)); + erts_move_msg_attached_data_to_heap(&factory, msgp); + erts_factory_close(&factory); ErtsGcQuickSanityCheck(p); } } } - *recl += adjust_after_fullsweep(p, size_before, need, objv, nobj); + adjust_after_fullsweep(p, need, objv, nobj); #ifdef HARDDEBUG disallow_heap_frag_ref_in_heap(p); @@ -1455,21 +1468,17 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) return 1; /* We are done. */ } -static Uint -adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int nobj) +static void +adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) { - Uint wanted, sz, size_after, need_after; + Uint wanted, sz, need_after; Uint stack_size = STACK_SZ_ON_HEAP(p); - Uint reclaimed_now; - - size_after = (HEAP_TOP(p) - HEAP_START(p)); - reclaimed_now = (size_before - size_after); /* * Resize the heap if needed. */ - need_after = size_after + need + stack_size; + need_after = (HEAP_TOP(p) - HEAP_START(p)) + need + stack_size; if (HEAP_SIZE(p) < need_after) { /* Too small - grow to match requested need */ sz = next_heap_size(p, need_after, 0); @@ -1492,8 +1501,6 @@ adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int shrink_new_heap(p, sz, objv, nobj); } } - - return reclaimed_now; } /* @@ -1870,6 +1877,21 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr if (!header_is_thing(gval)) { heap_ptr++; } else { + if (header_is_bin_matchstate(gval)) { + ErlBinMatchState *ms = (ErlBinMatchState*) heap_ptr; + ErlBinMatchBuffer *mb = &(ms->mb); + Eterm* origptr; + origptr = &(mb->orig); + ptr = boxed_val(*origptr); + val = *ptr; + if (IS_MOVED_BOXED(val)) { + *origptr = val; + mb->base = binary_bytes(*origptr); + } else if (in_area(ptr, src, src_size)) { + MOVE_BOXED(ptr,val,htop,origptr); + mb->base = binary_bytes(*origptr); + } + } heap_ptr += (thing_arityval(gval)+1); } break; @@ -1941,7 +1963,8 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, * until next GC. */ qb = MBUF(p); - while (qb != NULL) { + while (qb != NULL) { + ASSERT(!qb->off_heap.first); /* process fragments use the MSO(p) list */ frag_size = qb->used_size * sizeof(Eterm); if (frag_size != 0) { frag_begin = (char *) qb->mem; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index c676682463..ac1f00d2d8 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -385,6 +385,7 @@ erl_init(int ncpu, erts_init_bif_re(); erts_init_unicode(); /* after RE to get access to PCRE unicode */ erts_init_external(); + erts_init_map(); erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2); erts_late_init_process(); #if HAVE_ERTS_MSEG @@ -631,7 +632,7 @@ void erts_usage(void) erts_fprintf(stderr, "-v turn on chatty mode (GCs will be reported etc)\n"); - erts_fprintf(stderr, "-W<i|w> set error logger warnings mapping,\n"); + erts_fprintf(stderr, "-W<i|w|e> set error logger warnings mapping,\n"); erts_fprintf(stderr, " see error_logger documentation for details\n"); erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n"); erts_fprintf(stderr, " valid range is [1-%d]\n", INT_MAX/1024); @@ -1268,7 +1269,7 @@ erl_start(int argc, char **argv) verbose = DEBUG_DEFAULT; #endif - erts_error_logger_warnings = am_error; + erts_error_logger_warnings = am_warning; while (i < argc) { if (argv[i][0] != '-') { @@ -2006,11 +2007,12 @@ erl_start(int argc, char **argv) case 'i': erts_error_logger_warnings = am_info; break; + case 'e': + erts_error_logger_warnings = am_error; + break; case 'w': erts_error_logger_warnings = am_warning; break; - case 'e': /* The default */ - erts_error_logger_warnings = am_error; default: erts_fprintf(stderr, "unrecognized warning_map option %s\n", arg); erts_usage(); diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index a1bd39dbc8..95a10daa67 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -32,6 +32,7 @@ #include "erl_process.h" #include "error.h" #include "bif.h" +#include "erl_binary.h" #include "erl_map.h" @@ -79,8 +80,13 @@ typedef struct { static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB); -static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args); -static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB); +static BIF_RETTYPE map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args); +struct HashmapMergeContext_; +static BIF_RETTYPE hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB, int swap_args, + struct HashmapMergeContext_*); +static Export hashmap_merge_trap_export; +static BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1); +static Uint hashmap_subtree_size(Eterm node); static Eterm hashmap_to_list(Process *p, Eterm map); static Eterm hashmap_keys(Process *p, Eterm map); static Eterm hashmap_values(Process *p, Eterm map); @@ -95,6 +101,15 @@ static Eterm hashmap_bld_tuple_uint(Uint **hpp, Uint *szp, Uint n, Uint nums[]); static int hxnodecmp(hxnode_t* a, hxnode_t* b); static int hxnodecmpkey(hxnode_t* a, hxnode_t* b); + +void erts_init_map(void) { + erts_init_trap_export(&hashmap_merge_trap_export, + am_maps, am_merge_trap, 1, + &maps_merge_trap_1); + return; +} + + /* erlang:map_size/1 * the corresponding instruction is implemented in: * beam/erl_bif_guard.c @@ -410,8 +425,9 @@ static Eterm hashmap_from_validated_list(Process *p, Eterm list, Uint size) { } UnUseTmpHeap(2,p); - factory.p = p; + erts_factory_proc_init(&factory, p); res = hashmap_from_unsorted_array(&factory, hxns, size, 0); + erts_factory_close(&factory); erts_free(ERTS_ALC_T_TMP, (void *) hxns); ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); @@ -515,8 +531,9 @@ Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n hxns[i].i = i; } - factory.p = p; + erts_factory_proc_init(&factory, p); res = hashmap_from_unsorted_array(&factory, hxns, sz, 0); + erts_factory_close(&factory); erts_free(ERTS_ALC_T_TMP, (void *) hxns); ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); @@ -942,15 +959,15 @@ BIF_RETTYPE maps_merge_2(BIF_ALIST_2) { BIF_RET(flatmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2)); } else if (is_hashmap(BIF_ARG_2)) { /* Will always become a tree */ - BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_1, BIF_ARG_2, 0)); + return map_merge_mixed(BIF_P, BIF_ARG_1, BIF_ARG_2, 0); } BIF_P->fvalue = BIF_ARG_2; } else if (is_hashmap(BIF_ARG_1)) { if (is_hashmap(BIF_ARG_2)) { - BIF_RET(hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2)); + return hashmap_merge(BIF_P, BIF_ARG_1, BIF_ARG_2, 0, NULL); } else if (is_flatmap(BIF_ARG_2)) { /* Will always become a tree */ - BIF_RET(map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1)); + return map_merge_mixed(BIF_P, BIF_ARG_2, BIF_ARG_1, 1); } BIF_P->fvalue = BIF_ARG_2; } else { @@ -1063,8 +1080,9 @@ static Eterm flatmap_merge(Process *p, Eterm nodeA, Eterm nodeB) { hxns[i].i = i; } - factory.p = p; + erts_factory_proc_init(&factory, p); res = hashmap_from_unsorted_array(&factory, hxns, n, 0); + erts_factory_close(&factory); erts_free(ERTS_ALC_T_TMP, (void *) hxns); ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); @@ -1107,36 +1125,71 @@ static Eterm map_merge_mixed(Process *p, Eterm flat, Eterm tree, int swap_args) hxns[i].i = i; } - factory.p = p; + erts_factory_proc_init(&factory, p); res = hashmap_from_unsorted_array(&factory, hxns, n, 0); + erts_factory_close(&factory); erts_free(ERTS_ALC_T_TMP, (void *) hxns); ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); - return swap_args ? hashmap_merge(p, tree, res) : hashmap_merge(p, res, tree); + return hashmap_merge(p, res, tree, swap_args, NULL); +} + +#define PSTACK_TYPE struct HashmapMergePStackType +struct HashmapMergePStackType { + Eterm nodeA, nodeB; + Eterm *srcA, *srcB; + Uint32 abm, bbm, rbm; /* node bitmaps */ + int mix; /* &1: there are unique A stuff in node + * &2: there are unique B stuff in node */ + int ix; + Eterm array[16]; /* temp node construction area */ +}; + +typedef struct HashmapMergeContext_ { + Uint size; /* total key-value counter */ + unsigned int lvl; + Eterm trap_bin; + ErtsPStack pstack; +#ifdef DEBUG + Eterm dbg_map_A, dbg_map_B; +#endif +} HashmapMergeContext; + +static void hashmap_merge_ctx_destructor(Binary* ctx_bin) +{ + HashmapMergeContext* ctx = (HashmapMergeContext*) ERTS_MAGIC_BIN_DATA(ctx_bin); + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor); + + PSTACK_DESTROY_SAVED(&ctx->pstack); +} + +BIF_RETTYPE maps_merge_trap_1(BIF_ALIST_1) { + Binary* ctx_bin = ((ProcBin *) binary_val(BIF_ARG_1))->val; + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(ctx_bin) == hashmap_merge_ctx_destructor); + + return hashmap_merge(BIF_P, NIL, NIL, 0, + (HashmapMergeContext*) ERTS_MAGIC_BIN_DATA(ctx_bin)); } #define HALLOC_EXTRA 200 +#define MAP_MERGE_LOOP_FACTOR 8 -static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) { +static BIF_RETTYPE hashmap_merge(Process *p, Eterm map_A, Eterm map_B, + int swap_args, HashmapMergeContext* ctx) { #define PSTACK_TYPE struct HashmapMergePStackType - struct HashmapMergePStackType { - Eterm *srcA, *srcB; - Uint32 abm, bbm, rbm; /* node bitmaps */ - int keepA; - int ix; - Eterm array[16]; - }; PSTACK_DECLARE(s, 4); - struct HashmapMergePStackType* sp = PSTACK_PUSH(s); - Eterm *hp, *nhp; + HashmapMergeContext local_ctx; + struct HashmapMergePStackType* sp; + Uint32 hx; + Eterm res = THE_NON_VALUE; Eterm hdrA, hdrB; - Uint32 ahx, bhx; - Uint size; /* total key-value counter */ - int keepA = 0; - unsigned int lvl = 0; + Eterm *hp, *nhp; + Eterm trap_ret; + Sint initial_reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * MAP_MERGE_LOOP_FACTOR); + Sint reds = initial_reds; DeclareTmpHeap(th,2,p); - Eterm res = THE_NON_VALUE; UseTmpHeap(2,p); /* @@ -1144,152 +1197,139 @@ static Eterm hashmap_merge(Process *p, Eterm nodeA, Eterm nodeB) { * and merge each pair of nodes. */ - { - hashmap_head_t* a = (hashmap_head_t*) hashmap_val(nodeA); - hashmap_head_t* b = (hashmap_head_t*) hashmap_val(nodeB); - size = a->size + b->size; + PSTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_SAVED_ESTACK); + + if (ctx == NULL) { /* first call */ + hashmap_head_t* a = (hashmap_head_t*) hashmap_val(map_A); + hashmap_head_t* b = (hashmap_head_t*) hashmap_val(map_B); + + sp = PSTACK_PUSH(s); + sp->srcA = swap_args ? &map_B : &map_A; + sp->srcB = swap_args ? &map_A : &map_B; + sp->mix = 0; + local_ctx.size = a->size + b->size; + local_ctx.lvl = 0; + #ifdef DEBUG + local_ctx.dbg_map_A = map_A; + local_ctx.dbg_map_B = map_B; + local_ctx.trap_bin = THE_NON_VALUE; + #endif + ctx = &local_ctx; + } + else { + PSTACK_RESTORE(s, &ctx->pstack); + sp = PSTACK_TOP(s); + goto resume_from_trap; } recurse: - if (primary_tag(nodeA) == TAG_PRIMARY_BOXED && - primary_tag(nodeB) == TAG_PRIMARY_LIST) { - /* Avoid implementing this combination by switching places */ - Eterm tmp = nodeA; - nodeA = nodeB; - nodeB = tmp; - keepA = !keepA; - } - - switch (primary_tag(nodeA)) { - case TAG_PRIMARY_LIST: { - sp->srcA = list_val(nodeA); - switch (primary_tag(nodeB)) { - case TAG_PRIMARY_LIST: { /* LEAF + LEAF */ - sp->srcB = list_val(nodeB); - - if (EQ(CAR(sp->srcA), CAR(sp->srcB))) { - --size; - res = keepA ? nodeA : nodeB; - } else { - ahx = hashmap_restore_hash(th, lvl, CAR(sp->srcA)); - bhx = hashmap_restore_hash(th, lvl, CAR(sp->srcB)); - sp->abm = 1 << hashmap_index(ahx); - sp->bbm = 1 << hashmap_index(bhx); + sp->nodeA = *sp->srcA; + sp->nodeB = *sp->srcB; - sp->srcA = &nodeA; - sp->srcB = &nodeB; - } - break; - } - case TAG_PRIMARY_BOXED: { /* LEAF + NODE */ - sp->srcB = boxed_val(nodeB); - ASSERT(is_header(*sp->srcB)); - hdrB = *sp->srcB++; - - ahx = hashmap_restore_hash(th, lvl, CAR(sp->srcA)); - sp->abm = 1 << hashmap_index(ahx); - sp->srcA = &nodeA; - switch(hdrB & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_ARRAY: - sp->srcB++; - sp->bbm = 0xffff; - break; + if (sp->nodeA == sp->nodeB) { + res = sp->nodeA; + ctx->size -= is_list(sp->nodeB) ? 1 : hashmap_subtree_size(sp->nodeB); + } + else { + if (is_list(sp->nodeA)) { /* A is LEAF */ + Eterm keyA = CAR(list_val(sp->nodeA)); + + if (is_list(sp->nodeB)) { /* LEAF + LEAF */ + Eterm keyB = CAR(list_val(sp->nodeB)); + + if (EQ(keyA, keyB)) { + --ctx->size; + res = sp->nodeB; + sp->mix = 2; /* We assume values differ. + + Don't spend time comparing big values. + - Might waste some heap space for internal + nodes that could otherwise be reused. */ + goto merge_nodes; + } + } + hx = hashmap_restore_hash(th, ctx->lvl, keyA); + sp->abm = 1 << hashmap_index(hx); + /* keep srcA pointing at the leaf */ + } + else { /* A is NODE */ + sp->srcA = boxed_val(sp->nodeA); + hdrA = *sp->srcA++; + ASSERT(is_header(hdrA)); + switch (hdrA & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: { + sp->srcA++; + sp->abm = 0xffff; + break; + } + case HAMT_SUBTAG_HEAD_BITMAP: sp->srcA++; + case HAMT_SUBTAG_NODE_BITMAP: { + sp->abm = MAP_HEADER_VAL(hdrA); + break; + } + default: + erl_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrA); + } + } - case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++; - case HAMT_SUBTAG_NODE_BITMAP: - sp->bbm = MAP_HEADER_VAL(hdrB); - break; + if (is_list(sp->nodeB)) { /* B is LEAF */ + Eterm keyB = CAR(list_val(sp->nodeB)); - default: - erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK); - break; - } - break; - } - default: - erl_exit(1, "bad primary tag %ld\r\n", nodeB); - } - break; - } - case TAG_PRIMARY_BOXED: { /* NODE + NODE */ - sp->srcA = boxed_val(nodeA); - hdrA = *sp->srcA++; - ASSERT(is_header(hdrA)); - switch (hdrA & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_ARRAY: { - sp->srcA++; - ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED); - sp->abm = 0xffff; - sp->srcB = boxed_val(nodeB); - hdrB = *sp->srcB++; - ASSERT(is_header(hdrB)); - switch (hdrB & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_ARRAY: - sp->srcB++; - sp->bbm = 0xffff; - break; - case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++; - case HAMT_SUBTAG_NODE_BITMAP: - sp->bbm = MAP_HEADER_VAL(hdrB); - break; - default: - erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK); - } - break; - } - case HAMT_SUBTAG_HEAD_BITMAP: sp->srcA++; - case HAMT_SUBTAG_NODE_BITMAP: { - ASSERT(primary_tag(nodeB) == TAG_PRIMARY_BOXED); - sp->abm = MAP_HEADER_VAL(hdrA); - sp->srcB = boxed_val(nodeB); - hdrB = *sp->srcB++; - ASSERT(is_header(hdrB)); - switch (hdrB & _HEADER_MAP_SUBTAG_MASK) { - case HAMT_SUBTAG_HEAD_ARRAY: + hx = hashmap_restore_hash(th, ctx->lvl, keyB); + sp->bbm = 1 << hashmap_index(hx); + /* keep srcB pointing at the leaf */ + } + else { /* B is NODE */ + sp->srcB = boxed_val(sp->nodeB); + hdrB = *sp->srcB++; + ASSERT(is_header(hdrB)); + switch (hdrB & _HEADER_MAP_SUBTAG_MASK) { + case HAMT_SUBTAG_HEAD_ARRAY: { sp->srcB++; - sp->bbm = 0xffff; - break; - case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++; - case HAMT_SUBTAG_NODE_BITMAP: - sp->bbm = MAP_HEADER_VAL(hdrB); - break; - - default: - erl_exit(1, "bad header tag %ld\r\n", *sp->srcB & _HEADER_MAP_SUBTAG_MASK); - } - break; - } - default: - erl_exit(1, "bad primary tag %ld\r\n", nodeA); - } - break; - } - default: - erl_exit(1, "bad primary tag %ld\r\n", nodeA); + sp->bbm = 0xffff; + break; + } + case HAMT_SUBTAG_HEAD_BITMAP: sp->srcB++; + case HAMT_SUBTAG_NODE_BITMAP: { + sp->bbm = MAP_HEADER_VAL(hdrB); + break; + } + default: + erl_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrB); + } + } } +merge_nodes: + for (;;) { if (is_value(res)) { /* We have a complete (sub-)tree or leaf */ - if (lvl == 0) + int child_mix; + if (ctx->lvl == 0) break; /* Pop from stack and continue build parent node */ - lvl--; + ctx->lvl--; + child_mix = sp->mix; sp = PSTACK_POP(s); sp->array[sp->ix++] = res; + sp->mix |= child_mix; res = THE_NON_VALUE; if (sp->rbm) { sp->srcA++; sp->srcB++; - keepA = sp->keepA; } } else { /* Start build a node */ sp->ix = 0; sp->rbm = sp->abm | sp->bbm; - ASSERT(!(sp->rbm == 0 && lvl > 0)); + ASSERT(!(sp->rbm == 0 && ctx->lvl > 0)); } + if (--reds <= 0) { + goto trap; + } +resume_from_trap: + while (sp->rbm) { Uint32 next = sp->rbm & (sp->rbm-1); Uint32 bit = sp->rbm ^ next; @@ -1297,43 +1337,123 @@ recurse: if (sp->abm & bit) { if (sp->bbm & bit) { /* Bit clash. Push and resolve by recursive merge */ - if (sp->rbm) { - sp->keepA = keepA; - } - nodeA = *sp->srcA; - nodeB = *sp->srcB; - lvl++; + Eterm* srcA = sp->srcA; + Eterm* srcB = sp->srcB; + ctx->lvl++; sp = PSTACK_PUSH(s); + sp->srcA = srcA; + sp->srcB = srcB; + sp->mix = 0; goto recurse; } else { sp->array[sp->ix++] = *sp->srcA++; + sp->mix |= 1; } } else { ASSERT(sp->bbm & bit); sp->array[sp->ix++] = *sp->srcB++; + sp->mix |= 2; } } - ASSERT(sp->ix == hashmap_bitcount(sp->abm | sp->bbm)); - if (lvl == 0) { - nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(sp->ix), HALLOC_EXTRA); - hp = nhp; - *hp++ = (sp->ix == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY - : MAP_HEADER_HAMT_HEAD_BITMAP(sp->abm | sp->bbm)); - *hp++ = size; - } else { - nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(sp->ix), HALLOC_EXTRA); - hp = nhp; - *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm); - } - memcpy(hp, sp->array, sp->ix * sizeof(Eterm)); - res = make_boxed(nhp); + switch (sp->mix) { + case 0: /* Nodes A and B contain the *EXACT* same sub-trees + => fall through and reuse nodeA */ + + case 1: /* Only unique A stuff => reuse nodeA */ + res = sp->nodeA; + break; + + case 2: /* Only unique B stuff => reuse nodeB */ + res = sp->nodeB; + break; + + case 3: /* We have a mix => must build new node */ + ASSERT(sp->ix == hashmap_bitcount(sp->abm | sp->bbm)); + if (ctx->lvl == 0) { + nhp = HAllocX(p, HAMT_HEAD_BITMAP_SZ(sp->ix), HALLOC_EXTRA); + hp = nhp; + *hp++ = (sp->ix == 16 ? MAP_HEADER_HAMT_HEAD_ARRAY + : MAP_HEADER_HAMT_HEAD_BITMAP(sp->abm | sp->bbm)); + *hp++ = ctx->size; + } else { + nhp = HAllocX(p, HAMT_NODE_BITMAP_SZ(sp->ix), HALLOC_EXTRA); + hp = nhp; + *hp++ = MAP_HEADER_HAMT_NODE_BITMAP(sp->abm | sp->bbm); + } + sys_memcpy(hp, sp->array, sp->ix * sizeof(Eterm)); + res = make_boxed(nhp); + break; + default: + erl_exit(ERTS_ABORT_EXIT, "strange mix %d\r\n", sp->mix); + } + } + + /* Done */ + +#ifdef DEBUG + { + Eterm *head = hashmap_val(res); + Uint size = head[1]; + Uint real_size = hashmap_subtree_size(res); + ASSERT(size == real_size); + } +#endif + + if (ctx != &local_ctx) { + ASSERT(ctx->trap_bin != THE_NON_VALUE); + ASSERT(p->flags & F_DISABLE_GC); + erts_set_gc_state(p, 1); + } + else { + ASSERT(ctx->trap_bin == THE_NON_VALUE); + ASSERT(!(p->flags & F_DISABLE_GC)); } PSTACK_DESTROY(s); UnUseTmpHeap(2,p); + BUMP_REDS(p, (initial_reds - reds) / MAP_MERGE_LOOP_FACTOR); return res; + +trap: /* Yield */ + + if (ctx == &local_ctx) { + Binary* ctx_b = erts_create_magic_binary(sizeof(HashmapMergeContext), + hashmap_merge_ctx_destructor); + ctx = ERTS_MAGIC_BIN_DATA(ctx_b); + sys_memcpy(ctx, &local_ctx, sizeof(HashmapMergeContext)); + hp = HAlloc(p, PROC_BIN_SIZE); + ASSERT(ctx->trap_bin == THE_NON_VALUE); + ctx->trap_bin = erts_mk_magic_binary_term(&hp, &MSO(p), ctx_b); + + erts_set_gc_state(p, 0); + } + else { + ASSERT(ctx->trap_bin != THE_NON_VALUE); + ASSERT(p->flags & F_DISABLE_GC); + } + + PSTACK_SAVE(s, &ctx->pstack); + + BUMP_ALL_REDS(p); + ERTS_BIF_PREP_TRAP1(trap_ret, &hashmap_merge_trap_export, + p, ctx->trap_bin); + UnUseTmpHeap(2,p); + return trap_ret; } +static Uint hashmap_subtree_size(Eterm node) { + DECLARE_WSTACK(stack); + Uint size = 0; + + hashmap_iterator_init(&stack, node, 0); + while (hashmap_iterator_next(&stack)) { + size++; + } + DESTROY_WSTACK(stack); + return size; +} + + static int hash_cmp(Uint32 ha, Uint32 hb) { int i; @@ -1756,10 +1876,11 @@ void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) { sz = 16; break; case HAMT_SUBTAG_HEAD_BITMAP: - sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + case HAMT_SUBTAG_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); break; default: - erl_exit(1, "bad header"); + erl_exit(ERTS_ABORT_EXIT, "bad header"); } WSTACK_PUSH3((*s), (UWord)THE_NON_VALUE, /* end marker */ @@ -1796,7 +1917,7 @@ Eterm* hashmap_iterator_next(ErtsWStack* s) { ASSERT(sz < 17); break; default: - erl_exit(1, "bad header"); + erl_exit(ERTS_ABORT_EXIT, "bad header"); } idx++; @@ -2504,6 +2625,9 @@ int erts_validate_and_sort_flatmap(flatmap_t* mp) return 1; } +#if 0 /* Can't get myself to remove this beautiful piece of code + for probabilistic overestimation of nr of nodes in a hashmap */ + /* Really rough estimate of sqrt(x) * Guaranteed not to be less than sqrt(x) */ @@ -2525,7 +2649,10 @@ static int int_sqrt_ceiling(Uint x) } } -Uint hashmap_over_estimated_heap_size(Uint k) +/* May not be enough if hashing is broken (not uniform) + * or if hell freezes over. + */ +Uint hashmap_overestimated_node_count(Uint k) { /* k is nr of key-value pairs. N(k) is expected nr of nodes in hamt. @@ -2539,12 +2666,9 @@ Uint hashmap_over_estimated_heap_size(Uint k) by 15 std.devs above the average, which gives a probability for overrun less than 1.0e-49 (same magnitude as a git SHA1 collision). */ - Uint max_nodes = 2*k/5 + (15/3)*int_sqrt_ceiling(k); - return (k*2 + /* leaf cons cells */ - k + /* leaf list terms */ - max_nodes*2); /* headers + parent boxed terms */ + return 2*k/5 + 1 + (15/3)*int_sqrt_ceiling(k); } - +#endif BIF_RETTYPE erts_debug_map_info_1(BIF_ALIST_1) { if (is_hashmap(BIF_ARG_1)) { diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index 2cc6768bfc..b5941c5c9a 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -91,7 +91,6 @@ Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value, Uint *upsz, struct ErtsEStack_ *sp); int erts_validate_and_sort_flatmap(flatmap_t* map); -Uint hashmap_over_estimated_heap_size(Uint n); void hashmap_iterator_init(struct ErtsWStack_* s, Eterm node, int reverse); Eterm* hashmap_iterator_next(struct ErtsWStack_* s); Eterm* hashmap_iterator_prev(struct ErtsWStack_* s); @@ -191,5 +190,18 @@ typedef struct hashmap_head_s { #define hashmap_index(hash) (((Uint32)hash) & 0xf) +/* hashmap heap size: + [one cons cell + one list term in parent node] per key + [one header + one boxed term in parent node] per inner node + [one header + one size word] for root node +*/ +#define HASHMAP_HEAP_SIZE(KEYS,NODES) ((KEYS)*3 + (NODES)*2) +#ifdef DEBUG +# define HASHMAP_ESTIMATED_NODE_COUNT(KEYS) (KEYS) +#else +# define HASHMAP_ESTIMATED_NODE_COUNT(KEYS) (2*(KEYS)/5) +#endif +#define HASHMAP_ESTIMATED_HEAP_SIZE(KEYS) \ + HASHMAP_HEAP_SIZE(KEYS,HASHMAP_ESTIMATED_NODE_COUNT(KEYS)) #endif diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index ccfc2e6458..f806d2c498 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -93,9 +93,6 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, #endif ErlHeapFragment* nbp; - /* ToDo: Make use of 'used_size' to avoid realloc - when shrinking just a few words */ - #ifdef DEBUG { Uint off_sz = size < bp->used_size ? size : bp->used_size; @@ -110,8 +107,10 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, } #endif - if (size == bp->used_size) + if (size >= (bp->used_size - bp->used_size / 16)) { + bp->used_size = size; return bp; + } #ifdef HARD_DEBUG dbg_brefs = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(Eterm *)*brefs_size); @@ -237,8 +236,7 @@ erts_msg_distext2heap(Process *pp, Eterm msg; Uint tok_sz = 0; Eterm *hp = NULL; - Eterm *hp_end = NULL; - ErlOffHeap *ohp; + ErtsHeapFactory factory; Sint sz; *bpp = NULL; @@ -250,36 +248,26 @@ erts_msg_distext2heap(Process *pp, tok_sz = heap_frag->used_size; sz += tok_sz; } - if (pp) + if (pp) { + ErlOffHeap *ohp; hp = erts_alloc_message_heap(sz, bpp, &ohp, pp, plcksp); + } else { *bpp = new_message_buffer(sz); hp = (*bpp)->mem; - ohp = &(*bpp)->off_heap; } - hp_end = hp + sz; - msg = erts_decode_dist_ext(&hp, ohp, dist_extp); + erts_factory_message_init(&factory, pp, hp, *bpp); + msg = erts_decode_dist_ext(&factory, dist_extp); if (is_non_value(msg)) goto decode_error; if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); - *tokenp = copy_struct(*tokenp, tok_sz, &hp, ohp); + hp = erts_produce_heap(&factory, tok_sz, 0); + *tokenp = copy_struct(*tokenp, tok_sz, &hp, factory.off_heap); erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); - if (hp_end != hp) { - if (!(*bpp)) { - HRelease(pp, hp_end, hp); - } - else { - Uint final_size = hp - &(*bpp)->mem[0]; - Eterm brefs[2] = {msg, *tokenp}; - ASSERT(sz - (hp_end - hp) == final_size); - *bpp = erts_resize_message_buffer(*bpp, final_size, &brefs[0], 2); - msg = brefs[0]; - *tokenp = brefs[1]; - } - } + erts_factory_close(&factory); return msg; decode_error: @@ -288,13 +276,7 @@ erts_msg_distext2heap(Process *pp, erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); - if (*bpp) { - free_message_buffer(*bpp); - *bpp = NULL; - } - else if (hp) { - HRelease(pp, hp_end, hp); - } + *bpp = NULL; return THE_NON_VALUE; } @@ -851,10 +833,11 @@ erts_msg_attached_data_size_aux(ErlMessage *msg) } void -erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *msg) +erts_move_msg_attached_data_to_heap(ErtsHeapFactory* factory, + ErlMessage *msg) { if (is_value(ERL_MESSAGE_TERM(msg))) - erts_move_msg_mbuf_to_heap(hpp, ohp, msg); + erts_move_msg_mbuf_to_heap(&factory->hp, factory->off_heap, msg); else if (msg->data.dist_ext) { ASSERT(msg->data.dist_ext->heap_size >= 0); if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { @@ -862,12 +845,11 @@ erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *ms heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg), heap_frag->used_size, - hpp, - ohp); + &factory->hp, + factory->off_heap); erts_cleanup_offheap(&heap_frag->off_heap); } - ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(hpp, - ohp, + ERL_MESSAGE_TERM(msg) = erts_decode_dist_ext(factory, msg->data.dist_ext); erts_free_dist_ext_copy(msg->data.dist_ext); msg->data.dist_ext = NULL; @@ -1134,15 +1116,281 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, } } +void erts_factory_proc_init(ErtsHeapFactory* factory, + Process* p) +{ + erts_factory_proc_prealloc_init(factory, p, HEAP_LIMIT(p) - HEAP_TOP(p)); +} + +void erts_factory_proc_prealloc_init(ErtsHeapFactory* factory, + Process* p, + Sint size) +{ + factory->mode = FACTORY_HALLOC; + factory->p = p; + factory->hp_start = HAlloc(p, size); + factory->hp = factory->hp_start; + factory->hp_end = factory->hp_start + size; + factory->off_heap = &p->off_heap; + factory->off_heap_saved.first = p->off_heap.first; + factory->off_heap_saved.overhead = p->off_heap.overhead; + factory->heap_frags_saved = p->mbuf; + factory->heap_frags = NULL; /* not used */ + factory->alloc_type = 0; /* not used */ +} + +void erts_factory_message_init(ErtsHeapFactory* factory, + Process* rp, + Eterm* hp, + ErlHeapFragment* bp) +{ + if (bp) { + factory->mode = FACTORY_HEAP_FRAGS; + factory->p = NULL; + factory->hp_start = bp->mem; + factory->hp = hp ? hp : bp->mem; + factory->hp_end = bp->mem + bp->alloc_size; + factory->off_heap = &bp->off_heap; + factory->heap_frags = bp; + factory->heap_frags_saved = bp; + factory->alloc_type = ERTS_ALC_T_HEAP_FRAG; + ASSERT(!bp->next); + } + else { + factory->mode = FACTORY_HALLOC; + factory->p = rp; + factory->hp_start = hp; + factory->hp = hp; + factory->hp_end = HEAP_TOP(rp); + factory->off_heap = &rp->off_heap; + factory->heap_frags_saved = rp->mbuf; + factory->heap_frags = NULL; /* not used */ + factory->alloc_type = 0; /* not used */ + } + factory->off_heap_saved.first = factory->off_heap->first; + factory->off_heap_saved.overhead = factory->off_heap->overhead; + + ASSERT(factory->hp >= factory->hp_start && factory->hp <= factory->hp_end); +} + +void erts_factory_static_init(ErtsHeapFactory* factory, + Eterm* hp, + Uint size, + ErlOffHeap* off_heap) +{ + factory->mode = FACTORY_STATIC; + factory->hp_start = hp; + factory->hp = hp; + factory->hp_end = hp + size; + factory->off_heap = off_heap; + factory->off_heap_saved.first = factory->off_heap->first; + factory->off_heap_saved.overhead = factory->off_heap->overhead; +} + +/* When we know the term is an immediate and need no heap. +*/ +void erts_factory_dummy_init(ErtsHeapFactory* factory) +{ + factory->mode = FACTORY_CLOSED; +} + +static void reserve_heap(ErtsHeapFactory*, Uint need, Uint xtra); + Eterm* erts_produce_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) { Eterm* res; - if (factory->p) { - res = HAllocX(factory->p, need, xtra); - } else { - res = factory->hp; - factory->hp += need; + + ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED); + if (factory->hp + need > factory->hp_end) { + reserve_heap(factory, need, xtra); } + res = factory->hp; + factory->hp += need; return res; } +Eterm* erts_reserve_heap(ErtsHeapFactory* factory, Uint need) +{ + ASSERT((unsigned int)factory->mode > (unsigned int)FACTORY_CLOSED); + if (factory->hp + need > factory->hp_end) { + reserve_heap(factory, need, 200); + } + return factory->hp; +} + +static void reserve_heap(ErtsHeapFactory* factory, Uint need, Uint xtra) +{ + ErlHeapFragment* bp; + + switch (factory->mode) { + case FACTORY_HALLOC: + HRelease(factory->p, factory->hp_end, factory->hp); + factory->hp = HAllocX(factory->p, need, xtra); + factory->hp_end = factory->hp + need; + return; + + case FACTORY_HEAP_FRAGS: + bp = factory->heap_frags; + + if (bp) { + ASSERT(factory->hp > bp->mem); + ASSERT(factory->hp <= factory->hp_end); + ASSERT(factory->hp_end == bp->mem + bp->alloc_size); + + bp->used_size = factory->hp - bp->mem; + } + bp = (ErlHeapFragment*) ERTS_HEAP_ALLOC(factory->alloc_type, + ERTS_HEAP_FRAG_SIZE(need+xtra)); + bp->next = factory->heap_frags; + factory->heap_frags = bp; + bp->alloc_size = need + xtra; + bp->used_size = need; + bp->off_heap.first = NULL; + bp->off_heap.overhead = 0; + + factory->hp = bp->mem; + factory->hp_end = bp->mem + bp->alloc_size; + return; + + case FACTORY_STATIC: + case FACTORY_CLOSED: + default: + ASSERT(!"Invalid factory mode"); + } +} + +void erts_factory_close(ErtsHeapFactory* factory) +{ + ErlHeapFragment* bp; + + switch (factory->mode) { + case FACTORY_HALLOC: + HRelease(factory->p, factory->hp_end, factory->hp); + break; + + case FACTORY_HEAP_FRAGS: + bp = factory->heap_frags; + + if (bp) { + ASSERT(factory->hp >= bp->mem); + ASSERT(factory->hp <= factory->hp_end); + ASSERT(factory->hp_end == bp->mem + bp->alloc_size); + + bp->used_size = factory->hp - bp->mem; + } + break; + case FACTORY_STATIC: break; + case FACTORY_CLOSED: break; + default: + ASSERT(!"Invalid factory mode"); + } + factory->mode = FACTORY_CLOSED; +} + +void erts_factory_trim_and_close(ErtsHeapFactory* factory, + Eterm *brefs, Uint brefs_size) +{ + if (factory->mode == FACTORY_HEAP_FRAGS) { + ErlHeapFragment* bp = factory->heap_frags; + if (bp->next == NULL) { + Uint used_sz = factory->hp - bp->mem; + ASSERT(used_sz <= bp->alloc_size); + factory->heap_frags = erts_resize_message_buffer(bp, used_sz, + brefs, brefs_size); + factory->mode = FACTORY_CLOSED; + return; + } + /*else we don't trim multi fragmented messages for now */ + } + erts_factory_close(factory); +} + +void erts_factory_undo(ErtsHeapFactory* factory) +{ + ErlHeapFragment* bp; + struct erl_off_heap_header *hdr, **hdr_nextp; + + switch (factory->mode) { + case FACTORY_HALLOC: + case FACTORY_STATIC: + /* Cleanup off-heap + */ + hdr_nextp = NULL; + for (hdr = factory->off_heap->first; + hdr != factory->off_heap_saved.first; + hdr = hdr->next) { + + hdr_nextp = &hdr->next; + } + + if (hdr_nextp != NULL) { + *hdr_nextp = NULL; + erts_cleanup_offheap(factory->off_heap); + factory->off_heap->first = factory->off_heap_saved.first; + factory->off_heap->overhead = factory->off_heap_saved.overhead; + } + + if (factory->mode == FACTORY_HALLOC) { + /* Free heap frags + */ + bp = factory->p->mbuf; + if (bp != factory->heap_frags_saved) { + do { + ErlHeapFragment *next_bp = bp->next; + ASSERT(bp->off_heap.first == NULL); + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->alloc_size)); + bp = next_bp; + } while (bp != factory->heap_frags_saved); + + factory->p->mbuf = bp; + } + + /* Rollback heap top + */ + if (factory->heap_frags_saved == NULL) { /* No heap frags when we started */ + ASSERT(factory->hp_start >= HEAP_START(factory->p)); + ASSERT(factory->hp_start <= HEAP_LIMIT(factory->p)); + + HEAP_TOP(factory->p) = factory->hp_start; + } + else { + ASSERT(factory->heap_frags_saved == factory->p->mbuf); + if (factory->hp_start == factory->heap_frags_saved->mem) { + factory->p->mbuf = factory->p->mbuf->next; + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, factory->heap_frags_saved, + ERTS_HEAP_FRAG_SIZE(factory->heap_frags_saved->alloc_size)); + } + else if (factory->hp_start != factory->hp_end) { + unsigned remains = factory->hp_start - factory->heap_frags_saved->mem; + ASSERT(remains > 0 && remains < factory->heap_frags_saved->used_size); + factory->heap_frags_saved->used_size = remains; + } + } + } + break; + + case FACTORY_HEAP_FRAGS: + bp = factory->heap_frags; + do { + ErlHeapFragment* next_bp = bp->next; + + erts_cleanup_offheap(&bp->off_heap); + ERTS_HEAP_FREE(factory->alloc_type, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->size)); + bp = next_bp; + }while (bp != NULL); + break; + + case FACTORY_CLOSED: break; + default: + ASSERT(!"Invalid factory mode"); + } + factory->mode = FACTORY_CLOSED; +#ifdef DEBUG + factory->p = NULL; + factory->hp = NULL; + factory->heap_frags = NULL; +#endif +} + diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 1e1dafee90..705ba5e506 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -51,6 +51,45 @@ typedef struct erl_off_heap { (OHP)->first = NULL; \ (OHP)->overhead = 0; \ } while (0) + +typedef struct { + enum { + FACTORY_CLOSED = 0, + FACTORY_HALLOC, + FACTORY_HEAP_FRAGS, + FACTORY_STATIC + } mode; + Process* p; + Eterm* hp_start; + Eterm* hp; + Eterm* hp_end; + struct erl_heap_fragment* heap_frags; + struct erl_heap_fragment* heap_frags_saved; + ErlOffHeap* off_heap; + ErlOffHeap off_heap_saved; + Uint32 alloc_type; +} ErtsHeapFactory; + +void erts_factory_proc_init(ErtsHeapFactory*, Process*); +void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size); +void erts_factory_message_init(ErtsHeapFactory*, Process*, Eterm* hp, struct erl_heap_fragment*); +void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*); +void erts_factory_dummy_init(ErtsHeapFactory*); + +Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); +Eterm* erts_reserve_heap(ErtsHeapFactory*, Uint need); +void erts_factory_close(ErtsHeapFactory*); +void erts_factory_trim_and_close(ErtsHeapFactory*,Eterm *brefs, Uint brefs_size); +void erts_factory_undo(ErtsHeapFactory*); + +#ifdef CHECK_FOR_HOLES +# define ERTS_FACTORY_HOLE_CHECK(f) do { \ + /*if ((f)->p) erts_check_for_holes((f)->p);*/ \ + } while (0) +#else +# define ERTS_FACTORY_HOLE_CHECK(p) +#endif + #include "external.h" #include "erl_process.h" @@ -68,21 +107,6 @@ struct erl_heap_fragment { Eterm mem[1]; /* Data */ }; -typedef struct { - Process* p; - Eterm* hp; -} ErtsHeapFactory; - -Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra); -#ifdef CHECK_FOR_HOLES -# define ERTS_FACTORY_HOLE_CHECK(f) do { \ - if ((f)->p) erts_check_for_holes((f)->p); \ - } while (0) -#else -# define ERTS_FACTORY_HOLE_CHECK(p) -#endif - - typedef struct erl_mesg { struct erl_mesg* next; /* Next message */ union { @@ -139,7 +163,7 @@ typedef struct { *(p)->msg.last = (mp); \ (p)->msg.last = &(mp)->next; \ (p)->msg.len++; \ -} while(0) +} while (0) #ifdef ERTS_SMP @@ -212,17 +236,23 @@ do { \ do { \ if ((M)->data.attached) { \ Uint need__ = erts_msg_attached_data_size((M)); \ + { SWPO ; } \ if ((ST) - (HT) >= need__) { \ - Uint *htop__ = (HT); \ - erts_move_msg_attached_data_to_heap(&htop__, &MSO((P)), (M));\ - ASSERT(htop__ - (HT) <= need__); \ - (HT) = htop__; \ + ErtsHeapFactory factory__; \ + erts_factory_proc_prealloc_init(&factory__, (P), need__); \ + erts_move_msg_attached_data_to_heap(&factory__, (M)); \ + erts_factory_close(&factory__); \ + if ((P)->mbuf != NULL) { \ + /* Heap was exhausted by messages. This is a rare case */ \ + /* that can currently (OTP 18) only happen if hamts are */ \ + /* far exceeding the estimated heap size. Do GC. */ \ + (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ + } \ } \ else { \ - { SWPO ; } \ (FC) -= erts_garbage_collect((P), 0, NULL, 0); \ - { SWPI ; } \ } \ + { SWPI ; } \ ASSERT(!(M)->data.attached); \ } \ } while (0) @@ -266,23 +296,21 @@ void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); void erts_move_msg_mbuf_to_heap(Eterm**, ErlOffHeap*, ErlMessage *); Uint erts_msg_attached_data_size_aux(ErlMessage *msg); -void erts_move_msg_attached_data_to_heap(Eterm **, ErlOffHeap *, ErlMessage *); - +void erts_move_msg_attached_data_to_heap(ErtsHeapFactory*, ErlMessage *); Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **, Eterm *, ErtsDistExternal *); void erts_cleanup_offheap(ErlOffHeap *offheap); -ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg); +ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*); ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg) +ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp) { - const ErlHeapFragment *bp; Uint sz = 0; - for (bp = msg->data.heap_frag; bp!=NULL; bp=bp->next) { + for ( ; bp!=NULL; bp=bp->next) { sz += bp->used_size; } return sz; @@ -292,7 +320,7 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg) { ASSERT(msg->data.attached); if (is_value(ERL_MESSAGE_TERM(msg))) - return erts_msg_used_frag_sz(msg); + return erts_used_frag_sz(msg->data.heap_frag); else if (msg->data.dist_ext->heap_size < 0) return erts_msg_attached_data_size_aux(msg); else { diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 426a00304e..27f6c6f00d 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -445,7 +445,7 @@ int enif_is_list(ErlNifEnv* env, ERL_NIF_TERM term) int enif_is_exception(ErlNifEnv* env, ERL_NIF_TERM term) { - return term == THE_NON_VALUE; + return env->exception_thrown && term == THE_NON_VALUE; } int enif_is_number(ErlNifEnv* env, ERL_NIF_TERM term) @@ -737,12 +737,21 @@ Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, Eterm enif_make_badarg(ErlNifEnv* env) { + return enif_raise_exception(env, am_badarg); +} + +Eterm enif_raise_exception(ErlNifEnv* env, ERL_NIF_TERM reason) +{ env->exception_thrown = 1; - BIF_ERROR(env->proc, BADARG); + env->proc->fvalue = reason; + BIF_ERROR(env->proc, EXC_ERROR); } -int enif_has_pending_exception(ErlNifEnv* env) +int enif_has_pending_exception(ErlNifEnv* env, ERL_NIF_TERM* reason) { + if (env->exception_thrown && reason != NULL) { + *reason = env->proc->fvalue; + } return env->exception_thrown; } @@ -1199,7 +1208,11 @@ typedef struct enif_resource_t struct enif_resource_type_t* type; #ifdef DEBUG erts_refc_t nif_refc; +# ifdef ARCH_32 + byte align__[4]; +# endif #endif + char data[1]; }ErlNifResource; @@ -1375,7 +1388,7 @@ static void rollback_opened_resource_types(void) static void nif_resource_dtor(Binary* bin) { - ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(bin); + ErlNifResource* resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ErlNifResourceType* type = resource->type; ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); @@ -1396,8 +1409,10 @@ static void nif_resource_dtor(Binary* bin) void* enif_alloc_resource(ErlNifResourceType* type, size_t size) { - Binary* bin = erts_create_magic_binary(SIZEOF_ErlNifResource(size), &nif_resource_dtor); - ErlNifResource* resource = ERTS_MAGIC_BIN_DATA(bin); + Binary* bin = erts_create_magic_binary_x(SIZEOF_ErlNifResource(size), + &nif_resource_dtor, + 1); /* unaligned */ + ErlNifResource* resource = ERTS_MAGIC_BIN_UNALIGNED_DATA(bin); ASSERT(type->owner && type->next && type->prev); /* not allowed in load/upgrade */ resource->type = type; @@ -1412,7 +1427,7 @@ void* enif_alloc_resource(ErlNifResourceType* type, size_t size) void enif_release_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); #ifdef DEBUG @@ -1426,7 +1441,7 @@ void enif_release_resource(void* obj) void enif_keep_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); #ifdef DEBUG @@ -1438,7 +1453,7 @@ void enif_keep_resource(void* obj) ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource); Eterm* hp = alloc_heap(env,PROC_BIN_SIZE); return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); } @@ -1467,7 +1482,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ return 0; / * Or should we allow "resource binaries" as handles? * / }*/ mbin = pb->val; - resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(mbin); + resource = (ErlNifResource*) ERTS_MAGIC_BIN_UNALIGNED_DATA(mbin); if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor || resource->type != type) { return 0; @@ -1479,8 +1494,8 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ size_t enif_sizeof_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); - Binary* bin = &ERTS_MAGIC_BIN_FROM_DATA(resource)->binary; - return ERTS_MAGIC_BIN_DATA_SIZE(bin) - offsetof(ErlNifResource,data); + Binary* bin = &ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(resource)->binary; + return ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(bin) - offsetof(ErlNifResource,data); } @@ -1538,12 +1553,13 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent) * NIF exports need a few more items than the Export struct provides, * including the erl_module_nif* and a NIF function pointer, so the * NifExport below adds those. The Export member must be first in the - * struct. The saved_mfa, saved_argc, nif_level, alloced_argv_sz and argv - * members are used to track the MFA and arguments of the top NIF in case a - * chain of one or more enif_schedule_nif() calls results in an exception, - * since in that case the original MFA and registers have to be restored - * before returning to Erlang to ensure stacktrace information associated - * with the exception is correct. + * struct. The saved_mfa, exception_thrown, saved_argc, rootset_extra, and + * rootset members are used to track the MFA, any pending exception, and + * arguments of the top NIF in case a chain of one or more + * enif_schedule_nif() calls results in an exception, since in that case + * the original MFA and registers have to be restored before returning to + * Erlang to ensure stacktrace information associated with the exception is + * correct. */ typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]); @@ -1552,25 +1568,28 @@ typedef struct { struct erl_module_nif* m; NativeFunPtr fp; Eterm saved_mfa[3]; + int exception_thrown; int saved_argc; - int alloced_argv_sz; - Eterm argv[1]; + int rootset_extra; + Eterm rootset[1]; } NifExport; /* * If a process has saved arguments, they need to be part of the GC * rootset. The function below is called from setup_rootset() in - * erl_gc.c. This function is declared in erl_process.h. + * erl_gc.c. This function is declared in erl_process.h. Any exception term + * saved in the NifExport is also made part of the GC rootset here; it + * always resides in rootset[0]. */ int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj) { NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - int gc = (ep && ep->saved_argc > 0); + int gc = ep && (ep->saved_argc > 0 || ep->rootset[0] != NIL); if (gc) { - *objv = ep->argv; - *nobj = ep->saved_argc; + *objv = ep->rootset; + *nobj = 1 + ep->saved_argc; } return gc; } @@ -1582,14 +1601,14 @@ static NifExport* allocate_nif_sched_data(Process* proc, int argc) { NifExport* ep; - size_t argv_extra, total; + size_t total; int i; - argv_extra = argc > 1 ? sizeof(Eterm)*(argc-1) : 0; - total = sizeof(NifExport) + argv_extra; + total = sizeof(NifExport) + argc*sizeof(Eterm); ep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, total); sys_memset((void*) ep, 0, total); - ep->alloced_argv_sz = argc; + ep->rootset_extra = argc; + ep->rootset[0] = NIL; for (i=0; i<ERTS_NUM_CODE_IX; i++) { ep->exp.addressv[i] = &ep->exp.code[3]; } @@ -1630,15 +1649,22 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); if (!ep) ep = allocate_nif_sched_data(proc, argc); - else if (need_save && ep->alloced_argv_sz < argc) { + else if (need_save && ep->rootset_extra < argc) { NifExport* new_ep = allocate_nif_sched_data(proc, argc); destroy_nif_export(ep); ep = new_ep; } + if (env->exception_thrown) { + ep->exception_thrown = 1; + ep->rootset[0] = env->proc->fvalue; + } else { + ep->exception_thrown = 0; + ep->rootset[0] = NIL; + } ERTS_VBUMP_ALL_REDS(proc); for (i = 0; i < argc; i++) { if (need_save) - ep->argv[i] = reg[i]; + ep->rootset[i+1] = reg[i]; reg[i] = (Eterm) argv[i]; } if (need_save) { @@ -1674,7 +1700,7 @@ restore_nif_mfa(Process* proc, NifExport* ep, int exception) proc->current[2] = ep->saved_mfa[2]; if (exception) for (i = 0; i < ep->saved_argc; i++) - reg[i] = ep->argv[i]; + reg[i] = ep->rootset[i+1]; ep->saved_argc = 0; ep->saved_mfa[0] = THE_NON_VALUE; } @@ -1699,6 +1725,7 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ASSERT(!ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data)); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); + ASSERT(!ep->exception_thrown); if (ep->fp) restore_nif_mfa(proc, ep, 0); return argv[0]; @@ -1716,9 +1743,10 @@ dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ASSERT(!ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data)); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); + ASSERT(ep->exception_thrown); if (ep->fp) restore_nif_mfa(proc, ep, 1); - return enif_make_badarg(env); + return enif_raise_exception(env, ep->rootset[0]); } /* @@ -1843,6 +1871,7 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) NifExport* ep; ERL_NIF_TERM result; + ASSERT(!env->exception_thrown); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); ep->fp = NULL; @@ -1855,7 +1884,7 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) * which case we need to restore the original NIF MFA. */ if (ep->fp == NULL) - restore_nif_mfa(proc, ep, is_non_value(result) && proc->freason != TRAP); + restore_nif_mfa(proc, ep, env->exception_thrown); return result; } @@ -2024,8 +2053,8 @@ int enif_map_iterator_create(ErlNifEnv *env, size_t offset; switch (entry) { - case ERL_NIF_MAP_ITERATOR_HEAD: offset = 0; break; - case ERL_NIF_MAP_ITERATOR_TAIL: offset = flatmap_get_size(mp) - 1; break; + case ERL_NIF_MAP_ITERATOR_FIRST: offset = 0; break; + case ERL_NIF_MAP_ITERATOR_LAST: offset = flatmap_get_size(mp) - 1; break; default: goto error; } @@ -2048,12 +2077,12 @@ int enif_map_iterator_create(ErlNifEnv *env, WSTACK_INIT(iter->u.hash.wstack, ERTS_ALC_T_NIF); switch (entry) { - case ERL_NIF_MAP_ITERATOR_HEAD: + case ERL_NIF_MAP_ITERATOR_FIRST: iter->idx = 1; hashmap_iterator_init(&iter->u.hash.wstack->ws, map, 0); iter->u.hash.kv = hashmap_iterator_next(&iter->u.hash.wstack->ws); break; - case ERL_NIF_MAP_ITERATOR_TAIL: + case ERL_NIF_MAP_ITERATOR_LAST: iter->idx = hashmap_size(map); hashmap_iterator_init(&iter->u.hash.wstack->ws, map, 1); iter->u.hash.kv = hashmap_iterator_prev(&iter->u.hash.wstack->ws); @@ -2689,6 +2718,8 @@ erts_unload_nif(struct erl_module_nif* lib) void erl_nif_init() { + ERTS_CT_ASSERT((offsetof(ErlNifResource,data) % 8) == ERTS_MAGIC_BIN_BYTES_TO_ALIGN); + resource_type_list.next = &resource_type_list; resource_type_list.prev = &resource_type_list; resource_type_list.dtor = NULL; diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index c4fdfd4187..af806736fd 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -218,8 +218,12 @@ typedef struct /* All fields all internal and may change */ } ErlNifMapIterator; typedef enum { - ERL_NIF_MAP_ITERATOR_HEAD = 1, - ERL_NIF_MAP_ITERATOR_TAIL = 2 + ERL_NIF_MAP_ITERATOR_FIRST = 1, + ERL_NIF_MAP_ITERATOR_LAST = 2, + + /* deprecated synonyms (undocumented in 17 and 18-rc) */ + ERL_NIF_MAP_ITERATOR_HEAD = ERL_NIF_MAP_ITERATOR_FIRST, + ERL_NIF_MAP_ITERATOR_TAIL = ERL_NIF_MAP_ITERATOR_LAST } ErlNifMapIteratorEntry; #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index bdcbb32c46..e38a016958 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -156,7 +156,8 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_next, (ErlNifEnv *env, ErlNifMapIte ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_prev, (ErlNifEnv *env, ErlNifMapIterator *iter)); ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMapIterator *iter, ERL_NIF_TERM *key, ERL_NIF_TERM *value)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[])); -ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env)); +ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env, ERL_NIF_TERM* reason)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_raise_exception, (ErlNifEnv *env, ERL_NIF_TERM reason)); /* ** ADD NEW ENTRIES HERE (before this comment) !!! @@ -307,6 +308,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*)); # define enif_map_iterator_get_pair ERL_NIF_API_FUNC_MACRO(enif_map_iterator_get_pair) # define enif_schedule_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_nif) # define enif_has_pending_exception ERL_NIF_API_FUNC_MACRO(enif_has_pending_exception) +# define enif_raise_exception ERL_NIF_API_FUNC_MACRO(enif_raise_exception) /* ** ADD NEW ENTRIES HERE (before this comment) diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index c2acaea0bc..0950d7e7ef 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -758,7 +758,7 @@ erts_set_this_node(Eterm sysname, Uint creation) erts_this_node->sysname = sysname; erts_this_node->creation = creation; erts_this_node_sysname = erts_this_node_sysname_BUFFER; - erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname), + erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), "%T", sysname); (void) hash_put(&erts_node_table, (void *) erts_this_node); @@ -864,7 +864,7 @@ void erts_init_node_tables(int dd_sec) erts_this_node->creation = 0; erts_this_node->dist_entry = erts_this_dist_entry; erts_this_node_sysname = erts_this_node_sysname_BUFFER; - erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname), + erts_snprintf(erts_this_node_sysname, sizeof(erts_this_node_sysname_BUFFER), "%T", erts_this_node->sysname); (void) hash_put(&erts_node_table, (void *) erts_this_node); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 410f0214b9..43e84a1be1 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -2988,22 +2988,29 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) } if (aux_work) { - flgs = erts_smp_atomic32_read_acqb(&ssi->flags); - current_time = erts_get_monotonic_time(esdp); - if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_bump_timers(esdp->timer_wheel, current_time); } - erts_bump_timers(esdp->timer_wheel, current_time); } } else { ErtsMonotonicTime timeout_time; - timeout_time = erts_check_next_timeout_time(esdp); - current_time = erts_get_monotonic_time(esdp); - if (current_time >= timeout_time) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { + int do_timeout = 0; + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + timeout_time = erts_check_next_timeout_time(esdp); + current_time = erts_get_monotonic_time(esdp); + do_timeout = (current_time >= timeout_time); + } else + timeout_time = ERTS_MONOTONIC_TIME_MAX; + if (do_timeout) { + if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); } @@ -3027,23 +3034,28 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) int res; ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); ASSERT(flgs & ERTS_SSI_FLG_WAITING); - current_time = erts_get_monotonic_time(esdp); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); do { Sint64 timeout; if (current_time >= timeout_time) break; - timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time - - current_time - - 1) + 1; + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time + - current_time + - 1) + 1; + } else + timeout = -1; res = erts_tse_twait(ssi->event, timeout); - current_time = erts_get_monotonic_time(esdp); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); } while (res == EINTR); } } if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) erts_thr_progress_finalize_wait(esdp); } - if (current_time >= timeout_time) + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time) erts_bump_timers(esdp->timer_wheel, current_time); } @@ -3111,9 +3123,11 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) ASSERT(!erts_port_task_have_outstanding_io_tasks()); erl_sys_schedule(1); /* Might give us something to do */ - current_time = erts_get_monotonic_time(esdp); - if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) - erts_bump_timers(esdp->timer_wheel, current_time); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) + erts_bump_timers(esdp->timer_wheel, current_time); + } sys_aux_work: #ifndef ERTS_SMP @@ -3122,15 +3136,18 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work) { - if (!working) - sched_wall_time_change(esdp, working = 1); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (!working) + sched_wall_time_change(esdp, working = 1); #ifdef ERTS_SMP - if (!thr_prgr_active) - erts_thr_progress_active(esdp, thr_prgr_active = 1); + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); #endif + } aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); #ifdef ERTS_SMP - if (aux_work && erts_thr_progress_update(esdp)) + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && aux_work && + erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); #endif } @@ -3228,7 +3245,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erl_sys_schedule(0); - { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { ErtsMonotonicTime current_time = erts_get_monotonic_time(esdp); if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) erts_bump_timers(esdp->timer_wheel, current_time); @@ -6892,7 +6909,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) } } - (void) erts_get_monotonic_time(esdp); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + (void) erts_get_monotonic_time(esdp); erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); @@ -7008,7 +7026,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) & ERTS_RUNQ_FLGS_QMASK); aux_work = erts_atomic32_read_acqb(&ssi->aux_work); if (aux_work|qmask) { - if (!thr_prgr_active) { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); } @@ -7016,7 +7034,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); - if (aux_work && erts_thr_progress_update(esdp)) + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && aux_work && + erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); if (qmask) { erts_smp_runq_lock(esdp->run_queue); @@ -7026,32 +7045,40 @@ suspend_scheduler(ErtsSchedulerData *esdp) } if (aux_work) { - current_time = erts_get_monotonic_time(esdp); - if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_bump_timers(esdp->timer_wheel, current_time); } - erts_bump_timers(esdp->timer_wheel, current_time); } } else { ErtsMonotonicTime timeout_time; - timeout_time = erts_check_next_timeout_time(esdp); - current_time = erts_get_monotonic_time(esdp); - - if (current_time >= timeout_time) { + int do_timeout = 0; + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + timeout_time = erts_check_next_timeout_time(esdp); + current_time = erts_get_monotonic_time(esdp); + do_timeout = (current_time >= timeout_time); + } else + timeout_time = ERTS_MONOTONIC_TIME_MAX; + if (do_timeout) { if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); } } - else { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); + else { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(esdp); } - erts_thr_progress_prepare_wait(esdp); flgs = sched_spin_suspended(ssi, ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); if (flgs == (ERTS_SSI_FLG_SLEEPING @@ -7064,23 +7091,29 @@ suspend_scheduler(ErtsSchedulerData *esdp) | ERTS_SSI_FLG_SUSPENDED)) { int res; - current_time = erts_get_monotonic_time(esdp); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); do { Sint64 timeout; if (current_time >= timeout_time) break; - timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time - - current_time - - 1) + 1; + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time + - current_time + - 1) + 1; + } else + timeout = -1; res = erts_tse_twait(ssi->event, timeout); - current_time = erts_get_monotonic_time(esdp); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); } while (res == EINTR); } } - erts_thr_progress_finalize_wait(esdp); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + erts_thr_progress_finalize_wait(esdp); } - if (current_time >= timeout_time) + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time) erts_bump_timers(esdp->timer_wheel, current_time); } @@ -9298,13 +9331,15 @@ Process *schedule(Process *p, int calls) ERTS_SMP_CHK_NO_PROC_LOCKS; - if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS) - (void) erts_get_monotonic_time(esdp); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS) + (void) erts_get_monotonic_time(esdp); - if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { - erts_smp_runq_unlock(rq); - erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time); - erts_smp_runq_lock(rq); + if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + erts_smp_runq_unlock(rq); + erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time); + erts_smp_runq_lock(rq); + } } BM_STOP_TIMER(system); @@ -12552,38 +12587,6 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) { erts_print(to, to_arg, "%T", esdp->current_port->common.id); erts_print(to, to_arg, "\n"); - p = esdp->current_process; - erts_print(to, to_arg, "Current Process: "); - if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) { - flg = erts_smp_atomic32_read_dirty(&p->state); - erts_print(to, to_arg, "%T\n", p->common.id); - - erts_print(to, to_arg, "Current Process State: "); - erts_dump_process_state(to, to_arg, flg); - - erts_print(to, to_arg, "Current Process Internal State: "); - erts_dump_extended_process_state(to, to_arg, flg); - - erts_print(to, to_arg, "Current Process Program counter: %p (", p->i); - print_function_from_pc(to, to_arg, p->i); - erts_print(to, to_arg, ")\n"); - erts_print(to, to_arg, "Current Process CP: %p (", p->cp); - print_function_from_pc(to, to_arg, p->cp); - erts_print(to, to_arg, ")\n"); - - /* Getting this stacktrace can segfault if we are very very - unlucky if called while a process is being garbage collected. - Therefore we only call this on other schedulers if we either - have protection against segfaults, or we know that the process - is not garbage collecting. It *should* always be safe to call - on a process owned by us, even if it is currently being garbage - collected. - */ - erts_print(to, to_arg, "Current Process Limited Stack Trace:\n"); - erts_limited_stack_trace(to, to_arg, p); - } else - erts_print(to, to_arg, "\n"); - for (i = 0; i < ERTS_NO_PROC_PRIO_LEVELS; i++) { erts_print(to, to_arg, "Run Queue "); switch (i) { @@ -12670,6 +12673,40 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) { } } erts_print(to, to_arg, "\n"); + + /* This *MUST* to be the last information in scheduler block */ + p = esdp->current_process; + erts_print(to, to_arg, "Current Process: "); + if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) { + flg = erts_smp_atomic32_read_dirty(&p->state); + erts_print(to, to_arg, "%T\n", p->common.id); + + erts_print(to, to_arg, "Current Process State: "); + erts_dump_process_state(to, to_arg, flg); + + erts_print(to, to_arg, "Current Process Internal State: "); + erts_dump_extended_process_state(to, to_arg, flg); + + erts_print(to, to_arg, "Current Process Program counter: %p (", p->i); + print_function_from_pc(to, to_arg, p->i); + erts_print(to, to_arg, ")\n"); + erts_print(to, to_arg, "Current Process CP: %p (", p->cp); + print_function_from_pc(to, to_arg, p->cp); + erts_print(to, to_arg, ")\n"); + + /* Getting this stacktrace can segfault if we are very very + unlucky if called while a process is being garbage collected. + Therefore we only call this on other schedulers if we either + have protection against segfaults, or we know that the process + is not garbage collecting. It *should* always be safe to call + on a process owned by us, even if it is currently being garbage + collected. + */ + erts_print(to, to_arg, "Current Process Limited Stack Trace:\n"); + erts_limited_stack_trace(to, to_arg, p); + } else + erts_print(to, to_arg, "\n"); + } /* diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index fff267ff2a..b28bc498f6 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -1003,7 +1003,9 @@ erts_pid2proc_opt(Process *c_p, void erts_proc_lock_init(Process *p) { +#if ERTS_PROC_LOCK_OWN_IMPL || defined(ERTS_PROC_LOCK_DEBUG) int i; +#endif #if ERTS_PROC_LOCK_OWN_IMPL /* We always start with all locks locked */ #if ERTS_PROC_LOCK_ATOMIC_IMPL diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index bc04d7b78e..565528193e 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -128,10 +128,10 @@ FUNTY checked_##FUN(ARGTY x, const char *file, unsigned line) \ return _unchecked_##FUN(x); \ } -ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_taggable_pointer); +ET_DEFINE_CHECKED(Eterm,make_boxed,const Eterm*,_is_taggable_pointer); ET_DEFINE_CHECKED(int,is_boxed,Eterm,!is_header); ET_DEFINE_CHECKED(Eterm*,boxed_val,Wterm,_boxed_precond); -ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_taggable_pointer); +ET_DEFINE_CHECKED(Eterm,make_list,const Eterm*,_is_taggable_pointer); ET_DEFINE_CHECKED(int,is_not_list,Eterm,!is_header); ET_DEFINE_CHECKED(Eterm*,list_val,Wterm,_list_precond); ET_DEFINE_CHECKED(Uint,unsigned_val,Eterm,is_small); diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 602aab46dc..7b15b34da1 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -198,7 +198,7 @@ struct erl_node_; /* Declared in erl_node_tables.h */ #endif #define _is_aligned(x) (((Uint)(x) & 0x3) == 0) #define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED) -_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*) +_ET_DECLARE_CHECKED(Eterm,make_boxed,const Eterm*) #define make_boxed(x) _ET_APPLY(make_boxed,(x)) #if 1 #define _is_not_boxed(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_BOXED)) @@ -214,7 +214,7 @@ _ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm) /* cons cell ("list") access methods */ #define _unchecked_make_list(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST) -_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*) +_ET_DECLARE_CHECKED(Eterm,make_list,const Eterm*) #define make_list(x) _ET_APPLY(make_list,(x)) #if 1 #define _unchecked_is_not_list(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_LIST)) diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index fe48298155..0a69172980 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -94,9 +94,9 @@ static Uint is_external_string(Eterm obj, int* p_is_string); static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); struct B2TContext_t; -static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*, struct B2TContext_t*); +static byte* dec_term(ErtsDistExternal*, ErtsHeapFactory*, byte*, Eterm*, struct B2TContext_t*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); -static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); +static byte* dec_pid(ErtsDistExternal *, ErtsHeapFactory*, byte*, Eterm*); static Sint decoded_size(byte *ep, byte* endp, int internal_tags, struct B2TContext_t*); static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1); @@ -930,8 +930,7 @@ Sint erts_decode_ext_size_ets(byte *ext, Uint size) ** on return hpp is updated to point after allocated data */ Eterm -erts_decode_dist_ext(Eterm** hpp, - ErlOffHeap* off_heap, +erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *edep) { Eterm obj; @@ -951,7 +950,7 @@ erts_decode_dist_ext(Eterm** hpp, goto error; ep++; } - ep = dec_term(edep, hpp, ep, off_heap, &obj, NULL); + ep = dec_term(edep, factory, ep, &obj, NULL); if (!ep) goto error; @@ -960,19 +959,22 @@ erts_decode_dist_ext(Eterm** hpp, return obj; error: + erts_factory_undo(factory); bad_dist_ext(edep); return THE_NON_VALUE; } -Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext) +Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext) { Eterm obj; byte *ep = *ext; - if (*ep++ != VERSION_MAGIC) + if (*ep++ != VERSION_MAGIC) { + erts_factory_undo(factory); return THE_NON_VALUE; - ep = dec_term(NULL, hpp, ep, off_heap, &obj, NULL); + } + ep = dec_term(NULL, factory, ep, &obj, NULL); if (!ep) { #ifdef DEBUG bin_write(ERTS_PRINT_STDERR,NULL,*ext,500); @@ -983,10 +985,10 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext) return obj; } -Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext) +Eterm erts_decode_ext_ets(ErtsHeapFactory* factory, byte *ext) { Eterm obj; - ext = dec_term(NULL, hpp, ext, off_heap, &obj, NULL); + ext = dec_term(NULL, factory, ext, &obj, NULL); ASSERT(ext); return obj; } @@ -995,9 +997,8 @@ Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext) BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) { + ErtsHeapFactory factory; Eterm res; - Eterm *hp; - Eterm *hendp; Sint hsz; ErtsDistExternal ede; Eterm *tp; @@ -1044,12 +1045,9 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) if (hsz < 0) goto badarg; - hp = HAlloc(BIF_P, (Uint) hsz); - hendp = hp + hsz; - - res = erts_decode_dist_ext(&hp, &MSO(BIF_P), &ede); - - HRelease(BIF_P, hendp, hp); + erts_factory_proc_prealloc_init(&factory, BIF_P, hsz); + res = erts_decode_dist_ext(&factory, &ede); + erts_factory_close(&factory); if (is_value(res)) BIF_RET(res); @@ -1177,13 +1175,11 @@ typedef struct { byte* ep; Eterm res; Eterm* next; - Eterm* hp_start; - Eterm* hp; - Eterm* hp_end; + ErtsHeapFactory factory; int remaining_n; char* remaining_bytes; Eterm* maps_list; - struct dec_term_hamt_placeholder* hamt_list; + ErtsPStack hamt_array; } B2TDecodeContext; typedef struct { @@ -1307,10 +1303,12 @@ binary2term_abort(ErtsBinary2TermState *state) } static ERTS_INLINE Eterm -binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp) +binary2term_create(ErtsDistExternal *edep, ErtsBinary2TermState *state, + ErtsHeapFactory* factory) { Eterm res; - if (!dec_term(edep, hpp, state->extp, ohp, &res, NULL)) + + if (!dec_term(edep, factory, state->extp, &res, NULL)) res = THE_NON_VALUE; if (state->exttmp) { state->exttmp = 0; @@ -1343,9 +1341,9 @@ erts_binary2term_abort(ErtsBinary2TermState *state) } Eterm -erts_binary2term_create(ErtsBinary2TermState *state, Eterm **hpp, ErlOffHeap *ohp) +erts_binary2term_create(ErtsBinary2TermState *state, ErtsHeapFactory* factory) { - return binary2term_create(NULL,state, hpp, ohp); + return binary2term_create(NULL,state, factory); } static void b2t_destroy_context(B2TContext* context) @@ -1354,8 +1352,21 @@ static void b2t_destroy_context(B2TContext* context) ERTS_ALC_T_EXT_TERM_DATA); context->aligned_alloc = NULL; binary2term_abort(&context->b2ts); - if (context->state == B2TUncompressChunk) { + switch (context->state) { + case B2TUncompressChunk: erl_zlib_inflate_finish(&context->u.uc.stream); + break; + case B2TDecode: + case B2TDecodeList: + case B2TDecodeTuple: + case B2TDecodeString: + case B2TDecodeBinary: + if (context->u.dc.hamt_array.pstart) { + erts_free(context->u.dc.hamt_array.alloc_type, + context->u.dc.hamt_array.pstart); + } + break; + default:; } } @@ -1506,11 +1517,9 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar ctx->u.dc.ep = ctx->b2ts.extp; ctx->u.dc.res = (Eterm) (UWord) NULL; ctx->u.dc.next = &ctx->u.dc.res; - ctx->u.dc.hp_start = HAlloc(p, ctx->heap_size); - ctx->u.dc.hp = ctx->u.dc.hp_start; - ctx->u.dc.hp_end = ctx->u.dc.hp_start + ctx->heap_size; + erts_factory_proc_prealloc_init(&ctx->u.dc.factory, p, ctx->heap_size); ctx->u.dc.maps_list = NULL; - ctx->u.dc.hamt_list = NULL; + ctx->u.dc.hamt_array.pstart = NULL; ctx->state = B2TDecode; /*fall through*/ case B2TDecode: @@ -1520,11 +1529,10 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar case B2TDecodeBinary: { ErtsDistExternal fakedep; fakedep.flags = ctx->flags; - dec_term(&fakedep, NULL, NULL, &MSO(p), NULL, ctx); + dec_term(&fakedep, NULL, NULL, NULL, ctx); break; } case B2TDecodeFail: - HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp_start); /*fall through*/ case B2TBadArg: BUMP_REDS(p, (initial_reds - ctx->reds) / B2T_BYTES_PER_REDUCTION); @@ -1549,11 +1557,11 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar case B2TDone: b2t_destroy_context(ctx); - if (ctx->u.dc.hp > ctx->u.dc.hp_end) { + if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) { erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n", - __FILE__, __LINE__, ctx->u.dc.hp - ctx->u.dc.hp_end); + __FILE__, __LINE__, ctx->u.dc.factory.hp - ctx->u.dc.factory.hp_end); } - HRelease(p, ctx->u.dc.hp_end, ctx->u.dc.hp); + erts_factory_close(&ctx->u.dc.factory); if (!is_first_call) { erts_set_gc_state(p, 1); @@ -2247,7 +2255,7 @@ static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation) } static byte* -dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp) +dec_pid(ErtsDistExternal *edep, ErtsHeapFactory* factory, byte* ep, Eterm* objp) { Eterm sysname; Uint data; @@ -2286,15 +2294,15 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete if(node == erts_this_node) { *objp = make_internal_pid(data); } else { - ExternalThing *etp = (ExternalThing *) *hpp; - *hpp += EXTERNAL_THING_HEAD_SIZE + 1; + ExternalThing *etp = (ExternalThing *) factory->hp; + factory->hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_pid_header(1); - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; etp->data.ui[0] = data; - off_heap->first = (struct erl_off_heap_header*) etp; + factory->off_heap->first = (struct erl_off_heap_header*) etp; *objp = make_external_pid(etp); } return ep; @@ -2905,69 +2913,43 @@ is_external_string(Eterm list, int* p_is_string) return len; } -/* Assumes that the ones to undo are preluding the list. */ -static void -undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) -{ - const Uint area_sz = (end - start) * sizeof(Eterm); - struct erl_off_heap_header* hdr; - struct erl_off_heap_header** hdr_nextp = NULL; - - for (hdr = off_heap->first; ; hdr=hdr->next) { - if (!in_area(hdr, start, area_sz)) { - if (hdr_nextp != NULL) { - *hdr_nextp = NULL; - erts_cleanup_offheap(off_heap); - off_heap->first = hdr; - } - break; - } - hdr_nextp = &hdr->next; - } - - /* Assert that the ones to undo were indeed preluding the list. */ -#ifdef DEBUG - for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) { - ASSERT(!in_area(hdr, start, area_sz)); - } -#endif /* DEBUG */ -} -struct dec_term_hamt_placeholder +struct dec_term_hamt { - struct dec_term_hamt_placeholder* next; Eterm* objp; /* write result here */ Uint size; /* nr of leafs */ - Eterm leafs[1]; + Eterm* leaf_array; }; -#define DEC_TERM_HAMT_PLACEHOLDER_SIZE \ - (offsetof(struct dec_term_hamt_placeholder, leafs) / sizeof(Eterm)) /* Decode term from external format into *objp. -** On failure return NULL and *hpp will be unchanged. +** On failure calls erts_factory_undo() and returns NULL */ static byte* -dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, - Eterm* objp, B2TContext* ctx) +dec_term(ErtsDistExternal *edep, + ErtsHeapFactory* factory, + byte* ep, + Eterm* objp, + B2TContext* ctx) { - Eterm* hp_saved; +#define PSTACK_TYPE struct dec_term_hamt + PSTACK_DECLARE(hamt_array, 5); int n; ErtsAtomEncoding char_enc; register Eterm* hp; /* Please don't take the address of hp */ Eterm *maps_list; /* for preprocessing of small maps */ - struct dec_term_hamt_placeholder* hamt_list; /* for preprocessing of big maps */ Eterm* next; SWord reds; +#ifdef DEBUG + Eterm* dbg_resultp = ctx ? &ctx->u.dc.res : objp; +#endif if (ctx) { - hp_saved = ctx->u.dc.hp_start; reds = ctx->reds; next = ctx->u.dc.next; ep = ctx->u.dc.ep; - hpp = &ctx->u.dc.hp; + factory = &ctx->u.dc.factory; maps_list = ctx->u.dc.maps_list; - hamt_list = ctx->u.dc.hamt_list; if (ctx->state != B2TDecode) { int n_limit = reds; @@ -3012,7 +2994,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, break; case B2TDecodeString: - hp = *hpp; + hp = factory->hp; hp[-1] = make_list(hp); /* overwrite the premature NIL */ while (n-- > 0) { hp[0] = make_small(*ep++); @@ -3020,7 +3002,7 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, hp += 2; } hp[-1] = NIL; - *hpp = hp; + factory->hp = hp; break; case B2TDecodeBinary: @@ -3042,16 +3024,18 @@ dec_term(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, return NULL; } } + PSTACK_CHANGE_ALLOCATOR(hamt_array, ERTS_ALC_T_SAVED_ESTACK); + if (ctx->u.dc.hamt_array.pstart) { + PSTACK_RESTORE(hamt_array, &ctx->u.dc.hamt_array); + } } else { - hp_saved = *hpp; reds = ERTS_SWORD_MAX; next = objp; *next = (Eterm) (UWord) NULL; maps_list = NULL; - hamt_list = NULL; } - hp = *hpp; + hp = factory->hp; while (next != NULL) { @@ -3288,9 +3272,9 @@ dec_term_atom_common: break; } case PID_EXT: - *hpp = hp; - ep = dec_pid(edep, hpp, ep, off_heap, objp); - hp = *hpp; + factory->hp = hp; + ep = dec_pid(edep, factory, ep, objp); + hp = factory->hp; if (ep == NULL) { goto error; } @@ -3324,11 +3308,11 @@ dec_term_atom_common: hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_port_header(1); - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; etp->data.ui[0] = num; - off_heap->first = (struct erl_off_heap_header*)etp; + factory->off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_port(etp); } @@ -3408,10 +3392,10 @@ dec_term_atom_common: #else etp->header = make_external_ref_header(ref_words); #endif - etp->next = off_heap->first; + etp->next = factory->off_heap->first; etp->node = node; - off_heap->first = (struct erl_off_heap_header*)etp; + factory->off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_ref(etp); ref_num = &(etp->data.ui32[0]); } @@ -3451,9 +3435,9 @@ dec_term_atom_common: hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; - OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -3503,9 +3487,9 @@ dec_term_atom_common: pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; - OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -3557,9 +3541,9 @@ dec_term_atom_common: if ((ep = dec_atom(edep, ep, &name)) == NULL) { goto error; } - *hpp = hp; - ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL); - hp = *hpp; + factory->hp = hp; + ep = dec_term(edep, factory, ep, &temp, NULL); + hp = factory->hp; if (ep == NULL) { goto error; } @@ -3631,15 +3615,11 @@ dec_term_atom_common: } } else { /* Make hamt */ - struct dec_term_hamt_placeholder* holder = - (struct dec_term_hamt_placeholder*) hp; - - holder->next = hamt_list; - hamt_list = holder; - holder->objp = objp; - holder->size = size; + struct dec_term_hamt* hamt = PSTACK_PUSH(hamt_array); - hp += DEC_TERM_HAMT_PLACEHOLDER_SIZE; + hamt->objp = objp; + hamt->size = size; + hamt->leaf_array = hp; for (n = size; n; n--) { CDR(hp) = (Eterm) COMPRESS_POINTER(next); @@ -3681,9 +3661,9 @@ dec_term_atom_common: if ((ep = dec_atom(edep, ep, &module)) == NULL) { goto error; } - *hpp = hp; + factory->hp = hp; /* Index */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3692,7 +3672,7 @@ dec_term_atom_common: old_index = unsigned_val(temp); /* Uniq */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3704,8 +3684,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)funp; + funp->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)funp; funp->fe = erts_put_fun_entry2(module, old_uniq, old_index, uniq, index, arity); @@ -3716,7 +3696,7 @@ dec_term_atom_common: } funp->native_address = funp->fe->native_address; #endif - hp = *hpp; + hp = factory->hp; /* Environment */ for (i = num_free-1; i >= 0; i--) { @@ -3742,14 +3722,14 @@ dec_term_atom_common: ep += 4; hp += ERL_FUN_SIZE; hp += num_free; - *hpp = hp; + factory->hp = hp; funp->thing_word = HEADER_FUN; funp->num_free = num_free; *objp = make_fun(funp); /* Creator pid */ if (*ep != PID_EXT - || (ep = dec_pid(edep, hpp, ++ep, off_heap, + || (ep = dec_pid(edep, factory, ++ep, &funp->creator))==NULL) { goto error; } @@ -3760,7 +3740,7 @@ dec_term_atom_common: } /* Index */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3769,7 +3749,7 @@ dec_term_atom_common: old_index = unsigned_val(temp); /* Uniq */ - if ((ep = dec_term(edep, hpp, ep, off_heap, &temp, NULL)) == NULL) { + if ((ep = dec_term(edep, factory, ep, &temp, NULL)) == NULL) { goto error; } if (!is_small(temp)) { @@ -3780,8 +3760,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)funp; + funp->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)funp; old_uniq = unsigned_val(temp); funp->fe = erts_put_fun_entry(module, old_uniq, old_index); @@ -3789,7 +3769,7 @@ dec_term_atom_common: #ifdef HIPE funp->native_address = funp->fe->native_address; #endif - hp = *hpp; + hp = factory->hp; /* Environment */ for (i = num_free-1; i >= 0; i--) { @@ -3823,9 +3803,9 @@ dec_term_atom_common: erts_refc_inc(&pb->val->refc, 1); hp += PROC_BIN_SIZE; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; - OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->flags = 0; *objp = make_binary(pb); break; @@ -3841,9 +3821,9 @@ dec_term_atom_common: erts_refc_inc(&pb->val->refc, 1); hp += PROC_BIN_SIZE; - pb->next = off_heap->first; - off_heap->first = (struct erl_off_heap_header*)pb; - OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); pb->flags = 0; sub = (ErlSubBin*)hp; @@ -3869,9 +3849,11 @@ dec_term_atom_common: if (next || ctx->state != B2TDecode) { ctx->u.dc.ep = ep; ctx->u.dc.next = next; - ctx->u.dc.hp = hp; + ctx->u.dc.factory.hp = hp; ctx->u.dc.maps_list = maps_list; - ctx->u.dc.hamt_list = hamt_list; + if (!PSTACK_IS_EMPTY(hamt_array)) { + PSTACK_SAVE(hamt_array, &ctx->u.dc.hamt_array); + } ctx->reds = 0; return NULL; } @@ -3894,40 +3876,36 @@ dec_term_atom_common: maps_list = next; } - /* Iterate through all the hamts and build tree nodes. + ASSERT(hp <= factory->hp_end + || (factory->mode == FACTORY_CLOSED && is_immed(*dbg_resultp))); + factory->hp = hp; + /* + * From here on factory may produce (more) heap fragments */ - if (hamt_list) { - ErtsHeapFactory factory; - factory.p = NULL; - factory.hp = hp; - /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */ + if (!PSTACK_IS_EMPTY(hamt_array)) { + do { + struct dec_term_hamt* hamt = PSTACK_TOP(hamt_array); - do { - struct dec_term_hamt_placeholder* hamt = hamt_list; - *hamt->objp = erts_hashmap_from_array(&factory, - hamt->leafs, + *hamt->objp = erts_hashmap_from_array(factory, + hamt->leaf_array, hamt->size, 1); if (is_non_value(*hamt->objp)) - goto error; + goto error_hamt; - hamt_list = hamt->next; - - /* Yes, we waste a couple of heap words per hamt - for the temporary placeholder */ - *(Eterm*)hamt = make_pos_bignum_header(DEC_TERM_HAMT_PLACEHOLDER_SIZE-1); - } while (hamt_list); - - hp = factory.hp; + (void) PSTACK_POP(hamt_array); + } while (!PSTACK_IS_EMPTY(hamt_array)); + PSTACK_DESTROY(hamt_array); } + ASSERT((Eterm*)EXPAND_POINTER(*dbg_resultp) != NULL); + if (ctx) { ctx->state = B2TDone; ctx->reds = reds; } - *hpp = hp; return ep; error: @@ -3935,11 +3913,12 @@ error: * Must unlink all off-heap objects that may have been * linked into the process. */ - if (hp < *hpp) { /* Sometimes we used hp and sometimes *hpp */ - hp = *hpp; /* the largest must be the freshest */ + if (factory->hp < hp) { /* Sometimes we used hp and sometimes factory->hp */ + factory->hp = hp; /* the largest must be the freshest */ } - undo_offheap_in_area(off_heap, hp_saved, hp); - *hpp = hp_saved; +error_hamt: + erts_factory_undo(factory); + PSTACK_DESTROY(hamt_array); if (ctx) { ctx->state = B2TDecodeFail; ctx->reds = reds; @@ -4465,7 +4444,7 @@ init_done: if (n <= MAP_SMALL_MAP_LIMIT) { heap_size += 3 + n + 1 + n; } else { - heap_size += hashmap_over_estimated_heap_size(n); + heap_size += HASHMAP_ESTIMATED_HEAP_SIZE(n); } break; case STRING_EXT: diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index 50fcfa04d6..508ab8dc17 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -148,6 +148,7 @@ typedef struct { byte *extp; int exttmp; Uint extsize; + Uint heap_size; } ErtsBinary2TermState; @@ -185,18 +186,18 @@ void erts_destroy_dist_ext_copy(ErtsDistExternal *); int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint, DistEntry *, ErtsAtomCache *); Sint erts_decode_dist_ext_size(ErtsDistExternal *); -Eterm erts_decode_dist_ext(Eterm **, ErlOffHeap *, ErtsDistExternal *); +Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *); Sint erts_decode_ext_size(byte*, Uint); Sint erts_decode_ext_size_ets(byte*, Uint); -Eterm erts_decode_ext(Eterm **, ErlOffHeap *, byte**); -Eterm erts_decode_ext_ets(Eterm **, ErlOffHeap *, byte*); +Eterm erts_decode_ext(ErtsHeapFactory*, byte**); +Eterm erts_decode_ext_ets(ErtsHeapFactory*, byte*); Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags); Sint erts_binary2term_prepare(ErtsBinary2TermState *, byte *, Sint); void erts_binary2term_abort(ErtsBinary2TermState *); -Eterm erts_binary2term_create(ErtsBinary2TermState *, Eterm **hpp, ErlOffHeap *); +Eterm erts_binary2term_create(ErtsBinary2TermState *, ErtsHeapFactory*); int erts_debug_max_atom_out_cache_index(void); int erts_debug_atom_to_out_cache_index(Eterm); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 340c7033ab..14d42599a1 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -230,9 +230,23 @@ typedef struct { ERTS_INTERNAL_BINARY_FIELDS SWord orig_size; void (*destructor)(Binary *); - char magic_bin_data[1]; + union { + struct { + ERTS_BINARY_STRUCT_ALIGNMENT + char data[1]; + } aligned; + struct { + char data[1]; + } unaligned; + } u; } ErtsMagicBinary; +#ifdef ARCH_32 +#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4 +#else +#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0 +#endif + typedef union { Binary binary; ErtsMagicBinary magic_binary; @@ -252,15 +266,30 @@ typedef union { #define ERTS_MAGIC_BIN_DESTRUCTOR(BP) \ ((ErtsBinary *) (BP))->magic_binary.destructor #define ERTS_MAGIC_BIN_DATA(BP) \ - ((void *) ((ErtsBinary *) (BP))->magic_binary.magic_bin_data) -#define ERTS_MAGIC_BIN_DATA_SIZE(BP) \ - ((BP)->orig_size - sizeof(void (*)(Binary *))) + ((void *) ((ErtsBinary *) (BP))->magic_binary.u.aligned.data) +#define ERTS_MAGIC_DATA_OFFSET \ + (offsetof(ErtsMagicBinary,u.aligned.data) - offsetof(Binary,orig_bytes)) #define ERTS_MAGIC_BIN_ORIG_SIZE(Sz) \ - (sizeof(void (*)(Binary *)) + (Sz)) + (ERTS_MAGIC_DATA_OFFSET + (Sz)) #define ERTS_MAGIC_BIN_SIZE(Sz) \ - (offsetof(ErtsMagicBinary,magic_bin_data) + (Sz)) -#define ERTS_MAGIC_BIN_FROM_DATA(DATA) \ - ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,magic_bin_data))) + (offsetof(ErtsMagicBinary,u.aligned.data) + (Sz)) + +/* On 32-bit arch these macro variants will save memory + by not forcing 8-byte alignment for the magic payload. +*/ +#define ERTS_MAGIC_BIN_UNALIGNED_DATA(BP) \ + ((void *) ((ErtsBinary *) (BP))->magic_binary.u.unaligned.data) +#define ERTS_MAGIC_UNALIGNED_DATA_OFFSET \ + (offsetof(ErtsMagicBinary,u.unaligned.data) - offsetof(Binary,orig_bytes)) +#define ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(BP) \ + ((BP)->orig_size - ERTS_MAGIC_UNALIGNED_DATA_OFFSET) +#define ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(Sz) \ + (ERTS_MAGIC_UNALIGNED_DATA_OFFSET + (Sz)) +#define ERTS_MAGIC_BIN_UNALIGNED_SIZE(Sz) \ + (offsetof(ErtsMagicBinary,u.unaligned.data) + (Sz)) +#define ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(DATA) \ + ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,u.unaligned.data))) + #define Binary2ErlDrvBinary(B) (&((ErtsBinary *) (B))->driver.binary) #define ErlDrvBinary2Binary(D) ((Binary *) \ @@ -748,6 +777,15 @@ ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */ \ ERTS_ALC_T_ESTACK /* alloc_type */ \ } +#define PSTACK_CHANGE_ALLOCATOR(s,t) \ +do { \ + if (s.pstart != (byte*)PSTK_DEF_STACK(s)) { \ + erl_exit(1, "Internal error - trying to change allocator " \ + "type of active pstack\n"); \ + } \ + s.alloc_type = (t); \ + } while (0) + #define PSTACK_DESTROY(s) \ do { \ if (s.pstart != (byte*)PSTK_DEF_STACK(s)) { \ @@ -757,6 +795,8 @@ do { \ #define PSTACK_IS_EMPTY(s) (s.psp < s.pstart) +#define PSTACK_COUNT(s) (((PSTACK_TYPE*)s.psp + 1) - (PSTACK_TYPE*)s.pstart) + #define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), (PSTACK_TYPE*)(s.psp)) #define PSTACK_PUSH(s) \ @@ -767,6 +807,45 @@ do { \ #define PSTACK_POP(s) ((PSTACK_TYPE*) (s.psp -= sizeof(PSTACK_TYPE))) +/* + * Do not free the stack after this, it may have pointers into what + * was saved in 'dst'. + */ +#define PSTACK_SAVE(s,dst)\ +do {\ + if (s.pstart == (byte*)PSTK_DEF_STACK(s)) {\ + UWord _pbytes = PSTACK_COUNT(s) * sizeof(PSTACK_TYPE);\ + (dst)->pstart = erts_alloc(s.alloc_type,\ + sizeof(PSTK_DEF_STACK(s)));\ + sys_memcpy((dst)->pstart, s.pstart, _pbytes);\ + (dst)->psp = (dst)->pstart + _pbytes - sizeof(PSTACK_TYPE);\ + (dst)->pend = (dst)->pstart + sizeof(PSTK_DEF_STACK(s));\ + (dst)->alloc_type = s.alloc_type;\ + } else\ + *(dst) = s;\ + } while (0) + +/* + * Use on empty stack, only the allocator can be changed before this. + * The src stack is reset to NULL. + */ +#define PSTACK_RESTORE(s, src) \ +do { \ + ASSERT(s.pstart == (byte*)PSTK_DEF_STACK(s)); \ + s = *(src); /* struct copy */ \ + (src)->pstart = NULL; \ + ASSERT(s.psp >= (s.pstart - sizeof(PSTACK_TYPE))); \ + ASSERT(s.psp < s.pend); \ +} while (0) + +#define PSTACK_DESTROY_SAVED(pstack)\ +do {\ + if ((pstack)->pstart) {\ + erts_free((pstack)->alloc_type, (pstack)->pstart);\ + (pstack)->pstart = NULL;\ + }\ +} while(0) + /* binary.c */ @@ -1055,6 +1134,9 @@ Sint erts_binary_set_loop_limit(Sint limit); /* external.c */ void erts_init_external(void); +/* erl_map.c */ +void erts_init_map(void); + /* erl_unicode.c */ void erts_init_unicode(void); Sint erts_unicode_set_loop_limit(Sint limit); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index ccc7da265e..23f208c2eb 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -1395,31 +1395,22 @@ finalize_force_imm_drv_call(ErtsTryImmDrvCallState *sp) static ERTS_INLINE void queue_port_sched_op_reply(Process *rp, ErtsProcLocks *rp_locksp, - Eterm *hp_start, - Eterm *hp, - Uint h_size, - ErlHeapFragment* bp, + ErtsHeapFactory* factory, Uint32 *ref_num, Eterm msg) { - Eterm ref = make_internal_ref(hp); + Eterm* hp = erts_produce_heap(factory, ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE, 0); + Eterm ref; + + ref= make_internal_ref(hp); write_ref_thing(hp, ref_num[0], ref_num[1], ref_num[2]); hp += REF_THING_SIZE; msg = TUPLE2(hp, ref, msg); - hp += 3; - if (!bp) { - HRelease(rp, hp_start + h_size, hp); - } - else { - Uint used_h_size = hp - hp_start; - ASSERT(h_size >= used_h_size); - if (h_size > used_h_size) - bp = erts_resize_message_buffer(bp, used_h_size, &msg, 1); - } + erts_factory_trim_and_close(factory, &msg, 1); - erts_queue_message(rp, rp_locksp, bp, msg, NIL); + erts_queue_message(rp, rp_locksp, factory->heap_frags, msg, NIL); } static void @@ -1429,9 +1420,10 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) if (rp) { ErlOffHeap *ohp; ErlHeapFragment* bp; + ErtsHeapFactory factory; Eterm msg_copy; Uint hsz, msg_sz; - Eterm *hp, *hp_start; + Eterm *hp; ErtsProcLocks rp_locks = 0; hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; @@ -1442,22 +1434,22 @@ port_sched_op_reply(Eterm to, Uint32 *ref_num, Eterm msg) hsz += msg_sz; } - hp_start = hp = erts_alloc_message_heap(hsz, + hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks); + erts_factory_message_init(&factory, rp, hp, bp); if (is_immed(msg)) msg_copy = msg; - else + else { msg_copy = copy_struct(msg, msg_sz, &hp, ohp); + factory.hp = hp; + } queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, ref_num, msg_copy); @@ -3896,12 +3888,13 @@ port_sig_control(Port *prt, if (res == ERTS_PORT_OP_DONE) { Eterm msg; - Eterm *hp, *hp_start; + Eterm *hp; ErlHeapFragment *bp; ErlOffHeap *ohp; + ErtsHeapFactory factory; Process *rp; ErtsProcLocks rp_locks = 0; - Uint hsz; + Uint hsz, rsz; int control_flags; rp = erts_proc_lookup_raw(sigdp->caller); @@ -3910,17 +3903,19 @@ port_sig_control(Port *prt, control_flags = prt->control_flags; - hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; - hsz += port_control_result_size(control_flags, + rsz = port_control_result_size(control_flags, resp_bufp, &resp_size, &resp_buf[0]); + hsz = rsz + ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; - hp_start = hp = erts_alloc_message_heap(hsz, + + hp = erts_alloc_message_heap(hsz, &bp, &ohp, rp, &rp_locks); + erts_factory_message_init(&factory, rp, hp, bp); msg = write_port_control_result(control_flags, resp_bufp, @@ -3929,13 +3924,11 @@ port_sig_control(Port *prt, &hp, bp, ohp); + factory.hp = hp; queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, sigdp->ref, msg); @@ -4222,8 +4215,6 @@ port_sig_call(Port *prt, if (res == ERTS_PORT_OP_DONE) { Eterm msg; Eterm *hp; - ErlHeapFragment *bp; - ErlOffHeap *ohp; Process *rp; ErtsProcLocks rp_locks = 0; Sint hsz; @@ -4234,29 +4225,31 @@ port_sig_call(Port *prt, hsz = erts_decode_ext_size((byte *) resp_bufp, resp_size); if (hsz >= 0) { - Eterm *hp_start; + ErlHeapFragment* bp; + ErlOffHeap* ohp; + ErtsHeapFactory factory; byte *endp; hsz += 3; /* ok tuple */ hsz += ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; - hp_start = hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - rp, - &rp_locks); + hp = erts_alloc_message_heap(hsz, + &bp, + &ohp, + rp, + &rp_locks); endp = (byte *) resp_bufp; - msg = erts_decode_ext(&hp, ohp, &endp); + erts_factory_message_init(&factory, rp, hp, bp); + msg = erts_decode_ext(&factory, &endp); if (is_value(msg)) { + hp = erts_produce_heap(&factory, + 3, + ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE); msg = TUPLE2(hp, am_ok, msg); - hp += 3; queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, sigdp->ref, msg); @@ -4264,8 +4257,6 @@ port_sig_call(Port *prt, erts_smp_proc_unlock(rp, rp_locks); goto done; } - if (bp) - free_message_buffer(bp); if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); } @@ -4342,10 +4333,11 @@ erts_port_call(Process* c_p, try_call_res = try_imm_drv_call(&try_call_state); switch (try_call_res) { case ERTS_TRY_IMM_DRV_CALL_OK: { - Eterm *hp, *hp_end; + ErtsHeapFactory factory; Sint hsz; unsigned ret_flags = 0U; Eterm term; + Eterm* hp; res = call_driver_call(c_p->common.id, prt, @@ -4365,15 +4357,14 @@ erts_port_call(Process* c_p, if (hsz < 0) return ERTS_PORT_OP_BADARG; hsz += 3; - hp = HAlloc(c_p, hsz); - hp_end = hp + hsz; + erts_factory_proc_prealloc_init(&factory, c_p, hsz); endp = (byte *) resp_bufp; - term = erts_decode_ext(&hp, &MSO(c_p), &endp); + term = erts_decode_ext(&factory, &endp); if (term == THE_NON_VALUE) return ERTS_PORT_OP_BADARG; + hp = erts_produce_heap(&factory,3,0); *retvalp = TUPLE2(hp, am_ok, term); - hp += 3; - HRelease(c_p, hp_end, hp); + erts_factory_close(&factory); if (resp_bufp != &resp_buf[0] && !(ret_flags & DRIVER_CALL_KEEP_BUFFER)) driver_free(resp_bufp); @@ -4508,12 +4499,11 @@ port_sig_info(Port *prt, prt, sigdp->u.info.item); if (is_value(value)) { + ErtsHeapFactory factory; + erts_factory_message_init(&factory, NULL, hp, bp); queue_port_sched_op_reply(rp, &rp_locks, - hp_start, - hp, - hsz, - bp, + &factory, sigdp->ref, value); } @@ -5106,22 +5096,26 @@ cleanup_b2t_states(struct b2t_states__ *b2tsp) static int driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) { +#define HEAP_EXTRA 200 #define ERTS_DDT_FAIL do { res = -1; goto done; } while (0) Uint need = 0; int depth = 0; int res; - Eterm *hp = NULL, *hp_start = NULL, *hp_end = NULL; ErlDrvTermData* ptr; ErlDrvTermData* ptr_end; DECLARE_ESTACK(stack); - Eterm mess = NIL; /* keeps compiler happy */ + Eterm mess; Process* rp = NULL; - ErlHeapFragment *bp = NULL; - ErlOffHeap *ohp; + ErtsHeapFactory factory; ErtsProcLocks rp_locks = 0; struct b2t_states__ b2t; - int scheduler = 1; /* Silence erroneous warning... */ + int scheduler; + int is_heap_need_limited = 1; + ERTS_UNDEF(mess,NIL); + ERTS_UNDEF(scheduler,1); + + factory.mode = FACTORY_CLOSED; init_b2t_states(&b2t); /* @@ -5285,19 +5279,24 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) #ifdef DEBUG b2t.org_ext[b2t.ix] = ext; #endif - hsz = erts_binary2term_prepare(&b2t.state[b2t.ix++], ext, size); + hsz = erts_binary2term_prepare(&b2t.state[b2t.ix], ext, size); if (hsz < 0) ERTS_DDT_FAIL; /* Invalid data */ + b2t.state[b2t.ix++].heap_size = hsz; need += hsz; ptr += 2; depth++; + if (size > MAP_SMALL_MAP_LIMIT*3) { /* may contain big map */ + is_heap_need_limited = 0; + } break; } case ERL_DRV_MAP: { /* int */ ERTS_DDT_CHK_ENOUGH_ARGS(1); if ((int) ptr[0] < 0) ERTS_DDT_FAIL; if (ptr[0] > MAP_SMALL_MAP_LIMIT) { - need += hashmap_over_estimated_heap_size(ptr[0]); + need += HASHMAP_ESTIMATED_HEAP_SIZE(ptr[0]); + is_heap_need_limited = 0; } else { need += MAP_HEADER_FLATMAP_SZ + 1 + 2*ptr[0]; } @@ -5336,8 +5335,17 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) goto done; } - hp_start = hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks); - hp_end = hp + need; + /* Try copy directly to destination heap if we know there are no big maps */ + if (is_heap_need_limited) { + ErlOffHeap *ohp; + ErlHeapFragment* bp; + Eterm* hp = erts_alloc_message_heap(need, &bp, &ohp, rp, &rp_locks); + erts_factory_message_init(&factory, rp, hp, bp); + } + else { + erts_factory_message_init(&factory, NULL, NULL, + new_message_buffer(need)); + } /* * Interpret the instructions and build the term. @@ -5358,13 +5366,15 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_INT: /* signed int argument */ #if HALFWORD_HEAP - mess = erts_bld_sint64(&hp, NULL, (Sint64)ptr[0]); + erts_reserve_heap(&factory, BIG_NEED_SIZE(2)); + mess = erts_bld_sint64(&factory.hp, NULL, (Sint64)ptr[0]); #else + erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE); if (IS_SSMALL((Sint)ptr[0])) mess = make_small((Sint)ptr[0]); else { - mess = small_to_big((Sint)ptr[0], hp); - hp += BIG_UINT_HEAP_SIZE; + mess = small_to_big((Sint)ptr[0], factory.hp); + factory.hp += BIG_UINT_HEAP_SIZE; } #endif ptr++; @@ -5372,25 +5382,29 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_UINT: /* unsigned int argument */ #if HALFWORD_HEAP - mess = erts_bld_uint64(&hp, NULL, (Uint64)ptr[0]); + erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64)); + mess = erts_bld_uint64(&factory.hp, NULL, (Uint64)ptr[0]); #else + erts_reserve_heap(&factory, BIG_UINT_HEAP_SIZE); if (IS_USMALL(0, (Uint)ptr[0])) mess = make_small((Uint)ptr[0]); else { - mess = uint_to_big((Uint)ptr[0], hp); - hp += BIG_UINT_HEAP_SIZE; + mess = uint_to_big((Uint)ptr[0], factory.hp); + factory.hp += BIG_UINT_HEAP_SIZE; } #endif ptr++; break; case ERL_DRV_INT64: /* pointer to unsigned 64-bit int argument */ - mess = erts_bld_sint64(&hp, NULL, *((Sint64 *) ptr[0])); + erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64)); + mess = erts_bld_sint64(&factory.hp, NULL, *((Sint64 *) ptr[0])); ptr++; break; case ERL_DRV_UINT64: /* pointer to unsigned 64-bit int argument */ - mess = erts_bld_uint64(&hp, NULL, *((Uint64 *) ptr[0])); + erts_reserve_heap(&factory, BIG_NEED_FOR_BITS(64)); + mess = erts_bld_uint64(&factory.hp, NULL, *((Uint64 *) ptr[0])); ptr++; break; @@ -5407,8 +5421,8 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size); if (size <= ERL_ONHEAP_BIN_LIMIT) { - ErlHeapBin* hbp = (ErlHeapBin *) hp; - hp += heap_bin_size(size); + ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory, + heap_bin_size(size), HEAP_EXTRA); hbp->thing_word = header_heap_bin(size); hbp->size = size; if (size > 0) { @@ -5417,18 +5431,18 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) mess = make_binary(hbp); } else { - ProcBin* pb = (ProcBin *) hp; + ProcBin* pb = (ProcBin *) erts_produce_heap(&factory, + PROC_BIN_SIZE, HEAP_EXTRA); driver_binary_inc_refc(b); /* caller will free binary */ pb->thing_word = HEADER_PROC_BIN; pb->size = size; - pb->next = ohp->first; - ohp->first = (struct erl_off_heap_header*)pb; + pb->next = factory.off_heap->first; + factory.off_heap->first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(b); pb->bytes = ((byte*) b->orig_bytes) + offset; pb->flags = 0; mess = make_binary(pb); - hp += PROC_BIN_SIZE; - OH_OVERHEAD(ohp, pb->size / sizeof(Eterm)); + OH_OVERHEAD(factory.off_heap, pb->size / sizeof(Eterm)); } ptr += 3; break; @@ -5441,8 +5455,9 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) size); if (size <= ERL_ONHEAP_BIN_LIMIT) { - ErlHeapBin* hbp = (ErlHeapBin *) hp; - hp += heap_bin_size(size); + ErlHeapBin* hbp = (ErlHeapBin *) erts_produce_heap(&factory, + heap_bin_size(size), + HEAP_EXTRA); hbp->thing_word = header_heap_bin(size); hbp->size = size; if (size > 0) { @@ -5457,16 +5472,16 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) ASSERT(bufp); erts_refc_init(&bp->refc, 1); sys_memcpy((void *) bp->orig_bytes, (void *) bufp, size); - pbp = (ProcBin *) hp; - hp += PROC_BIN_SIZE; + pbp = (ProcBin *) erts_produce_heap(&factory, + PROC_BIN_SIZE, HEAP_EXTRA); pbp->thing_word = HEADER_PROC_BIN; pbp->size = size; - pbp->next = ohp->first; - ohp->first = (struct erl_off_heap_header*)pbp; + pbp->next = factory.off_heap->first; + factory.off_heap->first = (struct erl_off_heap_header*)pbp; pbp->val = bp; pbp->bytes = (byte*) bp->orig_bytes; pbp->flags = 0; - OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm)); + OH_OVERHEAD(factory.off_heap, pbp->size / sizeof(Eterm)); mess = make_binary(pbp); } ptr += 2; @@ -5475,13 +5490,15 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_STRING: /* char*, length */ erts_smp_atomic_add_nob(&erts_bytes_in, (erts_aint_t) ptr[1]); - mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], NIL); + erts_reserve_heap(&factory, 2*ptr[1]); + mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], NIL); ptr += 2; break; case ERL_DRV_STRING_CONS: /* char*, length */ mess = ESTACK_POP(stack); - mess = buf_to_intlist(&hp, (char*)ptr[0], ptr[1], mess); + erts_reserve_heap(&factory, 2*ptr[1]); + mess = buf_to_intlist(&factory.hp, (char*)ptr[0], ptr[1], mess); ptr += 2; break; @@ -5490,11 +5507,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) mess = ESTACK_POP(stack); i--; + erts_reserve_heap(&factory, 2*i); while(i > 0) { Eterm hd = ESTACK_POP(stack); - mess = CONS(hp, hd, mess); - hp += 2; + mess = CONS(factory.hp, hd, mess); + factory.hp += 2; i--; } ptr++; @@ -5503,13 +5521,12 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_TUPLE: { /* int */ int size = (int)ptr[0]; - Eterm* tp = hp; + Eterm* tp = erts_produce_heap(&factory, size+1, HEAP_EXTRA); *tp = make_arityval(size); mess = make_tuple(tp); tp += size; /* point at last element */ - hp = tp+1; /* advance "heap" pointer */ while(size--) { *tp-- = ESTACK_POP(stack); @@ -5525,20 +5542,22 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) case ERL_DRV_FLOAT: { /* double * */ FloatDef f; + Eterm* fp = erts_produce_heap(&factory, FLOAT_SIZE_OBJECT, HEAP_EXTRA); - mess = make_float(hp); + mess = make_float(fp); f.fd = *((double *) ptr[0]); if (!erts_isfinite(f.fd)) ERTS_DDT_FAIL; - PUT_DOUBLE(f, hp); - hp += FLOAT_SIZE_OBJECT; + PUT_DOUBLE(f, fp); ptr++; break; } case ERL_DRV_EXT2TERM: /* char *ext, int size */ ASSERT(b2t.org_ext[b2t.ix] == (byte *) ptr[0]); - mess = erts_binary2term_create(&b2t.state[b2t.ix++], &hp, ohp); + + erts_reserve_heap(&factory, b2t.state[b2t.ix].heap_size); + mess = erts_binary2term_create(&b2t.state[b2t.ix++], &factory); if (mess == THE_NON_VALUE) ERTS_DDT_FAIL; ptr += 2; @@ -5548,41 +5567,32 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) int size = (int)ptr[0]; if (size > MAP_SMALL_MAP_LIMIT) { int ix = 2*size; - ErtsHeapFactory factory; - Eterm* leafs = hp; - - hp += 2*size; - while(ix--) { *--hp = ESTACK_POP(stack); } + Eterm* leafs; - hp += 2*size; - factory.p = NULL; - factory.hp = hp; - /* We assume heap will suffice (see hashmap_over_estimated_heap_size) */ + erts_produce_heap(&factory, ix, HEAP_EXTRA); + leafs = factory.hp; + while(ix--) { *--leafs = ESTACK_POP(stack); } mess = erts_hashmap_from_array(&factory, leafs, size, 1); - if (is_non_value(mess)) ERTS_DDT_FAIL; - - hp = factory.hp; } else { - Eterm* tp = hp; Eterm* vp; flatmap_t *mp; + Eterm* tp = erts_produce_heap(&factory, + 2*size + 1 + MAP_HEADER_FLATMAP_SZ, + HEAP_EXTRA); *tp = make_arityval(size); - hp += 1 + size; - mp = (flatmap_t*)hp; + mp = (flatmap_t*) (tp + 1 + size); mp->thing_word = MAP_HEADER_FLATMAP; mp->size = size; mp->keys = make_tuple(tp); mess = make_flatmap(mp); - hp += MAP_HEADER_FLATMAP_SZ + size; - tp += size; /* point at last key */ - vp = hp - 1; /* point at last value */ + vp = factory.hp - 1; /* point at last value */ while(size--) { *vp-- = ESTACK_POP(stack); @@ -5605,25 +5615,16 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) if (res > 0) { mess = ESTACK_POP(stack); /* get resulting value */ - if (bp) - bp = erts_resize_message_buffer(bp, hp - hp_start, &mess, 1); - else { - ASSERT(hp); - HRelease(rp, hp_end, hp); - } + erts_factory_close(&factory); /* send message */ - erts_queue_message(rp, &rp_locks, bp, mess, am_undefined); + erts_queue_message(rp, &rp_locks, factory.heap_frags, mess, am_undefined); } else { if (b2t.ix > b2t.used) b2t.used = b2t.ix; for (b2t.ix = 0; b2t.ix < b2t.used; b2t.ix++) erts_binary2term_abort(&b2t.state[b2t.ix]); - if (bp) - free_message_buffer(bp); - else if (hp) { - HRelease(rp, hp_end, hp); - } + erts_factory_undo(&factory); } if (rp) { if (rp_locks) @@ -5635,6 +5636,7 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len) DESTROY_ESTACK(stack); return res; #undef ERTS_DDT_FAIL +#undef HEAP_EXTRA } static ERTS_INLINE int diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 965de748c9..cecd88197e 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1140,7 +1140,7 @@ make_hash2(Eterm term) ERTS_UNDEF(hash_xor_pairs, 0); -/* (HCONST * {2, ..., 16}) mod 2^32 */ +/* (HCONST * {2, ..., 22}) mod 2^32 */ #define HCONST_2 0x3c6ef372UL #define HCONST_3 0xdaa66d2bUL #define HCONST_4 0x78dde6e4UL @@ -1161,6 +1161,7 @@ make_hash2(Eterm term) #define HCONST_19 0xbe1e08bbUL #define HCONST_20 0x5c558274UL #define HCONST_21 0xfa8cfc2dUL +#define HCONST_22 0x98c475e6UL #define HASH_MAP_TAIL (_make_header(1,_TAG_HEADER_REF)) #define HASH_MAP_PAIR (_make_header(2,_TAG_HEADER_REF)) @@ -1645,8 +1646,9 @@ make_internal_hash(Eterm term) break; ptr = list_val(term); } - if (c > 0) - UINT32_HASH(sh, HCONST_4); + if (c > 0) + UINT32_HASH_2(sh, (Uint32)c, HCONST_22); + if (is_list(term)) { tmp = CDR(ptr); CONST_HASH(HCONST_17); /* Hash CAR in cons cell */ diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 5196eb51c6..10ef20fc82 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -293,6 +293,10 @@ static BOOL (WINAPI *fpSetHandleInformation)(HANDLE,DWORD,DWORD); static unsigned long zero_value = 0; static unsigned long one_value = 1; +#define TCP_SHUT_WR SD_SEND +#define TCP_SHUT_RD SD_RECEIVE +#define TCP_SHUT_RDWR SD_BOTH + #elif defined (__OSE__) /* @@ -421,6 +425,10 @@ typedef unsigned long u_long; inet_driver_select((d), (flags), (onoff)); \ } while(0) +#define TCP_SHUT_WR SHUT_WR +#define TCP_SHUT_RD SHUT_RD +#define TCP_SHUT_RDWR SHUT_RDWR + #else /* !__OSE__ && !__WIN32__ */ #include <sys/time.h> @@ -691,6 +699,9 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) inet_driver_select((d)->port, (ErlDrvEvent)(long)(d)->event, (flags), (onoff)); \ } while(0) +#define TCP_SHUT_WR SHUT_WR +#define TCP_SHUT_RD SHUT_RD +#define TCP_SHUT_RDWR SHUT_RDWR #endif /* !__WIN32__ && !__OSE__ */ @@ -820,6 +831,10 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define TCP_ADDF_CLOSE_SENT 2 /* Close sent (active mode only) */ #define TCP_ADDF_DELAYED_CLOSE_RECV 4 /* If receive fails, report {error,closed} (passive mode) */ #define TCP_ADDF_DELAYED_CLOSE_SEND 8 /* If send fails, report {error,closed} (passive mode) */ +#define TCP_ADDF_PENDING_SHUT_WR 16 /* Call shutdown(sock, SHUT_WR) when queue empties */ +#define TCP_ADDF_PENDING_SHUT_RDWR 32 /* Call shutdown(sock, SHUT_RDWR) when queue empties */ +#define TCP_ADDF_PENDING_SHUTDOWN \ + (TCP_ADDF_PENDING_SHUT_WR | TCP_ADDF_PENDING_SHUT_RDWR) /* *_REQ_* replies */ #define INET_REP_ERROR 0 @@ -1407,6 +1422,8 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev); static int tcp_recv(tcp_descriptor* desc, int request_len); static int tcp_deliver(tcp_descriptor* desc, int len); +static int tcp_shutdown_error(tcp_descriptor* desc, int err); + static int tcp_inet_output(tcp_descriptor* desc, HANDLE event); static int tcp_inet_input(tcp_descriptor* desc, HANDLE event); @@ -9473,10 +9490,18 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd, return ctl_error(EINVAL, rbuf, rsize); } how = buf[0]; - if (sock_shutdown(INETP(desc)->s, how) == 0) { + if (how != TCP_SHUT_RD && driver_sizeq(desc->inet.port) > 0) { + if (how == TCP_SHUT_WR) { + desc->tcp_add_flags |= TCP_ADDF_PENDING_SHUT_WR; + } else if (how == TCP_SHUT_RDWR) { + desc->tcp_add_flags |= TCP_ADDF_PENDING_SHUT_RDWR; + } return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); - } else { + } + if (IS_SOCKET_ERROR(sock_shutdown(INETP(desc)->s, how))) { return ctl_error(sock_errno(), rbuf, rsize); + } else { + return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); } } default: @@ -9613,6 +9638,8 @@ static void tcp_inet_commandv(ErlDrvData e, ErlIOVec* ev) else inet_reply_error(INETP(desc), ENOTCONN); } + else if (desc->tcp_add_flags & TCP_ADDF_PENDING_SHUTDOWN) + tcp_shutdown_error(desc, EPIPE); else if (tcp_sendv(desc, ev) == 0) inet_reply_ok(INETP(desc)); DEBUGF(("tcp_inet_commandv(%ld) }\r\n", (long)desc->inet.port)); @@ -10506,7 +10533,7 @@ static int tcp_inet_input(tcp_descriptor* desc, HANDLE event) return ret; } -static int tcp_send_error(tcp_descriptor* desc, int err) +static int tcp_send_or_shutdown_error(tcp_descriptor* desc, int err) { /* * If the port is busy, we must do some clean-up before proceeding. @@ -10563,6 +10590,16 @@ static int tcp_send_error(tcp_descriptor* desc, int err) return -1; } +static int tcp_send_error(tcp_descriptor* desc, int err) +{ + return tcp_send_or_shutdown_error(desc, err); +} + +static int tcp_shutdown_error(tcp_descriptor* desc, int err) +{ + return tcp_send_or_shutdown_error(desc, err); +} + /* ** Send non-blocking vector data */ @@ -10763,6 +10800,19 @@ static int tcp_send(tcp_descriptor* desc, char* ptr, ErlDrvSizeT len) return 0; } +/* shutdown on the socket: +** Assume caller has confirmed TCP_ADDF_PENDING_SHUTDOWN is set. +*/ +static void tcp_shutdown_async(tcp_descriptor* desc) +{ + int how; + + how = (desc->tcp_add_flags & TCP_ADDF_PENDING_SHUT_WR) ? + TCP_SHUT_WR : TCP_SHUT_RDWR; + if (IS_SOCKET_ERROR(sock_shutdown(INETP(desc)->s, how))) + tcp_shutdown_error(desc, sock_errno()); +} + #ifdef __OSE__ static void tcp_inet_drv_output_ose(ErlDrvData data, ErlDrvEvent event) @@ -10891,6 +10941,8 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event) if ((iov = driver_peekq(ix, &vsize)) == NULL) { sock_select(INETP(desc), FD_WRITE, 0); send_empty_out_q_msgs(INETP(desc)); + if (desc->tcp_add_flags & TCP_ADDF_PENDING_SHUTDOWN) + tcp_shutdown_async(desc); goto done; } vsize = vsize > MAX_VSIZE ? MAX_VSIZE : vsize; @@ -12120,6 +12172,8 @@ static MultiTimerData *add_multi_timer(MultiTimerData **first, ErlDrvPort port, void (*timeout_fun)(ErlDrvData drv_data, ErlDrvTermData caller)) { +#define eq_mega(a, b) ((a)->when.megasecs == (b)->when.megasecs) +#define eq_sec(a, b) ((a)->when.secs == (b)->when.secs) MultiTimerData *mtd, *p, *s; mtd = ALLOC(sizeof(MultiTimerData)); absolute_timeout(timeout, &(mtd->when)); @@ -12131,23 +12185,17 @@ static MultiTimerData *add_multi_timer(MultiTimerData **first, ErlDrvPort port, break; } } - if (!p || p->when.megasecs > mtd->when.megasecs) { - goto found; - } - for (; p!= NULL; s = p, p = p->next) { + for (; p!= NULL && eq_mega(p, mtd); s = p, p = p->next) { if (p->when.secs >= mtd->when.secs) { break; } } - if (!p || p->when.secs > mtd->when.secs) { - goto found; - } - for (; p!= NULL; s = p, p = p->next) { + for (; p!= NULL && eq_mega(p, mtd) && eq_sec(p, mtd); s = p, p = p->next) { if (p->when.microsecs >= mtd->when.microsecs) { break; } } - found: + if (!p) { if (!s) { *first = mtd; @@ -12173,6 +12221,8 @@ static MultiTimerData *add_multi_timer(MultiTimerData **first, ErlDrvPort port, } return mtd; } +#undef eq_mega +#undef eq_sec diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 099f4f90de..a4311d22e2 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -1745,13 +1745,11 @@ BIF_RETTYPE hipe_bifs_check_crc_1(BIF_ALIST_1) BIF_RET(am_false); } -BIF_RETTYPE hipe_bifs_system_crc_1(BIF_ALIST_1) +BIF_RETTYPE hipe_bifs_system_crc_0(BIF_ALIST_0) { Uint crc; - if (!term_to_Uint(BIF_ARG_1, &crc)) - BIF_ERROR(BIF_P, BADARG); - crc ^= (HIPE_SYSTEM_CRC ^ HIPE_LITERALS_CRC); + crc = HIPE_SYSTEM_CRC; BIF_RET(Uint_to_term(crc, BIF_P)); } diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index a3e04802df..4271a78de3 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -74,7 +74,7 @@ bif hipe_bifs:set_native_address_in_fe/2 bif hipe_bifs:find_na_or_make_stub/2 bif hipe_bifs:check_crc/1 -bif hipe_bifs:system_crc/1 +bif hipe_bifs:system_crc/0 bif hipe_bifs:get_rts_param/1 #bif hipe_bifs:tuple_to_float/1 diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index ed355ce264..49e8d39360 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -648,8 +648,7 @@ static int do_e(FILE *fp, const char* this_exe) fprintf(fp, "-define(HIPE_SYSTEM_CRC, %u).\n", system_crc); } else { - fprintf(fp, "-define(HIPE_SYSTEM_CRC, hipe_bifs:system_crc(%u)).\n", - literals_crc); + fprintf(fp, "-define(HIPE_SYSTEM_CRC, hipe_bifs:system_crc()).\n"); } return 0; } diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c index 5ad92dad02..d050748703 100644 --- a/erts/emulator/sys/unix/erl_child_setup.c +++ b/erts/emulator/sys/unix/erl_child_setup.c @@ -55,7 +55,7 @@ void sys_sigrelease(int sig) #endif /* !SIG_SIGSET */ #if defined(__ANDROID__) -int __system_properties_fd(void); +static int system_properties_fd(void); #endif /* __ANDROID__ */ #if defined(__ANDROID__) @@ -104,9 +104,12 @@ main(int argc, char *argv[]) #if defined(HAVE_CLOSEFROM) closefrom(from); #elif defined(__ANDROID__) - for (i = from; i <= to; i++) { - if (i!=__system_properties_fd) - (void) close(i); + if (from <= to) { + int spfd = system_properties_fd(); + for (i = from; i <= to; i++) { + if (i != spfd) + (void) close(i); + } } #else for (i = from; i <= to; i++) @@ -143,9 +146,9 @@ main(int argc, char *argv[]) } #if defined(__ANDROID__) -int __system_properties_fd(void) +static int system_properties_fd(void) { - int s, fd; + int fd; char *env; env = getenv("ANDROID_PROPERTY_WORKSPACE"); @@ -156,4 +159,3 @@ int __system_properties_fd(void) return fd; } #endif /* __ANDROID__ */ - diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl index 527b6987fa..b739250aad 100644 --- a/erts/emulator/test/map_SUITE.erl +++ b/erts/emulator/test/map_SUITE.erl @@ -2615,13 +2615,76 @@ t_erts_internal_order(_Config) when is_list(_Config) -> t_erts_internal_hash(_Config) when is_list(_Config) -> K1 = 0.0, K2 = 0.0/-1, + M = maps:from_list([{I,I}||I<-lists:seq(1,32)]), - M1 = (maps:from_list([{I,I}||I<-lists:seq(1,32)]))#{ K1 => a, K2 => b }, + M1 = M#{ K1 => a, K2 => b }, b = maps:get(K2,M1), - M2 = (maps:from_list([{I,I}||I<-lists:seq(1,32)]))#{ K2 => a, K1 => b }, + M2 = M#{ K2 => a, K1 => b }, b = maps:get(K1,M2), + %% test previously faulty hash list optimization + + M3 = M#{[0] => a, [0,0] => b, [0,0,0] => c, [0,0,0,0] => d}, + a = maps:get([0],M3), + b = maps:get([0,0],M3), + c = maps:get([0,0,0],M3), + d = maps:get([0,0,0,0],M3), + + M4 = M#{{[0]} => a, {[0,0]} => b, {[0,0,0]} => c, {[0,0,0,0]} => d}, + a = maps:get({[0]},M4), + b = maps:get({[0,0]},M4), + c = maps:get({[0,0,0]},M4), + d = maps:get({[0,0,0,0]},M4), + + M5 = M3#{[0,0,0] => e, [0,0,0,0] => f, [0,0,0,0,0] => g, + [0,0,0,0,0,0] => h, [0,0,0,0,0,0,0] => i, + [0,0,0,0,0,0,0,0] => j, [0,0,0,0,0,0,0,0,0] => k}, + + a = maps:get([0],M5), + b = maps:get([0,0],M5), + e = maps:get([0,0,0],M5), + f = maps:get([0,0,0,0],M5), + g = maps:get([0,0,0,0,0],M5), + h = maps:get([0,0,0,0,0,0],M5), + i = maps:get([0,0,0,0,0,0,0],M5), + j = maps:get([0,0,0,0,0,0,0,0],M5), + k = maps:get([0,0,0,0,0,0,0,0,0],M5), + + M6 = M4#{{[0,0,0]} => e, {[0,0,0,0]} => f, {[0,0,0,0,0]} => g, + {[0,0,0,0,0,0]} => h, {[0,0,0,0,0,0,0]} => i, + {[0,0,0,0,0,0,0,0]} => j, {[0,0,0,0,0,0,0,0,0]} => k}, + + a = maps:get({[0]},M6), + b = maps:get({[0,0]},M6), + e = maps:get({[0,0,0]},M6), + f = maps:get({[0,0,0,0]},M6), + g = maps:get({[0,0,0,0,0]},M6), + h = maps:get({[0,0,0,0,0,0]},M6), + i = maps:get({[0,0,0,0,0,0,0]},M6), + j = maps:get({[0,0,0,0,0,0,0,0]},M6), + k = maps:get({[0,0,0,0,0,0,0,0,0]},M6), + + M7 = maps:merge(M5,M6), + + a = maps:get([0],M7), + b = maps:get([0,0],M7), + e = maps:get([0,0,0],M7), + f = maps:get([0,0,0,0],M7), + g = maps:get([0,0,0,0,0],M7), + h = maps:get([0,0,0,0,0,0],M7), + i = maps:get([0,0,0,0,0,0,0],M7), + j = maps:get([0,0,0,0,0,0,0,0],M7), + k = maps:get([0,0,0,0,0,0,0,0,0],M7), + a = maps:get({[0]},M7), + b = maps:get({[0,0]},M7), + e = maps:get({[0,0,0]},M7), + f = maps:get({[0,0,0,0]},M7), + g = maps:get({[0,0,0,0,0]},M7), + h = maps:get({[0,0,0,0,0,0]},M7), + i = maps:get({[0,0,0,0,0,0,0]},M7), + j = maps:get({[0,0,0,0,0,0,0,0]},M7), + k = maps:get({[0,0,0,0,0,0,0,0,0]},M7), ok. t_pdict(_Config) -> diff --git a/erts/emulator/test/module_info_SUITE.erl b/erts/emulator/test/module_info_SUITE.erl index 1125cf3072..25ba6d1787 100644 --- a/erts/emulator/test/module_info_SUITE.erl +++ b/erts/emulator/test/module_info_SUITE.erl @@ -24,7 +24,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, - exports/1,functions/1,native/1,info/1]). + exports/1,functions/1,deleted/1,native/1,info/1]). %%-compile(native). @@ -51,9 +51,8 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - modules() -> - [exports, functions, native, info]. + [exports, functions, deleted, native, info]. init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog = ?t:timetrap(?t:minutes(3)), @@ -81,43 +80,66 @@ all_functions() -> %% Test that the list of exported functions from this module is correct. exports(Config) when is_list(Config) -> - ?line All = all_exported(), - ?line All = lists:sort(?MODULE:module_info(exports)), - ?line (catch ?MODULE:foo()), - ?line All = lists:sort(?MODULE:module_info(exports)), + All = all_exported(), + All = lists:sort(?MODULE:module_info(exports)), + (catch ?MODULE:foo()), + All = lists:sort(?MODULE:module_info(exports)), ok. %% Test that the list of exported functions from this module is correct. functions(Config) when is_list(Config) -> - ?line All = all_functions(), - ?line All = lists:sort(?MODULE:module_info(functions)), + All = all_functions(), + All = lists:sort(?MODULE:module_info(functions)), + ok. + +%% Test that deleted modules cause badarg +deleted(Config) when is_list(Config) -> + Data = ?config(data_dir, Config), + File = filename:join(Data, "module_info_test"), + {ok,module_info_test,Code} = compile:file(File, [binary]), + {module,module_info_test} = erlang:load_module(module_info_test, Code), + 17 = module_info_test:f(), + [_|_] = erlang:get_module_info(module_info_test, attributes), + [_|_] = erlang:get_module_info(module_info_test), + + %% first delete it + true = erlang:delete_module(module_info_test), + {'EXIT',{undef, _}} = (catch module_info_test:f()), + {'EXIT',{badarg, _}} = (catch erlang:get_module_info(module_info_test,attributes)), + {'EXIT',{badarg, _}} = (catch erlang:get_module_info(module_info_test)), + + %% then purge it + true = erlang:purge_module(module_info_test), + {'EXIT',{undef, _}} = (catch module_info_test:f()), + {'EXIT',{badarg, _}} = (catch erlang:get_module_info(module_info_test,attributes)), + {'EXIT',{badarg, _}} = (catch erlang:get_module_info(module_info_test)), ok. %% Test that the list of exported functions from this module is correct. %% Verify that module_info(native) works. native(Config) when is_list(Config) -> - ?line All = all_functions(), - ?line case ?MODULE:module_info(native_addresses) of - [] -> - ?line false = ?MODULE:module_info(native), - {comment,"no native functions"}; - L -> - ?line true = ?MODULE:module_info(native), - %% Verify that all functions have unique addresses. - ?line S0 = sofs:set(L, [{name,arity,addr}]), - ?line S1 = sofs:projection({external,fun ?MODULE:native_proj/1}, S0), - ?line S2 = sofs:relation_to_family(S1), - ?line S3 = sofs:family_specification(fun ?MODULE:native_filter/1, S2), - ?line 0 = sofs:no_elements(S3), - ?line S4 = sofs:range(S1), - - %% Verify that the set of function with native addresses - %% is a subset of all functions in the module. - ?line AllSet = sofs:set(All, [{name,arity}]), - ?line true = sofs:is_subset(S4, AllSet), - - {comment,integer_to_list(sofs:no_elements(S0))++" native functions"} - end. + All = all_functions(), + case ?MODULE:module_info(native_addresses) of + [] -> + false = ?MODULE:module_info(native), + {comment,"no native functions"}; + L -> + true = ?MODULE:module_info(native), + %% Verify that all functions have unique addresses. + S0 = sofs:set(L, [{name,arity,addr}]), + S1 = sofs:projection({external,fun ?MODULE:native_proj/1}, S0), + S2 = sofs:relation_to_family(S1), + S3 = sofs:family_specification(fun ?MODULE:native_filter/1, S2), + 0 = sofs:no_elements(S3), + S4 = sofs:range(S1), + + %% Verify that the set of function with native addresses + %% is a subset of all functions in the module. + AllSet = sofs:set(All, [{name,arity}]), + true = sofs:is_subset(S4, AllSet), + + {comment,integer_to_list(sofs:no_elements(S0))++" native functions"} + end. native_proj({Name,Arity,Addr}) -> {Addr,{Name,Arity}}. diff --git a/erts/emulator/test/module_info_SUITE_data/module_info_test.erl b/erts/emulator/test/module_info_SUITE_data/module_info_test.erl new file mode 100644 index 0000000000..f045f38464 --- /dev/null +++ b/erts/emulator/test/module_info_SUITE_data/module_info_test.erl @@ -0,0 +1,24 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(module_info_test). +-export([f/0]). + +f() -> + 17. diff --git a/erts/emulator/test/monitor_SUITE.erl b/erts/emulator/test/monitor_SUITE.erl index dc215b1529..7326dfceb1 100644 --- a/erts/emulator/test/monitor_SUITE.erl +++ b/erts/emulator/test/monitor_SUITE.erl @@ -665,98 +665,86 @@ list_cleanup(Config) when is_list(Config) -> mixer(doc) -> "Test mixing of internal and external monitors."; mixer(Config) when is_list(Config) -> - ?line PA = filename:dirname(code:which(?MODULE)), - ?line NN = [j0,j1,j2,j3], -% ?line NN = [j0,j1], - ?line NL0 = [begin - {ok, J} = test_server:start_node - (X, slave, [{args, "-pa " ++ PA}]), - J - end || X <- NN], - ?line NL1 = lists:duplicate(2,node()) ++ NL0, - ?line Perm = perm(NL1), - ?line lists:foreach( - fun(NL) -> - ?line Js = [ start_jeeves({[],M}) || M <- (NL ++ NL) ], - ?line [ask_jeeves(P,{monitor_process,self()}) || P <- Js], - ?line {monitored_by,MB} = - process_info(self(),monitored_by), - ?line MBL = lists:sort(MB), - ?line JsL = lists:sort(Js), - ?line MBL = JsL, - ?line {monitors,[]} = process_info(self(),monitors), - ?line [tell_jeeves(P,{exit,flaff}) || P <- Js], - ?line wait_for_m([],[],200) - end, - Perm), - ?line lists:foreach( - fun(NL) -> - ?line Js = [ start_jeeves({[],M}) || M <- (NL ++ NL) ], - ?line Rs = [begin - {monitor_process,Ref} = - ask_jeeves(P,{monitor_process,self()}), - {P,Ref} - end - || P <- Js], - ?line {monitored_by,MB} = - process_info(self(),monitored_by), - ?line MBL = lists:sort(MB), - ?line JsL = lists:sort(Js), - ?line MBL = JsL, - ?line {monitors,[]} = process_info(self(),monitors), - ?line [ask_jeeves(P,{demonitor,Ref}) || {P,Ref} <- Rs], - ?line wait_for_m([],[],200), - ?line [tell_jeeves(P,{exit,flaff}) || P <- Js] - end, - Perm), - ?line lists:foreach( - fun(NL) -> - ?line Js = [ start_jeeves({[],M}) || M <- (NL ++ NL) ], - ?line [ask_jeeves(P,{monitor_process,self()}) || P <- Js], - ?line [erlang:monitor(process,P) || P <- Js], - ?line {monitored_by,MB} = - process_info(self(),monitored_by), - ?line MBL = lists:sort(MB), - ?line JsL = lists:sort(Js), - ?line MBL = JsL, - ?line {monitors,M} = - process_info(self(),monitors), - ?line ML = lists:sort([P||{process,P} <- M]), - ?line ML = JsL, - ?line [begin - tell_jeeves(P,{exit,flaff}), - receive {'DOWN',_,process,P,_} -> ok end - end || P <- Js], - ?line wait_for_m([],[],200) - end, - Perm), - ?line lists:foreach( - fun(NL) -> - ?line Js = [ start_jeeves({[],M}) || M <- (NL ++ NL) ], - ?line Rs = [begin - {monitor_process,Ref} = - ask_jeeves(P,{monitor_process,self()}), - {P,Ref} - end - || P <- Js], - ?line R2s = [{P,erlang:monitor(process,P)} || P <- Js], - ?line {monitored_by,MB} = - process_info(self(),monitored_by), - ?line MBL = lists:sort(MB), - ?line JsL = lists:sort(Js), - ?line MBL = JsL, - ?line {monitors,M} = - process_info(self(),monitors), - ?line ML = lists:sort([P||{process,P} <- M]), - ?line ML = JsL, - ?line [ask_jeeves(P,{demonitor,Ref}) || {P,Ref} <- Rs], - ?line wait_for_m(lists:sort(M),[],200), - ?line [erlang:demonitor(Ref) || {_P,Ref} <- R2s], - ?line wait_for_m([],[],200), - ?line [tell_jeeves(P,{exit,flaff}) || P <- Js] - end, - Perm), - [test_server:stop_node(K) || K <- NL0 ], + PA = filename:dirname(code:which(?MODULE)), + NN = [j0,j1,j2], + NL0 = [begin + {ok, J} = test_server:start_node(X,slave,[{args, "-pa " ++ PA}]), + J + end || X <- NN], + NL1 = lists:duplicate(2,node()) ++ NL0, + Perm = perm(NL1), + lists:foreach( + fun(NL) -> + Js = [start_jeeves({[],M}) || M <- (NL ++ NL)], + [ask_jeeves(P,{monitor_process,self()}) || P <- Js], + {monitored_by,MB} = process_info(self(),monitored_by), + MBL = lists:sort(MB), + JsL = lists:sort(Js), + MBL = JsL, + {monitors,[]} = process_info(self(),monitors), + [tell_jeeves(P,{exit,flaff}) || P <- Js], + wait_for_m([],[],200) + end, + Perm), + lists:foreach( + fun(NL) -> + Js = [start_jeeves({[],M}) || M <- (NL ++ NL)], + Rs = [begin + {monitor_process,Ref} = ask_jeeves(P,{monitor_process,self()}), + {P,Ref} + end || P <- Js], + {monitored_by,MB} = process_info(self(),monitored_by), + MBL = lists:sort(MB), + JsL = lists:sort(Js), + MBL = JsL, + {monitors,[]} = process_info(self(),monitors), + [ask_jeeves(P,{demonitor,Ref}) || {P,Ref} <- Rs], + wait_for_m([],[],200), + [tell_jeeves(P,{exit,flaff}) || P <- Js] + end, + Perm), + lists:foreach( + fun(NL) -> + Js = [start_jeeves({[],M}) || M <- (NL ++ NL)], + [ask_jeeves(P,{monitor_process,self()}) || P <- Js], + [erlang:monitor(process,P) || P <- Js], + {monitored_by,MB} = process_info(self(),monitored_by), + MBL = lists:sort(MB), + JsL = lists:sort(Js), + MBL = JsL, + {monitors,M} = process_info(self(),monitors), + ML = lists:sort([P||{process,P} <- M]), + ML = JsL, + [begin + tell_jeeves(P,{exit,flaff}), + receive {'DOWN',_,process,P,_} -> ok end + end || P <- Js], + wait_for_m([],[],200) + end, + Perm), + lists:foreach( + fun(NL) -> + Js = [start_jeeves({[],M}) || M <- (NL ++ NL)], + Rs = [begin + {monitor_process,Ref} = ask_jeeves(P,{monitor_process,self()}), + {P,Ref} + end || P <- Js], + R2s = [{P,erlang:monitor(process,P)} || P <- Js], + {monitored_by,MB} = process_info(self(),monitored_by), + MBL = lists:sort(MB), + JsL = lists:sort(Js), + MBL = JsL, + {monitors,M} = process_info(self(),monitors), + ML = lists:sort([P||{process,P} <- M]), + ML = JsL, + [ask_jeeves(P,{demonitor,Ref}) || {P,Ref} <- Rs], + wait_for_m(lists:sort(M),[],200), + [erlang:demonitor(Ref) || {_P,Ref} <- R2s], + wait_for_m([],[],200), + [tell_jeeves(P,{exit,flaff}) || P <- Js] + end, + Perm), + [test_server:stop_node(K) || K <- NL0], ok. named_down(doc) -> ["Test that DOWN message for a named monitor isn't" diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index c35c71dd5b..778f6fd087 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -39,8 +39,9 @@ get_length/1, make_atom/1, make_string/1, reverse_list_test/1, otp_9828/1, otp_9668/1, consume_timeslice/1, dirty_nif/1, dirty_nif_send/1, - dirty_nif_exception/1, nif_schedule/1, - nif_exception/1, nif_nan_and_inf/1, nif_atom_too_long/1 + dirty_nif_exception/1, call_dirty_nif_exception/1, nif_schedule/1, + nif_exception/1, call_nif_exception/1, + nif_nan_and_inf/1, nif_atom_too_long/1 ]). -export([many_args_100/100]). @@ -1387,7 +1388,7 @@ is_checks(Config) when is_list(Config) -> self(), hd(erlang:ports()), [], [1,9,9,8], {hejsan, "hejsan", [$h,"ejs",<<"an">>]}, -18446744073709551616.2e2), try - ?line error = check_is_exception(), + ?line check_is_exception(), ?line throw(expected_badarg) catch error:badarg -> @@ -1599,9 +1600,9 @@ dirty_nif_exception(Config) when is_list(Config) -> N when is_integer(N) -> ensure_lib_loaded(Config), try - %% this checks that the expected exception - %% occurs when the NIF returns the result - %% of enif_make_badarg directly + %% this checks that the expected exception occurs when the + %% dirty NIF returns the result of enif_make_badarg + %% directly call_dirty_nif_exception(1), ?t:fail(expected_badarg) catch @@ -1611,10 +1612,9 @@ dirty_nif_exception(Config) when is_list(Config) -> ok end, try - %% this checks that the expected exception - %% occurs when the NIF calls enif_make_badarg - %% at some point but then returns a value that - %% isn't an exception + %% this checks that the expected exception occurs when the + %% dirty NIF calls enif_make_badarg at some point but then + %% returns a value that isn't an exception call_dirty_nif_exception(0), ?t:fail(expected_badarg) catch @@ -1622,7 +1622,10 @@ dirty_nif_exception(Config) when is_list(Config) -> [{?MODULE,call_dirty_nif_exception,[0],_}|_] = erlang:get_stacktrace(), ok - end + end, + %% this checks that a dirty NIF can raise various terms as + %% exceptions + ok = nif_raise_exceptions(call_dirty_nif_exception) catch error:badarg -> {skipped,"No dirty scheduler support"} @@ -1631,12 +1634,18 @@ dirty_nif_exception(Config) when is_list(Config) -> nif_exception(Config) when is_list(Config) -> ensure_lib_loaded(Config), try - call_nif_exception(), + %% this checks that the expected exception occurs when the NIF + %% calls enif_make_badarg at some point but then tries to return a + %% value that isn't an exception + call_nif_exception(0), ?t:fail(expected_badarg) catch error:badarg -> ok - end. + end, + %% this checks that a NIF can raise various terms as exceptions + ok = nif_raise_exceptions(call_nif_exception), + ok. nif_nan_and_inf(Config) when is_list(Config) -> ensure_lib_loaded(Config), @@ -1758,7 +1767,20 @@ check(Exp,Got,Line) -> io:format("CHECK at ~p: Expected ~p but got ~p\n",[Line,Exp,Got]), Got end. - + +nif_raise_exceptions(NifFunc) -> + ExcTerms = [{error, test}, "a string", <<"a binary">>, + 42, [1,2,3,4,5], [{p,1},{p,2},{p,3}]], + lists:foldl(fun(Term, ok) -> + try + erlang:apply(?MODULE,NifFunc,[Term]), + ?t:fail({expected,Term}) + catch + error:Term -> + [{?MODULE,NifFunc,[Term],_}|_] = erlang:get_stacktrace(), + ok + end + end, ok, ExcTerms). %% The NIFs: lib_version() -> undefined. @@ -1814,7 +1836,7 @@ call_dirty_nif(_,_,_) -> ?nif_stub. send_from_dirty_nif(_) -> ?nif_stub. call_dirty_nif_exception(_) -> ?nif_stub. call_dirty_nif_zero_args() -> ?nif_stub. -call_nif_exception() -> ?nif_stub. +call_nif_exception(_) -> ?nif_stub. call_nif_nan_or_inf(_) -> ?nif_stub. call_nif_atom_too_long(_) -> ?nif_stub. diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index 3cc9f51ef8..1639e47d61 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -883,16 +883,19 @@ static ERL_NIF_TERM check_is(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[] * * This function is separate from check_is because it calls enif_make_badarg * and so it must return the badarg exception as its return value. Thus, the - * badarg exception indicates success. Failure is indicated by returning an - * error atom. + * badarg exception indicates success. */ static ERL_NIF_TERM check_is_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + ERL_NIF_TERM badarg, exc_term; ERL_NIF_TERM error_atom = enif_make_atom(env, "error"); - ERL_NIF_TERM badarg = enif_make_badarg(env); - if (enif_is_exception(env, error_atom)) return error_atom; - if (!enif_is_exception(env, badarg)) return error_atom; - if (!enif_has_pending_exception(env)) return error_atom; + ERL_NIF_TERM badarg_atom = enif_make_atom(env, "badarg"); + assert(!enif_is_exception(env, error_atom)); + badarg = enif_make_badarg(env); + assert(enif_is_exception(env, badarg)); + assert(enif_has_pending_exception(env, NULL)); + assert(enif_has_pending_exception(env, &exc_term)); + assert(enif_is_identical(exc_term, badarg_atom)); return badarg; } @@ -1536,9 +1539,12 @@ static ERL_NIF_TERM nif_sched1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv static ERL_NIF_TERM call_nif_schedule(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + ERL_NIF_TERM result; if (argc != 2) return enif_make_atom(env, "false"); - return enif_schedule_nif(env, "nif_sched1", 0, nif_sched1, argc, argv); + result = enif_schedule_nif(env, "nif_sched1", 0, nif_sched1, argc, argv); + assert(!enif_is_exception(env, result)); + return result; } #ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT @@ -1612,13 +1618,18 @@ static ERL_NIF_TERM call_dirty_nif_exception(ErlNifEnv* env, int argc, const ERL { switch (argc) { case 1: { - ERL_NIF_TERM args[255]; - int i; - args[0] = argv[0]; - for (i = 1; i < 255; i++) - args[i] = enif_make_int(env, i); - return enif_schedule_nif(env, "call_dirty_nif_exception", ERL_NIF_DIRTY_JOB_CPU_BOUND, - call_dirty_nif_exception, 255, argv); + int arg; + if (enif_get_int(env, argv[0], &arg) && arg < 2) { + ERL_NIF_TERM args[255]; + int i; + args[0] = argv[0]; + for (i = 1; i < 255; i++) + args[i] = enif_make_int(env, i); + return enif_schedule_nif(env, "call_dirty_nif_exception", ERL_NIF_DIRTY_JOB_CPU_BOUND, + call_dirty_nif_exception, 255, args); + } else { + return enif_raise_exception(env, argv[0]); + } } case 2: { int return_badarg_directly; @@ -1651,14 +1662,32 @@ static ERL_NIF_TERM call_dirty_nif_zero_args(ErlNifEnv* env, int argc, const ERL #endif /* - * Call enif_make_badarg, but don't return its return value. Instead, - * return ok. Result should still be a badarg exception for the erlang - * caller. + * If argv[0] is the integer 0, call enif_make_badarg, but don't return its + * return value. Instead, return ok. Result should still be a badarg + * exception for the erlang caller. + * + * For any other value of argv[0], use it as an exception term and return + * the exception. */ static ERL_NIF_TERM call_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - /* ignore return value */ enif_make_badarg(env); - return enif_make_atom(env, "ok"); + ERL_NIF_TERM exc_term; + ERL_NIF_TERM badarg_atom = enif_make_atom(env, "badarg"); + int arg; + + if (enif_get_int(env, argv[0], &arg) && arg == 0) { + /* ignore return value */ enif_make_badarg(env); + assert(enif_has_pending_exception(env, NULL)); + assert(enif_has_pending_exception(env, &exc_term)); + assert(enif_is_identical(badarg_atom, exc_term)); + return enif_make_atom(env, "ok"); + } else { + ERL_NIF_TERM exc_retval = enif_raise_exception(env, argv[0]); + assert(enif_has_pending_exception(env, NULL)); + assert(enif_has_pending_exception(env, &exc_term)); + assert(enif_is_identical(argv[0], exc_term)); + return exc_retval; + } } #if !defined(NAN) || !defined(INFINITY) @@ -1693,7 +1722,7 @@ static ERL_NIF_TERM call_nif_nan_or_inf(ErlNifEnv* env, int argc, const ERL_NIF_ } res = enif_make_double(env, val); assert(enif_is_exception(env, res)); - assert(enif_has_pending_exception(env)); + assert(enif_has_pending_exception(env, NULL)); if (strcmp(arg, "tuple") == 0) { return enif_make_tuple2(env, argv[0], res); } else { @@ -1802,7 +1831,7 @@ static ERL_NIF_TERM sorted_list_from_maps_nif(ErlNifEnv* env, int argc, const ER if (argc != 1 && !enif_is_map(env, map)) return enif_make_int(env, __LINE__); - if(!enif_map_iterator_create(env, map, &iter_f, ERL_NIF_MAP_ITERATOR_HEAD)) + if(!enif_map_iterator_create(env, map, &iter_f, ERL_NIF_MAP_ITERATOR_FIRST)) return enif_make_int(env, __LINE__); cnt = 0; @@ -1817,7 +1846,7 @@ static ERL_NIF_TERM sorted_list_from_maps_nif(ErlNifEnv* env, int argc, const ER if (cnt && next_ret) return enif_make_int(env, __LINE__); - if(!enif_map_iterator_create(env, map, &iter_b, ERL_NIF_MAP_ITERATOR_TAIL)) + if(!enif_map_iterator_create(env, map, &iter_b, ERL_NIF_MAP_ITERATOR_LAST)) return enif_make_int(env, __LINE__); cnt = 0; @@ -1913,7 +1942,7 @@ static ErlNifFunc nif_funcs[] = {"call_dirty_nif_exception", 1, call_dirty_nif_exception, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"call_dirty_nif_zero_args", 0, call_dirty_nif_zero_args, ERL_NIF_DIRTY_JOB_CPU_BOUND}, #endif - {"call_nif_exception", 0, call_nif_exception}, + {"call_nif_exception", 1, call_nif_exception}, {"call_nif_nan_or_inf", 1, call_nif_nan_or_inf}, {"call_nif_atom_too_long", 1, call_nif_atom_too_long}, {"is_map_nif", 1, is_map_nif}, diff --git a/erts/emulator/test/send_term_SUITE.erl b/erts/emulator/test/send_term_SUITE.erl index 8e1f8df43a..4deae63ce7 100644 --- a/erts/emulator/test/send_term_SUITE.erl +++ b/erts/emulator/test/send_term_SUITE.erl @@ -279,7 +279,10 @@ generate_external_terms_files(BaseDir) -> {4444444444444444,-44444, [[[[[[[[[[[5]]]]]]]]]]], make_ref()}, {444444444444444444444,-44444, {{{{{{{{{{{{6}}}}}}}}}}}}, make_ref()}, {444444444444444,-44444, {{{{{{{{{{{{7}}}}}}}}}}}}, make_ref()}, - {4444444444444444444,-44444, {{{{{{{{{{{{8}}}}}}}}}}}}, make_ref()}], + {4444444444444444444,-44444, {{{{{{{{{{{{8}}}}}}}}}}}}, make_ref()}, + #{}, + #{1 => 11, 2 => 22, 3 => 33}, + maps:from_list([{K,K*11} || K <- lists:seq(1,100)])], ok = file:write_file(filename:join([BaseDir, "send_term_SUITE_data", "ext_terms.bin"]), diff --git a/erts/emulator/test/send_term_SUITE_data/ext_terms.bin b/erts/emulator/test/send_term_SUITE_data/ext_terms.bin Binary files differindex b239284323..5ff0b2ccf1 100644 --- a/erts/emulator/test/send_term_SUITE_data/ext_terms.bin +++ b/erts/emulator/test/send_term_SUITE_data/ext_terms.bin diff --git a/erts/emulator/test/send_term_SUITE_data/ext_terms.h b/erts/emulator/test/send_term_SUITE_data/ext_terms.h index 08134f3b05..5585585ec3 100644 --- a/erts/emulator/test/send_term_SUITE_data/ext_terms.h +++ b/erts/emulator/test/send_term_SUITE_data/ext_terms.h @@ -24,9 +24,9 @@ #ifndef EXT_TERMS_H__ #define EXT_TERMS_H__ static struct { - unsigned char ext[162]; + unsigned char ext[637]; int ext_size; - unsigned char cext[162]; + unsigned char cext[637]; int cext_size; } ext_terms[] = { {{131,104,3,98,0,0,18,103,98,255,255,237,153,108,0,0,0,2,100,0,7,97,110,95,97,116,111,109,107,0,6,97,32,108,105,115,116,106}, @@ -37,26 +37,26 @@ static struct { 46, {131,108,0,0,0,4,110,9,0,0,0,160,222,197,173,201,53,54,110,7,1,199,113,21,183,140,242,3,107,0,6,98,108,117,112,112,33,100,0,5,98,108,105,112,112,106}, 46}, - {{131,104,5,103,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,38,0,0,0,0,3,104,2,114,0,3,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,40,0,0,0,0,0,0,0,0,102,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,1,3,103,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,0,0,0,37,0,0,0,0,3,102,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,0,0,0,1,3,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,59,0,0,0,0,0,0,0,0}, - 162, - {131,80,0,0,0,161,120,156,203,96,77,79,97,224,77,140,207,203,79,73,117,72,207,47,74,74,76,103,96,96,80,3,98,6,230,12,166,34,6,102,116,89,102,160,140,6,3,20,164,97,209,203,200,12,52,145,39,17,85,80,21,108,96,26,166,4,35,51,216,14,20,97,144,21,214,48,43,0,1,209,36,52}, - 82}, - {{131,104,5,104,0,106,106,112,0,0,0,79,0,21,87,190,182,1,38,106,214,65,228,1,52,27,227,2,212,0,0,0,1,0,0,0,0,100,0,15,115,101,110,100,95,116,101,114,109,95,83,85,73,84,69,97,1,98,0,184,11,180,103,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,0,0,0,37,0,0,0,0,3,109,0,0,0,31,104,101,106,32,104,111,112,112,32,116,114,97,108,108,97,108,97,97,97,97,97,97,97,97,97,97,97,97,97,97,97}, - 123, - {131,80,0,0,0,122,120,156,203,96,205,96,200,202,42,96,96,96,240,103,16,13,223,183,141,81,45,235,154,227,19,70,19,233,199,76,87,128,130,140,64,204,144,194,192,95,156,154,151,18,95,146,90,148,27,31,28,234,25,226,154,200,152,196,176,131,123,75,122,10,3,79,98,94,126,74,170,67,122,126,81,82,98,58,80,173,42,72,3,115,46,144,144,207,72,205,82,200,200,47,40,80,40,41,74,204,201,73,204,73,68,5,0,18,237,35,68}, - 117}, + {{131,104,5,103,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,41,0,0,0,0,2,104,2,114,0,3,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,2,0,0,0,42,0,0,0,3,0,0,0,0,102,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,2,103,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,40,0,0,0,0,3,102,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,3,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,56,0,0,0,3,0,0,0,0}, + 204, + {131,80,0,0,0,203,120,156,203,96,77,79,97,16,73,140,207,203,79,73,117,72,205,169,48,54,201,200,171,52,50,210,53,76,98,96,96,208,4,98,6,166,12,166,34,6,102,28,138,152,128,10,180,128,152,25,164,50,13,183,73,12,76,64,107,132,179,19,115,114,48,229,52,64,242,204,105,56,229,25,152,193,246,99,147,5,89,107,1,179,30,0,103,37,46,144}, + 96}, + {{131,104,5,104,0,106,106,112,0,0,0,86,0,123,56,104,225,98,55,108,63,185,201,160,64,191,31,210,203,0,0,0,2,0,0,0,0,100,0,15,115,101,110,100,95,116,101,114,109,95,83,85,73,84,69,97,2,98,3,217,195,71,103,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,40,0,0,0,0,3,109,0,0,0,31,104,101,106,32,104,111,112,112,32,116,114,97,108,108,97,108,97,97,97,97,97,97,97,97,97,97,97,97,97,97,97}, + 130, + {131,80,0,0,0,129,120,156,203,96,205,96,200,202,42,96,96,96,8,99,168,182,200,120,152,100,158,99,191,243,228,2,135,253,242,151,78,3,5,153,128,152,33,133,129,191,56,53,47,37,190,36,181,40,55,62,56,212,51,196,53,145,41,137,249,230,97,247,244,20,6,225,236,196,156,156,84,135,212,156,10,99,147,140,188,74,35,35,93,195,36,160,22,13,144,62,230,92,32,33,159,145,154,165,144,145,95,80,160,80,82,4,84,154,152,147,136,10,0,219,221,39,33}, + 123}, {{131,108,0,0,0,4,110,10,0,28,199,113,166,118,185,145,86,105,9,110,5,1,28,103,24,89,10,107,0,2,98,33,100,0,10,98,108,105,112,112,112,112,112,112,112,106}, 46, {131,108,0,0,0,4,110,10,0,28,199,113,166,118,185,145,86,105,9,110,5,1,28,103,24,89,10,107,0,2,98,33,100,0,10,98,108,105,112,112,112,112,112,112,112,106}, 46}, - {{131,104,5,98,0,0,18,103,103,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,38,0,0,0,0,3,104,2,114,0,3,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,40,0,0,0,0,0,0,0,0,102,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,1,3,98,255,255,237,153,108,0,0,0,2,100,0,7,97,110,95,97,116,111,109,107,0,6,97,32,108,105,115,116,106}, - 120, - {131,80,0,0,0,119,120,156,203,96,77,98,96,16,74,79,79,97,224,77,140,207,203,79,73,117,72,207,47,74,74,76,103,96,96,80,3,98,6,230,12,166,34,6,102,116,89,102,160,140,6,3,20,164,97,209,203,200,156,244,255,255,219,153,57,64,38,83,10,3,123,98,94,124,98,73,126,110,54,3,91,162,66,78,102,113,73,22,0,167,192,30,158}, - 93}, - {{131,104,4,103,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,38,0,0,0,0,3,104,2,114,0,3,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,40,0,0,0,0,0,0,0,0,102,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,1,3,103,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,0,0,0,0,0,0,0,0,3,102,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,0,0,0,1,3}, - 131, - {131,80,0,0,0,130,120,156,203,96,73,79,97,224,77,140,207,203,79,73,117,72,207,47,74,74,76,103,96,96,80,3,98,6,230,12,166,34,6,102,116,89,102,160,140,6,3,20,164,97,209,203,200,12,52,145,39,17,85,16,12,152,211,48,37,24,153,1,215,214,30,50}, - 72}, + {{131,104,5,98,0,0,18,103,103,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,41,0,0,0,0,2,104,2,114,0,3,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,2,0,0,0,42,0,0,0,3,0,0,0,0,102,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,2,98,255,255,237,153,108,0,0,0,2,100,0,7,97,110,95,97,116,111,109,107,0,6,97,32,108,105,115,116,106}, + 141, + {131,80,0,0,0,140,120,156,203,96,77,98,96,16,74,79,79,97,16,73,140,207,203,79,73,117,72,205,169,48,54,201,200,171,52,50,210,53,4,202,49,104,2,49,3,83,6,83,17,3,51,14,69,76,64,5,90,64,204,12,82,153,134,219,36,6,166,164,255,255,223,206,204,1,177,82,24,216,19,243,226,19,75,242,115,179,25,216,18,21,114,50,139,75,178,0,77,99,35,202}, + 100}, + {{131,104,4,103,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,41,0,0,0,0,2,104,2,114,0,3,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,2,0,0,0,42,0,0,0,3,0,0,0,0,102,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,2,103,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,0,0,0,0,3,102,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,3}, + 166, + {131,80,0,0,0,165,120,156,203,96,73,79,97,16,73,140,207,203,79,73,117,72,205,169,48,54,201,200,171,52,50,210,53,76,98,96,96,208,4,98,6,166,12,166,34,6,102,28,138,152,128,10,180,128,152,25,164,50,13,183,73,12,76,64,107,132,179,19,115,114,176,200,129,0,115,26,110,121,102,0,219,33,38,209}, + 84}, {{131,104,3,98,0,0,18,103,98,255,255,237,153,108,0,0,0,2,100,0,7,97,110,95,97,116,111,109,107,0,6,97,32,108,105,115,116,106}, 38, {131,104,3,98,0,0,18,103,98,255,255,237,153,108,0,0,0,2,100,0,7,97,110,95,97,116,111,109,107,0,6,97,32,108,105,115,116,106}, @@ -65,46 +65,58 @@ static struct { 33, {131,104,3,98,0,0,18,103,98,255,255,237,153,108,0,0,0,2,100,0,4,97,116,111,109,107,0,4,108,105,115,116,106}, 33}, - {{131,104,4,103,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,38,0,0,0,0,3,104,2,114,0,3,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,40,0,0,0,0,0,0,0,0,102,100,0,13,97,95,110,111,100,101,64,103,111,114,98,97,103,0,0,0,1,3,103,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,0,0,0,0,0,0,0,0,3,102,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,0,0,0,1,3}, - 131, - {131,80,0,0,0,130,120,156,203,96,73,79,97,224,77,140,207,203,79,73,117,72,207,47,74,74,76,103,96,96,80,3,98,6,230,12,166,34,6,102,116,89,102,160,140,6,3,20,164,97,209,203,200,12,52,145,39,17,85,16,12,152,211,48,37,24,153,1,215,214,30,50}, + {{131,104,4,103,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,41,0,0,0,0,2,104,2,114,0,3,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,2,0,0,0,42,0,0,0,3,0,0,0,0,102,100,0,20,97,95,110,111,100,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,2,103,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,0,0,0,0,3,102,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,0,0,0,0,3}, + 166, + {131,80,0,0,0,165,120,156,203,96,73,79,97,16,73,140,207,203,79,73,117,72,205,169,48,54,201,200,171,52,50,210,53,76,98,96,96,208,4,98,6,166,12,166,34,6,102,28,138,152,128,10,180,128,152,25,164,50,13,183,73,12,76,64,107,132,179,19,115,114,176,200,129,0,115,26,110,121,102,0,219,33,38,209}, + 84}, + {{131,104,4,110,8,0,28,199,17,175,172,214,173,61,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,0,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,57,0,0,0,3,0,0,0,0}, + 81, + {131,80,0,0,0,80,120,156,203,96,201,227,96,144,57,46,184,126,205,181,181,182,73,255,255,7,165,100,48,98,133,12,69,12,204,41,12,194,217,137,57,57,169,14,169,57,21,198,38,25,121,149,70,70,186,134,73,204,12,12,12,150,64,12,162,25,0,231,161,20,138}, + 71}, + {{131,104,4,110,9,0,28,199,241,98,116,219,231,23,24,98,255,255,82,100,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,107,0,1,1,106,106,106,106,106,106,106,106,106,106,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,58,0,0,0,3,0,0,0,0}, + 122, + {131,80,0,0,0,121,120,156,203,96,201,227,100,144,57,254,49,169,228,246,115,113,137,164,255,255,131,82,114,24,24,24,24,73,34,178,25,24,25,179,224,160,136,129,57,133,65,56,59,49,39,39,213,33,53,167,194,216,36,35,175,210,200,72,215,48,137,25,168,214,10,136,65,52,3,0,142,142,25,0}, + 80}, + {{131,104,4,110,8,0,28,199,129,17,222,251,42,6,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,2,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,59,0,0,0,3,0,0,0,0}, + 83, + {131,80,0,0,0,82,120,156,203,96,201,227,96,144,57,222,40,120,239,183,22,91,210,255,255,65,41,25,140,216,97,34,83,17,3,115,10,131,112,118,98,78,78,170,67,106,78,133,177,73,70,94,165,145,145,174,97,18,51,3,3,131,53,16,131,104,6,0,233,167,20,95}, 72}, - {{131,104,4,110,8,0,28,199,17,175,172,214,173,61,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,0,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,60,0,0,0,0,0,0,0,0}, - 74, - {131,80,0,0,0,73,120,156,203,96,201,227,96,144,57,46,184,126,205,181,181,182,73,255,255,7,165,100,48,98,133,12,69,12,204,41,12,60,137,121,249,41,169,14,233,249,69,73,137,233,204,12,12,12,54,12,80,0,0,73,17,18,208}, - 63}, - {{131,104,4,110,9,0,28,199,241,98,116,219,231,23,24,98,255,255,82,100,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,107,0,1,1,106,106,106,106,106,106,106,106,106,106,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,61,0,0,0,0,0,0,0,0}, - 115, - {131,80,0,0,0,114,120,156,203,96,201,227,100,144,57,254,49,169,228,246,115,113,137,164,255,255,131,82,114,24,24,24,24,73,34,178,25,24,25,179,224,160,136,129,57,133,129,39,49,47,63,37,213,33,61,191,40,41,49,157,25,168,200,150,1,10,0,208,188,23,70}, + {{131,104,4,110,9,0,28,199,113,221,139,146,14,239,240,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,3,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,60,0,0,0,3,0,0,0,0}, + 84, + {131,80,0,0,0,83,120,156,203,96,201,227,100,144,57,94,120,183,123,18,223,251,15,73,255,255,7,165,100,48,98,135,137,204,69,12,204,41,12,194,217,137,57,57,169,14,169,57,21,198,38,25,121,149,70,70,186,134,73,204,12,12,12,54,64,12,162,25,0,106,11,22,31}, + 73}, + {{131,104,4,110,9,0,28,199,177,214,190,98,202,104,2,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,4,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,61,0,0,0,3,0,0,0,0}, + 84, + {131,80,0,0,0,83,120,156,203,96,201,227,100,144,57,190,241,218,190,164,83,25,76,73,255,255,7,165,100,48,98,135,137,44,69,12,204,41,12,194,217,137,57,57,169,14,169,57,21,198,38,25,121,149,70,70,186,134,73,204,12,12,12,182,64,12,162,25,0,74,151,21,164}, + 73}, + {{131,104,4,110,7,0,28,199,85,220,50,202,15,98,255,255,82,100,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,107,0,1,5,106,106,106,106,106,106,106,106,106,106,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,62,0,0,0,3,0,0,0,0}, + 120, + {131,80,0,0,0,119,120,156,203,96,201,99,103,144,57,30,122,199,232,20,127,210,255,255,65,41,57,12,12,12,140,36,17,217,12,140,172,89,112,80,196,192,156,194,32,156,157,152,147,147,234,144,154,83,97,108,146,145,87,105,100,164,107,152,196,12,84,107,7,196,32,154,1,0,225,225,23,138}, + 78}, + {{131,104,4,110,9,0,28,199,241,98,116,219,231,23,24,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,6,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,63,0,0,0,3,0,0,0,0}, + 84, + {131,80,0,0,0,83,120,156,203,96,201,227,100,144,57,254,49,169,228,246,115,113,137,164,255,255,131,82,50,24,177,195,68,182,34,6,230,20,6,225,236,196,156,156,84,135,212,156,10,99,147,140,188,74,35,35,93,195,36,102,6,6,6,123,32,6,209,12,0,64,205,21,133}, + 73}, + {{131,104,4,110,7,0,28,199,59,73,56,148,1,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,7,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,64,0,0,0,3,0,0,0,0}, + 82, + {131,80,0,0,0,81,120,156,203,96,201,99,103,144,57,110,237,105,49,133,49,233,255,255,160,148,12,70,236,48,145,189,136,129,57,133,65,56,59,49,39,39,213,33,53,167,194,216,36,35,175,210,200,72,215,48,137,153,129,129,193,1,136,65,52,3,0,137,143,19,30}, 71}, - {{131,104,4,110,8,0,28,199,129,17,222,251,42,6,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,2,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,62,0,0,0,0,0,0,0,0}, - 76, - {131,80,0,0,0,75,120,156,203,96,201,227,96,144,57,222,40,120,239,183,22,91,210,255,255,65,41,25,140,216,97,34,83,17,3,115,10,3,79,98,94,126,74,170,67,122,126,81,82,98,58,51,3,3,131,29,3,20,0,0,76,82,18,165}, - 64}, - {{131,104,4,110,9,0,28,199,113,221,139,146,14,239,240,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,3,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,63,0,0,0,0,0,0,0,0}, - 77, - {131,80,0,0,0,76,120,156,203,96,201,227,100,144,57,94,120,183,123,18,223,251,15,73,255,255,7,165,100,48,98,135,137,204,69,12,204,41,12,60,137,121,249,41,169,14,233,249,69,73,137,233,204,12,12,12,246,12,80,0,0,192,110,20,101}, - 65}, - {{131,104,4,110,9,0,28,199,177,214,190,98,202,104,2,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,4,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,64,0,0,0,0,0,0,0,0}, - 77, - {131,80,0,0,0,76,120,156,203,96,201,227,100,144,57,190,241,218,190,164,83,25,76,73,255,255,7,165,100,48,98,135,137,44,69,12,204,41,12,60,137,121,249,41,169,14,233,249,69,73,137,233,204,12,12,12,14,12,80,0,0,164,94,19,234}, - 65}, - {{131,104,4,110,7,0,28,199,85,220,50,202,15,98,255,255,82,100,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,108,0,0,0,1,107,0,1,5,106,106,106,106,106,106,106,106,106,106,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,65,0,0,0,0,0,0,0,0}, - 113, - {131,80,0,0,0,112,120,156,203,96,201,99,103,144,57,30,122,199,232,20,127,210,255,255,65,41,57,12,12,12,140,36,17,217,12,140,172,89,112,80,196,192,156,194,192,147,152,151,159,146,234,144,158,95,148,148,152,206,12,84,228,200,0,5,0,46,116,21,208}, - 69}, - {{131,104,4,110,9,0,28,199,241,98,116,219,231,23,24,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,6,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,66,0,0,0,0,0,0,0,0}, - 77, - {131,80,0,0,0,76,120,156,203,96,201,227,100,144,57,254,49,169,228,246,115,113,137,164,255,255,131,82,50,24,177,195,68,182,34,6,230,20,6,158,196,188,252,148,84,135,244,252,162,164,196,116,102,6,6,6,39,6,40,0,0,155,123,19,203}, - 65}, - {{131,104,4,110,7,0,28,199,59,73,56,148,1,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,7,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,67,0,0,0,0,0,0,0,0}, - 75, - {131,80,0,0,0,74,120,156,203,96,201,99,103,144,57,110,237,105,49,133,49,233,255,255,160,148,12,70,236,48,145,189,136,129,57,133,129,39,49,47,63,37,213,33,61,191,40,41,49,157,153,129,129,193,153,1,10,0,245,21,17,100}, - 62}, - {{131,104,4,110,8,0,28,199,17,175,172,214,173,61,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,8,114,0,3,100,0,12,97,110,111,100,101,64,103,111,114,98,97,103,3,0,0,0,68,0,0,0,0,0,0,0,0}, - 76, - {131,80,0,0,0,75,120,156,203,96,201,227,96,144,57,46,184,126,205,181,181,182,73,255,255,7,165,100,48,98,135,137,28,69,12,204,41,12,60,137,121,249,41,169,14,233,249,69,73,137,233,204,12,12,12,46,12,80,0,0,112,226,19,66}, - 64} + {{131,104,4,110,8,0,28,199,17,175,172,214,173,61,98,255,255,82,100,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,104,1,97,8,114,0,3,100,0,19,107,97,108,108,101,64,101,108,120,51,52,104,110,121,50,50,45,49,98,3,0,0,0,65,0,0,0,3,0,0,0,0}, + 83, + {131,80,0,0,0,82,120,156,203,96,201,227,96,144,57,46,184,126,205,181,181,182,73,255,255,7,165,100,48,98,135,137,28,69,12,204,41,12,194,217,137,57,57,169,14,169,57,21,198,38,25,121,149,70,70,186,134,73,204,12,12,12,142,64,12,162,25,0,18,103,20,252}, + 72}, + {{131,116,0,0,0,0}, + 6, + {131,116,0,0,0,0}, + 6}, + {{131,116,0,0,0,3,97,1,97,11,97,2,97,22,97,3,97,33}, + 18, + {131,116,0,0,0,3,97,1,97,11,97,2,97,22,97,3,97,33}, + 18}, + {{131,116,0,0,0,100,97,48,98,0,0,2,16,97,62,98,0,0,2,170,97,11,97,121,97,39,98,0,0,1,173,97,83,98,0,0,3,145,97,63,98,0,0,2,181,97,34,98,0,0,1,118,97,68,98,0,0,2,236,97,26,98,0,0,1,30,97,78,98,0,0,3,90,97,52,98,0,0,2,60,97,15,97,165,97,64,98,0,0,2,192,97,75,98,0,0,3,57,97,81,98,0,0,3,123,97,71,98,0,0,3,13,97,20,97,220,97,50,98,0,0,2,38,97,17,97,187,97,25,98,0,0,1,19,97,65,98,0,0,2,203,97,98,98,0,0,4,54,97,79,98,0,0,3,101,97,13,97,143,97,44,98,0,0,1,228,97,8,97,88,97,99,98,0,0,4,65,97,36,98,0,0,1,140,97,67,98,0,0,2,225,97,7,97,77,97,66,98,0,0,2,214,97,85,98,0,0,3,167,97,76,98,0,0,3,68,97,1,97,11,97,32,98,0,0,1,96,97,69,98,0,0,2,247,97,37,98,0,0,1,151,97,35,98,0,0,1,129,97,84,98,0,0,3,156,97,3,97,33,97,82,98,0,0,3,134,97,45,98,0,0,1,239,97,55,98,0,0,2,93,97,6,97,66,97,2,97,22,97,94,98,0,0,4,10,97,49,98,0,0,2,27,97,41,98,0,0,1,195,97,91,98,0,0,3,233,97,87,98,0,0,3,189,97,33,98,0,0,1,107,97,42,98,0,0,1,206,97,74,98,0,0,3,46,97,60,98,0,0,2,148,97,43,98,0,0,1,217,97,10,97,110,97,70,98,0,0,3,2,97,9,97,99,97,72,98,0,0,3,24,97,86,98,0,0,3,178,97,19,97,209,97,56,98,0,0,2,104,97,95,98,0,0,4,21,97,57,98,0,0,2,115,97,51,98,0,0,2,49,97,14,97,154,97,5,97,55,97,54,98,0,0,2,82,97,18,97,198,97,61,98,0,0,2,159,97,31,98,0,0,1,85,97,22,97,242,97,29,98,0,0,1,63,97,97,98,0,0,4,43,97,21,97,231,97,89,98,0,0,3,211,97,27,98,0,0,1,41,97,24,98,0,0,1,8,97,47,98,0,0,2,5,97,100,98,0,0,4,76,97,40,98,0,0,1,184,97,96,98,0,0,4,32,97,73,98,0,0,3,35,97,90,98,0,0,3,222,97,30,98,0,0,1,74,97,58,98,0,0,2,126,97,80,98,0,0,3,112,97,88,98,0,0,3,200,97,59,98,0,0,2,137,97,77,98,0,0,3,79,97,23,97,253,97,28,98,0,0,1,52,97,46,98,0,0,1,250,97,92,98,0,0,3,244,97,53,98,0,0,2,71,97,93,98,0,0,3,255,97,16,97,176,97,38,98,0,0,1,162,97,4,97,44,97,12,97,132}, + 637, + {131,80,0,0,2,124,120,156,21,143,123,100,87,113,24,135,207,126,219,218,165,86,171,109,181,182,118,107,181,181,123,171,181,181,218,90,91,91,171,93,90,91,173,221,219,158,93,24,145,40,145,49,70,98,140,68,68,70,196,196,136,137,24,137,136,68,68,70,140,68,68,34,34,35,34,117,158,191,158,247,115,121,223,239,57,55,130,32,152,228,224,120,16,68,146,57,33,150,217,204,45,10,195,49,234,41,23,66,68,223,163,193,224,57,123,53,111,210,172,250,65,134,42,155,115,86,6,169,210,172,99,27,75,156,116,124,69,187,65,45,221,98,134,86,145,68,42,159,56,100,94,192,118,94,176,219,27,41,52,234,188,99,60,68,76,53,93,86,167,72,226,46,165,230,95,137,167,159,9,195,70,246,233,44,112,202,141,47,196,209,73,147,227,71,122,221,122,66,135,104,38,42,252,139,92,171,99,180,152,255,102,191,234,1,249,98,142,139,214,22,137,38,143,30,199,59,148,25,252,164,198,246,8,155,104,34,194,78,46,251,106,34,149,186,153,20,217,121,205,144,27,223,233,19,47,201,211,188,66,177,120,79,155,102,57,117,46,220,167,68,115,157,68,174,114,218,32,66,2,19,156,113,76,231,146,120,70,10,31,56,106,125,154,81,95,75,163,86,117,157,195,162,146,173,60,36,150,26,170,149,61,236,224,13,245,142,143,200,241,122,111,248,149,191,200,114,108,0,15,148,144,198,55,6,188,190,70,166,65,17,233,34,158,10,23,99,153,180,214,193,1,205,85,198,84,185,156,117,33,159,65,241,153,108,179,54,142,185,48,203,121,205,107,244,139,183,28,215,156,167,83,213,197,46,254,178,199,118,21,229,226,15,195,6,27,28,177,214,202,136,234,31,201,172,80,96,254,152,24,74,217,194,237,255,248,120,145,79}, + 418} }; -#define NO_OF_EXT_TERMS 19 +#define NO_OF_EXT_TERMS 22 #endif diff --git a/erts/emulator/test/trace_meta_SUITE.erl b/erts/emulator/test/trace_meta_SUITE.erl index 45987cc319..25a59e0b07 100644 --- a/erts/emulator/test/trace_meta_SUITE.erl +++ b/erts/emulator/test/trace_meta_SUITE.erl @@ -72,7 +72,7 @@ config(priv_dir,_) -> info/1, tracer/1, combo/1, nosilent/1]). init_per_testcase(_Case, Config) -> - ?line Dog=test_server:timetrap(test_server:minutes(5)), + Dog=test_server:timetrap(test_server:minutes(5)), [{watchdog, Dog}|Config]. end_per_testcase(_Case, Config) -> @@ -179,107 +179,112 @@ nosilent(Config) when is_list(Config) -> %%% basic_test() -> - ?line Pid = setup(), - ?line erlang:trace_pattern({?MODULE,'_','_'},[],[meta]), - ?line [1,1,1,0] = apply_slave(?MODULE,exported_wrap,[1]), - ?line ?CTT(Pid,{?MODULE,exported_wrap,[1]}) = receive_next(), - ?line ?CTT(Pid,{?MODULE,exported,[1]}) = receive_next(), - ?line ?CTT(Pid,{?MODULE,local,[1]}) = receive_next(), - ?line ?CTT(Pid,{?MODULE,local2,[1]}) = receive_next(), - ?line ?CTT(Pid,{?MODULE,local_tail,[1]}) = receive_next(), - ?line erlang:trace_pattern({?MODULE,'_','_'},false,[meta]), - ?line [1,1,1,0] = apply_slave(?MODULE,exported_wrap,[1]), - ?line ?NM, - ?line [1,1,1,0] = lambda_slave(fun() -> - exported_wrap(1) - end), - ?line ?NM, - ?line erlang:trace_pattern({?MODULE,'_','_'},[],[meta]), - ?line [1,1,1,0] = lambda_slave(fun() -> - exported_wrap(1) - end), - ?line ?CTT(Pid,{?MODULE,_,_}) = receive_next(), %% The fun - ?line ?CTT(Pid,{?MODULE,exported_wrap,[1]}) = receive_next(), - ?line ?CTT(Pid,{?MODULE,exported,[1]}) = receive_next(), - ?line ?CTT(Pid,{?MODULE,local,[1]}) = receive_next(), - ?line ?CTT(Pid,{?MODULE,local2,[1]}) = receive_next(), - ?line ?CTT(Pid,{?MODULE,local_tail,[1]}) = receive_next(), - ?line erlang:trace_pattern({?MODULE,'_','_'},false,[meta]), - ?line shutdown(), - ?line ?NM, + Pid = setup(), + erlang:trace_pattern({?MODULE,'_','_'},[],[meta]), + [1,1,1,0] = apply_slave(?MODULE,exported_wrap,[1]), + ?CTT(Pid,{?MODULE,exported_wrap,[1]}) = receive_next(), + ?CTT(Pid,{?MODULE,exported,[1]}) = receive_next(), + ?CTT(Pid,{?MODULE,local,[1]}) = receive_next(), + ?CTT(Pid,{?MODULE,local2,[1]}) = receive_next(), + ?CTT(Pid,{?MODULE,local_tail,[1]}) = receive_next(), + erlang:trace_pattern({?MODULE,'_','_'},false,[meta]), + [1,1,1,0] = apply_slave(?MODULE,exported_wrap,[1]), + ?NM, + [1,1,1,0] = lambda_slave(fun() -> + exported_wrap(1) + end), + ?NM, + erlang:trace_pattern({?MODULE,'_','_'},[],[meta]), + [1,1,1,0] = lambda_slave(fun() -> + exported_wrap(1) + end), + ?CTT(Pid,{?MODULE,_,_}) = receive_next(), %% The fun + ?CTT(Pid,{?MODULE,exported_wrap,[1]}) = receive_next(), + ?CTT(Pid,{?MODULE,exported,[1]}) = receive_next(), + ?CTT(Pid,{?MODULE,local,[1]}) = receive_next(), + ?CTT(Pid,{?MODULE,local2,[1]}) = receive_next(), + ?CTT(Pid,{?MODULE,local_tail,[1]}) = receive_next(), + erlang:trace_pattern({?MODULE,'_','_'},false,[meta]), + shutdown(), + ?NM, ok. %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% return_test() -> - ?line Pid = setup(), - ?line erlang:trace_pattern({?MODULE,'_','_'},[{'_',[],[{return_trace}]}], + Pid = setup(), + erlang:trace_pattern({?MODULE,'_','_'},[{'_',[],[{return_trace}]}], [meta]), - ?line erlang:trace_pattern({erlang,phash2,'_'},[{'_',[],[{return_trace}]}], + erlang:trace_pattern({erlang,phash2,'_'},[{'_',[],[{return_trace}]}], [meta]), - ?line [1,1,1,0] = apply_slave(?MODULE,exported_wrap,[1]), - ?line ?CTT(Pid,{?MODULE,exported_wrap,[1]}) = receive_next(), - ?line ?CTT(Pid,{?MODULE,exported,[1]}) = receive_next(), - ?line ?CTT(Pid,{?MODULE,local,[1]}) = receive_next(), - ?line ?CTT(Pid,{?MODULE,local2,[1]}) = receive_next(), - ?line ?CTT(Pid,{?MODULE,local_tail,[1]}) = receive_next(), - ?line ?CTT(Pid,{erlang,phash2,[1,1]}) = receive_next(), - ?line ?RFT(Pid,{erlang,phash2,2},0) = receive_next(), - ?line ?RFT(Pid,{?MODULE,local_tail,1},[1,0]) = receive_next(), - ?line ?RFT(Pid,{?MODULE,local2,1},[1,0]) = receive_next(), - ?line ?RFT(Pid,{?MODULE,local,1},[1,1,0]) = receive_next(), - ?line ?RFT(Pid,{?MODULE,exported,1},[1,1,1,0]) = receive_next(), - ?line ?RFT(Pid,{?MODULE,exported_wrap,1},[1,1,1,0]) = receive_next(), - ?line shutdown(), - ?line ?NM, + [1,1,1,0] = apply_slave(?MODULE,exported_wrap,[1]), + ?CTT(Pid,{?MODULE,exported_wrap,[1]}) = receive_next(), + ?CTT(Pid,{?MODULE,exported,[1]}) = receive_next(), + ?CTT(Pid,{?MODULE,local,[1]}) = receive_next(), + ?CTT(Pid,{?MODULE,local2,[1]}) = receive_next(), + ?CTT(Pid,{?MODULE,local_tail,[1]}) = receive_next(), + ?CTT(Pid,{erlang,phash2,[1,1]}) = receive_next(), + ?RFT(Pid,{erlang,phash2,2},0) = receive_next(), + ?RFT(Pid,{?MODULE,local_tail,1},[1,0]) = receive_next(), + ?RFT(Pid,{?MODULE,local2,1},[1,0]) = receive_next(), + ?RFT(Pid,{?MODULE,local,1},[1,1,0]) = receive_next(), + ?RFT(Pid,{?MODULE,exported,1},[1,1,1,0]) = receive_next(), + ?RFT(Pid,{?MODULE,exported_wrap,1},[1,1,1,0]) = receive_next(), + shutdown(), + ?NM, ok. %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% on_and_off_test() -> - ?line Pid = setup(), - ?line 1 = erlang:trace_pattern({?MODULE,local_tail,1},[],[meta]), - ?line LocalTail = fun() -> - local_tail(1) - end, - ?line [1,0] = lambda_slave(LocalTail), - ?line ?CTT(Pid,{?MODULE,local_tail,[1]}) = receive_next(), - ?line 0 = erlang:trace_pattern({?MODULE,local_tail,1},[],[global]), - ?line [1,0] = lambda_slave(LocalTail), - ?line ?NM, - ?line 1 = erlang:trace_pattern({?MODULE,exported_wrap,1},[],[meta]), - ?line [1,1,1,0] = apply_slave(?MODULE,exported_wrap,[1]), - ?line ?CTT(Pid,{?MODULE,exported_wrap,[1]}) = receive_next(), - ?line 1 = erlang:trace_pattern({erlang,phash2,2},[],[meta]), - ?line [1,1,1,0] = apply_slave(?MODULE,exported_wrap,[1]), - ?line ?CTT(Pid,{?MODULE,exported_wrap,[1]}) = receive_next(), - ?line ?CTT(Pid,{erlang,phash2,[1,1]}) = receive_next(), - ?line shutdown(), - ?line erlang:trace_pattern({'_','_','_'},false,[meta]), - ?line N = erlang:trace_pattern({erlang,'_','_'},true,[meta]), - ?line case erlang:trace_pattern({erlang,'_','_'},false,[meta]) of - N -> - ok; - Else -> - exit({number_mismatch, {expected, N}, {got, Else}}) - end, - ?line case erlang:trace_pattern({erlang,'_','_'},false,[meta]) of - N -> - ok; - Else2 -> - exit({number_mismatch, {expected, N}, {got, Else2}}) - end, - ?line ?NM, + Pid = setup(), + 1 = erlang:trace_pattern({?MODULE,local_tail,1},[],[meta]), + LocalTail = fun() -> + local_tail(1) + end, + [1,0] = lambda_slave(LocalTail), + ?CTT(Pid,{?MODULE,local_tail,[1]}) = receive_next(), + 0 = erlang:trace_pattern({?MODULE,local_tail,1},[],[global]), + [1,0] = lambda_slave(LocalTail), + ?NM, + 1 = erlang:trace_pattern({?MODULE,exported_wrap,1},[],[meta]), + [1,1,1,0] = apply_slave(?MODULE,exported_wrap,[1]), + ?CTT(Pid,{?MODULE,exported_wrap,[1]}) = receive_next(), + 1 = erlang:trace_pattern({erlang,phash2,2},[],[meta]), + [1,1,1,0] = apply_slave(?MODULE,exported_wrap,[1]), + ?CTT(Pid,{?MODULE,exported_wrap,[1]}) = receive_next(), + ?CTT(Pid,{erlang,phash2,[1,1]}) = receive_next(), + shutdown(), + erlang:trace_pattern({'_','_','_'},false,[meta]), + N = erlang:trace_pattern({erlang,'_','_'},true,[meta]), + case erlang:trace_pattern({erlang,'_','_'},false,[meta]) of + N -> ok; + Else -> + exit({number_mismatch, {expected, N}, {got, Else}}) + end, + case erlang:trace_pattern({erlang,'_','_'},false,[meta]) of + N -> ok; + Else2 -> + exit({number_mismatch, {expected, N}, {got, Else2}}) + end, + %% ?NM, + %% Can't check for erlang:*/* stuff since common_test or test_server + %% will likely call list_to_binary in the logger. just drain any potential + %% message + ok = flush(), ok. - + +flush() -> + receive _ -> flush() after 0 -> ok end. + %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% stack_grow_test() -> - ?line Pid = setup(), - ?line 1 = erlang:trace_pattern({?MODULE,loop,4}, + Pid = setup(), + 1 = erlang:trace_pattern({?MODULE,loop,4}, [{'_',[],[{return_trace}]}],[meta]), - ?line Num = 1 bsl 15, - ?line Surface = + Num = 1 bsl 15, + Surface = fun (This, ?RFT(P,{?MODULE,loop,4},N), N) when P == Pid-> if N == Num -> ?NM, @@ -288,7 +293,7 @@ stack_grow_test() -> This(This, receive_next(), N+1) end end, - ?line Dive = + Dive = fun (This, ?CTT(P,{?MODULE,loop,[{hej,hopp},[a,b,c],4.5,N]}), N) when P == Pid-> if N == 0 -> @@ -297,272 +302,263 @@ stack_grow_test() -> This(This, receive_next(), N-1) end end, - ?line apply_slave(?MODULE,loop,[{hej,hopp},[a,b,c],4.5,Num]), -% ?line apply_slave_async(?MODULE,loop,[{hej,hopp},[a,b,c],4.5,Num]), -% ?line List = collect(test_server:seconds(5)), - ?line ok = Dive(Dive, receive_next(), Num), - ?line ?NM, + apply_slave(?MODULE,loop,[{hej,hopp},[a,b,c],4.5,Num]), +% apply_slave_async(?MODULE,loop,[{hej,hopp},[a,b,c],4.5,Num]), +% List = collect(test_server:seconds(5)), + ok = Dive(Dive, receive_next(), Num), + ?NM, ok. %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% info_test() -> - ?line setup(), - ?line Prog = [{['$1'],[{is_integer,'$1'}],[{message, false}]}, - {'_',[],[]}], - ?line Self = self(), - ?line GoOn = make_ref(), - - ?line Pid = + setup(), + Prog = [{['$1'],[{is_integer,'$1'}],[{message, false}]}, {'_',[],[]}], + Self = self(), + GoOn = make_ref(), + Pid = spawn_link( fun () -> erlang:trace_pattern({?MODULE,exported_wrap,1}, - Prog, [{meta, Self}]), + Prog, [{meta, Self}]), Self ! {self(), GoOn} end), - ?line receive {Pid, GoOn} -> ok end, - ?line {traced,false} = erlang:trace_info({?MODULE,exported_wrap,1}, traced), - ?line {match_spec, false} = + receive {Pid, GoOn} -> ok end, + {traced,false} = erlang:trace_info({?MODULE,exported_wrap,1}, traced), + {match_spec, false} = erlang:trace_info({?MODULE,exported_wrap,1}, match_spec), - ?line {meta, Self} = erlang:trace_info({?MODULE,exported_wrap,1}, meta), - ?line {meta_match_spec, MMS} = + {meta, Self} = erlang:trace_info({?MODULE,exported_wrap,1}, meta), + {meta_match_spec, MMS} = erlang:trace_info({?MODULE,exported_wrap,1}, meta_match_spec), - ?line case MMS of - Prog -> - ok; - Wrong -> - exit({bad_result, {erlang,trace_info, - [{?MODULE,exported_wrap,1}, - meta_match_spec]}, - {expected, Prog}, {got, Wrong}}) - end, - ?line erlang:garbage_collect(self()), - ?line receive - after 1 -> - ok - end, - ?line io:format("~p~n",[MMS]), - ?line {meta_match_spec,MMS2} = + case MMS of + Prog -> + ok; + Wrong -> + exit({bad_result, {erlang,trace_info, + [{?MODULE,exported_wrap,1}, + meta_match_spec]}, + {expected, Prog}, {got, Wrong}}) + end, + erlang:garbage_collect(self()), + receive + after 1 -> + ok + end, + io:format("~p~n",[MMS]), + {meta_match_spec,MMS2} = erlang:trace_info({?MODULE,exported_wrap,1}, meta_match_spec), - ?line io:format("~p~n",[MMS2]), - ?line case MMS2 of - Prog -> - ok; - Wrong2 -> - exit({bad_result, {erlang,trace_info, - [{?MODULE,exported_wrap,1}, - meta_match_spec]}, - {expected, Prog}, {got, Wrong2}}) - end, - ?line {all, [_|_]=L} = erlang:trace_info({?MODULE,exported_wrap,1}, all), - ?line {value, {meta, Self}} = - lists:keysearch(meta, 1, L), - ?line {value, {meta_match_spec, MMS}} = - lists:keysearch(meta_match_spec, 1, L), - - ?line erlang:trace_pattern({?MODULE,exported_wrap,1}, true, [meta]), - ?line {meta_match_spec, []} = + io:format("~p~n",[MMS2]), + case MMS2 of + Prog -> + ok; + Wrong2 -> + exit({bad_result, {erlang,trace_info, + [{?MODULE,exported_wrap,1}, + meta_match_spec]}, + {expected, Prog}, {got, Wrong2}}) + end, + {all, [_|_]=L} = erlang:trace_info({?MODULE,exported_wrap,1}, all), + {value, {meta, Self}} = lists:keysearch(meta, 1, L), + {value, {meta_match_spec, MMS}} = lists:keysearch(meta_match_spec, 1, L), + + erlang:trace_pattern({?MODULE,exported_wrap,1}, true, [meta]), + {meta_match_spec, []} = erlang:trace_info({?MODULE,exported_wrap,1}, meta_match_spec), - - ?line erlang:trace_pattern({?MODULE,exported_wrap,1}, false, [meta]), - ?line {meta, false} = erlang:trace_info({?MODULE,exported_wrap,1}, meta), - ?line {meta_match_spec, false} = + + erlang:trace_pattern({?MODULE,exported_wrap,1}, false, [meta]), + {meta, false} = erlang:trace_info({?MODULE,exported_wrap,1}, meta), + {meta_match_spec, false} = erlang:trace_info({?MODULE,exported_wrap,1}, meta_match_spec), - ?line {all, false} = erlang:trace_info({?MODULE,exported_wrap,1}, all), - - ?line shutdown(), - ?line ?NM, + {all, false} = erlang:trace_info({?MODULE,exported_wrap,1}, all), + shutdown(), + ?NM, ok. %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% tracer_test() -> - ?line Slave = setup(), - ?line Self = self(), - - ?line MatchSpec = [{'_',[],[{return_trace}]}], - ?line Tracer1 = spawn_link(fun () -> relay_n(3, Self) end), - ?line Setter = - spawn_link( - fun () -> - erlang:trace_pattern({?MODULE,receiver,1}, - MatchSpec, - [{meta,Tracer1}]), - erlang:trace_pattern({erlang,phash2,2}, - MatchSpec, - [{meta,Tracer1}]), - Self ! {self(), done} - end), - ?line receive {Setter, done} -> ok end, - ?line Ref = make_ref(), - ?line apply_slave_async(?MODULE, receiver, [Ref]), - ?line {Tracer1,?CTT(Slave,{?MODULE,receiver,[Ref]})} = receive_next(100), - ?line {Tracer1,?CTT(Slave,{erlang,phash2,[1,1]})} = receive_next(100), - ?line {Tracer1,?RFT(Slave,{erlang,phash2,2},0)} = receive_next(100), + Slave = setup(), + Self = self(), + + MatchSpec = [{'_',[],[{return_trace}]}], + Tracer1 = spawn_link(fun () -> relay_n(3, Self) end), + Setter = spawn_link( + fun () -> + erlang:trace_pattern({?MODULE,receiver,1}, + MatchSpec, + [{meta,Tracer1}]), + erlang:trace_pattern({erlang,phash2,2}, + MatchSpec, + [{meta,Tracer1}]), + Self ! {self(), done} + end), + receive {Setter, done} -> ok end, + Ref = make_ref(), + apply_slave_async(?MODULE, receiver, [Ref]), + {Tracer1,?CTT(Slave,{?MODULE,receiver,[Ref]})} = receive_next(100), + {Tracer1,?CTT(Slave,{erlang,phash2,[1,1]})} = receive_next(100), + {Tracer1,?RFT(Slave,{erlang,phash2,2},0)} = receive_next(100), %% Initiate a return_trace that will fail since the tracer just stopped - ?line Slave ! Ref, - ?line receive_no_next(100), + Slave ! Ref, + receive_no_next(100), %% The breakpoint has not been hit since the tracer stopped - ?line {meta,Tracer1} = + {meta,Tracer1} = erlang:trace_info({?MODULE,receiver,1}, meta), - ?line {meta_match_spec, MatchSpec} = + {meta_match_spec, MatchSpec} = erlang:trace_info({?MODULE,receiver,1}, meta_match_spec), - ?line {meta,Tracer1} = + {meta,Tracer1} = erlang:trace_info({erlang,phash2,2}, meta), - ?line {meta_match_spec, MatchSpec} = + {meta_match_spec, MatchSpec} = erlang:trace_info({erlang,phash2,2}, meta_match_spec), %% Initiate trace messages that will fail - ?line Ref2 = make_ref(), - ?line apply_slave_async(?MODULE, receiver, [Ref2]), - ?line Slave ! Ref2, - ?line receive_no_next(100), - ?line {meta,[]} = + Ref2 = make_ref(), + apply_slave_async(?MODULE, receiver, [Ref2]), + Slave ! Ref2, + receive_no_next(100), + {meta,[]} = erlang:trace_info({?MODULE,receiver,1}, meta), - ?line {meta_match_spec, MatchSpec} = + {meta_match_spec, MatchSpec} = erlang:trace_info({?MODULE,receiver,1}, meta_match_spec), - ?line {meta,[]} = + {meta,[]} = erlang:trace_info({erlang,phash2,2}, meta), - ?line {meta_match_spec, MatchSpec} = + {meta_match_spec, MatchSpec} = erlang:trace_info({erlang,phash2,2}, meta_match_spec), %% Change tracer - ?line Tracer2 = spawn_link(fun () -> relay_n(4, Self) end), - ?line erlang:trace_pattern({?MODULE,receiver,1}, - MatchSpec, - [{meta,Tracer2}]), - ?line erlang:trace_pattern({erlang,phash2,2}, - MatchSpec, - [{meta,Tracer2}]), - ?line Ref3 = make_ref(), - ?line apply_slave_async(?MODULE, receiver, [Ref3]), - ?line {Tracer2,?CTT(Slave,{?MODULE,receiver,[Ref3]})} = receive_next(), - ?line {Tracer2,?CTT(Slave,{erlang,phash2,[1,1]})} = receive_next(), - ?line {Tracer2,?RFT(Slave,{erlang,phash2,2},0)} = receive_next(), + Tracer2 = spawn_link(fun () -> relay_n(4, Self) end), + erlang:trace_pattern({?MODULE,receiver,1}, + MatchSpec, + [{meta,Tracer2}]), + erlang:trace_pattern({erlang,phash2,2}, + MatchSpec, + [{meta,Tracer2}]), + Ref3 = make_ref(), + apply_slave_async(?MODULE, receiver, [Ref3]), + {Tracer2,?CTT(Slave,{?MODULE,receiver,[Ref3]})} = receive_next(), + {Tracer2,?CTT(Slave,{erlang,phash2,[1,1]})} = receive_next(), + {Tracer2,?RFT(Slave,{erlang,phash2,2},0)} = receive_next(), %% Change tracer between call trace and return trace - ?line Tracer3 = spawn_link(fun () -> relay_n(4, Self) end), - ?line erlang:trace_pattern({?MODULE,receiver,1}, - MatchSpec, - [{meta,Tracer3}]), - ?line erlang:trace_pattern({erlang,phash2,2}, - MatchSpec, - [{meta,Tracer3}]), - ?line Slave ! Ref3, + Tracer3 = spawn_link(fun () -> relay_n(4, Self) end), + erlang:trace_pattern({?MODULE,receiver,1}, + MatchSpec, + [{meta,Tracer3}]), + erlang:trace_pattern({erlang,phash2,2}, + MatchSpec, + [{meta,Tracer3}]), + Slave ! Ref3, %% The return trace should still come from Tracer2 - ?line {Tracer2,?RFT(Slave,{?MODULE,receiver,1},Ref3)} = receive_next(), - ?line Ref4 = make_ref(), + {Tracer2,?RFT(Slave,{?MODULE,receiver,1},Ref3)} = receive_next(), + Ref4 = make_ref(), %% Now should Tracer3 be used - ?line apply_slave_async(?MODULE, receiver, [Ref4]), - ?line Slave ! Ref4, - ?line {Tracer3,?CTT(Slave,{?MODULE,receiver,[Ref4]})} = receive_next(), - ?line {Tracer3,?CTT(Slave,{erlang,phash2,[1,1]})} = receive_next(), - ?line {Tracer3,?RFT(Slave,{erlang,phash2,2},0)} = receive_next(), - ?line {Tracer3,?RFT(Slave,{?MODULE,receiver,1},Ref4)} = receive_next(), + apply_slave_async(?MODULE, receiver, [Ref4]), + Slave ! Ref4, + {Tracer3,?CTT(Slave,{?MODULE,receiver,[Ref4]})} = receive_next(), + {Tracer3,?CTT(Slave,{erlang,phash2,[1,1]})} = receive_next(), + {Tracer3,?RFT(Slave,{erlang,phash2,2},0)} = receive_next(), + {Tracer3,?RFT(Slave,{?MODULE,receiver,1},Ref4)} = receive_next(), %% The breakpoint has not been hit since the tracer stopped - ?line {meta,Tracer3} = - erlang:trace_info({?MODULE,receiver,1}, meta), - ?line {meta_match_spec, MatchSpec} = + {meta,Tracer3} = erlang:trace_info({?MODULE,receiver,1}, meta), + {meta_match_spec, MatchSpec} = erlang:trace_info({?MODULE,receiver,1}, meta_match_spec), - ?line {meta,Tracer3} = + {meta,Tracer3} = erlang:trace_info({erlang,phash2,2}, meta), - ?line {meta_match_spec, MatchSpec} = + {meta_match_spec, MatchSpec} = erlang:trace_info({erlang,phash2,2}, meta_match_spec), - - ?line shutdown(), - ?line ?NM, + shutdown(), + ?NM, ok. %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% combo_test() -> - ?line Slave = setup(), - ?line Self = self(), - - ?line MatchSpec = [{'_',[],[{return_trace}]}], - ?line Flags = lists:sort([call, return_to]), - ?line LocalTracer = spawn_link(fun () -> relay_n(6, Self) end), - ?line MetaTracer = spawn_link(fun () -> relay_n(4, Self) end), - ?line 1 = erlang:trace_pattern({?MODULE,receiver,1}, - MatchSpec, - [local,{meta,MetaTracer}]), - ?line 1 = erlang:trace_pattern({erlang,phash2,2}, - MatchSpec, - [local,{meta,MetaTracer}]), - ?line 1 = erlang:trace(Slave, true, - [{tracer,LocalTracer} | Flags]), + Slave = setup(), + Self = self(), + + MatchSpec = [{'_',[],[{return_trace}]}], + Flags = lists:sort([call, return_to]), + LocalTracer = spawn_link(fun () -> relay_n(6, Self) end), + MetaTracer = spawn_link(fun () -> relay_n(4, Self) end), + 1 = erlang:trace_pattern({?MODULE,receiver,1}, + MatchSpec, + [local,{meta,MetaTracer}]), + 1 = erlang:trace_pattern({erlang,phash2,2}, + MatchSpec, + [local,{meta,MetaTracer}]), + 1 = erlang:trace(Slave, true, + [{tracer,LocalTracer} | Flags]), %% - ?line {all, TraceInfo1} = + {all, TraceInfo1} = erlang:trace_info({?MODULE,receiver,1}, all), - ?line {meta,MetaTracer} = + {meta,MetaTracer} = erlang:trace_info({?MODULE,receiver,1}, meta), - ?line {value,{meta,MetaTracer}} = + {value,{meta,MetaTracer}} = lists:keysearch(meta, 1, TraceInfo1), - ?line {meta_match_spec,MatchSpec} = + {meta_match_spec,MatchSpec} = erlang:trace_info({?MODULE,receiver,1}, meta_match_spec), - ?line {value,{meta_match_spec,MatchSpec}} = + {value,{meta_match_spec,MatchSpec}} = lists:keysearch(meta_match_spec, 1, TraceInfo1), - ?line {traced,local} = + {traced,local} = erlang:trace_info({?MODULE,receiver,1}, traced), - ?line {value,{traced,local}} = + {value,{traced,local}} = lists:keysearch(traced, 1, TraceInfo1), - ?line {match_spec,MatchSpec} = + {match_spec,MatchSpec} = erlang:trace_info({?MODULE,receiver,1}, match_spec), - ?line {value,{match_spec,MatchSpec}} = + {value,{match_spec,MatchSpec}} = lists:keysearch(match_spec, 1, TraceInfo1), %% - ?line {all, TraceInfo2} = + {all, TraceInfo2} = erlang:trace_info({erlang,phash2,2}, all), - ?line {meta,MetaTracer} = + {meta,MetaTracer} = erlang:trace_info({erlang,phash2,2}, meta), - ?line {value,{meta,MetaTracer}} = + {value,{meta,MetaTracer}} = lists:keysearch(meta, 1, TraceInfo2), - ?line {meta_match_spec,MatchSpec} = + {meta_match_spec,MatchSpec} = erlang:trace_info({erlang,phash2,2}, meta_match_spec), - ?line {value,{meta_match_spec,MatchSpec}} = + {value,{meta_match_spec,MatchSpec}} = lists:keysearch(meta_match_spec, 1, TraceInfo2), - ?line {traced,local} = + {traced,local} = erlang:trace_info({erlang,phash2,2}, traced), - ?line {value,{traced,local}} = + {value,{traced,local}} = lists:keysearch(traced, 1, TraceInfo2), - ?line {match_spec,MatchSpec} = + {match_spec,MatchSpec} = erlang:trace_info({erlang,phash2,2}, match_spec), - ?line {value,{match_spec,MatchSpec}} = + {value,{match_spec,MatchSpec}} = lists:keysearch(match_spec, 1, TraceInfo2), %% - ?line {flags,Flags1} = erlang:trace_info(Slave, flags), - ?line Flags = lists:sort(Flags1), - ?line {tracer,LocalTracer} = erlang:trace_info(Slave, tracer), + {flags,Flags1} = erlang:trace_info(Slave, flags), + Flags = lists:sort(Flags1), + {tracer,LocalTracer} = erlang:trace_info(Slave, tracer), %% - ?line Ref = make_ref(), - ?line apply_slave_async(?MODULE, receiver, [Ref]), - ?line Slave ! Ref, - ?line ?CTT(Slave,{?MODULE,receiver,[Ref]}) = receive_next_bytag(MetaTracer), - ?line ?CTT(Slave,{erlang,phash2,[1,1]}) = receive_next_bytag(MetaTracer), - ?line ?RFT(Slave,{erlang,phash2,2},0) = receive_next_bytag(MetaTracer), - ?line ?RFT(Slave,{?MODULE,receiver,1},Ref) = receive_next_bytag(MetaTracer), - ?line ?CT(Slave,{?MODULE,receiver,[Ref]}) = receive_next_bytag(LocalTracer), - ?line ?CT(Slave,{erlang,phash2,[1,1]}) = receive_next_bytag(LocalTracer), - ?line case {receive_next_bytag(LocalTracer), + Ref = make_ref(), + apply_slave_async(?MODULE, receiver, [Ref]), + Slave ! Ref, + ?CTT(Slave,{?MODULE,receiver,[Ref]}) = receive_next_bytag(MetaTracer), + ?CTT(Slave,{erlang,phash2,[1,1]}) = receive_next_bytag(MetaTracer), + ?RFT(Slave,{erlang,phash2,2},0) = receive_next_bytag(MetaTracer), + ?RFT(Slave,{?MODULE,receiver,1},Ref) = receive_next_bytag(MetaTracer), + ?CT(Slave,{?MODULE,receiver,[Ref]}) = receive_next_bytag(LocalTracer), + ?CT(Slave,{erlang,phash2,[1,1]}) = receive_next_bytag(LocalTracer), + case {receive_next_bytag(LocalTracer), receive_next_bytag(LocalTracer)} of {?RF(Slave,{erlang,phash2,2},0), ?RT(Slave,{?MODULE,receiver,1})} -> - ?line ok; + ok; {?RT(Slave,{?MODULE,receiver,1}), ?RF(Slave,{erlang,phash2,2},0)} -> - ?line ok; + ok; Error1 -> ?t:fail({unexpected_message, Error1}) end, - ?line case {receive_next_bytag(LocalTracer), + case {receive_next_bytag(LocalTracer), receive_next_bytag(LocalTracer)} of {?RF(Slave,{?MODULE,receiver,1},Ref), ?RT(Slave,{?MODULE,slave,1})} -> - ?line ok; + ok; {?RT(Slave,{?MODULE,slave,1}), ?RF(Slave,{?MODULE,receiver,1},Ref)} -> - ?line ok; + ok; Error2 -> ?t:fail({unexpected_message, Error2}) end, - - ?line shutdown(), - ?line ?NM, + shutdown(), + ?NM, ok. %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -570,38 +566,38 @@ combo_test() -> %% Setup silent local call tracing, and start it using meta trace. nosilent_test() -> - ?line Pid = setup(), - ?line Trigger = {?MODULE,id,1}, - ?line TriggerMS = [{[start],[],[{silent,false}]}, - {[stop],[],[{silent,true},{return_trace}]}], - ?line 1 = erlang:trace(Pid, true, [call,silent,return_to]), - ?line erlang:trace_pattern({?MODULE,'_','_'},[],[local]), - ?line 1 = erlang:trace_pattern({?MODULE,local2,1}, - [{'_',[],[{return_trace}]}], - [local]), - ?line 1 = erlang:trace_pattern({?MODULE,slave,1},false,[local]), - ?line 1 = erlang:trace_pattern(Trigger,false,[local]), - ?line 1 = erlang:trace_pattern(Trigger,TriggerMS,[meta]), - ?line [1,1,1,0] = apply_slave(?MODULE,exported_wrap,[1]), - ?line receive_no_next(17), - ?line start = apply_slave(?MODULE, id, [start]), - ?line ?CTT(Pid,{?MODULE,id,[start]}) = receive_next(), - ?line [2,2,2,0] = apply_slave(?MODULE,exported_wrap,[2]), - ?line ?CT(Pid,{?MODULE,exported_wrap,[2]}) = receive_next(), - ?line ?CT(Pid,{?MODULE,exported,[2]}) = receive_next(), - ?line ?CT(Pid,{?MODULE,local,[2]}) = receive_next(), - ?line ?CT(Pid,{?MODULE,local2,[2]}) = receive_next(), - ?line ?CT(Pid,{?MODULE,local_tail,[2]}) = receive_next(), - ?line ?RF(Pid,{?MODULE,local2,1}, [2,0]) = receive_next(), - ?line ?RT(Pid,{?MODULE,local,1}) = receive_next(), - ?line ?RT(Pid,{?MODULE,exported,1}) = receive_next(), - ?line ?RT(Pid,{?MODULE,slave,1}) = receive_next(), - ?line stop = apply_slave(?MODULE, id, [stop]), - ?line ?CTT(Pid,{?MODULE,id,[stop]}) = receive_next(), - ?line ?RFT(Pid,{?MODULE,id,1}, stop) = receive_next(), - ?line [3,3,3,0] = apply_slave(?MODULE,exported_wrap,[3]), - ?line receive_no_next(17), - ?line shutdown(), + Pid = setup(), + Trigger = {?MODULE,id,1}, + TriggerMS = [{[start],[],[{silent,false}]}, + {[stop],[],[{silent,true},{return_trace}]}], + 1 = erlang:trace(Pid, true, [call,silent,return_to]), + erlang:trace_pattern({?MODULE,'_','_'},[],[local]), + 1 = erlang:trace_pattern({?MODULE,local2,1}, + [{'_',[],[{return_trace}]}], + [local]), + 1 = erlang:trace_pattern({?MODULE,slave,1},false,[local]), + 1 = erlang:trace_pattern(Trigger,false,[local]), + 1 = erlang:trace_pattern(Trigger,TriggerMS,[meta]), + [1,1,1,0] = apply_slave(?MODULE,exported_wrap,[1]), + receive_no_next(17), + start = apply_slave(?MODULE, id, [start]), + ?CTT(Pid,{?MODULE,id,[start]}) = receive_next(), + [2,2,2,0] = apply_slave(?MODULE,exported_wrap,[2]), + ?CT(Pid,{?MODULE,exported_wrap,[2]}) = receive_next(), + ?CT(Pid,{?MODULE,exported,[2]}) = receive_next(), + ?CT(Pid,{?MODULE,local,[2]}) = receive_next(), + ?CT(Pid,{?MODULE,local2,[2]}) = receive_next(), + ?CT(Pid,{?MODULE,local_tail,[2]}) = receive_next(), + ?RF(Pid,{?MODULE,local2,1}, [2,0]) = receive_next(), + ?RT(Pid,{?MODULE,local,1}) = receive_next(), + ?RT(Pid,{?MODULE,exported,1}) = receive_next(), + ?RT(Pid,{?MODULE,slave,1}) = receive_next(), + stop = apply_slave(?MODULE, id, [stop]), + ?CTT(Pid,{?MODULE,id,[stop]}) = receive_next(), + ?RFT(Pid,{?MODULE,id,1}, stop) = receive_next(), + [3,3,3,0] = apply_slave(?MODULE,exported_wrap,[3]), + receive_no_next(17), + shutdown(), ok. %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -645,9 +641,9 @@ slave(Sync) -> Sync ! sync, receive {From,apply, M, F, A} -> - ?line ?dbgformat("Apply: ~p:~p/~p (~p)~n",[M,F,length(A),A]), - ?line Res = apply(M,F,A), - ?line ?dbgformat("done Apply: ~p:~p/~p (~p)~n",[M,F,length(A),A]), + ?dbgformat("Apply: ~p:~p/~p (~p)~n",[M,F,length(A),A]), + Res = apply(M,F,A), + ?dbgformat("done Apply: ~p:~p/~p (~p)~n",[M,F,length(A),A]), From ! {apply, Res}, erlang:trace_pattern({?MODULE,slave,1},false,[meta]), slave(From); diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index dcf8ba2edd..50f4f0e8a6 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -1174,7 +1174,7 @@ usage_aux(void) "[+S NO_SCHEDULERS:NO_SCHEDULERS_ONLINE] " "[+SP PERCENTAGE_SCHEDULERS:PERCENTAGE_SCHEDULERS_ONLINE] " "[+T LEVEL] [+V] [+v] " - "[+W<i|w>] [+z MISC_OPTION] [args ...]\n"); + "[+W<i|w|e>] [+z MISC_OPTION] [args ...]\n"); exit(1); } diff --git a/erts/etc/ose/run_erl.c b/erts/etc/ose/run_erl.c index 8bc49a485e..a6499f2bf3 100644 --- a/erts/etc/ose/run_erl.c +++ b/erts/etc/ose/run_erl.c @@ -615,7 +615,7 @@ int run_erl(int argc,char **argv) { returns */ PROCESS main_pid; hunt_in_block("run_erl","main",&main_pid); - sig = alloc(sizeof(sig),ERTS_SIGNAL_RUN_ERL_DAEMON); + sig = alloc(sizeof(*sig),ERTS_SIGNAL_RUN_ERL_DAEMON); send(&sig,main_pid); sig = receive(sigsel); pid = sender(&sig); diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index 3ee092418e..c51b9e94ed 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -3634,7 +3634,7 @@ define etp-carrier-blocks printf "Free blocks: %u\n", $etp_fblk_cnt end if $etp_error_cnt - printf "%u ERRORs reported above\n", $etp-error-cnt + printf "%u ERRORs reported above\n", $etp_error_cnt end end diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src index 345a6ae3be..cf9a06599a 100644 --- a/erts/preloaded/src/erts.app.src +++ b/erts/preloaded/src/erts.app.src @@ -35,7 +35,7 @@ {registered, []}, {applications, []}, {env, []}, - {runtime_dependencies, ["stdlib-2.0", "kernel-3.0", "sasl-2.4"]} + {runtime_dependencies, ["stdlib-2.5", "kernel-4.0", "sasl-2.4"]} ]}. %% vim: ft=erlang diff --git a/erts/preloaded/src/prim_inet.erl b/erts/preloaded/src/prim_inet.erl index 79ff013c77..622e1be869 100644 --- a/erts/preloaded/src/prim_inet.erl +++ b/erts/preloaded/src/prim_inet.erl @@ -127,37 +127,18 @@ drv2protocol(_) -> undefined. %% TODO: shutdown equivalent for SCTP %% shutdown(S, read) when is_port(S) -> - shutdown_2(S, 0); + shutdown_1(S, 0); shutdown(S, write) when is_port(S) -> shutdown_1(S, 1); shutdown(S, read_write) when is_port(S) -> shutdown_1(S, 2). shutdown_1(S, How) -> - case subscribe(S, [subs_empty_out_q]) of - {ok,[{subs_empty_out_q,N}]} when N > 0 -> - shutdown_pend_loop(S, N); %% wait for pending output to be sent - _Other -> ok - end, - shutdown_2(S, How). - -shutdown_2(S, How) -> case ctl_cmd(S, ?TCP_REQ_SHUTDOWN, [How]) of {ok, []} -> ok; {error,_}=Error -> Error end. -shutdown_pend_loop(S, N0) -> - receive - {empty_out_q,S} -> ok - after ?INET_CLOSE_TIMEOUT -> - case getstat(S, [send_pend]) of - {ok,[{send_pend,N0}]} -> ok; - {ok,[{send_pend,N}]} -> shutdown_pend_loop(S, N); - _ -> ok - end - end. - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% CLOSE(insock()) -> ok diff --git a/lib/.gitignore b/lib/.gitignore index 4125111ebd..58c49adce0 100644 --- a/lib/.gitignore +++ b/lib/.gitignore @@ -546,6 +546,8 @@ snmp/doc/intex.html /syntax_tools/doc/src/erl_syntax.xml /syntax_tools/doc/src/erl_syntax_lib.xml /syntax_tools/doc/src/erl_tidy.xml +/syntax_tools/doc/src/merl.xml +/syntax_tools/doc/src/merl_transform.xml /syntax_tools/doc/src/igor.xml /syntax_tools/doc/src/prettypr.xml diff --git a/lib/asn1/doc/src/asn1ct.xml b/lib/asn1/doc/src/asn1ct.xml index 4e0bf055fc..30808a5ead 100644 --- a/lib/asn1/doc/src/asn1ct.xml +++ b/lib/asn1/doc/src/asn1ct.xml @@ -371,6 +371,15 @@ File3.asn</pre> representation of a value of the <c>ASN.1</c> type <c>Type</c>. The value is a random value and subsequent calls to this function will for most types return different values.</p> + <note> + <p>Currently, the <c>value</c> function has many limitations. + Essentially, it will mostly work for old specifications based + on the 1997 standard for ASN.1, but not for most modern-style + applications. Another limitation is that the <c>value</c> function + may not work if options that change code generations strategies + such as the options <c>macro_name_prefix</c> and + <c>record_name_prefix</c> have been used.</p> + </note> </desc> </func> @@ -391,6 +400,15 @@ File3.asn</pre> This function is useful during test to secure that the generated encode and decode functions as well as the general runtime support work as expected.</p> + <note> + <p>Currently, the <c>test</c> functions have many limitations. + Essentially, they will mostly work for old specifications based + on the 1997 standard for ASN.1, but not for most modern-style + applications. Another limitation is that the <c>test</c> functions + may not work if options that change code generations strategies + such as the options <c>macro_name_prefix</c> and + <c>record_name_prefix</c> have been used.</p> + </note> <list type="bulleted"> <item> <p><c>test/1</c> iterates over all types in <c>Module</c>.</p> diff --git a/lib/asn1/doc/src/notes.xml b/lib/asn1/doc/src/notes.xml index f73d21b9e3..9feb673c04 100644 --- a/lib/asn1/doc/src/notes.xml +++ b/lib/asn1/doc/src/notes.xml @@ -31,23 +31,6 @@ <p>This document describes the changes made to the asn1 application.</p> -<section><title>Asn1 4.0</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Many bugs have been eliminated in the the ASN.1 compiler - so that it can now successfully compile many more ASN.1 - specifications. Error messages have also been improved.</p> - <p> - Own Id: OTP-12395</p> - </item> - </list> - </section> - -</section> - <section><title>Asn1 3.0.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index 218f76d403..472e3b7833 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -32,79 +32,6 @@ <file>notes.xml</file> </header> -<section><title>Common_Test 1.11</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - The status of an aborted test due to test suite - compilation error has changed from 'auto_skipped' to - 'failed'. This affects both the textual log file, event - handling and CT hook callbacks. The logging of - compilation failures has also been improved, especially - in the case of multiple test suites failing compilation.</p> - <p> - Own Id: OTP-10816</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Earlier there was no way to add optional parameters like - default-operation to an edit-config request sent with - ct_netconfc:edit_config/3,4, you had to use - ct_netconfc:send_rpc/2,3. For simplicity and completion, - a new optional argument, OptParams, is now added to the - edit_config function.</p> - <p> - Own Id: OTP-10446 Aux Id: kunagi-266 [177] </p> - </item> - <item> - <p> - When running OTP tests using the ts interface, it is now - possible to specify so called test categories per OTP - application. A test category is represented by a CT test - specification and defines an arbitrary subset of existing - test suites, groups and cases. Examples of test - categories are 'smoke' (smoke tests) and 'bench' - (benchmarks). (Call ts:help() for more info). Also, - functions for reading terms from the current test - specification during test, ct:get_testspec_terms/0 and - ct:get_testspec_terms/1, have been implemented.</p> - <p> - Own Id: OTP-11962</p> - </item> - <item> - <p> - Obsolete scripts and make file operations have been - removed and the installation chapter in the Common Test - User's Guide has been updated.</p> - <p> - Own Id: OTP-12421</p> - </item> - <item> - <p> - The 'keep_alive' interval has been reduced to 8 seconds, - which is two seconds shorter than the default - 'idle_timeout' value for ct_telnet:expect/3. This way, - the telnet server receives a NOP message (which might - trigger an action) before the operation times out. Also - the TCP option 'nodelay' has been enabled per default for - all telnet connections, in order to reduce the risk for - communication timeouts.</p> - <p> - Own Id: OTP-12678 Aux Id: seq12818 </p> - </item> - </list> - </section> - -</section> - <section><title>Common_Test 1.10.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/common_test/src/ct_slave.erl b/lib/common_test/src/ct_slave.erl index 872c39de04..9ef6ec6e23 100644 --- a/lib/common_test/src/ct_slave.erl +++ b/lib/common_test/src/ct_slave.erl @@ -37,7 +37,7 @@ -record(options, {username, password, boot_timeout, init_timeout, startup_timeout, startup_functions, monitor_master, - kill_if_fail, erl_flags, env}). + kill_if_fail, erl_flags, env, ssh_port, ssh_opts}). %%%----------------------------------------------------------------- %%% @spec start(Node) -> Result @@ -254,11 +254,13 @@ fetch_options(Options) -> KillIfFail = get_option_value(kill_if_fail, Options, true), ErlFlags = get_option_value(erl_flags, Options, []), EnvVars = get_option_value(env, Options, []), + SSHPort = get_option_value(ssh_port, Options, []), + SSHOpts = get_option_value(ssh_opts, Options, []), #options{username=UserName, password=Password, boot_timeout=BootTimeout, init_timeout=InitTimeout, startup_timeout=StartupTimeout, startup_functions=StartupFunctions, monitor_master=Monitor, kill_if_fail=KillIfFail, - erl_flags=ErlFlags, env=EnvVars}. + erl_flags=ErlFlags, env=EnvVars, ssh_port=SSHPort, ssh_opts=SSHOpts}. % send a message when slave node is started % @hidden @@ -399,27 +401,18 @@ spawn_local_node(Node, Options) -> Cmd = get_cmd(Node, ErlFlags), open_port({spawn, Cmd}, [stream,{env,Env}]). -% start crypto and ssh if not yet started -check_for_ssh_running() -> - case application:get_application(crypto) of - undefined-> - application:start(crypto), - case application:get_application(ssh) of - undefined-> - application:start(ssh); - {ok, ssh}-> - ok - end; - {ok, crypto}-> - ok - end. - % spawn node remotely spawn_remote_node(Host, Node, Options) -> #options{username=Username, password=Password, erl_flags=ErlFlags, - env=Env} = Options, + env=Env, + ssh_port=MaybeSSHPort, + ssh_opts=SSHOpts} = Options, + SSHPort = case MaybeSSHPort of + [] -> 22; % Use default SSH port + A -> A + end, SSHOptions = case {Username, Password} of {[], []}-> []; @@ -427,14 +420,13 @@ spawn_remote_node(Host, Node, Options) -> [{user, Username}]; {_, _}-> [{user, Username}, {password, Password}] - end ++ [{silently_accept_hosts, true}], - check_for_ssh_running(), - {ok, SSHConnRef} = ssh:connect(atom_to_list(Host), 22, SSHOptions), + end ++ [{silently_accept_hosts, true}] ++ SSHOpts, + application:ensure_all_started(ssh), + {ok, SSHConnRef} = ssh:connect(atom_to_list(Host), SSHPort, SSHOptions), {ok, SSHChannelId} = ssh_connection:session_channel(SSHConnRef, infinity), ssh_setenv(SSHConnRef, SSHChannelId, Env), ssh_connection:exec(SSHConnRef, SSHChannelId, get_cmd(Node, ErlFlags), infinity). - ssh_setenv(SSHConnRef, SSHChannelId, [{Var, Value} | Vars]) when is_list(Var), is_list(Value) -> success = ssh_connection:setenv(SSHConnRef, SSHChannelId, diff --git a/lib/compiler/doc/src/compile.xml b/lib/compiler/doc/src/compile.xml index 5fccdcdcb5..a271729c82 100644 --- a/lib/compiler/doc/src/compile.xml +++ b/lib/compiler/doc/src/compile.xml @@ -32,15 +32,15 @@ <modulesummary>Erlang Compiler</modulesummary> <description> <p>This module provides an interface to the standard Erlang - compiler. It can generate either a new file which contains - the object code, or return a binary which can be loaded directly. + compiler. It can generate either a new file, which contains + the object code, or return a binary, which can be loaded directly. </p> </description> <funcs> <func> <name>file(File)</name> - <fsummary>Compile a file</fsummary> + <fsummary>Compiles a file.</fsummary> <desc> <p>Is the same as <c>file(File, [verbose,report_errors,report_warnings])</c>. @@ -50,7 +50,7 @@ <func> <name>file(File, Options) -> CompRet</name> - <fsummary>Compile a file</fsummary> + <fsummary>Compiles a file.</fsummary> <type> <v>CompRet = ModRet | BinRet | ErrRet</v> <v>ModRet = {ok,ModuleName} | {ok,ModuleName,Warnings}</v> @@ -64,39 +64,38 @@ <p>Returns <c>{ok,ModuleName}</c> if successful, or <c>error</c> if there are errors. An object code file is created if - the compilation succeeds with no errors. It is considered + the compilation succeeds without errors. It is considered to be an error if the module name in the source code is not the same as the basename of the output file.</p> - <p><marker id="type-option"/>Here follows first all elements of <c>Options</c> that in - some way control the behavior of the compiler.</p> + <p><marker id="type-option"/>Available options:</p> <taglist> <tag><c>basic_validation</c></tag> <item> - <p>This option is fast way to test whether a module will - compile successfully (mainly useful for code generators - that want to verify the code they emit). No code will + <p>This option is a fast way to test whether a module will + compile successfully. This is useful for code generators + that want to verify the code that they emit. No code is generated. If warnings are enabled, warnings generated by the <c>erl_lint</c> module (such as warnings for unused - variables and functions) will be returned too.</p> + variables and functions) are also returned.</p> - <p>Use the <c>strong_validation</c> option to generate all + <p>Use option <c>strong_validation</c> to generate all warnings that the compiler would generate.</p> </item> <tag><c>strong_validation</c></tag> <item> - <p>Similar to the <c>basic_validation</c> option, no code - will be generated, but more compiler passes will be run - to ensure also warnings generated by the optimization - passes are generated (such as clauses that will not match + <p>Similar to option <c>basic_validation</c>. No code + is generated, but more compiler passes are run + to ensure that warnings generated by the optimization + passes are generated (such as clauses that will not match, or expressions that are guaranteed to fail with an - exception at run-time).</p> + exception at runtime).</p> </item> <tag><c>binary</c></tag> <item> - <p>Causes the compiler to return the object code in a + <p>The compiler returns the object code in a binary instead of creating an object file. If successful, the compiler returns <c>{ok,ModuleName,Binary}</c>.</p> </item> @@ -105,7 +104,9 @@ <item> <p>The compiler will emit informational warnings about binary matching optimizations (both successful and unsuccessful). - See the <em>Efficiency Guide</em> for further information.</p> + For more information, see the section about + <seealso marker="doc/efficiency_guide:binaryhandling#bin_opt_info">bin_opt_info</seealso> + in the Efficiency Guide.</p> </item> <tag><c>compressed</c></tag> @@ -117,20 +118,19 @@ <tag><c>debug_info</c></tag> <item> <marker id="debug_info"></marker> - <p>Include debug information in the form of abstract code + <p>Includes debug information in the form of abstract code (see <seealso marker="erts:absform">The Abstract Format</seealso> in ERTS User's Guide) in the compiled beam module. Tools - such as Debugger, Xref and Cover require the debug - information to be included.</p> + such as <c>Debugger</c>, <c>Xref</c>, and <c>Cover</c> require + the debug information to be included.</p> <p><em>Warning</em>: Source code can be reconstructed from the debug information. Use encrypted debug information - (see below) to prevent this.</p> + (<c>encrypt_debug_info</c>) to prevent this.</p> - <p>See - <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso> - for details.</p> + <p>For details, see + <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso>.</p> </item> <tag><c>{debug_info_key,KeyString}</c></tag> @@ -138,65 +138,61 @@ <tag><c>{debug_info_key,{Mode,KeyString}}</c></tag> <item> <marker id="debug_info_key"></marker> - <p>Include debug information, but encrypt it, so that it + <p>Includes debug information, but encrypts it so that it cannot be accessed without supplying the key. (To give - the <c>debug_info</c> option as well is allowed, but is + option <c>debug_info</c> as well is allowed, but not necessary.) Using this option is a good way to always have the debug information available during testing, yet - protect the source code.</p> + protecting the source code.</p> <p><c>Mode</c> is the type of crypto algorithm to be used - for encrypting the debug information. The default type -- - and currently the only type -- is <c>des3_cbc</c>.</p> - <p>See - <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso> - for details.</p> + for encrypting the debug information. The default + (and currently the only) type is <c>des3_cbc</c>.</p> + <p>For details, see + <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso>.</p> </item> <tag><c>encrypt_debug_info</c></tag> <item> <marker id="encrypt_debug_info"></marker> - <p>Like the <c>debug_info_key</c> option above, except that - the key will be read from an <c>.erlang.crypt</c> file. + <p>Similar to the <c>debug_info_key</c> option, but + the key is read from an <c>.erlang.crypt</c> file. </p> - <p>See - <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso> - for details.</p> + <p>For details, see + <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso>.</p> </item> <tag><c>makedep</c></tag> <item> - <p>Produce a Makefile rule to track headers dependencies. + <p>Produces a Makefile rule to track headers dependencies. No object file is produced. </p> <p>By default, this rule is written to - <c><![CDATA[<File>.Pbeam]]></c>. However, if the option + <c><![CDATA[<File>.Pbeam]]></c>. However, if option <c>binary</c> is set, nothing is written and the rule is returned in <c>Binary</c>. </p> - <p>For instance, if one has the following module: + <p>For example, if you have the following module: </p> <code> -module(module). -include_lib("eunit/include/eunit.hrl"). --include("header.hrl"). - </code> - <p>Here is the Makefile rule generated by this option: +-include("header.hrl").</code> + <p>The Makefile rule generated by this option looks as follows: </p> <code> module.beam: module.erl \ /usr/local/lib/erlang/lib/eunit/include/eunit.hrl \ - header.hrl - </code> + header.hrl</code> </item> <tag><c>{makedep_output, Output}</c></tag> <item> - <p>Write generated rule(s) to <c>Output</c> instead of the + <p>Writes generated rules to <c>Output</c> instead of the default <c><![CDATA[<File>.Pbeam]]></c>. <c>Output</c> can be a filename or an <c>io_device()</c>. To write to - stdout, use <c>standard_io</c>. However if <c>binary</c> + stdout, use <c>standard_io</c>. However, if <c>binary</c> is set, nothing is written to <c>Output</c> and the result is returned to the caller with <c>{ok, ModuleName, Binary}</c>. @@ -205,7 +201,7 @@ module.beam: module.erl \ <tag><c>{makedep_target, Target}</c></tag> <item> - <p>Change the name of the rule emitted to <c>Target</c>. + <p>Changes the name of the rule emitted to <c>Target</c>. </p> </item> @@ -217,20 +213,20 @@ module.beam: module.erl \ <tag><c>makedep_add_missing</c></tag> <item> - <p>Consider missing headers as generated files and add them to the + <p>Considers missing headers as generated files and adds them to the dependencies. </p> </item> <tag><c>makedep_phony</c></tag> <item> - <p>Add a phony target for each dependency. + <p>Adds a phony target for each dependency. </p> </item> <tag><c>'P'</c></tag> <item> - <p>Produces a listing of the parsed code after preprocessing + <p>Produces a listing of the parsed code, after preprocessing and parse transforms, in the file <c><![CDATA[<File>.P]]></c>. No object file is produced. </p> @@ -238,7 +234,7 @@ module.beam: module.erl \ <tag><c>'E'</c></tag> <item> - <p>Produces a listing of the code after all source code + <p>Produces a listing of the code, after all source code transformations have been performed, in the file <c><![CDATA[<File>.E]]></c>. No object file is produced. </p> @@ -258,21 +254,21 @@ module.beam: module.erl \ <tag><c>report</c></tag> <item> - <p>This is a short form for both <c>report_errors</c> and + <p>A short form for both <c>report_errors</c> and <c>report_warnings</c>.</p> </item> <tag><c>return_errors</c></tag> <item> - <p>If this flag is set, then + <p>If this flag is set, <c>{error,ErrorList,WarningList}</c> is returned when there are errors.</p> </item> <tag><c>return_warnings</c></tag> <item> - <p>If this flag is set, then an extra field containing - <c>WarningList</c> is added to the tuples returned on + <p>If this flag is set, an extra field, containing + <c>WarningList</c>, is added to the tuples returned on success.</p> </item> @@ -284,13 +280,13 @@ module.beam: module.erl \ <tag><c>return</c></tag> <item> - <p>This is a short form for both <c>return_errors</c> and + <p>A short form for both <c>return_errors</c> and <c>return_warnings</c>.</p> </item> <tag><c>verbose</c></tag> <item> - <p>Causes more verbose information from the compiler + <p>Causes more verbose information from the compiler, describing what it is doing.</p> </item> @@ -314,7 +310,7 @@ module.beam: module.erl \ <tag><c>{i,Dir}</c></tag> <item> - <p>Add <c>Dir</c> to the list of directories to be searched + <p>Adds <c>Dir</c> to the list of directories to be searched when including a file. When encountering an <c>-include</c> or <c>-include_lib</c> directive, the compiler searches for header files in the following @@ -322,14 +318,14 @@ module.beam: module.erl \ <list type="ordered"> <item> <p><c>"."</c>, the current working directory of - the file server;</p> + the file server</p> </item> <item> - <p>the base name of the compiled file;</p> + <p>The base name of the compiled file</p> </item> <item> - <p>the directories specified using the <c>i</c> option. - The directory specified last is searched first.</p> + <p>The directories specified using option <c>i</c>; + the directory specified last is searched first</p> </item> </list> </item> @@ -353,15 +349,15 @@ module.beam: module.erl \ <tag><c>from_asm</c></tag> <item> <p>The input file is expected to be assembler code (default - file suffix ".S"). Note that the format of assembler files - is not documented, and may change between releases.</p> + file suffix ".S"). Notice that the format of assembler files + is not documented, and can change between releases.</p> </item> <tag><c>from_core</c></tag> <item> <p>The input file is expected to be core code (default - file suffix ".core"). Note that the format of core files - is not documented, and may change between releases.</p> + file suffix ".core"). Notice that the format of core files + is not documented, and can change between releases.</p> </item> <tag><c>no_strict_record_tests</c></tag> @@ -369,9 +365,9 @@ module.beam: module.erl \ <p>This option is not recommended.</p> <p>By default, the generated code for - the <c>Record#record_tag.field</c> operation verifies that - the tuple <c>Record</c> is of the correct size for - the record and that the first element is the tag + operation <c>Record#record_tag.field</c> verifies that + the tuple <c>Record</c> has the correct size for + the record, and that the first element is the tag <c>record_tag</c>. Use this option to omit the verification code.</p> </item> @@ -390,79 +386,87 @@ module.beam: module.erl \ <tag><c>{no_auto_import,[{F,A}, ...]}</c></tag> <item> <p>Makes the function <c>F/A</c> no longer being - auto-imported from the module <c>erlang</c>, which resolves - BIF name clashes. This option has to be used to resolve name - clashes with BIFs auto-imported before R14A, if one wants to + auto-imported from the <c>erlang</c> module, which resolves + BIF name clashes. This option must be used to resolve name + clashes with BIFs auto-imported before R14A, if it is needed to call the local function with the same name as an auto-imported BIF without module prefix.</p> <note> - <p>From R14A and forward, the compiler resolves calls + <p>As from R14A and forward, the compiler resolves calls without module prefix to local or imported functions before - trying auto-imported BIFs. If the BIF is to be + trying with auto-imported BIFs. If the BIF is to be called, use the <c>erlang</c> module prefix in the call, not - <c>{ no_auto_import,[{F,A}, ...]}</c></p> + <c>{ no_auto_import,[{F,A}, ...]}</c>.</p> </note> <p>If this option is written in the source code, as a <c>-compile</c> directive, the syntax <c>F/A</c> can be used instead - of <c>{F,A}</c>. Example:</p> + of <c>{F,A}</c>, for example:</p> <code>-compile({no_auto_import,[error/1]}).</code> </item> <tag><c>no_auto_import</c></tag> <item> - <p>Do not auto import any functions from the module <c>erlang</c>.</p> + <p>Do not auto-import any functions from <c>erlang</c> module.</p> </item> <tag><c>no_line_info</c></tag> <item> - <p>Omit line number information in order to produce a slightly + <p>Omits line number information to produce a slightly smaller output file. </p> </item> </taglist> - <p>If warnings are turned on (the <c>report_warnings</c> option - described above), the following options control what type of - warnings that will be generated. + <p>If warnings are turned on (option <c>report_warnings</c> + described earlier), the following options control what type of + warnings that are generated. <marker id="erl_lint_options"></marker> - With the exception of <c>{warn_format,Verbosity}</c> all - options below have two forms; one <c>warn_xxx</c> form to - turn on the warning and one <c>nowarn_xxx</c> form to turn off - the warning. In the description that follows, the form that - is used to change the default value is listed.</p> + Except from <c>{warn_format,Verbosity}</c>, the following options + have two forms:</p> + <list type="bulleted"> + <item>A <c>warn_xxx</c> form, to turn on the warning.</item> + <item>A <c>nowarn_xxx</c> form, to turn off the warning.</item> + </list> + <p>In the descriptions that follow, the form that is used to change + the default value are listed.</p> <taglist> <tag><c>{warn_format, Verbosity}</c></tag> <item> <p>Causes warnings to be emitted for malformed format strings as arguments to <c>io:format</c> and similar - functions. <c>Verbosity</c> selects the amount of - warnings: 0 = no warnings; 1 = warnings for invalid - format strings and incorrect number of arguments; 2 = - warnings also when the validity could not be checked - (for example, when the format string argument is a - variable). The default verbosity is 1. Verbosity 0 can - also be selected by the option <c>nowarn_format</c>.</p> + functions.</p> + <p><c>Verbosity</c> selects the number of warnings:</p> + <list type="bulleted"> + <item><c>0</c> = No warnings</item> + <item><c>1</c> = Warnings for invalid format strings and incorrect + number of arguments</item> + <item><c>2</c> = Warnings also when the validity cannot + be checked, for example, when the format string argument is a + variable.</item> + </list> + <p>The default verbosity is <c>1</c>. Verbosity <c>0</c> can + also be selected by option <c>nowarn_format</c>.</p> </item> <tag><c>nowarn_bif_clash</c></tag> <item> - <p>This option is removed, it will generate a fatal error if used.</p> + <p>This option is removed, it generates a fatal error if used.</p> <warning> - <p>Beginning with R14A, the compiler no longer calls the + <p>As from beginning with R14A, the compiler no longer calls the auto-imported BIF if the name clashes with a local or - explicitly imported function and a call without explicit - module name is issued. Instead the local or imported - function is called. Still accepting <c>nowarn_bif_clash</c> would makes a - module calling functions clashing with autoimported BIFs + explicitly imported function, and a call without explicit + module name is issued. Instead, the local or imported + function is called. Still accepting <c>nowarn_bif_clash</c> would + make a module calling functions clashing with auto-imported BIFs compile with both the old and new compilers, but with - completely different semantics, why the option was removed.</p> + completely different semantics. This is why the option is removed.</p> - <p>The use of this option has always been strongly discouraged. - From OTP R14A and forward it's an error to use it.</p> + <p>The use of this option has always been discouraged. + As from R14A, it is an error to use it.</p> <p>To resolve BIF clashes, use explicit module names or the <c>{no_auto_import,[F/A]}</c> compiler directive.</p> </warning> @@ -470,11 +474,11 @@ module.beam: module.erl \ <tag><c>{nowarn_bif_clash, FAs}</c></tag> <item> - <p>This option is removed, it will generate a fatal error if used.</p> + <p>This option is removed, it generates a fatal error if used.</p> <warning> - <p>The use of this option has always been strongly discouraged. - From OTP R14A and forward it's an error to use it.</p> + <p>The use of this option has always been discouraged. + As from R14A, it is an error to use it.</p> <p>To resolve BIF clashes, use explicit module names or the <c>{no_auto_import,[F/A]}</c> compiler directive.</p> </warning> @@ -482,35 +486,29 @@ module.beam: module.erl \ <tag><c>warn_export_all</c></tag> <item> - <p>Causes a warning to be emitted if the <c>export_all</c> - option has also been given.</p> + <p>Emits a warning if option <c>export_all</c> is also given.</p> </item> <tag><c>warn_export_vars</c></tag> <item> - <p>Causes warnings to be emitted for all implicitly - exported variables referred to after the primitives - where they were first defined. No warnings for exported - variables unless they are referred to in some pattern, - which is the default, can be selected by the option - <c>nowarn_export_vars</c>.</p> + <p>Emits warnings for all implicitly exported variables + referred to after the primitives where they were first defined. + By default, the compiler only emits warnings for exported + variables referred to in a pattern.</p> </item> - <tag><c>warn_shadow_vars</c></tag> + <tag><c>nowarn_shadow_vars</c></tag> <item> - <p>Causes warnings to be emitted for "fresh" variables - in functional objects or list comprehensions with the same - name as some already defined variable. The default is to - warn for such variables. No warnings for shadowed - variables can be selected by the option - <c>nowarn_shadow_vars</c>.</p> + <p>Turns off warnings for "fresh" variables + in functional objects or list comprehensions with the same + name as some already defined variable. Default is to + emit warnings for such variables.</p> </item> <tag><c>nowarn_unused_function</c></tag> <item> - <p>Turns off warnings for unused local functions. - By default (<c>warn_unused_function</c>), warnings are - emitted for all local functions that are not called + <p>Turns off warnings for unused local functions. Default + is to emit warnings for all local functions that are not called directly or indirectly by an exported function. The compiler does not include unused local functions in the generated beam file, but the warning is still useful @@ -519,148 +517,142 @@ module.beam: module.erl \ <tag><c>{nowarn_unused_function, FAs}</c></tag> <item> - <p>Turns off warnings for unused local functions as - <c>nowarn_unused_function</c> but only for the mentioned + <p>Turns off warnings for unused local functions like + <c>nowarn_unused_function</c> does, but only for the mentioned local functions. <c>FAs</c> is a tuple <c>{Name,Arity}</c> or a list of such tuples.</p> </item> <tag><c>nowarn_deprecated_function</c></tag> <item> - <p>Turns off warnings for calls to deprecated functions. By - default (<c>warn_deprecated_function</c>), warnings are - emitted for every call to a function known by the compiler - to be deprecated. Note that the compiler does not know - about the <c>-deprecated()</c> attribute but uses an + <p>Turns off warnings for calls to deprecated functions. Default + is to emit warnings for every call to a function known by the + compiler to be deprecated. Notice that the compiler does not know + about attribute <c>-deprecated()</c>, but uses an assembled list of deprecated functions in Erlang/OTP. To - do a more general check the <c>Xref</c> tool can be used. + do a more general check, the <c>Xref</c> tool can be used. See also <seealso marker="tools:xref#deprecated_function">xref(3)</seealso> and the function - <seealso marker="tools:xref#m/1">xref:m/1</seealso> also - accessible through - the <seealso marker="stdlib:c#xm/1">c:xm/1</seealso> - function.</p> + <seealso marker="tools:xref#m/1">xref:m/1</seealso>, also + accessible through the function + <seealso marker="stdlib:c#xm/1">c:xm/1</seealso>.</p> </item> <tag><c>{nowarn_deprecated_function, MFAs}</c></tag> <item> - <p>Turns off warnings for calls to deprecated functions as - <c>nowarn_deprecated_function</c> but only for + <p>Turns off warnings for calls to deprecated functions like + <c>nowarn_deprecated_function</c> does, but only for the mentioned functions. <c>MFAs</c> is a tuple <c>{Module,Name,Arity}</c> or a list of such tuples.</p> </item> <tag><c>nowarn_deprecated_type</c></tag> <item> - <p>Turns off warnings for uses of deprecated types. By - default (<c>warn_deprecated_type</c>), warnings are - emitted for every use of a type known by the compiler - to be deprecated.</p> + <p>Turns off warnings for use of deprecated types. Default + is to emit warnings for every use of a type known by the compiler + to be deprecated.</p> </item> <tag><c>warn_obsolete_guard</c></tag> <item> - <p>Causes warnings to be emitted for calls to old type - testing BIFs such as <c>pid/1</c> and <c>list/1</c>. See - the - <seealso marker="doc/reference_manual:expressions#guards">Erlang Reference Manual</seealso> + <p>Emits warnings for calls to old type testing BIFs, + such as <c>pid/1</c> and <c>list/1</c>. See the + <seealso marker="doc/reference_manual:expressions#guards">Erlang Reference Manual</seealso> for a complete list of type testing BIFs and their old - equivalents. No warnings for calls to old type testing - BIFs, which is the default, can be selected by the option - <c>nowarn_obsolete_guard</c>.</p> + equivalents. Default is to emit no warnings for calls to + old type testing BIFs.</p> </item> <tag><c>warn_unused_import</c></tag> <item> - <p>Causes warnings to be emitted for unused imported - functions. No warnings for unused imported functions, - which is the default, can be selected by the option - <c>nowarn_unused_import</c>. </p> + <p>Emits warnings for unused imported functions. + Default is to emit no warnings for unused imported functions.</p> </item> <tag><c>nowarn_unused_vars</c></tag> <item> - <p>By default, warnings are emitted for variables which - are not used, with the exception of variables beginning - with an underscore ("Prolog style warnings"). + <p>By default, warnings are emitted for unused variables, + except for variables beginning with an underscore + ("Prolog style warnings"). Use this option to turn off this kind of warnings.</p> </item> <tag><c>nowarn_unused_record</c></tag> <item> - <p>Turns off warnings for unused record types. By - default (<c>warn_unused_records</c>), warnings are - emitted for unused locally defined record types.</p> + <p>Turns off warnings for unused record types. Default is to + emit warnings for unused locally defined record types.</p> </item> </taglist> <p>Another class of warnings is generated by the compiler during optimization and code generation. They warn about patterns that will never match (such as <c>a=b</c>), guards - that will always evaluate to false, and expressions that will + that always evaluate to false, and expressions that always fail (such as <c>atom+42</c>).</p> - - <p>Note that the compiler does not warn for expressions that it - does not attempt to optimize. For instance, the compiler tries - to evaluate <c>1/0</c>, notices that it will cause an - exception and emits a warning. On the other hand, - the compiler is silent about the similar expression - <c>X/0</c>; because of the variable in it, the compiler does - not even try to evaluate and therefore it emits no warnings. - </p> - - <p>Currently, those warnings cannot be disabled (except by + <p>Those warnings cannot be disabled (except by disabling all warnings).</p> + <note> + <p>The compiler does not warn for expressions that it + does not attempt to optimize. For example, the compiler tries + to evaluate <c>1/0</c>, detects that it will cause an + exception, and emits a warning. However, + the compiler is silent about the similar expression, + <c>X/0</c>, because of the variable in it. Thus, the compiler does + not even try to evaluate and therefore it emits no warnings.</p> + </note> + <warning> - <p>Obviously, the absence of warnings does not mean that + <p>The absence of warnings does not mean that there are no remaining errors in the code.</p> </warning> - - <p>Note that all the options except the include path - (<c>{i,Dir}</c>) can also be given in the file with a - <c>-compile([Option,...])</c>. attribute. - The <c>-compile()</c> attribute is allowed after function + + <note> + <p>All options, except the include path + (<c>{i,Dir}</c>), can also be given in the file with attribute + <c>-compile([Option,...])</c>. + Attribute <c>-compile()</c> is allowed after the function definitions.</p> - - <p>Note also that the <c>{nowarn_unused_function, FAs}</c>, + </note> + + <note> + <p>The options <c>{nowarn_unused_function, FAs}</c>, <c>{nowarn_bif_clash, FAs}</c>, and - <c>{nowarn_deprecated_function, MFAs}</c> options are only + <c>{nowarn_deprecated_function, MFAs}</c> are only recognized when given in files. They are not affected by - the <c>warn_unused_function</c>, <c>warn_bif_clash</c>, or - <c>warn_deprecated_function</c> options.</p> + options <c>warn_unused_function</c>, <c>warn_bif_clash</c>, or + <c>warn_deprecated_function</c>.</p> + </note> <p>For debugging of the compiler, or for pure curiosity, the intermediate code generated by each compiler pass can be inspected. - A complete list of the options to produce list files can be - printed by typing <c>compile:options()</c> at the Erlang - shell prompt. - The options will be printed in order that the passes are + To print a complete list of the options to produce list files, + type <c>compile:options()</c> at the Erlang shell prompt. + The options are printed in the order that the passes are executed. If more than one listing option is used, the one representing the earliest pass takes effect.</p> - <p><em>Unrecognized options are ignored.</em></p> + <p>Unrecognized options are ignored.</p> <p>Both <c>WarningList</c> and <c>ErrorList</c> have the following format:</p> <code> -[{FileName,[ErrorInfo]}]. - </code> - - <p><c>ErrorInfo</c> is described below. The file name has been - included here as the compiler uses the Erlang pre-processor - <c>epp</c>, which allows the code to be included in other - files. For this reason, it is important to know to - <em>which</em> file an error or warning line number refers. +[{FileName,[ErrorInfo]}].</code> + + <p><c>ErrorInfo</c> is described later in this section. + The filename is included here, as the compiler uses the + Erlang pre-processor <c>epp</c>, which allows the code to be + included in other files. It is therefore important to know to + <em>which</em> file the line number of an error or a warning refers. </p> </desc> </func> <func> <name>forms(Forms)</name> - <fsummary>Compile a list of forms</fsummary> + <fsummary>Compiles a list of forms.</fsummary> <desc> <p>Is the same as <c>forms(File, [verbose,report_errors,report_warnings])</c>. @@ -670,7 +662,7 @@ module.beam: module.erl \ <func> <name>forms(Forms, Options) -> CompRet</name> - <fsummary>Compile a list of forms</fsummary> + <fsummary>Compiles a list of forms.</fsummary> <type> <v>Forms = [Form]</v> <v>CompRet = BinRet | ErrRet</v> @@ -681,48 +673,49 @@ module.beam: module.erl \ <desc> <p>Analogous to <c>file/1</c>, but takes a list of forms (in the Erlang abstract format representation) as first argument. - The option <c>binary</c> is implicit; i.e., no object code - file is produced. Options that would ordinarily produce a - listing file, such as 'E', will instead cause the internal - format for that compiler pass (an Erlang term; usually not a - binary) to be returned instead of a binary.</p> + Option <c>binary</c> is implicit, that is, no object code + file is produced. For options that normally produce a listing + file, such as 'E', the internal format for that compiler pass + (an Erlang term, usually not a binary) is returned instead of + a binary.</p> </desc> </func> <func> <name>format_error(ErrorDescriptor) -> chars()</name> - <fsummary>Format an error descriptor</fsummary> + <fsummary>Formats an error descriptor.</fsummary> <type> <v>ErrorDescriptor = errordesc()</v> </type> <desc> <p>Uses an <c>ErrorDescriptor</c> and returns a deep list of - characters which describes the error. This function is - usually called implicitly when an <c>ErrorInfo</c> structure - is processed. See below.</p> + characters that describes the error. This function is + usually called implicitly when an <c>ErrorInfo</c> structure + (described in section + <seealso marker="#error_information">Error Information</seealso>) is processed.</p> </desc> </func> <func> <name>output_generated(Options) -> true | false</name> - <fsummary>Determine whether the compile will generate an output file</fsummary> + <fsummary>Determines whether the compiler generates an output file.</fsummary> <type> <v>Options = [term()]</v> </type> <desc> - <p>Determines whether the compiler would generate a <c>beam</c> + <p>Determines whether the compiler generates a <c>beam</c> file with the given options. <c>true</c> means that a <c>beam</c> - file would be generated; <c>false</c> means that the compiler - would generate some listing file, return a binary, or merely - check the syntax of the source code.</p> + file is generated. <c>false</c> means that the compiler + generates some listing file, returns a binary, or merely + checks the syntax of the source code.</p> </desc> </func> <func> <name>noenv_file(File, Options) -> CompRet</name> - <fsummary>Compile a file (ignoring ERL_COMPILER_OPTIONS)</fsummary> + <fsummary>Compiles a file (ignoring <c>ERL_COMPILER_OPTIONS)</c>.</fsummary> <desc> - <p>Works exactly like <seealso marker="#file/2">file/2</seealso>, + <p>Works like <seealso marker="#file/2">file/2</seealso>, except that the environment variable <c>ERL_COMPILER_OPTIONS</c> is not consulted.</p> </desc> @@ -730,9 +723,9 @@ module.beam: module.erl \ <func> <name>noenv_forms(Forms, Options) -> CompRet</name> - <fsummary>Compile a list of forms (ignoring ERL_COMPILER_OPTIONS)</fsummary> + <fsummary>Compiles a list of forms (ignoring <c>ERL_COMPILER_OPTIONS)</c>.</fsummary> <desc> - <p>Works exactly like <seealso marker="#forms/2">forms/2</seealso>, + <p>Works like <seealso marker="#forms/2">forms/2</seealso>, except that the environment variable <c>ERL_COMPILER_OPTIONS</c> is not consulted.</p> </desc> @@ -740,12 +733,13 @@ module.beam: module.erl \ <func> <name>noenv_output_generated(Options) -> true | false</name> - <fsummary>Determine whether the compile will generate an output file (ignoring ERL_COMPILER_OPTIONS)</fsummary> + <fsummary>Determines whether the compiler generates an output file + (ignoring <c>ERL_COMPILER_OPTIONS)</c>.</fsummary> <type> <v>Options = [term()]</v> </type> <desc> - <p>Works exactly like + <p>Works like <seealso marker="#output_generated/1">output_generated/1</seealso>, except that the environment variable <c>ERL_COMPILER_OPTIONS</c> is not consulted.</p> @@ -755,14 +749,14 @@ module.beam: module.erl \ </funcs> <section> - <title>Default compiler options</title> + <title>Default Compiler Options</title> <p>The (host operating system) environment variable <c>ERL_COMPILER_OPTIONS</c> can be used to give default compiler options. Its value must be a valid Erlang term. If the value is a - list, it will be used as is. If it is not a list, it will be put + list, it is used as is. If it is not a list, it is put into a list.</p> - <p>The list will be appended to any options given to + <p>The list is appended to any options given to <seealso marker="#file/2">file/2</seealso>, <seealso marker="#forms/2">forms/2</seealso>, and <seealso marker="#output_generated/1">output_generated/2</seealso>. @@ -770,9 +764,9 @@ module.beam: module.erl \ <seealso marker="#noenv_file/2">noenv_file/2</seealso>, <seealso marker="#noenv_forms/2">noenv_forms/2</seealso>, or <seealso marker="#noenv_output_generated/1">noenv_output_generated/2</seealso> - if you don't want the environment variable to be consulted - (for instance, if you are calling the compiler recursively from - inside a parse transform).</p> + if you do not want the environment variable to be consulted, + for example, if you are calling the compiler recursively from + inside a parse transform.</p> </section> <section> @@ -781,31 +775,31 @@ module.beam: module.erl \ module. Inlining means that a call to a function is replaced with the function body with the arguments replaced with the actual values. The semantics are preserved, except if exceptions are - generated in the inlined code. Exceptions will be reported as + generated in the inlined code. Exceptions are reported as occurring in the function the body was inlined into. Also, - <c>function_clause</c> exceptions will be converted to similar + <c>function_clause</c> exceptions are converted to similar <c>case_clause</c> exceptions.</p> - <p>When a function is inlined, the original function will be + <p>When a function is inlined, the original function is kept if it is exported (either by an explicit export or if the - <c>export_all</c> option was given) or if not all calls to the - function were inlined.</p> + option <c>export_all</c> was given) or if not all calls to the + function are inlined.</p> <p>Inlining does not necessarily improve running time. - For instance, inlining may increase Beam stack usage which will - probably be detrimental to performance for recursive functions. + For example, inlining can increase Beam stack use, which + probably is detrimental to performance for recursive functions. </p> - <p>Inlining is never default; it must be explicitly enabled with a + <p>Inlining is never default. It must be explicitly enabled with a compiler option or a <c>-compile()</c> attribute in the source module.</p> - <p>To enable inlining, either use the <c>inline</c> option to - let the compiler decide which functions to inline or + <p>To enable inlining, either use the option <c>inline</c> to + let the compiler decide which functions to inline, or <c>{inline,[{Name,Arity},...]}</c> to have the compiler inline all calls to the given functions. If the option is given inside a <c>compile</c> directive in an Erlang module, <c>{Name,Arity}</c> - may be written as <c>Name/Arity</c>.</p> + can be written as <c>Name/Arity</c>.</p> <p>Example of explicit inlining:</p> @@ -817,33 +811,30 @@ pi() -> 3.1416. <p>Example of implicit inlining:</p> <pre> --compile(inline). - </pre> +-compile(inline).</pre> - <p>The <c>{inline_size,Size}</c> option controls how large functions - that are allowed to be inlined. Default is <c>24</c>, which will - keep the size of the inlined code roughly the same as - the un-inlined version (only relatively small functions will be + <p>The option <c>{inline_size,Size}</c> controls how large functions + that are allowed to be inlined. Default is <c>24</c>, which + keeps the size of the inlined code roughly the same as + the un-inlined version (only relatively small functions are inlined).</p> <p>Example:</p> <pre> %% Aggressive inlining - will increase code size. -compile(inline). --compile({inline_size,100}). - </pre> +-compile({inline_size,100}).</pre> </section> <section> - <title>Inlining of list functions</title> - <p>The compiler can also inline a variety of list manipulation functions - from the stdlib's lists module.</p> + <title>Inlining of List Functions</title> + <p>The compiler can also inline various list manipulation functions + from the module <c>list</c> in <c>STDLIB</c>.</p> <p>This feature must be explicitly enabled with a compiler option or a <c>-compile()</c> attribute in the source module.</p> - <p>To enable inlining of list functions, use the <c>inline_list_funcs</c> - option.</p> + <p>To enable inlining of list functions, use option <c>inline_list_funcs</c>.</p> <p>The following functions are inlined:</p> <list type="bulleted"> @@ -869,24 +860,23 @@ pi() -> 3.1416. </section> <section> + <marker id="error_information"></marker> <title>Error Information</title> - <p>The <c>ErrorInfo</c> mentioned above is the standard - <c>ErrorInfo</c> structure which is returned from all IO modules. + <p>The <c>ErrorInfo</c> mentioned earlier is the standard + <c>ErrorInfo</c> structure, which is returned from all I/O modules. It has the following format:</p> <code> -{ErrorLine, Module, ErrorDescriptor} - </code> +{ErrorLine, Module, ErrorDescriptor}</code> - <p><c>ErrorLine</c> will be the atom <c>none</c> if the error does - not correspond to a specific line (e.g. if the source file does - not exist).</p> + <p><c>ErrorLine</c> is the atom <c>none</c> if the error does + not correspond to a specific line, for example, if the source file does + not exist.</p> <p>A string describing the error is obtained with the following call:</p> <code> -Module:format_error(ErrorDescriptor) - </code> +Module:format_error(ErrorDescriptor)</code> </section> <section> diff --git a/lib/compiler/doc/src/notes.xml b/lib/compiler/doc/src/notes.xml index 0654738247..9b5b44f3e1 100644 --- a/lib/compiler/doc/src/notes.xml +++ b/lib/compiler/doc/src/notes.xml @@ -31,94 +31,6 @@ <p>This document describes the changes made to the Compiler application.</p> -<section><title>Compiler 6.0</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - The compiler optimizes away building of terms that are - never actually used. As a result, the compiler in OTP 18 - may produce more warnings for terms that are built but - not used than the compiler in OTP 17.</p> - <p> - Own Id: OTP-12453</p> - </item> - <item> - <p> - Using a map could incorrectly suppress warnings for - unused variables.</p> - <p> - Own Id: OTP-12515</p> - </item> - <item> - <p> - The compiler now properly reports unknown parse - transforms. That is, <c>undef</c> exceptions coming from - the parse transform itself is reported differently from - the absence of the parse transform.</p> - <p> - Own Id: OTP-12723</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - The <c>cerl</c> and <c>cerl_trees</c> modules in the - <c>compiler</c> application are now documented.</p> - <p> - Own Id: OTP-11978</p> - </item> - <item> - <p> - The deprecated '<c>asm</c>' option has been removed.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12100</p> - </item> - <item> - <p> - Support variables as Map keys in expressions and patterns</p> - <p>Erlang will accept any expression as keys in Map - expressions and it will accept literals or bound - variables as keys in Map patterns.</p> - <p> - Own Id: OTP-12218</p> - </item> - <item> - <p> - Infer Map type information in beam_type compiler - optimization pass.</p> - <p> - Own Id: OTP-12253</p> - </item> - <item> - <p> - Compiler optimizations have been improved.</p> - <p> - Own Id: OTP-12393</p> - </item> - <item> - <p> - Five undocumented functions in the module <c>core_lib</c> - have been deprecated and will be removed in the next - major release. The functions are: <c>get_anno/{1,2}</c>, - <c>is_literal/1</c>, <c>is_literal_list/1</c>, and - <c>literal_value</c>. Use the appropriate functions in - the <c>cerl</c> module instead.</p> - <p> - Own Id: OTP-12497</p> - </item> - </list> - </section> - -</section> - <section><title>Compiler 5.0.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/compiler/doc/src/ref_man.xml b/lib/compiler/doc/src/ref_man.xml index 6478ad4b11..6584e79c4e 100644 --- a/lib/compiler/doc/src/ref_man.xml +++ b/lib/compiler/doc/src/ref_man.xml @@ -29,7 +29,7 @@ <file>application.sgml</file> </header> <description> - <p>The <em>Compiler</em> application compiles Erlang + <p>The <c>Compiler</c> application compiles Erlang code to byte-code. The highly compact byte-code is executed by the Erlang emulator.</p> </description> diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile index 7c4cebdc28..78efc8dff0 100644 --- a/lib/compiler/src/Makefile +++ b/lib/compiler/src/Makefile @@ -70,6 +70,7 @@ MODULES = \ cerl \ cerl_clauses \ cerl_inline \ + cerl_sets \ cerl_trees \ compile \ core_lib \ diff --git a/lib/compiler/src/beam_bsm.erl b/lib/compiler/src/beam_bsm.erl index 2a15c1ddf3..ee3e88959d 100644 --- a/lib/compiler/src/beam_bsm.erl +++ b/lib/compiler/src/beam_bsm.erl @@ -242,6 +242,12 @@ btb_reaches_match_2([{bif,_,{f,F},Ss,Dst}=I|Is], Regs0, D0) -> Regs = btb_kill([Dst], Regs0), D = btb_follow_branch(F, Regs, D0), btb_reaches_match_1(Is, Regs, D); +btb_reaches_match_2([{get_map_elements,{f,F},Src,{list,Ls}}=I|Is], Regs0, D0) -> + {Ss,Ds} = beam_utils:split_even(Ls), + btb_ensure_not_used([Src|Ss], I, Regs0), + Regs = btb_kill(Ds, Regs0), + D = btb_follow_branch(F, Regs, D0), + btb_reaches_match_1(Is, Regs, D); btb_reaches_match_2([{test,bs_start_match2,{f,F},Live,[Ctx,_],Ctx}=I|Is], Regs0, D0) -> CtxRegs = btb_context_regs(Regs0), diff --git a/lib/compiler/src/beam_dead.erl b/lib/compiler/src/beam_dead.erl index 5932d8ce1d..bbe607cf19 100644 --- a/lib/compiler/src/beam_dead.erl +++ b/lib/compiler/src/beam_dead.erl @@ -96,7 +96,7 @@ move_move_into_block([], Acc) -> reverse(Acc). %%% forward(Is, Lc) -> - forward(Is, gb_trees:empty(), Lc, []). + forward(Is, #{}, Lc, []). forward([{move,_,_}=Move|[{label,L}|_]=Is], D, Lc, Acc) -> %% move/2 followed by jump/1 is optimized by backward/3. @@ -115,19 +115,20 @@ forward([{label,Lbl}=LblI,{block,[{set,[Dst],[Lit],move}|BlkIs]}=Blk|Is], D, Lc, %% cannot be reached in any other way than through the select_val/3 %% instruction (i.e. there can be no fallthrough to such label and %% it cannot be referenced by, for example, a jump/1 instruction). - Block = case gb_trees:lookup({Lbl,Dst}, D) of - {value,Lit} -> {block,BlkIs}; %Safe to remove move instruction. - _ -> Blk %Must keep move instruction. - end, + Key = {Lbl,Dst}, + Block = case D of + #{Key := Lit} -> {block,BlkIs}; %Safe to remove move instruction. + _ -> Blk %Must keep move instruction. + end, forward([Block|Is], D, Lc, [LblI|Acc]); forward([{label,Lbl}=LblI|[{move,Lit,Dst}|Is1]=Is0], D, Lc, Acc) -> %% Assumption: The target labels in a select_val/3 instruction %% cannot be reached in any other way than through the select_val/3 %% instruction (i.e. there can be no fallthrough to such label and %% it cannot be referenced by, for example, a jump/1 instruction). - Is = case gb_trees:lookup({Lbl,Dst}, D) of - {value,Lit} -> Is1; %Safe to remove move instruction. - _ -> Is0 %Keep move instruction. + Is = case maps:find({Lbl,Dst}, D) of + {ok,Lit} -> Is1; %Safe to remove move instruction. + _ -> Is0 %Keep move instruction. end, forward(Is, D, Lc, [LblI|Acc]); forward([{test,is_eq_exact,_,[Same,Same]}|Is], D, Lc, Acc) -> @@ -156,11 +157,11 @@ forward([], _, Lc, Acc) -> {Acc,Lc}. update_value_dict([Lit,{f,Lbl}|T], Reg, D0) -> Key = {Lbl,Reg}, - D = case gb_trees:lookup(Key, D0) of - none -> gb_trees:insert(Key, Lit, D0); %New. - {value,inconsistent} -> D0; %Inconsistent. - {value,_} -> gb_trees:update(Key, inconsistent, D0) - end, + D = case D0 of + #{Key := inconsistent} -> D0; + #{Key := _} -> D0#{Key := inconsistent}; + _ -> D0#{Key => Lit} + end, update_value_dict(T, Reg, D); update_value_dict([], _, D) -> D. diff --git a/lib/compiler/src/beam_dict.erl b/lib/compiler/src/beam_dict.erl index 68dc104dd3..b1aa98278e 100644 --- a/lib/compiler/src/beam_dict.erl +++ b/lib/compiler/src/beam_dict.erl @@ -31,22 +31,22 @@ -type index() :: non_neg_integer(). --type atom_tab() :: gb_trees:tree(atom(), index()). +-type atom_tab() :: #{atom() => index()}. -type import_tab() :: gb_trees:tree(mfa(), index()). --type fname_tab() :: gb_trees:tree(Name :: term(), index()). --type line_tab() :: gb_trees:tree({Fname :: index(), Line :: term()}, index()). +-type fname_tab() :: #{Name :: term() => index()}. +-type line_tab() :: #{{Fname :: index(), Line :: term()} => index()}. -type literal_tab() :: dict:dict(Literal :: term(), index()). -record(asm, - {atoms = gb_trees:empty() :: atom_tab(), + {atoms = #{} :: atom_tab(), exports = [] :: [{label(), arity(), label()}], locals = [] :: [{label(), arity(), label()}], imports = gb_trees:empty() :: import_tab(), strings = <<>> :: binary(), %String pool lambdas = [], %[{...}] literals = dict:new() :: literal_tab(), - fnames = gb_trees:empty() :: fname_tab(), - lines = gb_trees:empty() :: line_tab(), + fnames = #{} :: fname_tab(), + lines = #{} :: line_tab(), num_lines = 0 :: non_neg_integer(), %Number of line instructions next_import = 0 :: non_neg_integer(), string_offset = 0 :: non_neg_integer(), @@ -77,14 +77,12 @@ highest_opcode(#asm{highest_opcode=Op}) -> Op. %% atom(Atom, Dict) -> {Index,Dict'} -spec atom(atom(), bdict()) -> {pos_integer(), bdict()}. -atom(Atom, #asm{atoms=Atoms0}=Dict) when is_atom(Atom) -> - case gb_trees:lookup(Atom, Atoms0) of - {value,Index} -> - {Index,Dict}; - none -> - NextIndex = gb_trees:size(Atoms0) + 1, - Atoms = gb_trees:insert(Atom, NextIndex, Atoms0), - {NextIndex,Dict#asm{atoms=Atoms}} +atom(Atom, #asm{atoms=Atoms}=Dict) when is_atom(Atom) -> + case Atoms of + #{ Atom := Index} -> {Index,Dict}; + _ -> + NextIndex = maps:size(Atoms) + 1, + {NextIndex,Dict#asm{atoms=Atoms#{Atom=>NextIndex}}} end. %% Remembers an exported function. @@ -177,26 +175,22 @@ line([], #asm{num_lines=N}=Dict) -> %% No location available. Return the special pre-defined %% index 0. {0,Dict#asm{num_lines=N+1}}; -line([{location,Name,Line}], #asm{lines=Lines0,num_lines=N}=Dict0) -> +line([{location,Name,Line}], #asm{lines=Lines,num_lines=N}=Dict0) -> {FnameIndex,Dict1} = fname(Name, Dict0), - case gb_trees:lookup({FnameIndex,Line}, Lines0) of - {value,Index} -> - {Index,Dict1#asm{num_lines=N+1}}; - none -> - Index = gb_trees:size(Lines0) + 1, - Lines = gb_trees:insert({FnameIndex,Line}, Index, Lines0), - Dict = Dict1#asm{lines=Lines,num_lines=N+1}, - {Index,Dict} + Key = {FnameIndex,Line}, + case Lines of + #{Key := Index} -> {Index,Dict1#asm{num_lines=N+1}}; + _ -> + Index = maps:size(Lines) + 1, + {Index, Dict1#asm{lines=Lines#{Key=>Index},num_lines=N+1}} end. -fname(Name, #asm{fnames=Fnames0}=Dict) -> - case gb_trees:lookup(Name, Fnames0) of - {value,Index} -> - {Index,Dict}; - none -> - Index = gb_trees:size(Fnames0), - Fnames = gb_trees:insert(Name, Index, Fnames0), - {Index,Dict#asm{fnames=Fnames}} +fname(Name, #asm{fnames=Fnames}=Dict) -> + case Fnames of + #{Name := Index} -> {Index,Dict}; + _ -> + Index = maps:size(Fnames), + {Index,Dict#asm{fnames=Fnames#{Name=>Index}}} end. %% Returns the atom table. @@ -204,14 +198,12 @@ fname(Name, #asm{fnames=Fnames0}=Dict) -> -spec atom_table(bdict()) -> {non_neg_integer(), [[non_neg_integer(),...]]}. atom_table(#asm{atoms=Atoms}) -> - NumAtoms = gb_trees:size(Atoms), - Sorted = lists:keysort(2, gb_trees:to_list(Atoms)), - Fun = fun({A,_}) -> - L = atom_to_list(A), - [length(L)|L] - end, - AtomTab = lists:map(Fun, Sorted), - {NumAtoms,AtomTab}. + NumAtoms = maps:size(Atoms), + Sorted = lists:keysort(2, maps:to_list(Atoms)), + {NumAtoms,[begin + L = atom_to_list(A), + [length(L)|L] + end || {A,_} <- Sorted]}. %% Returns the table of local functions. %% local_table(Dict) -> {NumLocals, [{Function, Arity, Label}...]} @@ -273,11 +265,11 @@ my_term_to_binary(Term) -> non_neg_integer(),[{non_neg_integer(),non_neg_integer()}]}. line_table(#asm{fnames=Fnames0,lines=Lines0,num_lines=NumLineInstrs}) -> - NumFnames = gb_trees:size(Fnames0), - Fnames1 = lists:keysort(2, gb_trees:to_list(Fnames0)), + NumFnames = maps:size(Fnames0), + Fnames1 = lists:keysort(2, maps:to_list(Fnames0)), Fnames = [Name || {Name,_} <- Fnames1], - NumLines = gb_trees:size(Lines0), - Lines1 = lists:keysort(2, gb_trees:to_list(Lines0)), + NumLines = maps:size(Lines0), + Lines1 = lists:keysort(2, maps:to_list(Lines0)), Lines = [L || {L,_} <- Lines1], {NumLineInstrs,NumFnames,Fnames,NumLines,Lines}. diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl index 52b6464c7f..80b2998ddc 100644 --- a/lib/compiler/src/beam_jump.erl +++ b/lib/compiler/src/beam_jump.erl @@ -152,14 +152,14 @@ function({function,Name,Arity,CLabel,Asm0}) -> share(Is0) -> %% We will get more sharing if we never fall through to a label. Is = eliminate_fallthroughs(Is0, []), - share_1(Is, dict:new(), [], []). + share_1(Is, #{}, [], []). share_1([{label,_}=Lbl|Is], Dict, [], Acc) -> share_1(Is, Dict, [], [Lbl|Acc]); share_1([{label,L}=Lbl|Is], Dict0, Seq, Acc) -> - case dict:find(Seq, Dict0) of + case maps:find(Seq, Dict0) of error -> - Dict = dict:store(Seq, L, Dict0), + Dict = maps:put(Seq, L, Dict0), share_1(Is, Dict, [], [Lbl|Seq ++ Acc]); {ok,Label} -> share_1(Is, Dict0, [], [Lbl,{jump,{f,Label}}|Acc]) @@ -188,7 +188,7 @@ clean_non_sharable(Dict) -> %% a sequence inside the 'try' block is a sequence that ends %% with an instruction that causes an exception. Any sequence %% that causes an exception must contain a line/1 instruction. - dict:filter(fun(K, _V) -> sharable_with_try(K) end, Dict). + maps:filter(fun(K, _V) -> sharable_with_try(K) end, Dict). sharable_with_try([{line,_}|_]) -> %% This sequence may cause an exception and may potentially @@ -268,13 +268,13 @@ extract_seq_1(_, _) -> no. -record(st, {fc, %Label for function class errors. entry, %Entry label (must not be moved). mlbl, %Moved labels. - labels %Set of referenced labels. + labels :: cerl_sets:set() %Set of referenced labels. }). opt([{label,Fc}|_]=Is0, CLabel) -> Lbls = initial_labels(Is0), find_fixpoint(fun(Is) -> - St = #st{fc=Fc,entry=CLabel,mlbl=dict:new(), + St = #st{fc=Fc,entry=CLabel,mlbl=#{}, labels=Lbls}, opt(Is, [], St) end, Is0). @@ -320,11 +320,11 @@ opt([{test,_,{f,_}=Lbl,_,_,_}=I|Is], Acc, St) -> opt([{select,_,_R,Fail,Vls}=I|Is], Acc, St) -> skip_unreachable(Is, [I|Acc], label_used([Fail|Vls], St)); opt([{label,Lbl}=I|Is], Acc, #st{mlbl=Mlbl}=St0) -> - case dict:find(Lbl, Mlbl) of + case maps:find(Lbl, Mlbl) of {ok,Lbls} -> %% Essential to remove the list of labels from the dictionary, %% since we will rescan the inserted labels. We MUST rescan. - St = St0#st{mlbl=dict:erase(Lbl, Mlbl)}, + St = St0#st{mlbl=maps:remove(Lbl, Mlbl)}, insert_labels([Lbl|Lbls], Is, Acc, St); error -> opt(Is, [I|Acc], St0) end; @@ -339,7 +339,7 @@ opt([{jump,{f,L}=Lbl}=I|Is], Acc0, #st{mlbl=Mlbl0}=St0) -> St = case Lbls of [] -> St0; [_|_] -> - Mlbl = dict:append_list(L, Lbls, Mlbl0), + Mlbl = maps_append_list(L, Lbls, Mlbl0), St0#st{mlbl=Mlbl} end, skip_unreachable(Is, [I|Acc], label_used(Lbl, St)); @@ -363,14 +363,20 @@ opt([I|Is], Acc, #st{labels=Used0}=St0) -> end; opt([], Acc, #st{fc=Fc,mlbl=Mlbl}) -> Code = reverse(Acc), - case dict:find(Fc, Mlbl) of + case maps:find(Fc, Mlbl) of {ok,Lbls} -> insert_fc_labels(Lbls, Mlbl, Code); error -> Code end. +maps_append_list(K,Vs,M) -> + case M of + #{K:=Vs0} -> M#{K:=Vs0++Vs}; % same order as dict + _ -> M#{K => Vs} + end. + insert_fc_labels([L|Ls], Mlbl, Acc0) -> Acc = [{label,L}|Acc0], - case dict:find(L, Mlbl) of + case maps:find(L, Mlbl) of error -> insert_fc_labels(Ls, Mlbl, Acc); {ok,Lbls} -> @@ -434,7 +440,7 @@ skip_unreachable([], Acc, St) -> %% Add one or more label to the set of used labels. -label_used({f,L}, St) -> St#st{labels=gb_sets:add(L, St#st.labels)}; +label_used({f,L}, St) -> St#st{labels=cerl_sets:add_element(L,St#st.labels)}; label_used([H|T], St0) -> label_used(T, label_used(H, St0)); label_used([], St) -> St; label_used(_Other, St) -> St. @@ -442,7 +448,7 @@ label_used(_Other, St) -> St. %% Test if label is used. is_label_used(L, St) -> - gb_sets:is_member(L, St#st.labels). + cerl_sets:is_element(L, St#st.labels). %% is_unreachable_after(Instruction) -> boolean() %% Test whether the code after Instruction is unreachable. @@ -472,14 +478,14 @@ is_exit_instruction(_) -> false. %% (including inside blocks). is_label_used_in(Lbl, Is) -> - is_label_used_in_1(Is, Lbl, gb_sets:empty()). + is_label_used_in_1(Is, Lbl, cerl_sets:new()). is_label_used_in_1([{block,Block}|Is], Lbl, Empty) -> lists:any(fun(I) -> is_label_used_in_block(I, Lbl) end, Block) orelse is_label_used_in_1(Is, Lbl, Empty); is_label_used_in_1([I|Is], Lbl, Empty) -> Used = ulbl(I, Empty), - gb_sets:is_member(Lbl, Used) orelse is_label_used_in_1(Is, Lbl, Empty); + cerl_sets:is_element(Lbl, Used) orelse is_label_used_in_1(Is, Lbl, Empty); is_label_used_in_1([], _, _) -> false. is_label_used_in_block({set,_,_,Info}, Lbl) -> @@ -506,7 +512,7 @@ remove_unused_labels(Is) -> rem_unused(Is, Used, []). rem_unused([{label,Lbl}=I|Is0], Used, [Prev|_]=Acc) -> - case gb_sets:is_member(Lbl, Used) of + case cerl_sets:is_element(Lbl, Used) of false -> Is = case is_unreachable_after(Prev) of true -> drop_upto_label(Is0); @@ -528,7 +534,7 @@ initial_labels([{line,_}|Is], Acc) -> initial_labels([{label,Lbl}|Is], Acc) -> initial_labels(Is, [Lbl|Acc]); initial_labels([{func_info,_,_,_},{label,Lbl}|_], Acc) -> - gb_sets:from_list([Lbl|Acc]). + cerl_sets:from_list([Lbl|Acc]). drop_upto_label([{label,_}|_]=Is) -> Is; drop_upto_label([_|Is]) -> drop_upto_label(Is); @@ -576,10 +582,10 @@ ulbl({get_map_elements,Lbl,_Src,_List}, Used) -> ulbl(_, Used) -> Used. mark_used({f,0}, Used) -> Used; -mark_used({f,L}, Used) -> gb_sets:add(L, Used). +mark_used({f,L}, Used) -> cerl_sets:add_element(L, Used). mark_used_list([{f,L}|T], Used) -> - mark_used_list(T, gb_sets:add(L, Used)); + mark_used_list(T, cerl_sets:add_element(L, Used)); mark_used_list([_|T], Used) -> mark_used_list(T, Used); mark_used_list([], Used) -> Used. diff --git a/lib/compiler/src/beam_type.erl b/lib/compiler/src/beam_type.erl index 4731b5e78e..7ab548152e 100644 --- a/lib/compiler/src/beam_type.erl +++ b/lib/compiler/src/beam_type.erl @@ -554,10 +554,10 @@ flush(Rs, [{set,[_],[],{put_tuple,_}}|_]=Is0, Acc0) -> Acc = flush_all(Rs, Is0, Acc0), {[],Acc}; flush(Rs0, [{set,Ds,Ss,_Op}|_], Acc0) -> - Save = gb_sets:from_list(Ss), + Save = cerl_sets:from_list(Ss), Acc = save_regs(Rs0, Save, Acc0), Rs1 = foldl(fun(S, A) -> mark(S, A, clean) end, Rs0, Ss), - Kill = gb_sets:from_list(Ds), + Kill = cerl_sets:from_list(Ds), Rs = kill_regs(Rs1, Kill), {Rs,Acc}; flush(Rs0, Is, Acc0) -> @@ -580,7 +580,7 @@ save_regs(Rs, Save, Acc) -> foldl(fun(R, A) -> save_reg(R, Save, A) end, Acc, Rs). save_reg({I,V,dirty}, Save, Acc) -> - case gb_sets:is_member(V, Save) of + case cerl_sets:is_element(V, Save) of true -> [{set,[V],[{fr,I}],fmove}|checkerror(Acc)]; false -> Acc end; @@ -590,7 +590,7 @@ kill_regs(Rs, Kill) -> [kill_reg(R, Kill) || R <- Rs]. kill_reg({_,V,_}=R, Kill) -> - case gb_sets:is_member(V, Kill) of + case cerl_sets:is_element(V, Kill) of true -> free; false -> R end; diff --git a/lib/compiler/src/cerl_sets.erl b/lib/compiler/src/cerl_sets.erl new file mode 100644 index 0000000000..4df78dc432 --- /dev/null +++ b/lib/compiler/src/cerl_sets.erl @@ -0,0 +1,206 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2000-2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(cerl_sets). + +%% Standard interface. +-export([new/0,is_set/1,size/1,to_list/1,from_list/1]). +-export([is_element/2,add_element/2,del_element/2]). +-export([union/2,union/1,intersection/2,intersection/1]). +-export([is_disjoint/2]). +-export([subtract/2,is_subset/2]). +-export([fold/3,filter/2]). + +-export_type([set/0, set/1]). + +%%------------------------------------------------------------------------------ + +-type set() :: set(_). +-opaque set(Element) :: #{Element => 'ok'}. + +%%------------------------------------------------------------------------------ + +%% new() -> Set +-spec new() -> set(). + +new() -> #{}. + +%% is_set(Set) -> boolean(). +%% Return 'true' if Set is a set of elements, else 'false'. +-spec is_set(Set) -> boolean() when + Set :: term(). + +is_set(S) when is_map(S) -> true; +is_set(_) -> false. + +%% size(Set) -> int(). +%% Return the number of elements in Set. +-spec size(Set) -> non_neg_integer() when + Set :: set(). + +size(S) -> maps:size(S). + +%% to_list(Set) -> [Elem]. +%% Return the elements in Set as a list. +-spec to_list(Set) -> List when + Set :: set(Element), + List :: [Element]. + +to_list(S) -> maps:keys(S). + +%% from_list([Elem]) -> Set. +%% Build a set from the elements in List. +-spec from_list(List) -> Set when + List :: [Element], + Set :: set(Element). +from_list(Ls) -> maps:from_list([{K,ok}||K<-Ls]). + +%% is_element(Element, Set) -> boolean(). +%% Return 'true' if Element is an element of Set, else 'false'. +-spec is_element(Element, Set) -> boolean() when + Set :: set(Element). + +is_element(E,S) -> + case S of + #{E := _} -> true; + _ -> false + end. + +%% add_element(Element, Set) -> Set. +%% Return Set with Element inserted in it. +-spec add_element(Element, Set1) -> Set2 when + Set1 :: set(Element), + Set2 :: set(Element). + +add_element(E,S) -> S#{E=>ok}. + +-spec del_element(Element, Set1) -> Set2 when + Set1 :: set(Element), + Set2 :: set(Element). + +%% del_element(Element, Set) -> Set. +%% Return Set but with Element removed. +del_element(E,S) -> maps:remove(E,S). + +%% union(Set1, Set2) -> Set +%% Return the union of Set1 and Set2. +-spec union(Set1, Set2) -> Set3 when + Set1 :: set(Element), + Set2 :: set(Element), + Set3 :: set(Element). + +union(S1,S2) -> maps:merge(S1,S2). + +%% union([Set]) -> Set +%% Return the union of the list of sets. +-spec union(SetList) -> Set when + SetList :: [set(Element)], + Set :: set(Element). + +union([S1,S2|Ss]) -> + union1(union(S1, S2), Ss); +union([S]) -> S; +union([]) -> new(). + +union1(S1, [S2|Ss]) -> + union1(union(S1, S2), Ss); +union1(S1, []) -> S1. + +%% intersection(Set1, Set2) -> Set. +%% Return the intersection of Set1 and Set2. +-spec intersection(Set1, Set2) -> Set3 when + Set1 :: set(Element), + Set2 :: set(Element), + Set3 :: set(Element). + +intersection(S1, S2) -> + filter(fun (E) -> is_element(E, S1) end, S2). + +%% intersection([Set]) -> Set. +%% Return the intersection of the list of sets. +-spec intersection(SetList) -> Set when + SetList :: [set(Element),...], + Set :: set(Element). + +intersection([S1,S2|Ss]) -> + intersection1(intersection(S1, S2), Ss); +intersection([S]) -> S. + +intersection1(S1, [S2|Ss]) -> + intersection1(intersection(S1, S2), Ss); +intersection1(S1, []) -> S1. + +%% is_disjoint(Set1, Set2) -> boolean(). +%% Check whether Set1 and Set2 are disjoint. +-spec is_disjoint(Set1, Set2) -> boolean() when + Set1 :: set(Element), + Set2 :: set(Element). + +is_disjoint(S1, S2) when map_size(S1) < map_size(S2) -> + fold(fun (_, false) -> false; + (E, true) -> not is_element(E, S2) + end, true, S1); +is_disjoint(S1, S2) -> + fold(fun (_, false) -> false; + (E, true) -> not is_element(E, S1) + end, true, S2). + +%% subtract(Set1, Set2) -> Set. +%% Return all and only the elements of Set1 which are not also in +%% Set2. +-spec subtract(Set1, Set2) -> Set3 when + Set1 :: set(Element), + Set2 :: set(Element), + Set3 :: set(Element). + +subtract(S1, S2) -> + filter(fun (E) -> not is_element(E, S2) end, S1). + +%% is_subset(Set1, Set2) -> boolean(). +%% Return 'true' when every element of Set1 is also a member of +%% Set2, else 'false'. +-spec is_subset(Set1, Set2) -> boolean() when + Set1 :: set(Element), + Set2 :: set(Element). + +is_subset(S1, S2) -> + fold(fun (E, Sub) -> Sub andalso is_element(E, S2) end, true, S1). + +%% fold(Fun, Accumulator, Set) -> Accumulator. +%% Fold function Fun over all elements in Set and return Accumulator. +-spec fold(Function, Acc0, Set) -> Acc1 when + Function :: fun((Element, AccIn) -> AccOut), + Set :: set(Element), + Acc0 :: Acc, + Acc1 :: Acc, + AccIn :: Acc, + AccOut :: Acc. + +fold(F, Init, D) -> + lists:foldl(fun(E,Acc) -> F(E,Acc) end,Init,maps:keys(D)). + +%% filter(Fun, Set) -> Set. +%% Filter Set with Fun. +-spec filter(Pred, Set1) -> Set2 when + Pred :: fun((Element) -> boolean()), + Set1 :: set(Element), + Set2 :: set(Element). + +filter(F, D) -> + maps:from_list(lists:filter(fun({K,_}) -> F(K) end, maps:to_list(D))). diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 22810c910c..0158cf64db 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -342,10 +342,10 @@ run_tc({Name,Fun}, St) -> run_eprof({Name,Fun}, Name, St) -> io:format("~p: Running eprof\n", [Name]), - eprof:start_profiling([self()]), + c:appcall(tools, eprof, start_profiling, [[self()]]), Val = (catch Fun(St)), - eprof:stop_profiling(), - eprof:analyze(), + c:appcall(tools, eprof, stop_profiling, []), + c:appcall(tools, eprof, analyze, []), Val; run_eprof({_,Fun}, _, St) -> catch Fun(St). diff --git a/lib/compiler/src/compiler.app.src b/lib/compiler/src/compiler.app.src index 2a40c1c379..0bfd998301 100644 --- a/lib/compiler/src/compiler.app.src +++ b/lib/compiler/src/compiler.app.src @@ -45,6 +45,7 @@ cerl, cerl_clauses, cerl_inline, + cerl_sets, cerl_trees, compile, core_scan, @@ -69,5 +70,5 @@ {registered, []}, {applications, [kernel, stdlib]}, {env, []}, - {runtime_dependencies, ["stdlib-2.0","kernel-3.0","hipe-3.10.3","erts-7.0", - "crypto-3.3"]}]}. + {runtime_dependencies, ["stdlib-2.5","kernel-4.0","hipe-3.12","erts-7.0", + "crypto-3.6"]}]}. diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index 6f8279f65e..7f4184fd30 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -92,10 +92,10 @@ -endif. %% Variable value info. --record(sub, {v=[], %Variable substitutions - s=[], %Variables in scope - t=[], %Types - in_guard=false}). %In guard or not. +-record(sub, {v=[], %Variable substitutions + s=cerl_sets:new() :: cerl_sets:set(), %Variables in scope + t=#{} :: map(), %Types + in_guard=false}). %In guard or not. -type type_info() :: cerl:cerl() | 'bool' | 'integer'. -type yes_no_maybe() :: 'yes' | 'no' | 'maybe'. @@ -1123,18 +1123,30 @@ let_substs(Vs0, As0, Sub0) -> {Vs2,As1,Ss} = let_substs_1(Vs1, As0, Sub1), Sub2 = sub_add_scope([V || #c_var{name=V} <- Vs2], Sub1), {Vs2,As1, - foldl(fun ({V,S}, Sub) -> sub_set_name(V, S, Sub) end, Sub2, Ss)}. + foldl(fun ({V,S}, Sub) -> sub_set_name(V, S, Sub) end, Sub2, Ss)}. let_substs_1(Vs, #c_values{es=As}, Sub) -> let_subst_list(Vs, As, Sub); let_substs_1([V], A, Sub) -> let_subst_list([V], [A], Sub); let_substs_1(Vs, A, _) -> {Vs,A,[]}. -let_subst_list([V|Vs0], [A|As0], Sub) -> +let_subst_list([V|Vs0], [A0|As0], Sub) -> {Vs1,As1,Ss} = let_subst_list(Vs0, As0, Sub), - case is_subst(A) of - true -> {Vs1,As1,sub_subst_var(V, A, Sub) ++ Ss}; - false -> {[V|Vs1],[A|As1],Ss} + case is_subst(A0) of + true -> + A = case is_compiler_generated(V) andalso + not is_compiler_generated(A0) of + true -> + %% Propagate the 'compiler_generated' annotation + %% along with the value. + Ann = [compiler_generated|cerl:get_ann(A0)], + cerl:set_ann(A0, Ann); + false -> + A0 + end, + {Vs1,As1,sub_subst_var(V, A, Sub) ++ Ss}; + false -> + {[V|Vs1],[A0|As1],Ss} end; let_subst_list([], [], _) -> {[],[],[]}. @@ -1242,10 +1254,10 @@ is_subst(_) -> false. %% to force renaming if variables in the scope occurs as pattern %% variables. -sub_new() -> #sub{v=orddict:new(),s=gb_trees:empty(),t=[]}. +sub_new() -> #sub{v=orddict:new(),s=cerl_sets:new(),t=#{}}. sub_new(#sub{}=Sub) -> - Sub#sub{v=orddict:new(),t=[]}. + Sub#sub{v=orddict:new(),t=#{}}. sub_new_preserve_types(#sub{}=Sub) -> Sub#sub{v=orddict:new()}. @@ -1262,16 +1274,16 @@ sub_set_var(#c_var{name=V}, Val, Sub) -> sub_set_name(V, Val, #sub{v=S,s=Scope,t=Tdb0}=Sub) -> Tdb1 = kill_types(V, Tdb0), Tdb = copy_type(V, Val, Tdb1), - Sub#sub{v=orddict:store(V, Val, S),s=gb_sets:add(V, Scope),t=Tdb}. + Sub#sub{v=orddict:store(V, Val, S),s=cerl_sets:add_element(V, Scope),t=Tdb}. sub_del_var(#c_var{name=V}, #sub{v=S,s=Scope,t=Tdb}=Sub) -> %% Profiling shows that for programs with many record operations, %% sub_del_var/2 is a bottleneck. Since the scope contains all %% variables that are live, we know that V cannot be present in S %% if it is not in the scope. - case gb_sets:is_member(V, Scope) of + case cerl_sets:is_element(V, Scope) of false -> - Sub#sub{s=gb_sets:insert(V, Scope)}; + Sub#sub{s=cerl_sets:add_element(V, Scope)}; true -> Sub#sub{v=orddict:erase(V, S),t=kill_types(V, Tdb)} end. @@ -1282,12 +1294,12 @@ sub_subst_var(#c_var{name=V}, Val, #sub{v=S0}) -> sub_add_scope(Vs, #sub{s=Scope0}=Sub) -> Scope = foldl(fun(V, S) when is_integer(V); is_atom(V) -> - gb_sets:add(V, S) + cerl_sets:add_element(V, S) end, Scope0, Vs), Sub#sub{s=Scope}. sub_subst_scope(#sub{v=S0,s=Scope}=Sub) -> - S = [{-1,#c_var{name=Sv}} || Sv <- gb_sets:to_list(Scope)]++S0, + S = [{-1,#c_var{name=Sv}} || Sv <- cerl_sets:to_list(Scope)]++S0, Sub#sub{v=S}. sub_is_val(#c_var{name=V}, #sub{v=S,s=Scope}) -> @@ -1295,7 +1307,7 @@ sub_is_val(#c_var{name=V}, #sub{v=S,s=Scope}) -> %% became the new bottleneck. Since the scope contains all %% live variables, a variable V can only be the target for %% a substitution if it is in the scope. - gb_sets:is_member(V, Scope) andalso v_is_value(V, S). + cerl_sets:is_element(V, Scope) andalso v_is_value(V, S). v_is_value(Var, [{_,#c_var{name=Var}}|_]) -> true; v_is_value(Var, [_|T]) -> v_is_value(Var, T); @@ -1760,8 +1772,9 @@ case_opt_compiler_generated(Core) -> %% return Expr0 unchanged. %% case_expand_var(E, #sub{t=Tdb}) -> - case orddict:find(cerl:var_name(E), Tdb) of - {ok,T0} -> + Key = cerl:var_name(E), + case Tdb of + #{Key:=T0} -> case cerl:is_c_tuple(T0) of false -> E; @@ -1785,7 +1798,7 @@ case_expand_var(E, #sub{t=Tdb}) -> E end end; - error -> + _ -> E end. @@ -1899,8 +1912,8 @@ case_data_pat_alias(P, BindTo0, TypeSig, Bs0) -> %% Here we will need to actually build the data and bind %% it to the variable. {Type,Arity} = TypeSig, - Vars = make_vars([], Arity), Ann = [compiler_generated], + Vars = make_vars(Ann, Arity), Data = cerl:ann_make_data(Ann, Type, Vars), Bs = [{BindTo0,P},{P,Data}|Bs0], {Vars,Bs}; @@ -2147,7 +2160,7 @@ is_bool_expr_list([], _) -> true. %% functions, or is_record/2). %% is_safe_bool_expr(Core, Sub) -> - is_safe_bool_expr_1(Core, Sub, gb_sets:empty()). + is_safe_bool_expr_1(Core, Sub, cerl_sets:new()). is_safe_bool_expr_1(#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_record}, @@ -2193,7 +2206,7 @@ is_safe_bool_expr_1(#c_let{vars=Vars,arg=Arg,body=B}, Sub, BoolVars) -> true -> case {is_safe_bool_expr_1(Arg, Sub, BoolVars),Vars} of {true,[#c_var{name=V}]} -> - is_safe_bool_expr_1(B, Sub, gb_sets:add(V, BoolVars)); + is_safe_bool_expr_1(B, Sub, cerl_sets:add_element(V, BoolVars)); {false,_} -> is_safe_bool_expr_1(B, Sub, BoolVars) end; @@ -2202,7 +2215,7 @@ is_safe_bool_expr_1(#c_let{vars=Vars,arg=Arg,body=B}, Sub, BoolVars) -> is_safe_bool_expr_1(#c_literal{val=Val}, _Sub, _) -> is_boolean(Val); is_safe_bool_expr_1(#c_var{name=V}, _Sub, BoolVars) -> - gb_sets:is_element(V, BoolVars); + cerl_sets:is_element(V, BoolVars); is_safe_bool_expr_1(_, _, _) -> false. is_safe_bool_expr_list([C|Cs], Sub, BoolVars) -> @@ -2236,7 +2249,7 @@ move_let_into_expr(#c_let{vars=InnerVs0,body=InnerBody0}=Inner, %% in <InnerBody> %% Arg = body(Arg0, Sub0), - ScopeSub0 = sub_subst_scope(Sub0#sub{t=[]}), + ScopeSub0 = sub_subst_scope(Sub0#sub{t=#{}}), {OuterVs,ScopeSub} = pattern_list(OuterVs0, ScopeSub0), OuterBody = body(OuterBody0, ScopeSub), @@ -2275,15 +2288,15 @@ move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let, CaVars0 = Ca0#c_clause.pats, G0 = Ca0#c_clause.guard, B0 = Ca0#c_clause.body, - ScopeSub0 = sub_subst_scope(Sub0#sub{t=[]}), + ScopeSub0 = sub_subst_scope(Sub0#sub{t=#{}}), {CaVars,ScopeSub} = pattern_list(CaVars0, ScopeSub0), G = guard(G0, ScopeSub), B1 = body(B0, ScopeSub), {Lvs,B2,Sub1} = let_substs(Lvs0, B1, Sub0), - Sub2 = Sub1#sub{s=gb_sets:union(ScopeSub#sub.s, - Sub1#sub.s)}, + Sub2 = Sub1#sub{s=cerl_sets:union(ScopeSub#sub.s, + Sub1#sub.s)}, Lbody = body(Lbody0, Sub2), B = Let#c_let{vars=Lvs,arg=core_lib:make_values(B2),body=Lbody}, @@ -2392,8 +2405,9 @@ delay_build_1(Core0, TypeSig) -> try delay_build_expr(Core0, TypeSig) of Core -> {Type,Arity} = TypeSig, - Vars = make_vars([], Arity), - Data = cerl:ann_make_data([compiler_generated], Type, Vars), + Ann = [compiler_generated], + Vars = make_vars(Ann, Arity), + Data = cerl:ann_make_data(Ann, Type, Vars), {yes,Vars,Core,Data} catch throw:impossible -> @@ -2480,7 +2494,7 @@ opt_simple_let_2(Let0, Vs0, Arg0, Body, PrevBody, Ctxt, Sub) -> Arg1; false -> %% let <Var> = Arg in <OtherVar> ==> seq Arg OtherVar - Arg = maybe_suppress_warnings(Arg1, Vs0, PrevBody, Ctxt), + Arg = maybe_suppress_warnings(Arg1, Vs0, PrevBody), expr(#c_seq{arg=Arg,body=Body}, Ctxt, sub_new_preserve_types(Sub)) end; @@ -2488,7 +2502,7 @@ opt_simple_let_2(Let0, Vs0, Arg0, Body, PrevBody, Ctxt, Sub) -> %% No variables left. Body; {Vs,Arg1,#c_literal{}} -> - Arg = maybe_suppress_warnings(Arg1, Vs, PrevBody, Ctxt), + Arg = maybe_suppress_warnings(Arg1, Vs, PrevBody), E = case Ctxt of effect -> %% Throw away the literal body. @@ -2507,7 +2521,7 @@ opt_simple_let_2(Let0, Vs0, Arg0, Body, PrevBody, Ctxt, Sub) -> %% seq Arg BodyWithoutVar case is_any_var_used(Vs, Body) of false -> - Arg = maybe_suppress_warnings(Arg1, Vs, PrevBody, Ctxt), + Arg = maybe_suppress_warnings(Arg1, Vs, PrevBody), expr(#c_seq{arg=Arg,body=Body}, Ctxt, sub_new_preserve_types(Sub)); true -> @@ -2517,7 +2531,7 @@ opt_simple_let_2(Let0, Vs0, Arg0, Body, PrevBody, Ctxt, Sub) -> end end. -%% maybe_suppress_warnings(Arg, [#c_var{}], PreviousBody, Context) -> Arg' +%% maybe_suppress_warnings(Arg, [#c_var{}], PreviousBody) -> Arg' %% Try to suppress false warnings when a variable is not used. %% For instance, we don't expect a warning for useless building in: %% @@ -2528,10 +2542,7 @@ opt_simple_let_2(Let0, Vs0, Arg0, Body, PrevBody, Ctxt, Sub) -> %% referenced in the original unoptimized code. If they were, we will %% consider the warning false and suppress it. -maybe_suppress_warnings(Arg, _, _, effect) -> - %% Don't suppress any warnings in effect context. - Arg; -maybe_suppress_warnings(Arg, Vs, PrevBody, value) -> +maybe_suppress_warnings(Arg, Vs, PrevBody) -> case should_suppress_warning(Arg) of true -> Arg; %Already suppressed. @@ -2555,8 +2566,16 @@ suppress_warning([H|T]) -> true -> suppress_warning(cerl:data_es(H) ++ T); false -> - Arg = cerl:set_ann(H, [compiler_generated]), - cerl:c_seq(Arg, suppress_warning(T)) + %% Some other thing, such as a function call. + %% This cannot be the compiler's fault, so the + %% warning should not be suppressed. We must + %% be careful not to destroy tail-recursion. + case T of + [] -> + H; + [_|_] -> + cerl:c_seq(H, suppress_warning(T)) + end end end; suppress_warning([]) -> void(). @@ -2574,7 +2593,7 @@ move_case_into_arg(#c_case{arg=#c_let{vars=OuterVars0,arg=OuterArg, %% let <OuterVars> = <OuterArg> %% in case <InnerArg> of <InnerClauses> end %% - ScopeSub0 = sub_subst_scope(Sub#sub{t=[]}), + ScopeSub0 = sub_subst_scope(Sub#sub{t=#{}}), {OuterVars,ScopeSub} = pattern_list(OuterVars0, ScopeSub0), InnerArg = body(InnerArg0, ScopeSub), Outer#c_let{vars=OuterVars,arg=OuterArg, @@ -2603,7 +2622,7 @@ move_case_into_arg(#c_case{arg=#c_case{arg=OuterArg, %% <OuterCb> %% end %% - ScopeSub0 = sub_subst_scope(Sub#sub{t=[]}), + ScopeSub0 = sub_subst_scope(Sub#sub{t=#{}}), {OuterPats,ScopeSub} = pattern_list(OuterPats0, ScopeSub0), OuterGuard = guard(OuterGuard0, ScopeSub), InnerArg = body(InnerArg0, ScopeSub), @@ -2688,9 +2707,9 @@ is_any_var_used([], _) -> false. -spec get_type(cerl:cerl(), #sub{}) -> type_info() | 'none'. get_type(#c_var{name=V}, #sub{t=Tdb}) -> - case orddict:find(V, Tdb) of - {ok,Type} -> Type; - error -> none + case Tdb of + #{V:=Type} -> Type; + _ -> none end; get_type(C, _) -> case cerl:type(C) of @@ -2805,35 +2824,38 @@ update_types_1(#c_var{name=V,anno=Anno}, Pat, Types) -> update_types_1(_, _, Types) -> Types. update_types_2(V, [#c_tuple{}=P], Types) -> - orddict:store(V, P, Types); + Types#{V=>P}; update_types_2(V, [#c_literal{val=Bool}], Types) when is_boolean(Bool) -> - orddict:store(V, bool, Types); + Types#{V=>bool}; update_types_2(V, [Type], Types) when is_atom(Type) -> - orddict:store(V, Type, Types); + Types#{V=>Type}; update_types_2(_, _, Types) -> Types. %% kill_types(V, Tdb) -> Tdb' %% Kill any entries that references the variable, %% either in the key or in the value. -kill_types(V, [{V,_}|Tdb]) -> - kill_types(V, Tdb); -kill_types(V, [{_,#c_tuple{}=Tuple}=Entry|Tdb]) -> +kill_types(V, Tdb) -> + maps:from_list(kill_types2(V,maps:to_list(Tdb))). + +kill_types2(V, [{V,_}|Tdb]) -> + kill_types2(V, Tdb); +kill_types2(V, [{_,#c_tuple{}=Tuple}=Entry|Tdb]) -> case core_lib:is_var_used(V, Tuple) of - false -> [Entry|kill_types(V, Tdb)]; - true -> kill_types(V, Tdb) + false -> [Entry|kill_types2(V, Tdb)]; + true -> kill_types2(V, Tdb) end; -kill_types(V, [{_,Atom}=Entry|Tdb]) when is_atom(Atom) -> - [Entry|kill_types(V, Tdb)]; -kill_types(_, []) -> []. +kill_types2(V, [{_,Atom}=Entry|Tdb]) when is_atom(Atom) -> + [Entry|kill_types2(V, Tdb)]; +kill_types2(_, []) -> []. %% copy_type(DestVar, SrcVar, Tdb) -> Tdb' %% If the SrcVar has a type, assign it to DestVar. %% copy_type(V, #c_var{name=Src}, Tdb) -> - case orddict:find(Src, Tdb) of - {ok,Type} -> orddict:store(V, Type, Tdb); - error -> Tdb + case Tdb of + #{Src:=Type} -> Tdb#{V=>Type}; + _ -> Tdb end; copy_type(_, _, Tdb) -> Tdb. @@ -3237,12 +3259,12 @@ format_error(bin_var_used_in_guard) -> verify_scope(E, #sub{s=Scope}) -> Free0 = cerl_trees:free_variables(E), Free = [V || V <- Free0, not is_tuple(V)], %Ignore function names. - case ordsets:is_subset(Free, gb_sets:to_list(Scope)) of + case ordsets:is_subset(Free, cerl_sets:to_list(Scope)) of true -> true; false -> io:format("~p\n", [E]), io:format("~p\n", [Free]), - io:format("~p\n", [gb_sets:to_list(Scope)]), + io:format("~p\n", [cerl_sets:to_list(Scope)]), false end. -endif. diff --git a/lib/compiler/src/v3_codegen.erl b/lib/compiler/src/v3_codegen.erl index aa2ebc0f85..c9b1a45cfc 100644 --- a/lib/compiler/src/v3_codegen.erl +++ b/lib/compiler/src/v3_codegen.erl @@ -43,7 +43,7 @@ -export([module/2]). -import(lists, [member/2,keymember/3,keysort/2,keydelete/3, - append/1,map/2,flatmap/2,filter/2,foldl/3,foldr/3,mapfoldl/3, + append/1,flatmap/2,filter/2,foldl/3,foldr/3,mapfoldl/3, sort/1,reverse/1,reverse/2]). -import(v3_life, [vdb_find/2]). @@ -57,8 +57,7 @@ break, %Break label recv, %Receive label is_top_block, %Boolean: top block or not - functable=gb_trees:empty(), %Gb tree of local functions: - % {{Name,Arity},Label} + functable=#{}, %Map of local functions: {Name,Arity}=>Label in_catch=false, %Inside a catch or not. need_frame, %Need a stack frame. ultimate_failure %Label for ultimate match failure. @@ -673,9 +672,7 @@ select_val_cg(Type, R, [Val, {f,Lbl}], Tf, Vf, [{label,Lbl}|Sis]) -> [{test,select_type_test(Type),{f,Tf},[R]}, {test,is_eq_exact,{f,Vf},[R,{Type,Val}]}|Sis]; select_val_cg(Type, R, Vls0, Tf, Vf, Sis) -> - Vls1 = map(fun ({f,_Lbl} = F) -> F; - (Value) -> {Type,Value} - end, Vls0), + Vls1 = [case Value of {f,_Lbl} -> Value; _ -> {Type,Value} end || Value <- Vls0], [{test,select_type_test(Type),{f,Tf},[R]}, {select_val,R,{f,Vf},{list,Vls1}}|Sis]. select_type_test(integer) -> is_integer; @@ -1080,7 +1077,7 @@ protected_cg(Ts, Rs, _Fail, I, Vdb, Bef, St0) -> St2#cg{bfail=Pfail}), %%ok = io:fwrite("cg ~w: ~p~n", [?LINE,{Rs,I,Vdb,Aft}]), %% Set return values to false. - Mis = map(fun ({var,V}) -> {move,{atom,false},fetch_var(V, Aft)} end, Rs), + Mis = [{move,{atom,false},fetch_var(V,Aft)}||{var,V} <- Rs], {Tis ++ [{jump,{f,Psucc}}, {label,Pfail}] ++ Mis ++ [{label,Psucc}], Aft,St3#cg{bfail=St0#cg.bfail}}. @@ -1263,13 +1260,12 @@ enter_line(_, _, _) -> local_func_label(Name, Arity, St) -> local_func_label({Name,Arity}, St). -local_func_label(Key, #cg{functable=Tab}=St0) -> - case gb_trees:lookup(Key, Tab) of - {value,Label} -> - {Label,St0}; - none -> +local_func_label(Key, #cg{functable=Map}=St0) -> + case Map of + #{Key := Label} -> {Label,St0}; + _ -> {Label,St} = new_label(St0), - {Label,St#cg{functable=gb_trees:insert(Key, Label, Tab)}} + {Label,St#cg{functable=Map#{Key => Label}}} end. %% need_stack_frame(State) -> State' @@ -1992,25 +1988,28 @@ clear_dead(Sr, Until, Vdb) -> stk=clear_dead_stk(Sr#sr.stk, Until, Vdb)}. clear_dead_reg(Sr, Until, Vdb) -> - Reg = map(fun ({_I,V} = IV) -> - case vdb_find(V, Vdb) of - {V,_,L} when L > Until -> IV; - _ -> free %Remove anything else - end; - ({reserved,_I,_V} = Reserved) -> Reserved; - (free) -> free - end, Sr#sr.reg), + Reg = [case R of + {_I,V} = IV -> + case vdb_find(V, Vdb) of + {V,_,L} when L > Until -> IV; + _ -> free %Remove anything else + end; + {reserved,_I,_V} = Reserved -> Reserved; + free -> free + end || R <- Sr#sr.reg], reserve(Sr#sr.res, Reg, Sr#sr.stk). clear_dead_stk(Stk, Until, Vdb) -> - map(fun ({V} = T) -> - case vdb_find(V, Vdb) of - {V,_,L} when L > Until -> T; - _ -> dead %Remove anything else - end; - (free) -> free; - (dead) -> dead - end, Stk). + [case S of + {V} = T -> + case vdb_find(V, Vdb) of + {V,_,L} when L > Until -> T; + _ -> dead %Remove anything else + end; + free -> free; + dead -> dead + end || S <- Stk]. + %% sr_merge(Sr1, Sr2) -> Sr. %% Merge two stack/register states keeping the longest of both stack diff --git a/lib/compiler/src/v3_kernel.erl b/lib/compiler/src/v3_kernel.erl index 7dff58582e..c21b2a1505 100644 --- a/lib/compiler/src/v3_kernel.erl +++ b/lib/compiler/src/v3_kernel.erl @@ -114,7 +114,7 @@ copy_anno(Kdst, Ksrc) -> ff, %Current function vcount=0, %Variable counter fcount=0, %Fun counter - ds=[], %Defined variables + ds=cerl_sets:new() :: cerl_sets:set(), %Defined variables funs=[], %Fun functions free=[], %Free variables ws=[] :: [warning()], %Warnings. @@ -148,7 +148,7 @@ include_attribute(_) -> true. function({#c_var{name={F,Arity}=FA},Body}, St0) -> try - St1 = St0#kern{func=FA,ff=undefined,vcount=0,fcount=0,ds=sets:new()}, + St1 = St0#kern{func=FA,ff=undefined,vcount=0,fcount=0,ds=cerl_sets:new()}, {#ifun{anno=Ab,vars=Kvs,body=B0},[],St2} = expr(Body, new_sub(), St1), {B1,_,St3} = ubody(B0, return, St2), %%B1 = B0, St3 = St2, %Null second pass @@ -715,15 +715,15 @@ force_variable(Ke, St0) -> %% handling. pattern(#c_var{anno=A,name=V}, _Isub, Osub, St0) -> - case sets:is_element(V, St0#kern.ds) of + case cerl_sets:is_element(V, St0#kern.ds) of true -> {New,St1} = new_var_name(St0), {#k_var{anno=A,name=New}, set_vsub(V, New, Osub), - St1#kern{ds=sets:add_element(New, St1#kern.ds)}}; + St1#kern{ds=cerl_sets:add_element(New, St1#kern.ds)}}; false -> {#k_var{anno=A,name=V},Osub, - St0#kern{ds=sets:add_element(V, St0#kern.ds)}} + St0#kern{ds=cerl_sets:add_element(V, St0#kern.ds)}} end; pattern(#c_literal{anno=A,val=Val}, _Isub, Osub, St) -> {#k_literal{anno=A,val=Val},Osub,St}; @@ -897,7 +897,7 @@ new_vars(0, St, Vs) -> {Vs,St}. make_vars(Vs) -> [ #k_var{name=V} || V <- Vs ]. add_var_def(V, St) -> - St#kern{ds=sets:add_element(V#k_var.name, St#kern.ds)}. + St#kern{ds=cerl_sets:add_element(V#k_var.name, St#kern.ds)}. %%add_vars_def(Vs, St) -> %% Ds = foldl(fun (#k_var{name=V}, Ds) -> add_element(V, Ds) end, diff --git a/lib/compiler/src/v3_life.erl b/lib/compiler/src/v3_life.erl index 4b1f1c3f71..ee0565efb6 100644 --- a/lib/compiler/src/v3_life.erl +++ b/lib/compiler/src/v3_life.erl @@ -411,7 +411,7 @@ is_gc_bif(Bif, Arity) -> %% must be sorted. init_vars(Vs) -> - sort([{V,0,0} || {var,V} <- Vs]). + vdb_new(Vs). new_vars([], _, Vdb) -> Vdb; new_vars([V], I, Vdb) -> vdb_store_new(V, {V,I,I}, Vdb); @@ -430,6 +430,16 @@ use_vars(Vs, I, Vdb) -> vdb_update_vars(Vs, Vdb, I). add_var(V, F, L, Vdb) -> vdb_store_new(V, {V,F,L}, Vdb). +%% is_in_guard() -> true|false. + +is_in_guard() -> + get(guard_refc) > 0. + +%% vdb + +vdb_new(Vs) -> + sort([{V,0,0} || {var,V} <- Vs]). + vdb_find(V, Vdb) -> case lists:keyfind(V, 1, Vdb) of false -> error; @@ -471,8 +481,3 @@ vdb_sub(Min, Max, Vdb) -> [ if L >= Max -> {V,F,locked}; true -> Vd end || {V,F,L}=Vd <- Vdb, F < Min, L >= Min ]. - -%% is_in_guard() -> true|false. - -is_in_guard() -> - get(guard_refc) > 0. diff --git a/lib/compiler/test/warnings_SUITE.erl b/lib/compiler/test/warnings_SUITE.erl index f6ba75577d..4e266875ee 100644 --- a/lib/compiler/test/warnings_SUITE.erl +++ b/lib/compiler/test/warnings_SUITE.erl @@ -38,7 +38,8 @@ -export([pattern/1,pattern2/1,pattern3/1,pattern4/1, guard/1,bad_arith/1,bool_cases/1,bad_apply/1, files/1,effect/1,bin_opt_info/1,bin_construction/1, - comprehensions/1,maps/1,redundant_boolean_clauses/1, + comprehensions/1,maps/1,maps_bin_opt_info/1, + redundant_boolean_clauses/1, latin1_fallback/1,underscore/1,no_warnings/1]). % Default timetrap timeout (set in init_per_testcase). @@ -64,6 +65,7 @@ groups() -> [pattern,pattern2,pattern3,pattern4,guard, bad_arith,bool_cases,bad_apply,files,effect, bin_opt_info,bin_construction,comprehensions,maps, + maps_bin_opt_info, redundant_boolean_clauses,latin1_fallback, underscore,no_warnings]}]. @@ -281,6 +283,7 @@ bad_arith(Config) when is_list(Config) -> {3,sys_core_fold,{eval_failure,badarith}}, {9,sys_core_fold,nomatch_guard}, {9,sys_core_fold,{eval_failure,badarith}}, + {9,sys_core_fold,{no_effect,{erlang,is_integer,1}}}, {10,sys_core_fold,nomatch_guard}, {10,sys_core_fold,{eval_failure,badarith}}, {15,sys_core_fold,{eval_failure,badarith}} @@ -369,7 +372,7 @@ files(Config) when is_list(Config) -> %% Test warnings for term construction and BIF calls in effect context. effect(Config) when is_list(Config) -> - Ts = [{lc, + Ts = [{effect, <<" t(X) -> case X of @@ -475,6 +478,19 @@ effect(Config) when is_list(Config) -> m9(Bs) -> [{B,ok} = {B,foo:bar(B)} || B <- Bs], ok. + + m10(ConfigTableSize) -> + case ConfigTableSize of + apa -> + CurrentConfig = {id(camel_phase3),id(sms)}, + case CurrentConfig of + {apa, bepa} -> ok; + _ -> ok + end + end, + ok. + + id(I) -> I. ">>, [], {warnings,[{5,sys_core_fold,{no_effect,{erlang,is_integer,1}}}, @@ -619,6 +635,19 @@ maps(Config) when is_list(Config) -> run(Config, Ts), ok. +maps_bin_opt_info(Config) when is_list(Config) -> + Ts = [{map_bsm, + <<" + t1(<<0:8,7:8,T/binary>>,#{val := I}=M) -> + t1(T, M#{val := I+1}); + t1(<<_:8>>,M) -> + M. + ">>, + [bin_opt_info], + {warnings,[{2,beam_bsm,bin_opt}]}}], + [] = run(Config, Ts), + ok. + redundant_boolean_clauses(Config) when is_list(Config) -> Ts = [{redundant_boolean_clauses, <<" @@ -739,6 +768,14 @@ no_warnings(Config) when is_list(Config) -> case R0 of {r,V1,_V2,V3} -> {r,V1,\"def\",V3} end. + + d(In0, Bool) -> + {In1,Int} = case id(Bool) of + false -> {In0,0} + end, + [In1,Int]. + + id(I) -> I. ">>, [], []}], diff --git a/lib/cosEvent/doc/src/notes.xml b/lib/cosEvent/doc/src/notes.xml index 229e0c9945..8f519447fc 100644 --- a/lib/cosEvent/doc/src/notes.xml +++ b/lib/cosEvent/doc/src/notes.xml @@ -32,23 +32,7 @@ <file>notes.xml</file> </header> - <section><title>cosEvent 2.2</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> Remove the usage of erlang:now() from all Corba - applications and use the new rand module instead of - random. </p> - <p> - Own Id: OTP-12687</p> - </item> - </list> - </section> - -</section> - -<section><title>cosEvent 2.1.15</title> + <section><title>cosEvent 2.1.15</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/cosEventDomain/doc/src/notes.xml b/lib/cosEventDomain/doc/src/notes.xml index 72408b52f4..2c3bf16411 100644 --- a/lib/cosEventDomain/doc/src/notes.xml +++ b/lib/cosEventDomain/doc/src/notes.xml @@ -31,23 +31,7 @@ <file>notes.xml</file> </header> - <section><title>cosEventDomain 1.2</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> Remove the usage of erlang:now() from all Corba - applications and use the new rand module instead of - random. </p> - <p> - Own Id: OTP-12687</p> - </item> - </list> - </section> - -</section> - -<section><title>cosEventDomain 1.1.14</title> + <section><title>cosEventDomain 1.1.14</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/cosFileTransfer/doc/src/notes.xml b/lib/cosFileTransfer/doc/src/notes.xml index a3cce2a7d8..1d0c826d40 100644 --- a/lib/cosFileTransfer/doc/src/notes.xml +++ b/lib/cosFileTransfer/doc/src/notes.xml @@ -30,23 +30,7 @@ <file>notes.xml</file> </header> - <section><title>cosFileTransfer 1.2</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> Remove the usage of erlang:now() from all Corba - applications and use the new rand module instead of - random. </p> - <p> - Own Id: OTP-12687</p> - </item> - </list> - </section> - -</section> - -<section><title>cosFileTransfer 1.1.16</title> + <section><title>cosFileTransfer 1.1.16</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/cosNotification/doc/src/notes.xml b/lib/cosNotification/doc/src/notes.xml index b16571a1cf..2e4f922142 100644 --- a/lib/cosNotification/doc/src/notes.xml +++ b/lib/cosNotification/doc/src/notes.xml @@ -31,23 +31,7 @@ <file>notes.xml</file> </header> - <section><title>cosNotification 1.2</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> Remove the usage of erlang:now() from all Corba - applications and use the new rand module instead of - random. </p> - <p> - Own Id: OTP-12687</p> - </item> - </list> - </section> - -</section> - -<section><title>cosNotification 1.1.21</title> + <section><title>cosNotification 1.1.21</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/cosNotification/src/cosNotification.app.src b/lib/cosNotification/src/cosNotification.app.src index 09bf8f01fc..52ce164d46 100644 --- a/lib/cosNotification/src/cosNotification.app.src +++ b/lib/cosNotification/src/cosNotification.app.src @@ -117,6 +117,6 @@ {applications, [orber, stdlib, kernel]}, {env, []}, {mod, {cosNotificationApp, []}}, - {runtime_dependencies, ["stdlib-2.0","orber-3.6.27","kernel-3.0","erts-7.0", + {runtime_dependencies, ["stdlib-2.5","orber-3.6.27","kernel-3.0","erts-7.0", "cosTime-1.1.14","cosEvent-2.1.15"]} ]}. diff --git a/lib/cosProperty/doc/src/notes.xml b/lib/cosProperty/doc/src/notes.xml index 1885a8fd6b..739f41617f 100644 --- a/lib/cosProperty/doc/src/notes.xml +++ b/lib/cosProperty/doc/src/notes.xml @@ -31,23 +31,7 @@ <file>notes.xml</file> </header> - <section><title>cosProperty 1.2</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> Remove the usage of erlang:now() from all Corba - applications and use the new rand module instead of - random. </p> - <p> - Own Id: OTP-12687</p> - </item> - </list> - </section> - -</section> - -<section><title>cosProperty 1.1.17</title> + <section><title>cosProperty 1.1.17</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/cosTime/doc/src/notes.xml b/lib/cosTime/doc/src/notes.xml index 6c948c8c2b..f218f19a6b 100644 --- a/lib/cosTime/doc/src/notes.xml +++ b/lib/cosTime/doc/src/notes.xml @@ -32,23 +32,7 @@ <file>notes.xml</file> </header> - <section><title>cosTime 1.2</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> Remove the usage of erlang:now() from all Corba - applications and use the new rand module instead of - random. </p> - <p> - Own Id: OTP-12687</p> - </item> - </list> - </section> - -</section> - -<section><title>cosTime 1.1.14</title> + <section><title>cosTime 1.1.14</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/cosTransactions/doc/src/notes.xml b/lib/cosTransactions/doc/src/notes.xml index ae3b9eb6f2..4b20f23efb 100644 --- a/lib/cosTransactions/doc/src/notes.xml +++ b/lib/cosTransactions/doc/src/notes.xml @@ -32,23 +32,7 @@ <file>notes.xml</file> </header> - <section><title>cosTransactions 1.3</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> Remove the usage of erlang:now() from all Corba - applications and use the new rand module instead of - random. </p> - <p> - Own Id: OTP-12687</p> - </item> - </list> - </section> - -</section> - -<section><title>cosTransactions 1.2.14</title> + <section><title>cosTransactions 1.2.14</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/crypto/doc/src/notes.xml b/lib/crypto/doc/src/notes.xml index 6ab109e349..a0ebc4b3dd 100644 --- a/lib/crypto/doc/src/notes.xml +++ b/lib/crypto/doc/src/notes.xml @@ -30,49 +30,6 @@ </header> <p>This document describes the changes made to the Crypto application.</p> -<section><title>Crypto 3.6</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Enhance crypto:generate_key to calculate ECC public keys - from private key.</p> - <p> - Own Id: OTP-12394</p> - </item> - <item> - <p> - Fix bug in <c>crypto:generate_key</c> for <c>ecdh</c> - that could cause VM crash for faulty input.</p> - <p> - Own Id: OTP-12733</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Use the EVP API for AES-CBC crypto to enables the use of - hardware acceleration for AES-CBC crypto on newer Intel - CPUs (AES-NI), among other platforms.</p> - <p> - Own Id: OTP-12380</p> - </item> - <item> - <p> - Add AES ECB block encryption.</p> - <p> - Own Id: OTP-12403</p> - </item> - </list> - </section> - -</section> - <section><title>Crypto 3.5</title> <section><title>Improvements and New Features</title> diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index 72944eea8e..ff7af1f2c1 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -1884,8 +1884,9 @@ dss_params() -> 18320614775012672475365915366944922415598782131828709277168615511695849821411624805195787607930033958243224786899641459701930253094446221381818858674389863050420226114787005820357372837321561754462061849169568607689530279303056075793886577588606958623645901271866346406773590024901668622321064384483571751669]. ec_key_named() -> - {D2_pub, D2_priv} = crypto:generate_key(ecdh, sect113r2), - {[D2_priv, sect113r2], [D2_pub, sect113r2]}. + Curve = secp112r2, + {D2_pub, D2_priv} = crypto:generate_key(ecdh, Curve), + {[D2_priv, Curve], [D2_pub, Curve]}. ec_msg() -> <<99,234,6,64,190,237,201,99,80,248,58,40,70,45,149,218,5,246,242,63>>. diff --git a/lib/crypto/test/old_crypto_SUITE.erl b/lib/crypto/test/old_crypto_SUITE.erl index 040edbf092..80306927c5 100644 --- a/lib/crypto/test/old_crypto_SUITE.erl +++ b/lib/crypto/test/old_crypto_SUITE.erl @@ -1887,9 +1887,9 @@ ec(Config) when is_list(Config) -> ec_do() -> %% test for a name curve - {D2_pub, D2_priv} = crypto:generate_key(ecdh, sect113r2), - PrivECDH = [D2_priv, sect113r2], - PubECDH = [D2_pub, sect113r2], + {D2_pub, D2_priv} = crypto:generate_key(ecdh, secp112r2), + PrivECDH = [D2_priv, secp112r2], + PubECDH = [D2_pub, secp112r2], %%TODO: find a published test case for a EC key %% test for a full specified curve and public key, diff --git a/lib/debugger/doc/src/notes.xml b/lib/debugger/doc/src/notes.xml index 2bf80597b5..7384189a6f 100644 --- a/lib/debugger/doc/src/notes.xml +++ b/lib/debugger/doc/src/notes.xml @@ -32,24 +32,6 @@ <p>This document describes the changes made to the Debugger application.</p> -<section><title>Debugger 4.1</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Support variables as Map keys in expressions and patterns</p> - <p>Erlang will accept any expression as keys in Map - expressions and it will accept literals or bound - variables as keys in Map patterns.</p> - <p> - Own Id: OTP-12218</p> - </item> - </list> - </section> - -</section> - <section><title>Debugger 4.0.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/debugger/src/dbg_wx_trace.erl b/lib/debugger/src/dbg_wx_trace.erl index 4438466bb0..ffdfc46496 100644 --- a/lib/debugger/src/dbg_wx_trace.erl +++ b/lib/debugger/src/dbg_wx_trace.erl @@ -140,7 +140,7 @@ init(Pid, Parent, Meta, TraceWin, BackTrace, Strings) -> int:meta(Meta, trace, State3#state.trace), - gui_enable_updown(stack_trace, {1,1}), + gui_enable_updown(State3#state.stack_trace, {1,1}), gui_enable_btrace(false, false), dbg_wx_trace_win:display(Win,idle), diff --git a/lib/debugger/src/debugger.app.src b/lib/debugger/src/debugger.app.src index f102385d39..a013c5c11f 100644 --- a/lib/debugger/src/debugger.app.src +++ b/lib/debugger/src/debugger.app.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -47,5 +47,5 @@ ]}, {registered, [dbg_iserver, dbg_wx_mon, dbg_wx_winman]}, {applications, [kernel, stdlib]}, - {runtime_dependencies, ["wx-1.2","stdlib-2.0","kernel-3.0","erts-6.0", + {runtime_dependencies, ["wx-1.2","stdlib-2.5","kernel-3.0","erts-6.0", "compiler-5.0"]}]}. diff --git a/lib/debugger/src/int.erl b/lib/debugger/src/int.erl index 33954ca82c..6f84ca3bca 100644 --- a/lib/debugger/src/int.erl +++ b/lib/debugger/src/int.erl @@ -365,7 +365,7 @@ stop() -> %% function will receive the following messages: %% {int, {interpret, Mod}} %% {int, {no_interpret, Mod}} -%% {int, {new_process, Pid, Function, Status, Info}} +%% {int, {new_process, {Pid, Function, Status, Info}}} %% {int, {new_status, Pid, Status, Info}} %% {int, {new_break, {Point, Options}}} %% {int, {delete_break, Point}} diff --git a/lib/debugger/test/map_SUITE.erl b/lib/debugger/test/map_SUITE.erl index 12fdd184b8..74847e161f 100644 --- a/lib/debugger/test/map_SUITE.erl +++ b/lib/debugger/test/map_SUITE.erl @@ -1308,7 +1308,7 @@ t_guard_receive(Config) when is_list(Config) -> done = call(Pid, done), ok. --define(t_guard_receive_large_procs, 150). +-define(t_guard_receive_large_procs, 50). t_guard_receive_large(Config) when is_list(Config) -> M = lists:foldl(fun(_,#{procs := Ps } = M) -> @@ -1326,7 +1326,7 @@ guard_receive_large_loop(M) -> receive #{pid := Pid, msg := hello} -> case M of - #{done := Count, procs := #{Pid := 150}} -> + #{done := Count, procs := #{Pid := 15}} -> Pid ! {self(), done}, guard_receive_large_loop(M#{done := Count + 1}); #{procs := #{Pid := Count} = Ps} -> diff --git a/lib/dialyzer/doc/src/dialyzer.xml b/lib/dialyzer/doc/src/dialyzer.xml index 2a8bf6edcc..fc076c24a6 100644 --- a/lib/dialyzer/doc/src/dialyzer.xml +++ b/lib/dialyzer/doc/src/dialyzer.xml @@ -70,7 +70,7 @@ [--build_plt] [--add_to_plt] [--remove_from_plt] [--check_plt] [--no_check_plt] [--plt_info] [--get_warnings] [--dump_callgraph file] [--no_native] [--fullpath] - [--statistics]</code> + [--statistics] [--no_native_cache]</code> <p>Options:</p> <taglist> <tag><c><![CDATA[files_or_dirs]]></c> (for backwards compatibility also @@ -198,6 +198,11 @@ heuristically performs when dialyzing many files; this avoids the compilation time but it may result in (much) longer analysis time.</item> + <tag><c><![CDATA[--no_native_cache]]></c></tag> + <item>By default, Dialyzer caches the results of native compilation in the + <c>$XDG_CACHE_HOME/erlang/dialyzer_hipe_cache</c> directory. + <c>XDG_CACHE_HOME</c> defaults to <c>$HOME/.cache</c>. + Use this option to disable caching.</item> <tag><c><![CDATA[--fullpath]]></c></tag> <item>Display the full path names of files for which warnings are emitted.</item> <tag><c><![CDATA[--gui]]></c></tag> @@ -368,6 +373,7 @@ Option :: {files, [Filename :: string()]} | {include_dirs, [DirName :: string()]} | {output_file, FileName :: string()} | {output_plt, FileName :: string()} + | {check_plt, boolean()}, | {analysis_type, 'succ_typings' | 'plt_add' | 'plt_build' | diff --git a/lib/dialyzer/doc/src/notes.xml b/lib/dialyzer/doc/src/notes.xml index d30882b5b9..8976679c1d 100644 --- a/lib/dialyzer/doc/src/notes.xml +++ b/lib/dialyzer/doc/src/notes.xml @@ -31,92 +31,6 @@ <p>This document describes the changes made to the Dialyzer application.</p> -<section><title>Dialyzer 2.8</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> The translation of Erlang forms to the type - representation used by Dialyzer has been improved in - several ways. The most important change is that deeply - nested records can be handled. </p> - <p> - Own Id: OTP-12350</p> - </item> - <item> - <p> Update the PLT properly when a module is changed. - (Thanks to James Fish for the bug report, and to Stavros - Aronis for fixing the bug.) </p> - <p> - Own Id: OTP-12637</p> - </item> - <item> - <p> - An argument of '*'/2 is not constraind if the other - operand can be zero.</p> - <p> - Own Id: OTP-12725</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> The <c>-dialyzer()</c> attribute can be used for - suppressing warnings in a module by specifying functions - or warning options. It can also be used for requesting - warnings in a module. </p> - <p> - Own Id: OTP-10280</p> - </item> - <item> - <p> The pre-defined types <c>array()</c>, <c>dict()</c>, - <c>digraph()</c>, <c>gb_set()</c>, <c>gb_tree()</c>, - <c>queue()</c>, <c>set()</c>, and <c>tid()</c> have been - removed. </p> - <p> - Own Id: OTP-11445 Aux Id: OTP-10342, OTP-9352 </p> - </item> - <item> - <p> A few type names that have been used for representing - certain predefined types can now be used for user-defined - types. This affects the types <c>product/_</c>, - <c>union/_</c>, and <c>range/2</c> as well as - <c>tuple/N</c> (N > 0), <c>map/N</c> (N > 0), - <c>atom/1</c>, <c>integer/1</c>, <c>binary/2</c>, - <c>record/_,</c> and <c>'fun'/_</c>. A consequence is - that, for example, it is no longer possible to refer to a - record type with <c>record(r)</c>; instead the usual - record notation, <c>#r{}</c>, is to be used. </p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-11851</p> - </item> - <item> - <p> When implementing user-defined behaviours it is now - possible to specify optional callback functions. See OTP - Design Principles User's Guide, Sys and Proc_Lib, - User-Defined Behaviours, for details. </p> - <p> - Own Id: OTP-11861</p> - </item> - <item> - <p>Add two options to the Dialyzer: - <c>no_missing_calls</c> suppresses warnings about calls - to missing or unexported functions; <c>unknown</c> lets - warnings about unknown functions or types affect the exit - status. See also dialyzer(3). </p> - <p> - Own Id: OTP-12682</p> - </item> - </list> - </section> - -</section> - <section><title>Dialyzer 2.7.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/dialyzer/src/dialyzer.app.src b/lib/dialyzer/src/dialyzer.app.src index 7b2e1d4a9d..b6b9173a84 100644 --- a/lib/dialyzer/src/dialyzer.app.src +++ b/lib/dialyzer/src/dialyzer.app.src @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2014. All Rights Reserved. +%% Copyright Ericsson AB 2006-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -45,6 +45,6 @@ {registered, []}, {applications, [compiler, gs, hipe, kernel, stdlib, wx]}, {env, []}, - {runtime_dependencies, ["wx-1.2","syntax_tools-1.6.14","stdlib-2.0", + {runtime_dependencies, ["wx-1.2","syntax_tools-1.6.14","stdlib-2.5", "kernel-3.0","hipe-3.10.3","erts-7.0", "compiler-5.0"]}]}. diff --git a/lib/dialyzer/src/dialyzer.erl b/lib/dialyzer/src/dialyzer.erl index c9e7da9ef0..c8537e3bd8 100644 --- a/lib/dialyzer/src/dialyzer.erl +++ b/lib/dialyzer/src/dialyzer.erl @@ -2,7 +2,7 @@ %%----------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2014. All Rights Reserved. +%% Copyright Ericsson AB 2006-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -162,14 +162,7 @@ run(Opts) -> {error, Msg} -> throw({dialyzer_error, Msg}); OptsRecord -> - case OptsRecord#options.check_plt of - true -> - case cl_check_init(OptsRecord) of - {ok, ?RET_NOTHING_SUSPICIOUS} -> ok; - {error, ErrorMsg1} -> throw({dialyzer_error, ErrorMsg1}) - end; - false -> ok - end, + ok = check_init(OptsRecord), case dialyzer_cl:start(OptsRecord) of {?RET_DISCREPANCIES, Warnings} -> Warnings; {?RET_NOTHING_SUSPICIOUS, _} -> [] @@ -179,6 +172,16 @@ run(Opts) -> erlang:error({dialyzer_error, lists:flatten(ErrorMsg)}) end. +check_init(#options{analysis_type = plt_check}) -> + ok; +check_init(#options{check_plt = true} = OptsRecord) -> + case cl_check_init(OptsRecord) of + {ok, _} -> ok; + {error, Msg} -> throw({dialyzer_error, Msg}) + end; +check_init(#options{check_plt = false}) -> + ok. + internal_gui(OptsRecord) -> F = fun() -> dialyzer_gui_wx:start(OptsRecord), @@ -199,17 +202,13 @@ gui(Opts) -> throw({dialyzer_error, Msg}); OptsRecord -> ok = check_gui_options(OptsRecord), - case cl_check_init(OptsRecord) of - {ok, ?RET_NOTHING_SUSPICIOUS} -> - F = fun() -> - dialyzer_gui_wx:start(OptsRecord) - end, - case doit(F) of - {ok, _} -> ok; - {error, Msg} -> throw({dialyzer_error, Msg}) - end; - {error, ErrorMsg1} -> - throw({dialyzer_error, ErrorMsg1}) + ok = check_init(OptsRecord), + F = fun() -> + dialyzer_gui_wx:start(OptsRecord) + end, + case doit(F) of + {ok, _} -> ok; + {error, Msg} -> throw({dialyzer_error, Msg}) end catch throw:{dialyzer_error, ErrorMsg} -> diff --git a/lib/dialyzer/src/dialyzer_cl.erl b/lib/dialyzer/src/dialyzer_cl.erl index 4386a8d52a..55fcd15641 100644 --- a/lib/dialyzer/src/dialyzer_cl.erl +++ b/lib/dialyzer/src/dialyzer_cl.erl @@ -512,32 +512,82 @@ hipe_compile(Files, #options{erlang_mode = ErlangMode} = Options) -> dialyzer_worker], report_native_comp(Options), {T1, _} = statistics(wall_clock), - native_compile(Mods), + Cache = (get(dialyzer_options_native_cache) =/= false), + native_compile(Mods, Cache), {T2, _} = statistics(wall_clock), report_elapsed_time(T1, T2, Options) end end. -native_compile(Mods) -> +native_compile(Mods, Cache) -> case dialyzer_utils:parallelism() > ?MIN_PARALLELISM of true -> Parent = self(), - Pids = [spawn(fun () -> Parent ! {self(), hc(M)} end) || M <- Mods], + Pids = [spawn(fun () -> Parent ! {self(), hc(M, Cache)} end) || M <- Mods], lists:foreach(fun (Pid) -> receive {Pid, Res} -> Res end end, Pids); false -> - lists:foreach(fun (Mod) -> hc(Mod) end, Mods) + lists:foreach(fun (Mod) -> hc(Mod, Cache) end, Mods) end. -hc(Mod) -> +hc(Mod, Cache) -> {module, Mod} = code:ensure_loaded(Mod), case code:is_module_native(Mod) of true -> ok; false -> %% io:format(" ~w", [Mod]), - {ok, Mod} = hipe:c(Mod), - ok + case Cache of + false -> + {ok, Mod} = hipe:c(Mod), + ok; + true -> + hc_cache(Mod) + end end. +hc_cache(Mod) -> + CacheBase = cache_base_dir(), + %% Use HiPE architecture and version in directory name, to avoid + %% clashes between incompatible binaries. + HipeArchVersion = + lists:concat( + [erlang:system_info(hipe_architecture), "-", + hipe:version(), "-", + hipe_bifs:system_crc()]), + CacheDir = filename:join(CacheBase, HipeArchVersion), + OrigBeamFile = code:which(Mod), + {ok, {Mod, <<Checksum:128>>}} = beam_lib:md5(OrigBeamFile), + CachedBeamFile = filename:join(CacheDir, lists:concat([Mod, "-", Checksum, ".beam"])), + ok = filelib:ensure_dir(CachedBeamFile), + ModBin = + case filelib:is_file(CachedBeamFile) of + true -> + {ok, BinFromFile} = file:read_file(CachedBeamFile), + BinFromFile; + false -> + {ok, Mod, CompiledBin} = compile:file(OrigBeamFile, [from_beam, native, binary]), + ok = file:write_file(CachedBeamFile, CompiledBin), + CompiledBin + end, + code:unstick_dir(filename:dirname(OrigBeamFile)), + {module, Mod} = code:load_binary(Mod, CachedBeamFile, ModBin), + true = code:is_module_native(Mod), + ok. + +cache_base_dir() -> + %% http://standards.freedesktop.org/basedir-spec/basedir-spec-0.7.html + %% If XDG_CACHE_HOME is set to an absolute path, use it as base. + XdgCacheHome = os:getenv("XDG_CACHE_HOME"), + CacheHome = + case is_list(XdgCacheHome) andalso filename:pathtype(XdgCacheHome) =:= absolute of + true -> + XdgCacheHome; + false -> + %% Otherwise, the default is $HOME/.cache. + {ok, [[Home]]} = init:get_argument(home), + filename:join(Home, ".cache") + end, + filename:join([CacheHome, "dialyzer_hipe_cache"]). + new_state() -> #cl_state{}. diff --git a/lib/dialyzer/src/dialyzer_cl_parse.erl b/lib/dialyzer/src/dialyzer_cl_parse.erl index 21fc424a1b..fae88ed6e8 100644 --- a/lib/dialyzer/src/dialyzer_cl_parse.erl +++ b/lib/dialyzer/src/dialyzer_cl_parse.erl @@ -75,6 +75,9 @@ cl(["-nn"|T]) -> cl(["--no_native"|T]) -> put(dialyzer_options_native, false), cl(T); +cl(["--no_native_cache"|T]) -> + put(dialyzer_options_native_cache, false), + cl(T); cl(["--plt_info"|T]) -> put(dialyzer_options_analysis_type, plt_info), cl(T); @@ -363,7 +366,7 @@ help_message() -> [--build_plt] [--add_to_plt] [--remove_from_plt] [--check_plt] [--no_check_plt] [--plt_info] [--get_warnings] [--dump_callgraph file] [--no_native] [--fullpath] - [--statistics] + [--statistics] [--no_native_cache] Options: files_or_dirs (for backwards compatibility also as: -c files_or_dirs) Use Dialyzer from the command line to detect defects in the @@ -468,6 +471,11 @@ Options: Bypass the native code compilation of some key files that Dialyzer heuristically performs when dialyzing many files; this avoids the compilation time but it may result in (much) longer analysis time. + --no_native_cache + By default, Dialyzer caches the results of native compilation in the + $XDG_CACHE_HOME/erlang/dialyzer_hipe_cache directory. + XDG_CACHE_HOME defaults to $HOME/.cache. Use this option to disable + caching. --fullpath Display the full path names of files for which warnings are emitted. --gui diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl index 4a1ba9c539..914a4c6d8f 100644 --- a/lib/dialyzer/src/dialyzer_contracts.erl +++ b/lib/dialyzer/src/dialyzer_contracts.erl @@ -409,7 +409,7 @@ contract_from_form([{type, _, 'fun', [_, _]} = Form | Left], Module, RecDict, fun(ExpTypes, AllRecords) -> NewType = try - erl_types:t_from_form(Form, ExpTypes, Module, AllRecords) + from_form_with_check(Form, ExpTypes, Module, AllRecords) catch throw:{error, Msg} -> {File, Line} = FileLine, @@ -430,8 +430,8 @@ contract_from_form([{type, _L1, bounded_fun, fun(ExpTypes, AllRecords) -> {Constr1, VarDict} = process_constraints(Constr, Module, RecDict, ExpTypes, AllRecords), - NewType = erl_types:t_from_form(Form, ExpTypes, Module, AllRecords, - VarDict), + NewType = from_form_with_check(Form, ExpTypes, Module, AllRecords, + VarDict), NewTypeNoVars = erl_types:subst_all_vars_to_any(NewType), {NewTypeNoVars, Constr1} end, @@ -454,7 +454,7 @@ initialize_constraints([], _Module, _RecDict, _ExpTypes, _AllRecords, Acc) -> initialize_constraints([Constr|Rest], Module, RecDict, ExpTypes, AllRecords, Acc) -> case Constr of {type, _, constraint, [{atom, _, is_subtype}, [Type1, Type2]]} -> - T1 = final_form(Type1, Module, ExpTypes, AllRecords, dict:new()), + T1 = final_form(Type1, ExpTypes, Module, AllRecords, dict:new()), Entry = {T1, Type2}, initialize_constraints(Rest, Module, RecDict, ExpTypes, AllRecords, [Entry|Acc]); {type, _, constraint, [{atom,_,Name}, List]} -> @@ -483,7 +483,16 @@ constraints_fixpoint(OldVarDict, Module, Constrs, RecDict, ExpTypes, AllRecords) constraints_fixpoint(NewVarDict, Module, Constrs, RecDict, ExpTypes, AllRecords) end. -final_form(Form, Module, ExpTypes, AllRecords, VarDict) -> +final_form(Form, ExpTypes, Module, AllRecords, VarDict) -> + from_form_with_check(Form, ExpTypes, Module, AllRecords, VarDict). + +from_form_with_check(Form, ExpTypes, Module, AllRecords) -> + erl_types:t_check_record_fields(Form, ExpTypes, Module, AllRecords), + erl_types:t_from_form(Form, ExpTypes, Module, AllRecords). + +from_form_with_check(Form, ExpTypes, Module, AllRecords, VarDict) -> + erl_types:t_check_record_fields(Form, ExpTypes, Module, AllRecords, + VarDict), erl_types:t_from_form(Form, ExpTypes, Module, AllRecords, VarDict). constraints_to_dict(Constrs, Module, RecDict, ExpTypes, AllRecords, VarDict) -> @@ -495,7 +504,7 @@ constraints_to_subs([], _Module, _RecDict, _ExpTypes, _AllRecords, _VarDict, Acc Acc; constraints_to_subs([C|Rest], Module, RecDict, ExpTypes, AllRecords, VarDict, Acc) -> {T1, Form2} = C, - T2 = final_form(Form2, Module, ExpTypes, AllRecords, VarDict), + T2 = final_form(Form2, ExpTypes, Module, AllRecords, VarDict), NewAcc = [{subtype, T1, T2}|Acc], constraints_to_subs(Rest, Module, RecDict, ExpTypes, AllRecords, VarDict, NewAcc). diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl index e29fc3ba8b..dbfd20014c 100644 --- a/lib/dialyzer/src/dialyzer_utils.erl +++ b/lib/dialyzer/src/dialyzer_utils.erl @@ -64,15 +64,15 @@ print_types(RecDict) -> print_types1([], _) -> ok; print_types1([{type, _Name, _NArgs} = Key|T], RecDict) -> - {ok, {{_Mod, _Form, _Args}, Type}} = dict:find(Key, RecDict), + {ok, {{_Mod, _FileLine, _Form, _Args}, Type}} = dict:find(Key, RecDict), io:format("\n~w: ~w\n", [Key, Type]), print_types1(T, RecDict); print_types1([{opaque, _Name, _NArgs} = Key|T], RecDict) -> - {ok, {{_Mod, _Form, _Args}, Type}} = dict:find(Key, RecDict), + {ok, {{_Mod, _FileLine, _Form, _Args}, Type}} = dict:find(Key, RecDict), io:format("\n~w: ~w\n", [Key, Type]), print_types1(T, RecDict); print_types1([{record, _Name} = Key|T], RecDict) -> - {ok, [{_Arity, _Fields} = AF]} = dict:find(Key, RecDict), + {ok, {_FileLine, [{_Arity, _Fields} = AF]}} = dict:find(Key, RecDict), io:format("~w: ~w\n\n", [Key, AF]), print_types1(T, RecDict). -define(debug(D_), print_types(D_)). @@ -203,52 +203,53 @@ get_record_and_type_info(AbstractCode) -> {'ok', dict:dict()} | {'error', string()}. get_record_and_type_info(AbstractCode, Module, RecDict) -> - get_record_and_type_info(AbstractCode, Module, [], RecDict). + get_record_and_type_info(AbstractCode, Module, RecDict, "nofile"). -get_record_and_type_info([{attribute, _, record, {Name, Fields0}}|Left], - Module, Records, RecDict) -> +get_record_and_type_info([{attribute, A, record, {Name, Fields0}}|Left], + Module, RecDict, File) -> {ok, Fields} = get_record_fields(Fields0, RecDict), Arity = length(Fields), - NewRecDict = dict:store({record, Name}, [{Arity, Fields}], RecDict), - get_record_and_type_info(Left, Module, [{record, Name}|Records], NewRecDict); -get_record_and_type_info([{attribute, _, type, {{record, Name}, Fields0, []}} - |Left], Module, Records, RecDict) -> + FN = {File, erl_anno:line(A)}, + NewRecDict = dict:store({record, Name}, {FN, [{Arity,Fields}]}, RecDict), + get_record_and_type_info(Left, Module, NewRecDict, File); +get_record_and_type_info([{attribute, A, type, {{record, Name}, Fields0, []}} + |Left], Module, RecDict, File) -> %% This overrides the original record declaration. {ok, Fields} = get_record_fields(Fields0, RecDict), Arity = length(Fields), - NewRecDict = dict:store({record, Name}, [{Arity, Fields}], RecDict), - get_record_and_type_info(Left, Module, Records, NewRecDict); -get_record_and_type_info([{attribute, _, Attr, {Name, TypeForm}}|Left], - Module, Records, RecDict) when Attr =:= 'type'; - Attr =:= 'opaque' -> - try add_new_type(Attr, Name, TypeForm, [], Module, RecDict) of + FN = {File, erl_anno:line(A)}, + NewRecDict = dict:store({record, Name}, {FN, [{Arity, Fields}]}, RecDict), + get_record_and_type_info(Left, Module, NewRecDict, File); +get_record_and_type_info([{attribute, A, Attr, {Name, TypeForm}}|Left], + Module, RecDict, File) + when Attr =:= 'type'; Attr =:= 'opaque' -> + FN = {File, erl_anno:line(A)}, + try add_new_type(Attr, Name, TypeForm, [], Module, FN, RecDict) of NewRecDict -> - get_record_and_type_info(Left, Module, Records, NewRecDict) + get_record_and_type_info(Left, Module, NewRecDict, File) catch throw:{error, _} = Error -> Error end; -get_record_and_type_info([{attribute, _, Attr, {Name, TypeForm, Args}}|Left], - Module, Records, RecDict) when Attr =:= 'type'; - Attr =:= 'opaque' -> - try add_new_type(Attr, Name, TypeForm, Args, Module, RecDict) of +get_record_and_type_info([{attribute, A, Attr, {Name, TypeForm, Args}}|Left], + Module, RecDict, File) + when Attr =:= 'type'; Attr =:= 'opaque' -> + FN = {File, erl_anno:line(A)}, + try add_new_type(Attr, Name, TypeForm, Args, Module, FN, RecDict) of NewRecDict -> - get_record_and_type_info(Left, Module, Records, NewRecDict) + get_record_and_type_info(Left, Module, NewRecDict, File) catch throw:{error, _} = Error -> Error end; -get_record_and_type_info([_Other|Left], Module, Records, RecDict) -> - get_record_and_type_info(Left, Module, Records, RecDict); -get_record_and_type_info([], _Module, Records, RecDict) -> - case - check_type_of_record_fields(lists:reverse(Records), RecDict) - of - ok -> - {ok, RecDict}; - {error, Name, Error} -> - {error, flat_format(" Error while parsing #~w{}: ~s\n", [Name, Error])} - end. - -add_new_type(TypeOrOpaque, Name, TypeForm, ArgForms, Module, RecDict) -> +get_record_and_type_info([{attribute, _, file, {IncludeFile, _}}|Left], + Module, RecDict, _File) -> + get_record_and_type_info(Left, Module, RecDict, IncludeFile); +get_record_and_type_info([_Other|Left], Module, RecDict, File) -> + get_record_and_type_info(Left, Module, RecDict, File); +get_record_and_type_info([], _Module, RecDict, _File) -> + {ok, RecDict}. + +add_new_type(TypeOrOpaque, Name, TypeForm, ArgForms, Module, FN, + RecDict) -> Arity = length(ArgForms), case erl_types:type_is_defined(TypeOrOpaque, Name, Arity, RecDict) of true -> @@ -258,7 +259,7 @@ add_new_type(TypeOrOpaque, Name, TypeForm, ArgForms, Module, RecDict) -> try erl_types:t_var_names(ArgForms) of ArgNames -> dict:store({TypeOrOpaque, Name, Arity}, - {{Module, TypeForm, ArgNames}, + {{Module, FN, TypeForm, ArgNames}, erl_types:t_any()}, RecDict) catch _:_ -> @@ -280,32 +281,16 @@ get_record_fields([{typed_record_field, OrdRecField, TypeForm}|Left], end, get_record_fields(Left, RecDict, [{Name, TypeForm}|Acc]); get_record_fields([{record_field, _Line, Name}|Left], RecDict, Acc) -> - NewAcc = [{erl_parse:normalise(Name), {var, -1, '_'}}|Acc], + A = erl_anno:set_generated(true, erl_anno:new(1)), + NewAcc = [{erl_parse:normalise(Name), {var, A, '_'}}|Acc], get_record_fields(Left, RecDict, NewAcc); get_record_fields([{record_field, _Line, Name, _Init}|Left], RecDict, Acc) -> - NewAcc = [{erl_parse:normalise(Name), {var, -1, '_'}}|Acc], + A = erl_anno:set_generated(true, erl_anno:new(1)), + NewAcc = [{erl_parse:normalise(Name), {var, A, '_'}}|Acc], get_record_fields(Left, RecDict, NewAcc); get_record_fields([], _RecDict, Acc) -> lists:reverse(Acc). -%% Just check the local types. process_record_remote_types will add -%% the types later. -check_type_of_record_fields([], _RecDict) -> - ok; -check_type_of_record_fields([RecKey|Recs], RecDict) -> - {ok, [{_Arity, Fields}]} = dict:find(RecKey, RecDict), - try - [erl_types:t_from_form_without_remote(FieldTypeForm, RecDict) - || {_FieldName, FieldTypeForm, _} <- Fields] - of - L when is_list(L) -> - check_type_of_record_fields(Recs, RecDict) - catch - throw:{error, Error} -> - {record, Name} = RecKey, - {error, Name, Error} - end. - -spec process_record_remote_types(codeserver()) -> codeserver(). %% The field types are cached. Used during analysis when handling records. @@ -327,9 +312,10 @@ process_record_remote_types(CServer) -> TempRecords)} || {Name, Field, _} <- Fields] end, - orddict:map(FieldFun, Value); + {FileLine, Fields} = Value, + {FileLine, orddict:map(FieldFun, Fields)}; {opaque, _, _} -> - {{_Module, Form, _ArgNames}=F, _Type} = Value, + {{_Module, _FileLine, Form, _ArgNames}=F, _Type} = Value, Type = erl_types:t_from_form(Form, TempExpTypes, Module, TempRecords), {F, Type}; @@ -340,13 +326,53 @@ process_record_remote_types(CServer) -> end, try dict:map(ModuleFun, TempRecords) of NewRecords -> + ok = check_record_fields(NewRecords, TempExpTypes), CServer1 = dialyzer_codeserver:finalize_records(NewRecords, CServer), dialyzer_codeserver:finalize_exported_types(TempExpTypes, CServer1) catch - throw:{error, _RecName, _Error} = Error-> + throw:{error, _RecName, _Error} = Error -> Error end. +check_record_fields(Records, TempExpTypes) -> + CheckFun = + fun({Module, Element}) -> + CheckForm = fun(F) -> + erl_types:t_check_record_fields(F, TempExpTypes, + Module, Records) + end, + ElemFun = + fun({Key, Value}) -> + case Key of + {record, _Name} -> + FieldFun = + fun({_Arity, Fields}) -> + _ = [ok = CheckForm(Field) || {_, Field, _} <- Fields], + ok + end, + {FileLine, Fields} = Value, + Fun = fun() -> lists:foreach(FieldFun, Fields) end, + msg_with_position(Fun, FileLine); + {_OpaqueOrType, _Name, _} -> + {{_Module, FileLine, Form, _ArgNames}, _Type} = Value, + Fun = fun() -> ok = CheckForm(Form) end, + msg_with_position(Fun, FileLine) + end + end, + lists:foreach(ElemFun, dict:to_list(Element)) + end, + lists:foreach(CheckFun, dict:to_list(Records)). + +msg_with_position(Fun, FileLine) -> + try Fun() + catch + throw:{error, Msg} -> + {File, Line} = FileLine, + BaseName = filename:basename(File), + NewMsg = io_lib:format("~s:~p: ~s", [BaseName, Line, Msg]), + throw({error, NewMsg}) + end. + -spec merge_records(dict:dict(), dict:dict()) -> dict:dict(). merge_records(NewRecords, OldRecords) -> @@ -385,11 +411,11 @@ get_optional_callbacks(Abs) -> %% - Constraint is of the form {subtype, T1, T2} where T1 and T2 %% are erl_types:erl_type() -get_spec_info([{attribute, Attr, Contract, {Id, TypeSpec}}|Left], +get_spec_info([{attribute, Anno, Contract, {Id, TypeSpec}}|Left], SpecDict, CallbackDict, RecordsDict, ModName, OptCb, File) when ((Contract =:= 'spec') or (Contract =:= 'callback')), is_list(TypeSpec) -> - Ln = erl_anno:line(Attr), + Ln = erl_anno:line(Anno), MFA = case Id of {_, _, _} = T -> T; {F, A} -> {ModName, F, A} diff --git a/lib/dialyzer/test/opaque_SUITE_data/results/para b/lib/dialyzer/test/opaque_SUITE_data/results/para index 3aaa238de6..8fe67e39ad 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/results/para +++ b/lib/dialyzer/test/opaque_SUITE_data/results/para @@ -7,15 +7,27 @@ para1.erl:38: Attempt to test for equality between a term of type para1_adt:t(in para1.erl:43: Attempt to test for equality between a term of type para1_adt:t() and a term of opaque type para1_adt:t(atom()) para1.erl:48: Attempt to test for equality between a term of type para1_adt:t(integer()) and a term of opaque type para1_adt:t() para1.erl:53: The test {3,2} =:= {'a','b'} can never evaluate to 'true' -para2.erl:103: Attempt to test for equality between a term of type para2_adt:circ({{integer(),integer()},{integer(),integer()}},{{integer(),integer()},{integer(),integer()}}) and a term of opaque type para2_adt:circ({{integer(),integer()},{integer(),integer()}}) +para2.erl:103: Attempt to test for equality between a term of type para2_adt:circ(integer(),integer()) and a term of opaque type para2_adt:circ(integer()) para2.erl:117: Attempt to test for equality between a term of type para2_adt:un(atom(),integer()) and a term of opaque type para2_adt:un(integer(),atom()) para2.erl:31: The test 'a' =:= 'b' can never evaluate to 'true' para2.erl:61: Attempt to test for equality between a term of type para2_adt:c2() and a term of opaque type para2_adt:c1() para2.erl:66: The test 'a' =:= 'b' can never evaluate to 'true' -para2.erl:88: The test para2:circ({{integer(),integer()},{integer(),integer()}}) =:= para2:circ({{integer(),integer()},{integer(),integer()}},{{integer(),integer()},{integer(),integer()}}) can never evaluate to 'true' +para2.erl:88: The test para2:circ(integer()) =:= para2:circ(integer(),integer()) can never evaluate to 'true' para3.erl:28: Invalid type specification for function para3:ot2/0. The success typing is () -> 'foo' para3.erl:36: The pattern {{{17}}} can never match the type {{{{{{_,_,_,_,_}}}}}} para3.erl:55: Invalid type specification for function para3:t2/0. The success typing is () -> 'foo' para3.erl:65: The attempt to match a term of type {{{{{para3_adt:ot1(_,_,_,_,_)}}}}} against the pattern {{{{{17}}}}} breaks the opaqueness of para3_adt:ot1(_,_,_,_,_) para3.erl:68: The pattern {{{{17}}}} can never match the type {{{{{para3_adt:ot1(_,_,_,_,_)}}}}} para3.erl:74: Invalid type specification for function para3:exp_adt/0. The success typing is () -> 3 +para4.erl:21: Invalid type specification for function para4:a/1. The success typing is (dict:dict(atom() | integer(),atom() | integer()) | para4:d_all()) -> [{atom() | integer(),atom() | integer()}] +para4.erl:26: Invalid type specification for function para4:i/1. The success typing is (dict:dict(atom() | integer(),atom() | integer()) | para4:d_all()) -> [{atom() | integer(),atom() | integer()}] +para4.erl:31: Invalid type specification for function para4:t/1. The success typing is (dict:dict(atom() | integer(),atom() | integer()) | para4:d_all()) -> [{atom() | integer(),atom() | integer()}] +para4.erl:59: Attempt to test for equality between a term of type para4_adt:t(atom() | integer()) and a term of opaque type para4_adt:t(integer()) +para4.erl:64: Attempt to test for equality between a term of type para4_adt:t(atom() | integer()) and a term of opaque type para4_adt:t(atom()) +para4.erl:69: Attempt to test for equality between a term of type para4_adt:int(1 | 2 | 3 | 4) and a term of opaque type para4_adt:int(1 | 2) +para4.erl:74: Attempt to test for equality between a term of type para4_adt:int(2 | 3 | 4) and a term of opaque type para4_adt:int(1 | 2) +para4.erl:79: Attempt to test for equality between a term of type para4_adt:int(2 | 3 | 4) and a term of opaque type para4_adt:int(5 | 6 | 7) +para4.erl:84: Attempt to test for equality between a term of type para4_adt:un(3 | 4) and a term of opaque type para4_adt:un(1 | 2) +para4.erl:89: Attempt to test for equality between a term of type para4_adt:tup({_,_}) and a term of opaque type para4_adt:tup(tuple()) +para5.erl:13: Attempt to test for inequality between a term of type para5_adt:dd(atom()) and a term of opaque type para5_adt:d() +para5.erl:8: The test para5_adt:d() =:= para5_adt:d() can never evaluate to 'true' diff --git a/lib/dialyzer/test/opaque_SUITE_data/results/simple b/lib/dialyzer/test/opaque_SUITE_data/results/simple index 29864d6065..1a7a139d6e 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/results/simple +++ b/lib/dialyzer/test/opaque_SUITE_data/results/simple @@ -28,11 +28,11 @@ rec_api.erl:99: Record construction #r2{f1::10} violates the declared type of fi simple1_api.erl:113: The test simple1_api:d1() =:= simple1_api:d2() can never evaluate to 'true' simple1_api.erl:118: Guard test simple1_api:d2() =:= A::simple1_api:d1() can never succeed simple1_api.erl:142: Attempt to test for equality between a term of type simple1_adt:o2() and a term of opaque type simple1_adt:o1() -simple1_api.erl:148: Guard test simple1_adt:o2() =:= A::simple1_adt:o1() contains an opaque term as 1st argument +simple1_api.erl:148: Guard test simple1_adt:o2() =:= A::simple1_adt:o1() contains opaque terms as 1st and 2nd arguments simple1_api.erl:154: Attempt to test for inequality between a term of type simple1_adt:o2() and a term of opaque type simple1_adt:o1() simple1_api.erl:160: Attempt to test for inequality between a term of type simple1_adt:o2() and a term of opaque type simple1_adt:o1() simple1_api.erl:165: Attempt to test for equality between a term of type simple1_adt:c2() and a term of opaque type simple1_adt:c1() -simple1_api.erl:181: Guard test A::simple1_adt:d1() =< B::simple1_adt:d2() contains an opaque term as 1st argument +simple1_api.erl:181: Guard test A::simple1_adt:d1() =< B::simple1_adt:d2() contains opaque terms as 1st and 2nd arguments simple1_api.erl:185: Guard test 'a' =< B::simple1_adt:d2() contains an opaque term as 2nd argument simple1_api.erl:189: Guard test A::simple1_adt:d1() =< 'd' contains an opaque term as 1st argument simple1_api.erl:197: The type test is_integer(A::simple1_adt:d1()) breaks the opaqueness of the term A::simple1_adt:d1() @@ -72,12 +72,12 @@ simple1_api.erl:499: The call 'foo':A(A::simple1_api:i()) requires that A is of simple1_api.erl:503: The call 'foo':A(A::simple1_adt:i()) requires that A is of type atom() not simple1_adt:i() simple1_api.erl:507: The call A:'foo'(A::simple1_api:i()) requires that A is of type atom() | tuple() not simple1_api:i() simple1_api.erl:511: The call A:'foo'(A::simple1_adt:i()) requires that A is of type atom() | tuple() not simple1_adt:i() -simple1_api.erl:519: Guard test A::simple1_adt:d2() == B::simple1_adt:d1() contains an opaque term as 1st argument +simple1_api.erl:519: Guard test A::simple1_adt:d2() == B::simple1_adt:d1() contains opaque terms as 1st and 2nd arguments simple1_api.erl:534: Guard test A::simple1_adt:d1() >= 3 contains an opaque term as 1st argument simple1_api.erl:536: Guard test A::simple1_adt:d1() == 3 contains an opaque term as 1st argument simple1_api.erl:538: Guard test A::simple1_adt:d1() =:= 3 contains an opaque term as 1st argument simple1_api.erl:548: The call erlang:'<'(A::simple1_adt:d1(),3) contains an opaque term as 1st argument when terms of different types are expected in these positions -simple1_api.erl:558: The call erlang:'=<'(A::simple1_adt:d1(),B::simple1_adt:d2()) contains an opaque term as 1st argument when terms of different types are expected in these positions +simple1_api.erl:558: The call erlang:'=<'(A::simple1_adt:d1(),B::simple1_adt:d2()) contains opaque terms as 1st and 2nd arguments when terms of different types are expected in these positions simple1_api.erl:565: Guard test {digraph:graph(),3} > {digraph:graph(),atom() | ets:tid()} contains an opaque term as 2nd argument simple1_api.erl:91: Invalid type specification for function simple1_api:tup/0. The success typing is () -> {'a','b'} simple2_api.erl:100: The call lists:flatten(A::simple1_adt:tuple1()) contains an opaque term as 1st argument when a structured term of type [any()] is expected diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_macros.hrl b/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_macros.hrl new file mode 100644 index 0000000000..07243f8d23 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_macros.hrl @@ -0,0 +1,215 @@ +%% -*- erlang-indent-level: 2 -*- +%%------------------------------------------------------------------------------ + +%%==================================================================== +%% Types +%%==================================================================== + +%% Code and Monitor servers' info. +-record(svs, { + code :: pid(), + monitor :: pid() +}). + +%% Tags of an AST's node. +-record(tags, { + this = undefined :: cuter_cerl:tag() | undefined, + next = undefined :: cuter_cerl:tag() | undefined +}). + +-type loaded_ret_atoms() :: cover_compiled | preloaded | non_existing. +-type servers() :: #svs{}. +-type ast_tags() :: #tags{}. + +%%==================================================================== +%% Directories +%%==================================================================== + +-define(RELATIVE_TMP_DIR, "temp"). +-define(PYTHON_CALL, ?PYTHON_PATH ++ " -u " ++ ?PRIV ++ "/cuter_interface.py"). + +%%==================================================================== +%% Prefixes +%%==================================================================== + +-define(DEPTH_PREFIX, '__conc_depth'). +-define(EXECUTION_PREFIX, '__conc_prefix'). +-define(SYMBOLIC_PREFIX, '__s'). +-define(CONCOLIC_PREFIX_MSG, '__concm'). +-define(ZIPPED_VALUE_PREFIX, '__czip'). +-define(CONCOLIC_PREFIX_PDICT, '__concp'). +-define(FUNCTION_PREFIX, '__cfunc'). +-define(UNBOUND_VAR_PREFIX, '__uboundvar'). +-define(BRANCH_TAG_PREFIX, '__branch_tag'). +-define(VISITED_TAGS_PREFIX, '__visited_tags'). +-define(EXECUTION_COUNTER_PREFIX, '__exec_counter'). + +%%==================================================================== +%% Flags & Default Values +%%==================================================================== + +-define(LOGGING_FLAG, ok). +-define(DELETE_TRACE, ok). +-define(LOG_UNSUPPORTED_MFAS, ok). +%%-define(VERBOSE_SCHEDULER, ok). +%%-define(VERBOSE_FILE_DELETION, ok). +%%-define(VERBOSE_SOLVING, ok). +%%-define(VERBOSE_MERGING, ok). +%%-define(VERBOSE_REPORTING, ok). +-define(USE_SPECS, ok). + +%%==================================================================== +%% Solver Responses +%%==================================================================== + +-define(RSP_MODEL_DELIMITER_START, <<"model_start">>). +-define(RSP_MODEL_DELIMITER_END, <<"model_end">>). + +%%==================================================================== +%% OpCodes for types in JSON objects +%%==================================================================== + +-define(JSON_TYPE_ANY, 0). +-define(JSON_TYPE_INT, 1). +-define(JSON_TYPE_FLOAT, 2). +-define(JSON_TYPE_ATOM, 3). +-define(JSON_TYPE_LIST, 4). +-define(JSON_TYPE_TUPLE, 5). +-define(JSON_TYPE_PID, 6). +-define(JSON_TYPE_REF, 7). + +%%==================================================================== +%% OpCodes for the commands to the solver +%%==================================================================== + +-define(JSON_CMD_LOAD_TRACE_FILE, 1). +-define(JSON_CMD_SOLVE, 2). +-define(JSON_CMD_GET_MODEL, 3). +-define(JSON_CMD_ADD_AXIOMS, 4). +-define(JSON_CMD_FIX_VARIABLE, 5). +-define(JSON_CMD_RESET_SOLVER, 6). +-define(JSON_CMD_STOP, 42). + +%%==================================================================== +%% OpCodes for constraint types +%%==================================================================== + +-define(CONSTRAINT_TRUE, 1). +-define(CONSTRAINT_FALSE, 2). +-define(NOT_CONSTRAINT, 3). + +-define(CONSTRAINT_TRUE_REPR, 84). %% $T +-define(CONSTRAINT_FALSE_REPR, 70). %% $F + +%%==================================================================== +%% OpCodes of constraints & built-in operations +%%==================================================================== + +%% Empty tag ID +-define(EMPTY_TAG_ID, 0). + +%% MFA's Parameters & Spec definitions. +-define(OP_PARAMS, 1). +-define(OP_SPEC, 2). +%% Constraints. +-define(OP_GUARD_TRUE, 3). +-define(OP_GUARD_FALSE, 4). +-define(OP_MATCH_EQUAL_TRUE, 5). +-define(OP_MATCH_EQUAL_FALSE, 6). +-define(OP_TUPLE_SZ, 7). +-define(OP_TUPLE_NOT_SZ, 8). +-define(OP_TUPLE_NOT_TPL, 9). +-define(OP_LIST_NON_EMPTY, 10). +-define(OP_LIST_EMPTY, 11). +-define(OP_LIST_NOT_LST, 12). +%% Information used for syncing & merging the traces of many processes. +-define(OP_SPAWN, 13). +-define(OP_SPAWNED, 14). +-define(OP_MSG_SEND, 15). +-define(OP_MSG_RECEIVE, 16). +-define(OP_MSG_CONSUME, 17). +%% Necessary operations for the evaluation of Core Erlang. +-define(OP_UNFOLD_TUPLE, 18). +-define(OP_UNFOLD_LIST, 19). +%% Bogus operation (operations interpreted as the identity function). +-define(OP_BOGUS, 48). +%% Type conversions. +-define(OP_FLOAT, 47). +-define(OP_LIST_TO_TUPLE, 52). +-define(OP_TUPLE_TO_LIST, 53). +%% Query types. +-define(OP_IS_INTEGER, 27). +-define(OP_IS_ATOM, 28). +-define(OP_IS_FLOAT, 29). +-define(OP_IS_LIST, 30). +-define(OP_IS_TUPLE, 31). +-define(OP_IS_BOOLEAN, 32). +-define(OP_IS_NUMBER, 33). +%% Arithmetic operations. +-define(OP_PLUS, 34). +-define(OP_MINUS, 35). +-define(OP_TIMES, 36). +-define(OP_RDIV, 37). +-define(OP_IDIV_NAT, 38). +-define(OP_REM_NAT, 39). +-define(OP_UNARY, 40). +%% Operations on atoms. +-define(OP_ATOM_NIL, 49). +-define(OP_ATOM_HEAD, 50). +-define(OP_ATOM_TAIL, 51). +%% Operations on lists. +-define(OP_HD, 25). +-define(OP_TL, 26). +-define(OP_CONS, 56). +%% Operations on tuples. +-define(OP_TCONS, 57). +%% Comparisons. +-define(OP_EQUAL, 41). +-define(OP_UNEQUAL, 42). +-define(OP_LT_INT, 54). +-define(OP_LT_FLOAT, 55). + +%% Maps MFAs to their JSON Opcodes +-define(OPCODE_MAPPING, + dict:from_list([ %% Simulated built-in operations + { {cuter_erlang, atom_to_list_bogus, 1}, ?OP_BOGUS } + , { {cuter_erlang, is_atom_nil, 1}, ?OP_ATOM_NIL } + , { {cuter_erlang, safe_atom_head, 1}, ?OP_ATOM_HEAD } + , { {cuter_erlang, safe_atom_tail, 1}, ?OP_ATOM_TAIL } + , { {cuter_erlang, safe_pos_div, 2}, ?OP_IDIV_NAT } + , { {cuter_erlang, safe_pos_rem, 2}, ?OP_REM_NAT } + , { {cuter_erlang, lt_int, 2}, ?OP_LT_INT } + , { {cuter_erlang, lt_float, 2}, ?OP_LT_FLOAT } + , { {cuter_erlang, safe_plus, 2}, ?OP_PLUS } + , { {cuter_erlang, safe_minus, 2}, ?OP_MINUS } + , { {cuter_erlang, safe_times, 2}, ?OP_TIMES } + , { {cuter_erlang, safe_rdiv, 2}, ?OP_RDIV } + , { {cuter_erlang, safe_float, 1}, ?OP_FLOAT } + , { {cuter_erlang, safe_list_to_tuple, 1}, ?OP_LIST_TO_TUPLE } + , { {cuter_erlang, safe_tuple_to_list, 1}, ?OP_TUPLE_TO_LIST } + , { {bogus_erlang, cons, 2}, ?OP_CONS } + %% Actual erlang BIFs + , { {erlang, hd, 1}, ?OP_HD } + , { {erlang, tl, 1}, ?OP_TL } + , { {erlang, is_integer, 1}, ?OP_IS_INTEGER } + , { {erlang, is_atom, 1}, ?OP_IS_ATOM } + , { {erlang, is_boolean, 1}, ?OP_IS_BOOLEAN } + , { {erlang, is_float, 1}, ?OP_IS_FLOAT } + , { {erlang, is_list, 1}, ?OP_IS_LIST } + , { {erlang, is_tuple, 1}, ?OP_IS_TUPLE } + , { {erlang, is_number, 1}, ?OP_IS_NUMBER } + , { {erlang, '-', 1}, ?OP_UNARY } + , { {erlang, '=:=', 2}, ?OP_EQUAL } + , { {erlang, '=/=', 2}, ?OP_UNEQUAL } + ])). + +%% All the MFAs that are supported for symbolic evaluation. +-define(SUPPORTED_MFAS, gb_sets:from_list(dict:fetch_keys(?OPCODE_MAPPING))). + +-define(UNSUPPORTED_MFAS, + gb_sets:from_list([ {cuter_erlang, unsupported_lt, 2} ])). + +%% The set of all the built-in operations that the solver can try to reverse. +-define (REVERSIBLE_OPERATIONS, + gb_sets:from_list([ ?OP_HD, ?OP_TL + ])). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_types.erl b/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_types.erl new file mode 100644 index 0000000000..e9561374cc --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_types.erl @@ -0,0 +1,607 @@ +%% -*- erlang-indent-level: 2 -*- +%%------------------------------------------------------------------------------ +-module(cuter_types). + +-export([parse_spec/3, retrieve_types/1, retrieve_specs/1, find_spec/2, get_kind/1]). + +-export([params_of_t_function_det/1, ret_of_t_function_det/1, atom_of_t_atom_lit/1, integer_of_t_integer_lit/1, + elements_type_of_t_list/1, elements_type_of_t_nonempty_list/1, elements_types_of_t_tuple/1, + elements_types_of_t_union/1, bounds_of_t_range/1, segment_size_of_bitstring/1]). + +-export_type([erl_type/0, erl_spec_clause/0, erl_spec/0, stored_specs/0, stored_types/0, stored_spec_value/0, t_range_limit/0]). + +-include("cuter_macros.hrl"). +-include("cuter_types.hrl"). + + +%% Define tags +-define(type_variable, vart). +-define(type_var, tvar). +-define(max_char, 16#10ffff). + +%% Pre-processed types. + +-type type_name() :: atom(). +-type type_arity() :: byte(). +-type type_var() :: {?type_var, atom()}. +-type remote_type() :: {module(), type_name(), type_arity()}. +-type record_name() :: atom(). +-type record_field_name() :: atom(). +-type record_field_type() :: {record_field_name(), raw_type()}. +-type dep() :: remote_type(). +-type deps() :: ordsets:ordset(remote_type()). +-record(t, { + kind, + rep, + deps = ordsets:new() :: deps() +}). +-type erl_type() :: t_any() % any() + | t_nil() % [] + | t_atom() % atom() + | t_atom_lit() % Erlang_Atom + | t_integer() % integer(), +infinity, -inifinity + | t_integer_lit() % Erlang_Integer + | t_float() % float() + | t_tuple() % tuple(), {TList} + | t_list() % list(Type) + | t_nonempty_list() % nonempty_list(Type) + | t_union() % Type1 | ... | TypeN + | t_range() % Erlang_Integer..Erlang_Integer + | t_bitstring() % <<_:M>> + | t_function() % function() | Fun | BoundedFun + . +-type raw_type() :: erl_type() + | t_local() % Local Type Usage + | t_remote() % Remote Type Usage + | t_record() % Record Usage + | t_type_var() % Type Variable + . + +-type t_any() :: #t{kind :: ?any_tag}. +-type t_nil() :: #t{kind :: ?nil_tag}. +-type t_atom() :: #t{kind :: ?atom_tag}. +-type t_atom_lit() :: #t{kind :: ?atom_lit_tag, rep :: atom()}. +-type t_integer() :: #t{kind :: ?integer_tag}. +-type t_integer_lit() :: #t{kind :: ?integer_lit_tag, rep :: integer()}. +-type t_float() :: #t{kind :: ?float_tag}. +-type t_tuple() :: #t{kind :: ?tuple_tag, rep :: [raw_type()]}. +-type t_list() :: #t{kind :: ?list_tag, rep :: raw_type()}. +-type t_nonempty_list() :: #t{kind :: ?nonempty_list_tag, rep :: raw_type()}. +-type t_union() :: #t{kind :: ?union_tag, rep :: [raw_type()]}. +-type t_range() :: #t{kind :: ?range_tag, rep :: {t_range_limit(), t_range_limit()}}. +-type t_range_limit() :: t_integer_lit() | t_integer_inf(). +-type t_integer_inf() :: t_integer_pos_inf() | t_integer_neg_inf(). +-type t_integer_pos_inf() :: #t{kind :: ?pos_inf}. +-type t_integer_neg_inf() :: #t{kind :: ?neg_inf}. +-type t_bitstring() :: #t{kind :: ?bitstring_tag, rep :: 1|8}. +-type t_function() :: #t{kind :: ?function_tag} | t_function_det(). +-type t_function_det() :: #t{kind :: ?function_tag, rep :: {[raw_type()], raw_type(), [t_constraint()]}, deps :: deps()}. +-type t_constraint() :: {t_type_var(), raw_type()}. +-type t_local() :: #t{kind :: ?local_tag, rep :: {type_name(), [raw_type()]}}. +-type t_remote() :: #t{kind :: ?remote_tag, rep :: {module(), type_name(), [raw_type()]}}. +-type t_record() :: #t{kind :: ?record_tag, rep :: {record_name(), [record_field_type()]}}. +-type t_type_var() :: #t{kind :: ?type_variable, rep :: type_var()}. + +%% How pre-processed types are stored. +-type stored_type_key() :: {record, record_name()} | {type, type_name(), type_arity()}. +-type stored_type_value() :: [record_field_type()] | {any(), [type_var()]}. % raw_type() +-type stored_types() :: dict:dict(stored_type_key(), stored_type_value()). + +-type stored_spec_key() :: {type_name(), type_arity()}. +-type stored_spec_value() :: [t_function_det()]. +-type stored_specs() :: dict:dict(stored_spec_key(), stored_spec_value()). + +-type type_var_env() :: dict:dict(type_var(), raw_type()). +-type erl_spec_clause() :: t_function_det(). +-type erl_spec() :: [erl_spec_clause()]. + +%% Pre-process the type & record declarations of a module. +-spec retrieve_types([cuter_cerl:cerl_attr_type()]) -> stored_types(). +retrieve_types(TypeAttrs) -> + lists:foldl(fun process_type_attr/2, dict:new(), TypeAttrs). + +-spec process_type_attr(cuter_cerl:cerl_recdef() | cuter_cerl:cerl_typedef(), stored_types()) -> stored_types(). +%% Declaration of a record. +process_type_attr({{record, Name}, Fields, []}, Processed) -> + Fs = [t_field_from_form(Field) || Field <- Fields], + Record = t_record(Name, Fs), + dict:store({record, Name}, Record, Processed); +%% Declaration of a type. +process_type_attr({Name, Repr, Vars}, Processed) -> + Type = safe_t_from_form(Repr), + Vs = [{?type_var, Var} || {var, _, Var} <- Vars], + dict:store({type, Name, length(Vs)}, {Type, Vs}, Processed). + +%% The fields of a declared record. +-spec t_field_from_form(cuter_cerl:cerl_record_field()) -> record_field_type(). +t_field_from_form({record_field, _, {atom, _, Name}}) -> + {Name, t_any()}; +t_field_from_form({record_field, _, {atom, _, Name}, _Default}) -> + {Name, t_any()}; +t_field_from_form({typed_record_field, {record_field, _, {atom, _, Name}}, Type}) -> + {Name, safe_t_from_form(Type)}; +t_field_from_form({typed_record_field, {record_field, _, {atom, _, Name}, _Default}, Type}) -> + {Name, safe_t_from_form(Type)}. + +%% Provision for unsupported types. +safe_t_from_form(Form) -> + try t_from_form(Form) + catch throw:{unsupported, Info} -> + cuter_pp:form_has_unsupported_type(Info), + t_any() + end. + +%% Parse a type. + +-spec t_from_form(cuter_cerl:cerl_type()) -> raw_type(). +%% Erlang_Atom +t_from_form({atom, _, Atom}) -> + t_atom_lit(Atom); +%% Erlang_Integer +t_from_form({integer, _, Integer}) -> + t_integer_lit(Integer); +%% integer() +t_from_form({type, _, integer, []}) -> + t_integer(); +%% nil +t_from_form({type, _, nil, []}) -> + t_nil(); +%% any() +t_from_form({type, _, any, []}) -> + t_any(); +%% term() +t_from_form({type, _, term, []}) -> + t_any(); +%% atom() +t_from_form({type, _, atom, []}) -> + t_atom(); +%% module() +t_from_form({type, _, module, []}) -> + t_module(); +%% float() +t_from_form({type, _, float, []}) -> + t_float(); +%% tuple() +t_from_form({type, _, tuple, any}) -> + t_tuple(); +%% {TList} +t_from_form({type, _, tuple, Types}) -> + Ts = [t_from_form(T) || T <- Types], + t_tuple(Ts); +%% list() +t_from_form({type, _, list, []}) -> + t_list(); +%% list(Type) +t_from_form({type, _, list, [Type]}) -> + T = t_from_form(Type), + t_list(T); +%% Type1 | ... | TypeN +t_from_form({type, _, union, Types}) -> + Ts = [t_from_form(T) || T <- Types], + t_union(Ts); +%% boolean() +t_from_form({type, _, boolean, []}) -> + t_union([t_atom_lit(true), t_atom_lit(false)]); +%% number() +t_from_form({type, _, number, []}) -> + t_union([t_integer(), t_float()]); +%% Erlang_Integer..Erlang_Integer +t_from_form({type, _, range, [{integer, _, I1}, {integer, _, I2}]}) -> + t_range(t_integer_lit(I1), t_integer_lit(I2)); +%% non_neg_integer() +t_from_form({type, _, non_neg_integer, []}) -> + t_range(t_integer_lit(0), t_pos_inf()); +%% pos_integer() +t_from_form({type, _, pos_integer, []}) -> + t_range(t_integer_lit(1), t_pos_inf()); +%% neg_integer() +t_from_form({type, _, neg_integer, []}) -> + t_range(t_neg_inf(), t_integer_lit(-1)); +%% char() +t_from_form({type, _, char, []}) -> + t_char(); +%% byte() +t_from_form({type, _, byte, []}) -> + t_byte(); +%% mfa() +t_from_form({type, _, mfa, []}) -> + t_tuple([t_module(), t_atom(), t_byte()]); +%% string() +t_from_form({type, _, string, []}) -> + t_list(t_char()); +%% nonempty_list() +t_from_form({type, _, nonempty_list, []}) -> + t_nonempty_list(); +%% nonempty_list(Type) +t_from_form({type, _, nonempty_list, [Type]}) -> + T = t_from_form(Type), + t_nonempty_list(T); +%% binary() +t_from_form({type, _, binary, []}) -> + t_bitstring(8); +%% bitstring() +t_from_form({type, _, bitstring, []}) -> + t_bitstring(1); +%% function() +t_from_form({type, _, function, []}) -> + t_function(); +%% fun((TList) -> Type) +t_from_form({type, _, 'fun', [_Product, _RetType]}=Fun) -> + t_function_from_form(Fun); +%% fun((TList) -> Type) (bounded_fun) +t_from_form({type, _, 'bounded_fun', [_Fun, _Cs]}=BoundedFun) -> + t_bounded_function_from_form(BoundedFun); +%% ann_type +t_from_form({ann_type, _, [_Var, Type]}) -> + t_from_form(Type); +%% paren_type +t_from_form({paren_type, _, [Type]}) -> + t_from_form(Type); +%% remote_type +t_from_form({remote_type, _, [{atom, _, M}, {atom, _, Name}, Types]}) -> + Ts = [t_from_form(T) || T <- Types], + t_remote(M, Name, Ts); +%% Record +t_from_form({type, _, record, [{atom, _, Name} | FieldTypes]}) -> + Fields = [t_bound_field_from_form(F) || F <- FieldTypes], + t_record(Name, Fields); +%% Map +t_from_form({type, _, map, _}=X) -> + throw({unsupported, X}); +%% local type +t_from_form({type, _, Name, Types}) -> + Ts = [t_from_form(T) || T <- Types], + t_local(Name, Ts); +%% Type Variable +t_from_form({var, _, Var}) -> + t_var(Var); +%% Unsupported forms +t_from_form(Type) -> + throw({unsupported, Type}). + +-spec t_bound_field_from_form(cuter_cerl:cerl_type_record_field()) -> record_field_type(). +%% Record Field. +t_bound_field_from_form({type, _, field_type, [{atom, _, Name}, Type]}) -> + {Name, t_from_form(Type)}. + +-spec t_function_from_form(cuter_cerl:cerl_func()) -> t_function_det(). +t_function_from_form({type, _, 'fun', [{type, _, 'product', Types}, RetType]}) -> + Ret = t_from_form(RetType), + Ts = [t_from_form(T) || T <- Types], + t_function(Ts, Ret). + +-spec t_bounded_function_from_form(cuter_cerl:cerl_bounded_func()) -> t_function_det(). +t_bounded_function_from_form({type, _, 'bounded_fun', [Fun, Constraints]}) -> + {type, _, 'fun', [{type, _, 'product', Types}, RetType]} = Fun, + Ret = t_from_form(RetType), + Ts = [t_from_form(T) || T <- Types], + Cs = [t_constraint_from_form(C) || C <- Constraints], + t_function(Ts, Ret, Cs). + +-spec t_constraint_from_form(cuter_cerl:cerl_constraint()) -> t_constraint(). +t_constraint_from_form({type, _, constraint, [{atom, _, is_subtype}, [{var, _, Var}, Type]]}) -> + {t_var(Var), t_from_form(Type)}. + + +%% Type constructors. + +-spec t_any() -> t_any(). +t_any() -> + #t{kind = ?any_tag}. + +-spec t_atom_lit(atom()) -> t_atom_lit(). +t_atom_lit(Atom) -> + #t{kind = ?atom_lit_tag, rep = Atom}. + +-spec t_atom() -> t_atom(). +t_atom() -> + #t{kind = ?atom_tag}. + +-spec t_module() -> t_atom(). +t_module() -> t_atom(). + +-spec t_integer_lit(integer()) -> t_integer_lit(). +t_integer_lit(Integer) -> + #t{kind = ?integer_lit_tag, rep = Integer}. + +-spec t_integer() -> t_integer(). +t_integer() -> + #t{kind = ?integer_tag}. + +-spec t_range(t_range_limit(), t_range_limit()) -> t_range(). +t_range(Int1, Int2) -> + #t{kind = ?range_tag, rep = {Int1, Int2}}. + +-spec t_pos_inf() -> t_integer_pos_inf(). +t_pos_inf() -> + #t{kind = ?pos_inf}. + +-spec t_neg_inf() -> t_integer_neg_inf(). +t_neg_inf() -> + #t{kind = ?neg_inf}. + +-spec t_char() -> t_range(). +t_char() -> + t_range(t_integer_lit(0), t_integer_lit(?max_char)). + +-spec t_nil() -> t_nil(). +t_nil() -> + #t{kind = ?nil_tag}. + +-spec t_float() -> t_float(). +t_float() -> + #t{kind = ?float_tag}. + +-spec t_list() -> t_list(). +t_list() -> + #t{kind = ?list_tag, rep = t_any()}. + +-spec t_list(raw_type()) -> t_list(). +t_list(Type) -> + #t{kind = ?list_tag, rep = Type, deps = get_deps(Type)}. + +-spec t_nonempty_list() -> t_nonempty_list(). +t_nonempty_list() -> + #t{kind = ?nonempty_list_tag, rep = t_any()}. + +-spec t_nonempty_list(raw_type()) -> t_nonempty_list(). +t_nonempty_list(Type) -> + #t{kind = ?nonempty_list_tag, rep = Type, deps = get_deps(Type)}. + +-spec t_tuple() -> t_tuple(). +t_tuple() -> + #t{kind = ?tuple_tag, rep = []}. + +-spec t_tuple([raw_type()]) -> t_tuple(). +t_tuple(Types) -> + #t{kind = ?tuple_tag, rep = Types, deps = unify_deps(Types)}. + +-spec t_union([raw_type()]) -> t_union(). +t_union(Types) -> + #t{kind = ?union_tag, rep = Types, deps = unify_deps(Types)}. + +-spec t_byte() -> t_range(). +t_byte() -> + t_range(t_integer_lit(0), t_integer_lit(255)). + +-spec t_local(type_name(), [raw_type()]) -> t_local(). +t_local(Name, Types) -> + Rep = {Name, Types}, + #t{kind = ?local_tag, rep = Rep, deps = unify_deps(Types)}. + +-spec t_remote(module(), type_name(), [raw_type()]) -> t_remote(). +t_remote(Mod, Name, Types) -> + Rep = {Mod, Name, Types}, + Dep = {Mod, Name, length(Types)}, + #t{kind = ?remote_tag, rep = Rep, deps = add_dep(Dep, unify_deps(Types))}. + +-spec t_var(atom()) -> t_type_var(). +t_var(Var) -> + #t{kind = ?type_variable, rep = {?type_var, Var}}. + +-spec t_record(record_name(), [record_field_type()]) -> t_record(). +t_record(Name, Fields) -> + Rep = {Name, Fields}, + Ts = [T || {_, T} <- Fields], + #t{kind = ?record_tag, rep = Rep, deps = unify_deps(Ts)}. + +-spec fields_of_t_record(t_record()) -> [record_field_type()]. +fields_of_t_record(Record) -> + Rep = Record#t.rep, + element(2, Rep). + +-spec t_bitstring(1 | 8) -> t_bitstring(). +t_bitstring(N) -> + #t{kind = ?bitstring_tag, rep = N}. + +-spec t_function() -> t_function(). +t_function() -> + #t{kind = ?function_tag}. + +-spec t_function([raw_type()], raw_type()) -> t_function_det(). +t_function(Types, Ret) -> + Rep = {Types, Ret, []}, + #t{kind = ?function_tag, rep = Rep, deps = unify_deps([Ret|Types])}. + +-spec t_function([raw_type()], raw_type(), [t_constraint()]) -> t_function_det(). +t_function(Types, Ret, Constraints) -> + Rep = {Types, Ret, Constraints}, + Ts = [T || {_V, T} <- Constraints], + #t{kind = ?function_tag, rep = Rep, deps = unify_deps([Ret|Types] ++ Ts)}. + +%% Accessors of representations. + +-spec params_of_t_function_det(t_function_det()) -> [raw_type()]. +params_of_t_function_det(#t{kind = ?function_tag, rep = {Params, _Ret, _Constraints}}) -> + Params. + +-spec ret_of_t_function_det(t_function_det()) -> raw_type(). +ret_of_t_function_det(#t{kind = ?function_tag, rep = {_Params, Ret, _Constraints}}) -> + Ret. + +-spec atom_of_t_atom_lit(t_atom_lit()) -> atom(). +atom_of_t_atom_lit(#t{kind = ?atom_lit_tag, rep = Atom}) -> + Atom. + +-spec integer_of_t_integer_lit(t_integer_lit()) -> integer(). +integer_of_t_integer_lit(#t{kind = ?integer_lit_tag, rep = Integer}) -> + Integer. + +-spec elements_type_of_t_list(t_list()) -> raw_type(). +elements_type_of_t_list(#t{kind = ?list_tag, rep = Type}) -> + Type. + +-spec elements_type_of_t_nonempty_list(t_nonempty_list()) -> raw_type(). +elements_type_of_t_nonempty_list(#t{kind = ?nonempty_list_tag, rep = Type}) -> + Type. + +-spec elements_types_of_t_tuple(t_tuple()) -> [raw_type()]. +elements_types_of_t_tuple(#t{kind = ?tuple_tag, rep = Types}) -> + Types. + +-spec elements_types_of_t_union(t_union()) -> [raw_type()]. +elements_types_of_t_union(#t{kind = ?union_tag, rep = Types}) -> + Types. + +-spec bounds_of_t_range(t_range()) -> {t_range_limit(), t_range_limit()}. +bounds_of_t_range(#t{kind = ?range_tag, rep = Limits}) -> + Limits. + +-spec segment_size_of_bitstring(t_bitstring()) -> integer(). +segment_size_of_bitstring(#t{kind = ?bitstring_tag, rep = Sz}) -> + Sz. + +-spec is_tvar_wild_card(t_type_var()) -> boolean(). +is_tvar_wild_card(#t{kind = ?type_variable, rep = {?type_var, Var}}) -> + Var =:= '_'. + +%% Helper functions for kinds. + +-spec get_kind(raw_type()) -> atom(). +get_kind(Type) -> + Type#t.kind. + +%% Helper functions for dependencies. + +-spec get_deps(raw_type()) -> deps(). +get_deps(Type) -> + Type#t.deps. + +-spec has_deps(raw_type()) -> boolean(). +has_deps(Type) -> + get_deps(Type) =/= ordsets:new(). + +-spec add_dep(dep(), deps()) -> deps(). +add_dep(Dep, Deps) -> + ordsets:add_element(Dep, Deps). + +-spec unify_deps([raw_type()]) -> deps(). +unify_deps(Types) -> + ordsets:union([T#t.deps || T <- Types]). + +%% Deal with specs. + +-spec retrieve_specs([cuter_cerl:cerl_attr_spec()]) -> stored_specs(). +retrieve_specs(SpecAttrs) -> + lists:foldl(fun process_spec_attr/2, dict:new(), SpecAttrs). + +-spec process_spec_attr(cuter_cerl:cerl_attr_spec(), stored_specs()) -> stored_specs(). +process_spec_attr({FA, Specs}, Processed) -> + Xs = [t_spec_from_form(Spec) || Spec <- Specs], + dict:store(FA, Xs, Processed). + +-spec t_spec_from_form(cuter_cerl:cerl_spec_func()) -> t_function_det(). +t_spec_from_form({type, _, 'fun', _}=Fun) -> + t_function_from_form(Fun); +t_spec_from_form({type, _, 'bounded_fun', _}=Fun) -> + t_bounded_function_from_form(Fun). + +-spec find_spec(stored_spec_key(), stored_specs()) -> {'ok', stored_spec_value()} | 'error'. +find_spec(FA, Specs) -> + dict:find(FA, Specs). + +%% Parse the spec of an MFA. + +-type spec_parse_reply() :: {error, has_remote_types | recursive_type} + | {error, unsupported_type, type_name()} + | {ok, erl_spec()}. + +-spec parse_spec(stored_spec_key(), stored_spec_value(), stored_types()) -> spec_parse_reply(). +parse_spec(FA, Spec, Types) -> + try parse_spec_clauses(FA, Spec, Types, []) of + {error, has_remote_types}=E -> E; + Parsed -> {ok, Parsed} + catch + throw:remote_type -> {error, has_remote_types}; + throw:recursive_type -> {error, recursive_type}; + throw:{unsupported, Name} -> {error, unsupported_type, Name} + end. + + +parse_spec_clauses(_FA, [], _Types, Acc) -> + lists:reverse(Acc); +parse_spec_clauses(FA, [Clause|Clauses], Types, Acc) -> + case has_deps(Clause) of + true -> {error, has_remote_types}; + false -> + Visited = ordsets:add_element(FA, ordsets:new()), + Simplified = simplify(Clause, Types, dict:new(), Visited), + parse_spec_clauses(FA, Clauses, Types, [Simplified|Acc]) + end. + +add_constraints_to_env([], Env) -> + Env; +add_constraints_to_env([{Var, Type}|Cs], Env) -> + F = fun(StoredTypes, E, Visited) -> simplify(Type, StoredTypes, E, Visited) end, + Env1 = dict:store(Var#t.rep, F, Env), + add_constraints_to_env(Cs, Env1). + +bind_parameters([], [], Env) -> + Env; +bind_parameters([P|Ps], [A|As], Env) -> + F = fun(StoredTypes, E, Visited) -> simplify(A, StoredTypes, E, Visited) end, + Env1 = dict:store(P, F, Env), + bind_parameters(Ps, As, Env1). + +-spec simplify(raw_type(), stored_types(), type_var_env(), ordsets:ordset(stored_spec_key())) -> raw_type(). +%% fun +simplify(#t{kind = ?function_tag, rep = {Params, Ret, Constraints}}=Raw, StoredTypes, Env, Visited) -> + Env1 = add_constraints_to_env(Constraints, Env), + ParamsSimplified = [simplify(P, StoredTypes, Env1, Visited) || P <- Params], + RetSimplified = simplify(Ret, StoredTypes, Env1, Visited), + Rep = {ParamsSimplified, RetSimplified, []}, + Raw#t{rep = Rep}; +%% tuple +simplify(#t{kind = ?tuple_tag, rep = Types}=Raw, StoredTypes, Env, Visited) -> + Rep = [simplify(T, StoredTypes, Env, Visited) || T <- Types], + Raw#t{rep = Rep}; +%% list / nonempty_list +simplify(#t{kind = Tag, rep = Type}=Raw, StoredTypes, Env, Visited) when Tag =:= ?list_tag; Tag =:= ?nonempty_list_tag -> + Rep = simplify(Type, StoredTypes, Env, Visited), + Raw#t{rep = Rep}; +%% union +simplify(#t{kind = ?union_tag, rep = Types}=Raw, StoredTypes, Env, Visited) -> + Rep = [simplify(T, StoredTypes, Env, Visited) || T <- Types], + Raw#t{rep = Rep}; +%% local type +simplify(#t{kind = ?local_tag, rep = {Name, Args}}, StoredTypes, Env, Visited) -> + Arity = length(Args), + TA = {Name, Arity}, + case ordsets:is_element(TA, Visited) of + true -> throw(recursive_type); + false -> + case dict:find({type, Name, Arity}, StoredTypes) of + error -> throw({unsupported, Name}); + {ok, {Type, Params}} -> + Env1 = bind_parameters(Params, Args, Env), + simplify(Type, StoredTypes, Env1, [TA|Visited]) + end + end; +%% type variable +simplify(#t{kind = ?type_variable, rep = TVar}=T, StoredTypes, Env, Visited) -> + case is_tvar_wild_card(T) of + true -> t_any(); + false -> + V = dict:fetch(TVar, Env), + V(StoredTypes, Env, Visited) + end; +simplify(#t{kind = ?remote_tag}, _StoredTypes, _Env, _Visited) -> + throw(remote_type); +%% record +simplify(#t{kind = ?record_tag, rep = {Name, OverridenFields}}, StoredTypes, Env, Visited) -> + RecordDecl = dict:fetch({record, Name}, StoredTypes), + Fields = fields_of_t_record(RecordDecl), + ActualFields = replace_record_fields(Fields, OverridenFields), + FinalFields = [{N, simplify(T, StoredTypes, Env, Visited)} || {N, T} <- ActualFields], + Simplified = [T || {_, T} <- FinalFields], + t_tuple([t_atom_lit(Name)|Simplified]); +%% all others +simplify(Raw, _StoredTypes, _Env, _Visited) -> + Raw. + +-spec replace_record_fields([record_field_type()], [record_field_type()]) -> [record_field_type()]. +replace_record_fields(Fields, []) -> + Fields; +replace_record_fields(Fields, [{Name, Type}|Rest]) -> + Replaced = lists:keyreplace(Name, 1, Fields, {Name, Type}), + replace_record_fields(Replaced, Rest). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_types.hrl b/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_types.hrl new file mode 100644 index 0000000000..4172184709 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/cuter/cuter_types.hrl @@ -0,0 +1,26 @@ +%% -*- erlang-indent-level: 2 -*- +%%------------------------------------------------------------------------------ + +%%==================================================================== +%% Tags for the kind of encoded types. +%%==================================================================== + +-define(atom_lit_tag, atom_lit). +-define(integer_lit_tag, integer_lit). +-define(integer_tag, integer). +-define(nil_tag, nil). +-define(any_tag, any). +-define(atom_tag, atom). +-define(float_tag, float). +-define(tuple_tag, tuple). +-define(list_tag, list). +-define(nonempty_list_tag, nonempty_list). +-define(union_tag, union). +-define(range_tag, range). +-define(bitstring_tag, bitstring). +-define(neg_inf, neg_inf). +-define(pos_inf, pos_inf). +-define(remote_tag, remote). +-define(local_tag, local). +-define(record_tag, record). +-define(function_tag, function). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para/para2.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para/para2.erl index 09b2235fa5..4461ff291c 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/src/para/para2.erl +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para/para2.erl @@ -85,7 +85,7 @@ ct2_adt() -> tcirc() -> A = circ1(), B = circ2(), - A =:= B. % can never evaluate to 'true' (but the types are not OK, or?) + A =:= B. % can never evaluate to 'true' -spec circ1() -> circ(integer()). @@ -100,7 +100,7 @@ circ2() -> tcirc_adt() -> A = circ1_adt(), B = circ2_adt(), - A =:= B. % opaque attempt (one would expect them to be the same...) + A =:= B. % opaque attempt (number of parameters differs) circ1_adt() -> para2_adt:circ1(). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para/para3.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para/para3.erl index 792ae40d39..102215b28d 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/src/para/para3.erl +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para/para3.erl @@ -71,7 +71,7 @@ t2_adt() -> -type exp() :: para3_adt:exp1(para3_adt:exp2()). --spec exp_adt() -> exp(). +-spec exp_adt() -> exp(). % invalid type spec exp_adt() -> 3. diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para/para4.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para/para4.erl new file mode 100644 index 0000000000..b9794672a9 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para/para4.erl @@ -0,0 +1,134 @@ +-module(para4). + +-compile(export_all). + +-export_type([d_atom/0, d_integer/0, d_tuple/0, d_all/0]). + +-export_type([t/1]). + +-type ai() :: atom() | integer(). + +-type d(T) :: dict:dict(T, T). + +-opaque d_atom() :: d(atom()). +-opaque d_integer() :: d(integer()). +-opaque d_tuple() :: d(tuple()). +-opaque d_all() :: d(ai()). + +b(D) -> + a(D) ++ i(D). + +-spec a(d_atom()) -> [{atom(), atom()}]. % Invalid type spec + +a(D) -> + c(D). + +-spec i(d_integer()) -> [{integer(), integer()}]. % Invalid type spec + +i(D) -> + c(D). + +-spec t(d_tuple()) -> [{tuple(), tuple()}]. % Invalid type spec. + +t(D) -> + c(D). + +-spec c(d_all()) -> [{ai(), ai()}]. + +c(D) -> + dict:to_list(D). + + + + +-opaque t(A) :: {A, A}. + +adt_tt5() -> + I1 = adt_y1(), + I2 = adt_y3(), + I1 =:= I2. + +adt_tt6() -> + I1 = adt_y2(), + I2 = adt_y3(), + I1 =:= I2. + +adt_tt7() -> + I1 = adt_t1(), + I2 = adt_t3(), + I1 =:= I2. % opaque attempt + +adt_tt8() -> + I1 = adt_t2(), + I2 = adt_t3(), + I1 =:= I2. % opaque attempt + +adt_tt9() -> + I1 = adt_int2(), + I2 = adt_int4(), + I1 =:= I2. % opaque attempt + +adt_tt10() -> + I1 = adt_int2(), + I2 = adt_int2_4(), + I1 =:= I2. % opaque attempt + +adt_tt11() -> + I1 = adt_int5_7(), + I2 = adt_int2_4(), + I1 =:= I2. % opaque attempt + +adt_tt12() -> + I1 = adt_un1_2(), + I2 = adt_un3_4(), + I1 =:= I2. % opaque attempt + +adt_tt13() -> + I1 = adt_tup(), + I2 = adt_tup2(), + I1 =:= I2. % opaque attempt + +y3() -> + {a, 3}. + +adt_t1() -> + para4_adt:t1(). + +adt_t2() -> + para4_adt:t2(). + +adt_t3() -> + para4_adt:t3(). + +adt_y1() -> + para4_adt:y1(). + +adt_y2() -> + para4_adt:y2(). + +adt_y3() -> + para4_adt:y3(). + +adt_int2() -> + para4_adt:int2(). + +adt_int4() -> + para4_adt:int4(). + +adt_int2_4() -> + para4_adt:int2_4(). + +adt_int5_7() -> + para4_adt:int5_7(). + +adt_un1_2() -> + para4_adt:un1_2(). + +adt_un3_4() -> + para4_adt:un3_4(). + +adt_tup() -> + para4_adt:tup(). + +adt_tup2() -> + para4_adt:tup2(). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para/para4_adt.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para/para4_adt.erl new file mode 100644 index 0000000000..407dd198a7 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para/para4_adt.erl @@ -0,0 +1,108 @@ +-module(para4_adt). + +-export([t1/0, t2/0, t3/0, y1/0, y2/0, y3/0]). + +-export([int2/0, int4/0, int2_4/0, int5_7/0]). + +-export([un1_2/0, un3_4/0]). + +-export([tup/0, tup2/0]). + +-export_type([t/1, y/1, int/1, tup/1, un/1]). + +-type ai() :: atom() | integer(). + +-opaque t(A) :: {A, A}. + +-type y(A) :: {A, A}. + +-opaque int(I) :: I. + +-opaque un(I) :: atom() | I. + +-opaque tup(T) :: T. + +-spec t1() -> t(integer()). + +t1() -> + {i(), i()}. + +-spec t2() -> t(atom()). + +t2() -> + {a(), a()}. + +-spec t3() -> t(ai()). + +t3() -> + {ai(), ai()}. + +-spec y1() -> y(integer()). + +y1() -> + {i(), i()}. + +-spec y2() -> y(atom()). + +y2() -> + {a(), a()}. + +-spec y3() -> y(ai()). + +y3() -> + {ai(), ai()}. + +-spec a() -> atom(). + +a() -> + foo:a(). + +-spec i() -> integer(). + +i() -> + foo:i(). + +-spec ai() -> ai(). + +ai() -> + foo:ai(). + +-spec int2() -> int(1..2). + +int2() -> + foo:int2(). + +-spec int4() -> int(1..4). + +int4() -> + foo:int4(). + +-spec int2_4() -> int(2..4). + +int2_4() -> + foo:int2_4(). + +-spec int5_7() -> int(5..7). + +int5_7() -> + foo:int5_7(). + +-spec un1_2() -> un(1..2). + +un1_2() -> + foo:un1_2(). + +-spec un3_4() -> un(3..4). + +un3_4() -> + foo:un3_4(). + +-spec tup() -> tup(tuple()). + +tup() -> + foo:tup(). + +-spec tup2() -> tup({_, _}). + +tup2() -> + foo:tup2(). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para/para5.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para/para5.erl new file mode 100644 index 0000000000..76ea3e76b5 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para/para5.erl @@ -0,0 +1,33 @@ +-module(para5). + +-export([d/0, dd/0, da1/0]). + +d() -> + I1 = adt_d1(), + I2 = adt_d2(), + I1 =:= I2. % can never evaluate to true + +dd() -> + I1 = adt_d1(), + I2 = adt_dd(), + I1 =/= I2. % incompatible opaque types + +da1() -> + I1 = adt_da1(), + I2 = adt_da2(), + I1 =:= I2. + +adt_d1() -> + para5_adt:d1(). + +adt_d2() -> + para5_adt:d2(). + +adt_dd() -> + para5_adt:dd(). + +adt_da1() -> + para5_adt:da1(). + +adt_da2() -> + para5_adt:da2(). diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/para/para5_adt.erl b/lib/dialyzer/test/opaque_SUITE_data/src/para/para5_adt.erl new file mode 100644 index 0000000000..a62e0488e0 --- /dev/null +++ b/lib/dialyzer/test/opaque_SUITE_data/src/para/para5_adt.erl @@ -0,0 +1,36 @@ +-module(para5_adt). + +-export([d1/0, d2/0, dd/0, da1/0, da2/0]). + +-export_type([d/0, dd/1, da/2]). + +-opaque d() :: 1 | 2. + +-spec d1() -> d(). + +d1() -> + 1. + +-spec d2() -> d(). + +d2() -> + 2. + +-opaque dd(A) :: A. + +-spec dd() -> dd(atom()). + +dd() -> + foo:atom(). + +-opaque da(A, B) :: {A, B}. + +-spec da1() -> da(any(), atom()). + +da1() -> + {3, a}. + +-spec da2() -> da(integer(), any()). + +da2() -> + {3, a}. diff --git a/lib/dialyzer/test/opaque_SUITE_data/src/simple/simple1_api.erl b/lib/dialyzer/test/opaque_SUITE_data/src/simple/simple1_api.erl index eef2074e0c..7db1100597 100644 --- a/lib/dialyzer/test/opaque_SUITE_data/src/simple/simple1_api.erl +++ b/lib/dialyzer/test/opaque_SUITE_data/src/simple/simple1_api.erl @@ -145,7 +145,7 @@ adt_t4() -> A = simple1_adt:n1(), B = simple1_adt:n2(), A = A, % OK - A = B. % opaque term + A = B. % opaque terms adt_t7() -> A = simple1_adt:n1(), @@ -178,7 +178,7 @@ c2(A, B) -> c2() -> A = simple1_adt:d1(), B = simple1_adt:d2(), - if A =< B -> ok end. % opaque term + if A =< B -> ok end. % opaque terms c3() -> B = simple1_adt:d2(), @@ -516,7 +516,7 @@ eq1() -> A = simple1_adt:d2(), B = simple1_adt:d1(), if - A == B -> % opaque term + A == B -> % opaque terms 0; A == A -> 1; @@ -555,7 +555,7 @@ c6(A, B) -> c7(A, B) -> A = simple1_adt:d1(), B = simple1_adt:d2(), - A =< B. % opaque term + A =< B. % opaque terms c8() -> D = digraph:new(), diff --git a/lib/dialyzer/test/plt_SUITE.erl b/lib/dialyzer/test/plt_SUITE.erl index ef4cdc57f0..ecbac14e5d 100644 --- a/lib/dialyzer/test/plt_SUITE.erl +++ b/lib/dialyzer/test/plt_SUITE.erl @@ -6,12 +6,13 @@ -include_lib("common_test/include/ct.hrl"). -include("dialyzer_test_constants.hrl"). --export([suite/0, all/0, build_plt/1, beam_tests/1, update_plt/1]). +-export([suite/0, all/0, build_plt/1, beam_tests/1, update_plt/1, + run_plt_check/1, run_succ_typings/1]). suite() -> [{timetrap, ?plt_timeout}]. -all() -> [build_plt, beam_tests, update_plt]. +all() -> [build_plt, beam_tests, update_plt, run_plt_check, run_succ_typings]. build_plt(Config) -> OutDir = ?config(priv_dir, Config), @@ -37,14 +38,76 @@ beam_tests(Config) when is_list(Config) -> ">>, Opts = [no_auto_import], {ok, BeamFile} = compile(Config, Prog, no_auto_import, Opts), - [] = run_dialyzer([BeamFile]), + [] = run_dialyzer(plt_build, [BeamFile], []), ok. -run_dialyzer(Files) -> - dialyzer:run([{analysis_type, plt_build}, - {files, Files}, - {from, byte_code}, - {check_plt, false}]). +run_plt_check(Config) when is_list(Config) -> + Mod1 = <<" + -module(run_plt_check1). + ">>, + + Mod2A = <<" + -module(run_plt_check2). + ">>, + + {ok, BeamFile1} = compile(Config, Mod1, run_plt_check1, []), + {ok, BeamFile2} = compile(Config, Mod2A, run_plt_check2, []), + [] = run_dialyzer(plt_build, [BeamFile1, BeamFile2], []), + + Mod2B = <<" + -module(run_plt_check2). + + -export([call/1]). + + call(X) -> run_plt_check1:call(X). + ">>, + + {ok, BeamFile2} = compile(Config, Mod2B, run_plt_check2, []), + + % callgraph warning as run_plt_check2:call/1 makes a call to unexported + % function run_plt_check1:call/1. + [_] = run_dialyzer(plt_check, [], []), + + ok. + +run_succ_typings(Config) when is_list(Config) -> + Mod1A = <<" + -module(run_succ_typings1). + + -export([call/0]). + + call() -> a. + ">>, + + {ok, BeamFile1} = compile(Config, Mod1A, run_succ_typings1, []), + [] = run_dialyzer(plt_build, [BeamFile1], []), + + Mod1B = <<" + -module(run_succ_typings1). + + -export([call/0]). + + call() -> b. + ">>, + + Mod2 = <<" + -module(run_succ_typings2). + + -export([call/0]). + + -spec call() -> b. + call() -> run_succ_typings1:call(). + ">>, + + {ok, BeamFile1} = compile(Config, Mod1B, run_succ_typings1, []), + {ok, BeamFile2} = compile(Config, Mod2, run_succ_typings2, []), + % contract types warning as run_succ_typings2:call/0 makes a call to + % run_succ_typings1:call/0, which returns a (not b) in the PLT. + [_] = run_dialyzer(succ_typings, [BeamFile2], [{check_plt, false}]), + % warning not returned as run_succ_typings1 is updated in the PLT. + [] = run_dialyzer(succ_typings, [BeamFile2], [{check_plt, true}]), + + ok. %%% [James Fish:] %%% If a function is removed from a module and the module has previously @@ -103,3 +166,9 @@ compile(Config, Prog, Module, CompileOpts) -> Opts = [{outdir, PrivDir}, debug_info | CompileOpts], {ok, Module} = compile:file(Filename, Opts), {ok, filename:join([PrivDir, lists:concat([Module, ".beam"])])}. + +run_dialyzer(Analysis, Files, Opts) -> + dialyzer:run([{analysis_type, Analysis}, + {files, Files}, + {from, byte_code} | + Opts]). diff --git a/lib/dialyzer/test/small_SUITE_data/src/remote_field.erl b/lib/dialyzer/test/small_SUITE_data/src/remote_field.erl index c34fa1b9dd..d83f2e3234 100644 --- a/lib/dialyzer/test/small_SUITE_data/src/remote_field.erl +++ b/lib/dialyzer/test/small_SUITE_data/src/remote_field.erl @@ -3,7 +3,7 @@ -type f(T) :: {ssl:sslsocket(), T}. -record(r1, { f1 :: f(_) }). --type r1(T) :: #r1{ f1 :: fun((ssl:sslsocket(), T) -> any()) }. +-type r1(T) :: #r1{ f1 :: {ssl:sslsocket(), T} }. -record(state, { r :: r1(T), diff --git a/lib/diameter/doc/src/notes.xml b/lib/diameter/doc/src/notes.xml index 6931788c83..c5df63a7f0 100644 --- a/lib/diameter/doc/src/notes.xml +++ b/lib/diameter/doc/src/notes.xml @@ -42,6 +42,36 @@ first.</p> <!-- ===================================================================== --> +<section><title>diameter 1.9.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix broken relay counters.</p> + <p> + OTP-12654 in OTP 17.5.3 broke counters in the case of + answer messages received in the relay application. + Counters were accumulated as unknown messages or + no_result_code instead of as relayed messages on the + intended Result-Code and 'Experimental-Result' tuples.</p> + <p> + Own Id: OTP-12741</p> + </item> + <item> + <p> + Fix diameter_sctp listener race.</p> + <p> + An oversight in OTP-12428 made it possible to start a + transport process that could not establish associations.</p> + <p> + Own Id: OTP-12744</p> + </item> + </list> + </section> + +</section> + <section><title>diameter 1.9.1</title> <section><title>Known Bugs and Problems</title> @@ -65,7 +95,7 @@ first.</p> received in an answer not setting the E-bit. The correct AVP is now extracted from the incoming message.</p> <p> - Own Id: OTP-12654 Aux Id: seq12851 </p> + Own Id: OTP-12654</p> </item> <item> <p> diff --git a/lib/diameter/src/base/diameter_traffic.erl b/lib/diameter/src/base/diameter_traffic.erl index ffd2c0afa2..eb4bbae931 100644 --- a/lib/diameter/src/base/diameter_traffic.erl +++ b/lib/diameter/src/base/diameter_traffic.erl @@ -131,11 +131,11 @@ peer_down(TPid) -> %% incr/4 %% --------------------------------------------------------------------------- -incr(Dir, #diameter_packet{header = H}, TPid, Dict) -> - incr(Dir, H, TPid, Dict); +incr(Dir, #diameter_packet{header = H}, TPid, AppDict) -> + incr(Dir, H, TPid, AppDict); -incr(Dir, #diameter_header{} = H, TPid, Dict) -> - incr(TPid, {msg_id(H, Dict), Dir}). +incr(Dir, #diameter_header{} = H, TPid, AppDict) -> + incr(TPid, {msg_id(H, AppDict), Dir}). %% --------------------------------------------------------------------------- %% incr_error/4 @@ -143,26 +143,26 @@ incr(Dir, #diameter_header{} = H, TPid, Dict) -> %% Identify messages using the application dictionary, not the encode %% dictionary, which may differ in the case of answer-message. -incr_error(Dir, T, Pid, {_Dict, AppDict}) -> +incr_error(Dir, T, Pid, {_MsgDict, AppDict}) -> incr_error(Dir, T, Pid, AppDict); %% Decoded message without errors. incr_error(recv, #diameter_packet{errors = []}, _, _) -> ok; -incr_error(recv = D, #diameter_packet{header = H}, TPid, Dict) -> - incr_error(D, H, TPid, Dict); +incr_error(recv = D, #diameter_packet{header = H}, TPid, AppDict) -> + incr_error(D, H, TPid, AppDict); %% Encoded message with errors and an identifiable header ... -incr_error(send = D, {_, _, #diameter_header{} = H}, TPid, Dict) -> - incr_error(D, H, TPid, Dict); +incr_error(send = D, {_, _, #diameter_header{} = H}, TPid, AppDict) -> + incr_error(D, H, TPid, AppDict); %% ... or not. incr_error(send = D, {_,_}, TPid, _) -> incr_error(D, unknown, TPid); -incr_error(Dir, #diameter_header{} = H, TPid, Dict) -> - incr_error(Dir, msg_id(H, Dict), TPid); +incr_error(Dir, #diameter_header{} = H, TPid, AppDict) -> + incr_error(Dir, msg_id(H, AppDict), TPid); incr_error(Dir, Id, TPid, _) -> incr_error(Dir, Id, TPid). @@ -179,18 +179,20 @@ incr_error(Dir, Id, TPid) -> | Reason when Pkt :: #diameter_packet{}, TPid :: pid(), - DictT :: module() | {module(), module(), module()}, + DictT :: module() | {MsgDict :: module(), + AppDict :: module(), + CommonDict:: module()}, Counter :: {'Result-Code', integer()} | {'Experimental-Result', integer(), integer()}, Reason :: atom(). -incr_rc(Dir, Pkt, TPid, {Dict, _, _} = DictT) -> +incr_rc(Dir, Pkt, TPid, {_, AppDict, _} = DictT) -> try incr_result(Dir, Pkt, TPid, DictT) catch exit: {E,_} when E == no_result_code; E == invalid_error_bit -> - incr(TPid, {msg_id(Pkt#diameter_packet.header, Dict), Dir, E}), + incr(TPid, {msg_id(Pkt#diameter_packet.header, AppDict), Dir, E}), E end; @@ -259,7 +261,8 @@ recv(false, #request{ref = Ref, handler = Pid} = Req, _, Pkt, Dict0, _) -> %% any others are discarded. %% ... or not. -recv(false, false, _, _, _, _) -> +recv(false, false, TPid, _, _, _) -> + incr(TPid, {{unknown, 0}, recv, discarded}), ok. %% spawn_request/4 @@ -307,14 +310,14 @@ recv_request(TPid, Pkt, Dict0, RecvData) -> %% from old code %% recv_R/5 -recv_R({#diameter_app{id = Id, dictionary = Dict} = App, Caps}, +recv_R({#diameter_app{id = Id, dictionary = AppDict} = App, Caps}, TPid, Pkt0, Dict0, RecvData) -> - incr(recv, Pkt0, TPid, Dict), - Pkt = errors(Id, diameter_codec:decode(Id, Dict, Pkt0)), - incr_error(recv, Pkt, TPid, Dict), + incr(recv, Pkt0, TPid, AppDict), + Pkt = errors(Id, diameter_codec:decode(Id, AppDict, Pkt0)), + incr_error(recv, Pkt, TPid, AppDict), {Caps, Pkt, App, recv_R(App, TPid, Dict0, Caps, RecvData, Pkt)}; %% Note that the decode is different depending on whether or not Id is %% ?APP_ID_RELAY. @@ -522,14 +525,17 @@ send_A(_, _, _, _) -> %% send_A/6 -send_A(T, TPid, DictT, ReqPkt, EvalPktFs, EvalFs) -> - reply(T, TPid, DictT, EvalPktFs, ReqPkt), +send_A(T, TPid, {AppDict, Dict0} = DictT0, ReqPkt, EvalPktFs, EvalFs) -> + {MsgDict, Pkt} = reply(T, TPid, DictT0, EvalPktFs, ReqPkt), + incr(send, Pkt, TPid, AppDict), + incr_rc(send, Pkt, TPid, {MsgDict, AppDict, Dict0}), %% count outgoing + send(TPid, Pkt), lists:foreach(fun diameter_lib:eval/1, EvalFs). %% answer/6 answer({reply, Ans}, _Caps, _Pkt, App, Dict0, _RecvData) -> - {dict(App#diameter_app.dictionary, Dict0, Ans), Ans}; + {msg_dict(App#diameter_app.dictionary, Dict0, Ans), Ans}; answer({call, Opts}, Caps, Pkt, App, Dict0, RecvData) -> #diameter_caps{origin_host = {OH,_}} @@ -552,27 +558,37 @@ answer({answer_message, RC} = T, Caps, Pkt, App, Dict0, _RecvData) -> orelse ?ERROR({invalid_return, T, handle_request, App}), answer_message(RC, Caps, Dict0, Pkt). -%% dict/3 +%% msg_dict/3 +%% +%% Return the dictionary defining the message grammar in question: the +%% application dictionary or the common dictionary. + +msg_dict(AppDict, Dict0, [Msg]) + when is_list(Msg); + is_tuple(Msg) -> + msg_dict(AppDict, Dict0, Msg); -%% An incoming answer, not yet decoded. -dict(Dict, Dict0, #diameter_packet{header - = #diameter_header{is_request = false, - is_error = E}, - msg = undefined}) -> - if E -> Dict0; true -> Dict end; +msg_dict(AppDict, Dict0, Msg) -> + choose(is_answer_message(Msg, Dict0), Dict0, AppDict). -dict(Dict, Dict0, [Msg]) -> - dict(Dict, Dict0, Msg); +%% Incoming, not yet decoded. +is_answer_message(#diameter_packet{header = #diameter_header{} = H, + msg = undefined}, + Dict0) -> + is_answer_message([H], Dict0); -dict(Dict, Dict0, #diameter_packet{msg = Msg}) -> - dict(Dict, Dict0, Msg); +is_answer_message(#diameter_packet{msg = Msg}, Dict0) -> + is_answer_message(Msg, Dict0); -dict(Dict, Dict0, Msg) -> - choose(is_answer_message(Msg, Dict0), Dict0, Dict). +%% Message sent as a header/avps list. +is_answer_message([#diameter_header{is_request = R, is_error = E} | _], _) -> + E andalso not R; +%% Message sent as a tagged avp/value list. is_answer_message([Name | _], _) -> Name == 'answer-message'; +%% Message sent as a record. is_answer_message(Rec, Dict) -> try 'answer-message' == Dict:rec2msg(element(1,Rec)) @@ -642,7 +658,7 @@ resend(false, %% %% Relay a reply to a relayed request. -%% Answer from the peer: reset the hop by hop identifier and send. +%% Answer from the peer: reset the hop by hop identifier. resend(#diameter_packet{bin = B} = Pkt, _Caps, @@ -681,13 +697,13 @@ is_loop(Code, Vid, OH, Dict0, Avps) -> %% reply/5 %% Local answer ... -reply({Dict, Ans}, TPid, {AppDict, Dict0}, Fs, ReqPkt) -> - local(Ans, TPid, {Dict, AppDict, Dict0}, Fs, ReqPkt); +reply({MsgDict, Ans}, TPid, {AppDict, Dict0}, Fs, ReqPkt) -> + local(Ans, TPid, {MsgDict, AppDict, Dict0}, Fs, ReqPkt); %% ... or relayed. -reply(#diameter_packet{} = Pkt, TPid, _Dict0, Fs, _ReqPkt) -> +reply(#diameter_packet{} = Pkt, _TPid, {AppDict, Dict0}, Fs, _ReqPkt) -> eval_packet(Pkt, Fs), - send(TPid, Pkt). + {msg_dict(AppDict, Dict0, Pkt), Pkt}. %% local/5 %% @@ -700,14 +716,12 @@ local([Msg], TPid, DictT, Fs, ReqPkt) is_tuple(Msg) -> local(Msg, TPid, DictT, Fs, ReqPkt#diameter_packet{errors = []}); -local(Msg, TPid, {Dict, AppDict, Dict0} = DictT, Fs, ReqPkt) -> - Pkt = encode({Dict, AppDict}, +local(Msg, TPid, {MsgDict, AppDict, Dict0}, Fs, ReqPkt) -> + Pkt = encode({MsgDict, AppDict}, TPid, - reset(make_answer_packet(Msg, ReqPkt), Dict, Dict0), + reset(make_answer_packet(Msg, ReqPkt), MsgDict, Dict0), Fs), - incr(send, Pkt, TPid, AppDict), - incr_rc(send, Pkt, TPid, DictT), %% count outgoing - send(TPid, Pkt). + {MsgDict, Pkt}. %% reset/3 @@ -1067,54 +1081,75 @@ find_avp(Code, VId, [_ | Avps]) -> %% Increment a stats counter for result codes in incoming and outgoing %% answers. +%% Message sent as a header/avps list. +incr_result(send = Dir, + #diameter_packet{msg = [#diameter_header{} = H | _]} + = Pkt, + TPid, + DictT) -> + incr_res(Dir, Pkt#diameter_packet{header = H}, TPid, DictT); + %% Outgoing message as binary: don't count. (Sending binaries is only %% partially supported.) -incr_result(_, #diameter_packet{msg = undefined = No}, _, _) -> +incr_result(send, #diameter_packet{header = undefined = No}, _, _) -> No; %% Incoming or outgoing. Outgoing with encode errors never gets here %% since encode fails. -incr_result(Dir, Pkt, TPid, {Dict, AppDict, Dict0}) -> - #diameter_packet{header = #diameter_header{is_error = E} - = Hdr, - errors = Es} - = Pkt, +incr_result(Dir, Pkt, TPid, DictT) -> + incr_res(Dir, Pkt, TPid, DictT). + +incr_res(Dir, + #diameter_packet{header = #diameter_header{is_error = E} + = Hdr, + errors = Es} + = Pkt, + TPid, + DictT) -> + {MsgDict, AppDict, Dict0} = DictT, Id = msg_id(Hdr, AppDict), + %% Could be {relay, 0}, in which case the R-bit is redundant since + %% only answers are being counted. Let it be however, so that the + %% same tuple is in both send/recv and result code counters. %% Count incoming decode errors. recv /= Dir orelse [] == Es orelse incr_error(Dir, Id, TPid, AppDict), %% Exit on a missing result code. - T = rc_counter(Dict, Dir, Pkt), - T == false andalso ?LOGX(no_result_code, {Dict, Dir, Hdr}), + T = rc_counter(MsgDict, Dir, Pkt), + T == false andalso ?LOGX(no_result_code, {MsgDict, Dir, Hdr}), {Ctr, RC, Avp} = T, %% Or on an inappropriate value. is_result(RC, E, Dict0) - orelse ?LOGX(invalid_error_bit, {Dict, Dir, Hdr, Avp}), + orelse ?LOGX(invalid_error_bit, {MsgDict, Dir, Hdr, Avp}), incr(TPid, {Id, Dir, Ctr}), Ctr. %% msg_id/2 -msg_id(#diameter_packet{header = H}, Dict) -> - msg_id(H, Dict); +msg_id(#diameter_packet{header = H}, AppDict) -> + msg_id(H, AppDict); %% Only count on known keys so as not to be vulnerable to attack: %% there are 2^32 (application ids) * 2^24 (command codes) = 2^56 %% pairs for an attacker to choose from. -msg_id(Hdr, Dict) -> +msg_id(Hdr, AppDict) -> {Aid, Code, R} = Id = diameter_codec:msg_id(Hdr), - if Aid == ?APP_ID_RELAY -> + case AppDict:id() of + ?APP_ID_RELAY -> {relay, R}; - true -> - choose(Aid /= Dict:id() orelse '' == Dict:msg_name(Code, 0 == R), - unknown, - Id) + A -> + unknown(A /= Aid orelse '' == AppDict:msg_name(Code, 0 == R), Id) end. +unknown(true, {_, _, R}) -> + {unknown, R}; +unknown(false, Id) -> + Id. + %% No E-bit: can't be 3xxx. is_result(RC, false, _Dict0) -> RC < 3000 orelse 4000 =< RC; @@ -1142,7 +1177,11 @@ incr(TPid, Counter) -> %% applications MUST include either one Result-Code AVP or one %% Experimental-Result AVP. -rc_counter(Dict, recv, #diameter_packet{header = H, avps = As}) -> +rc_counter(Dict, Dir, #diameter_packet{header = H, + avps = As, + msg = Msg}) + when Dir == recv; %% decoded incoming + Msg == undefined -> %% relayed outgoing rc_counter(Dict, [H|As]); rc_counter(Dict, _, #diameter_packet{msg = Msg}) -> @@ -1434,12 +1473,12 @@ fold_record(Rec, R) -> %% send_R/6 send_R(Pkt0, - {TPid, Caps, #diameter_app{dictionary = Dict} = App}, + {TPid, Caps, #diameter_app{dictionary = AppDict} = App}, Opts, {Pid, Ref}, SvcName, Fs) -> - Pkt = encode(Dict, TPid, Pkt0, Fs), + Pkt = encode(AppDict, TPid, Pkt0, Fs), #options{timeout = Timeout} = Opts, @@ -1452,7 +1491,7 @@ send_R(Pkt0, packet = Pkt0}, try - incr(send, Pkt, TPid, Dict), + incr(send, Pkt, TPid, AppDict), TRef = send_request(TPid, Pkt, Req, SvcName, Timeout), Pid ! Ref, %% tell caller a send has been attempted handle_answer(SvcName, @@ -1492,10 +1531,10 @@ handle_answer(SvcName, id = Id} = App, {answer, Req, Dict0, Pkt}) -> - Dict = dict(AppDict, Dict0, Pkt), - handle_A(errors(Id, diameter_codec:decode({Dict, AppDict}, Pkt)), + MsgDict = msg_dict(AppDict, Dict0, Pkt), + handle_A(errors(Id, diameter_codec:decode({MsgDict, AppDict}, Pkt)), SvcName, - Dict, + MsgDict, Dict0, App, Req). @@ -1765,19 +1804,19 @@ retransmit(T, {_, _, App}, _, _, _, _) -> ?ERROR({invalid_return, T, prepare_retransmit, App}). resend_request(Pkt0, - {TPid, Caps, #diameter_app{dictionary = Dict}}, + {TPid, Caps, #diameter_app{dictionary = AppDict}}, Req0, SvcName, Tmo, Fs) -> - Pkt = encode(Dict, TPid, Pkt0, Fs), + Pkt = encode(AppDict, TPid, Pkt0, Fs), Req = Req0#request{transport = TPid, packet = Pkt0, caps = Caps}, ?LOG(retransmission, Pkt#diameter_packet.header), - incr(TPid, {msg_id(Pkt, Dict), send, retransmission}), + incr(TPid, {msg_id(Pkt, AppDict), send, retransmission}), TRef = send_request(TPid, Pkt, Req, SvcName, Tmo), {TRef, Req}. @@ -1887,7 +1926,7 @@ get_avp(Dict, Name, [#diameter_header{} | Avps]) -> find_avp(Code, VId, Avps) of A -> - avp_decode(Dict, Name, ungroup(A)) + (avp_decode(Dict, Name, ungroup(A)))#diameter_avp{name = Name} catch error: _ -> undefined diff --git a/lib/diameter/src/diameter.appup.src b/lib/diameter/src/diameter.appup.src index 0ef0fd35a9..b89859ed24 100644 --- a/lib/diameter/src/diameter.appup.src +++ b/lib/diameter/src/diameter.appup.src @@ -65,11 +65,14 @@ {update, diameter_sup, supervisor}]}, {"1.9", [{load_module, diameter_codec}, %% 17.5 {load_module, diameter_traffic}, + {load_module, diameter_sctp}, {load_module, diameter_gen_base_rfc6733}, {load_module, diameter_gen_acct_rfc6733}, {load_module, diameter_gen_base_rfc3588}, {load_module, diameter_gen_base_accounting}, - {load_module, diameter_gen_relay}]} + {load_module, diameter_gen_relay}]}, + {"1.9.1", [{load_module, diameter_traffic}, %% 17.5.3 + {load_module, diameter_sctp}]} ], [ {"0.9", [{restart_application, diameter}]}, @@ -120,7 +123,10 @@ {load_module, diameter_gen_base_rfc3588}, {load_module, diameter_gen_acct_rfc6733}, {load_module, diameter_gen_base_rfc6733}, + {load_module, diameter_sctp}, {load_module, diameter_traffic}, - {load_module, diameter_codec}]} + {load_module, diameter_codec}]}, + {"1.9.1", [{load_module, diameter_sctp}, + {load_module, diameter_traffic}]} ] }. diff --git a/lib/diameter/src/transport/diameter_sctp.erl b/lib/diameter/src/transport/diameter_sctp.erl index 2c8d6f0a14..f80de0a816 100644 --- a/lib/diameter/src/transport/diameter_sctp.erl +++ b/lib/diameter/src/transport/diameter_sctp.erl @@ -223,9 +223,9 @@ init(T) -> i({listen, Ref, {Opts, Addrs}}) -> {[Matches], Rest} = proplists:split(Opts, [accept]), {LAs, Sock} = AS = open(Addrs, Rest, ?DEFAULT_PORT), - proc_lib:init_ack({ok, self(), LAs}), ok = gen_sctp:listen(Sock, true), true = diameter_reg:add_new({?MODULE, listener, {Ref, AS}}), + proc_lib:init_ack({ok, self(), LAs}), start_timer(#listener{ref = Ref, socket = Sock, accept = accept(Matches)}); diff --git a/lib/diameter/test/diameter_3xxx_SUITE.erl b/lib/diameter/test/diameter_3xxx_SUITE.erl index 44fc3a60aa..44cb0cc484 100644 --- a/lib/diameter/test/diameter_3xxx_SUITE.erl +++ b/lib/diameter/test/diameter_3xxx_SUITE.erl @@ -195,13 +195,13 @@ counters(_, _, _, _) -> stats(?CLIENT, E, rfc3588, L) when E == answer; E == answer_3xxx -> - [{{unknown,recv},2}, + [{{{unknown,0},recv},2}, {{{0,257,0},recv},1}, {{{0,257,1},send},1}, {{{0,275,0},recv},6}, {{{0,275,1},send},10}, - {{unknown,recv,{'Result-Code',3001}},1}, - {{unknown,recv,{'Result-Code',3007}},1}, + {{{unknown,0},recv,{'Result-Code',3001}},1}, + {{{unknown,0},recv,{'Result-Code',3007}},1}, {{{0,257,0},recv,{'Result-Code',2001}},1}, {{{0,275,0},recv,{'Result-Code',2001}},1}, {{{0,275,0},recv,{'Result-Code',3008}},2}, @@ -213,15 +213,15 @@ stats(?CLIENT, E, rfc3588, L) stats(?SERVER, E, rfc3588, L) when E == answer; E == answer_3xxx -> - [{{unknown,recv},1}, - {{unknown,send},2}, + [{{{unknown,0},send},2}, + {{{unknown,1},recv},1}, {{{0,257,0},send},1}, {{{0,257,1},recv},1}, {{{0,275,0},send},6}, {{{0,275,1},recv},8}, - {{unknown,recv,error},1}, - {{unknown,send,{'Result-Code',3001}},1}, - {{unknown,send,{'Result-Code',3007}},1}, + {{{unknown,0},send,{'Result-Code',3001}},1}, + {{{unknown,0},send,{'Result-Code',3007}},1}, + {{{unknown,1},recv,error},1}, {{{0,257,0},send,{'Result-Code',2001}},1}, {{{0,275,0},send,{'Result-Code',2001}},1}, {{{0,275,0},send,{'Result-Code',3008}},2}, @@ -232,13 +232,13 @@ stats(?SERVER, E, rfc3588, L) = L; stats(?CLIENT, answer, rfc6733, L) -> - [{{unknown,recv},2}, + [{{{unknown,0},recv},2}, {{{0,257,0},recv},1}, {{{0,257,1},send},1}, {{{0,275,0},recv},8}, {{{0,275,1},send},10}, - {{unknown,recv,{'Result-Code',3001}},1}, - {{unknown,recv,{'Result-Code',3007}},1}, + {{{unknown,0},recv,{'Result-Code',3001}},1}, + {{{unknown,0},recv,{'Result-Code',3007}},1}, {{{0,257,0},recv,{'Result-Code',2001}},1}, {{{0,275,0},recv,{'Result-Code',3008}},2}, {{{0,275,0},recv,{'Result-Code',3999}},1}, @@ -248,15 +248,15 @@ stats(?CLIENT, answer, rfc6733, L) -> = L; stats(?SERVER, answer, rfc6733, L) -> - [{{unknown,recv},1}, - {{unknown,send},2}, + [{{{unknown,0},send},2}, + {{{unknown,1},recv},1}, {{{0,257,0},send},1}, {{{0,257,1},recv},1}, {{{0,275,0},send},8}, {{{0,275,1},recv},8}, - {{unknown,recv,error},1}, - {{unknown,send,{'Result-Code',3001}},1}, - {{unknown,send,{'Result-Code',3007}},1}, + {{{unknown,0},send,{'Result-Code',3001}},1}, + {{{unknown,0},send,{'Result-Code',3007}},1}, + {{{unknown,1},recv,error},1}, {{{0,257,0},send,{'Result-Code',2001}},1}, {{{0,275,0},send,{'Result-Code',3008}},2}, {{{0,275,0},send,{'Result-Code',3999}},1}, @@ -267,13 +267,13 @@ stats(?SERVER, answer, rfc6733, L) -> = L; stats(?CLIENT, answer_3xxx, rfc6733, L) -> - [{{unknown,recv},2}, + [{{{unknown,0},recv},2}, {{{0,257,0},recv},1}, {{{0,257,1},send},1}, {{{0,275,0},recv},8}, {{{0,275,1},send},10}, - {{unknown,recv,{'Result-Code',3001}},1}, - {{unknown,recv,{'Result-Code',3007}},1}, + {{{unknown,0},recv,{'Result-Code',3001}},1}, + {{{unknown,0},recv,{'Result-Code',3007}},1}, {{{0,257,0},recv,{'Result-Code',2001}},1}, {{{0,275,0},recv,{'Result-Code',2001}},1}, {{{0,275,0},recv,{'Result-Code',3008}},2}, @@ -284,15 +284,15 @@ stats(?CLIENT, answer_3xxx, rfc6733, L) -> = L; stats(?SERVER, answer_3xxx, rfc6733, L) -> - [{{unknown,recv},1}, - {{unknown,send},2}, + [{{{unknown,0},send},2}, + {{{unknown,1},recv},1}, {{{0,257,0},send},1}, {{{0,257,1},recv},1}, {{{0,275,0},send},8}, {{{0,275,1},recv},8}, - {{unknown,recv,error},1}, - {{unknown,send,{'Result-Code',3001}},1}, - {{unknown,send,{'Result-Code',3007}},1}, + {{{unknown,0},send,{'Result-Code',3001}},1}, + {{{unknown,0},send,{'Result-Code',3007}},1}, + {{{unknown,1},recv,error},1}, {{{0,257,0},send,{'Result-Code',2001}},1}, {{{0,275,0},send,{'Result-Code',2001}},1}, {{{0,275,0},send,{'Result-Code',3008}},2}, @@ -304,12 +304,12 @@ stats(?SERVER, answer_3xxx, rfc6733, L) -> = L; stats(?CLIENT, callback, rfc3588, L) -> - [{{unknown,recv},1}, + [{{{unknown,0},recv},1}, {{{0,257,0},recv},1}, {{{0,257,1},send},1}, {{{0,275,0},recv},6}, {{{0,275,1},send},10}, - {{unknown,recv,{'Result-Code',3007}},1}, + {{{unknown,0},recv,{'Result-Code',3007}},1}, {{{0,257,0},recv,{'Result-Code',2001}},1}, {{{0,275,0},recv,{'Result-Code',2001}},2}, {{{0,275,0},recv,{'Result-Code',3999}},1}, @@ -318,14 +318,14 @@ stats(?CLIENT, callback, rfc3588, L) -> = L; stats(?SERVER, callback, rfc3588, L) -> - [{{unknown,recv},1}, - {{unknown,send},1}, + [{{{unknown,0},send},1}, + {{{unknown,1},recv},1}, {{{0,257,0},send},1}, {{{0,257,1},recv},1}, {{{0,275,0},send},6}, {{{0,275,1},recv},8}, - {{unknown,recv,error},1}, - {{unknown,send,{'Result-Code',3007}},1}, + {{{unknown,0},send,{'Result-Code',3007}},1}, + {{{unknown,1},recv,error},1}, {{{0,257,0},send,{'Result-Code',2001}},1}, {{{0,275,0},send,{'Result-Code',2001}},2}, {{{0,275,0},send,{'Result-Code',3999}},1}, @@ -335,12 +335,12 @@ stats(?SERVER, callback, rfc3588, L) -> = L; stats(?CLIENT, callback, rfc6733, L) -> - [{{unknown,recv},1}, + [{{{unknown,0},recv},1}, {{{0,257,0},recv},1}, {{{0,257,1},send},1}, {{{0,275,0},recv},8}, {{{0,275,1},send},10}, - {{unknown,recv,{'Result-Code',3007}},1}, + {{{unknown,0},recv,{'Result-Code',3007}},1}, {{{0,257,0},recv,{'Result-Code',2001}},1}, {{{0,275,0},recv,{'Result-Code',2001}},2}, {{{0,275,0},recv,{'Result-Code',3999}},1}, @@ -350,14 +350,14 @@ stats(?CLIENT, callback, rfc6733, L) -> = L; stats(?SERVER, callback, rfc6733, L) -> - [{{unknown,recv},1}, - {{unknown,send},1}, + [{{{unknown,0},send},1}, + {{{unknown,1},recv},1}, {{{0,257,0},send},1}, {{{0,257,1},recv},1}, {{{0,275,0},send},8}, {{{0,275,1},recv},8}, - {{unknown,recv,error},1}, - {{unknown,send,{'Result-Code',3007}},1}, + {{{unknown,0},send,{'Result-Code',3007}},1}, + {{{unknown,1},recv,error},1}, {{{0,257,0},send,{'Result-Code',2001}},1}, {{{0,275,0},send,{'Result-Code',2001}},2}, {{{0,275,0},send,{'Result-Code',3999}},1}, diff --git a/lib/diameter/test/diameter_config_SUITE.erl b/lib/diameter/test/diameter_config_SUITE.erl index bbdf672291..4bcaa8119f 100644 --- a/lib/diameter/test/diameter_config_SUITE.erl +++ b/lib/diameter/test/diameter_config_SUITE.erl @@ -50,7 +50,7 @@ {request_errors, RE}, {call_mutates_state, C}]] || D <- [diameter_gen_base_rfc3588, diameter_gen_base_rfc6733], - M <- [?MODULE, [?MODULE, now()]], + M <- [?MODULE, [?MODULE, diameter_lib:now()]], A <- [0, common, make_ref()], S <- [[], make_ref()], AE <- [report, callback, discard], diff --git a/lib/diameter/test/diameter_relay_SUITE.erl b/lib/diameter/test/diameter_relay_SUITE.erl index 735a908d97..7142239bbb 100644 --- a/lib/diameter/test/diameter_relay_SUITE.erl +++ b/lib/diameter/test/diameter_relay_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2013. All Rights Reserved. +%% Copyright Ericsson AB 2010-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -49,6 +49,7 @@ send_timeout_1/1, send_timeout_2/1, info/1, + counters/1, disconnect/1, stop_services/1, stop/1]). @@ -120,6 +121,7 @@ all() -> start_services, connect, {group, all}, + counters, {group, all, [parallel]}, disconnect, stop_services, @@ -201,8 +203,8 @@ send3(_Config) -> send4(_Config) -> call(?SERVER4). -%% Send an ASR that loops between the relays and expect the loop to -%% be detected. +%% Send an ASR that loops between the relays (RELAY1 -> RELAY2 -> +%% RELAY1) and expect the loop to be detected. send_loop(_Config) -> Req = ['ASR', {'Destination-Realm', realm(?SERVER1)}, {'Destination-Host', ?SERVER1}, @@ -227,8 +229,103 @@ send_timeout(Tmo) -> call(Req, [{filter, realm}, {timeout, Tmo}]). info(_Config) -> + %% Wait for RELAY1 to have answered all requests, so that the + %% suite doesn't end before all answers are sent and counted. + receive after 6000 -> ok end, [] = ?util:info(). +counters(_Config) -> + [] = ?util:run([[fun counters/2, K, S] + || K <- [statistics, transport, connections], + S <- ?SERVICES]). + +counters(Key, Svc) -> + counters(Key, Svc, [_|_] = diameter:service_info(Svc, Key)). + +counters(statistics, Svc, Stats) -> + stats(Svc, lists:foldl(fun({K,N},D) -> orddict:update_counter(K, N, D) end, + orddict:new(), + lists:append([L || {P,L} <- Stats, is_pid(P)]))); + +counters(_, _, _) -> + todo. + +stats(?CLIENT, L) -> + [{{{0,257,0},recv},2}, %% CEA + {{{0,257,1},send},2}, %% CER + {{{0,258,0},recv},1}, %% RAA (send_timeout_1) + {{{0,258,1},send},2}, %% RAR (send_timeout_[12]) + {{{0,274,0},recv},1}, %% ASA (send_loop) + {{{0,274,1},send},1}, %% ASR (send_loop) + {{{0,275,0},recv},4}, %% STA (send[1-4]) + {{{0,275,1},send},4}, %% STR (send[1-4]) + {{{unknown,0},recv,discarded},1}, %% RAR (send_timeout_2) + {{{0,257,0},recv,{'Result-Code',2001}},2}, %% CEA + {{{0,258,0},recv,{'Result-Code',3002}},1}, %% RAA (send_timeout_1) + {{{0,274,0},recv,{'Result-Code',3005}},1}, %% ASA (send_loop) + {{{0,275,0},recv,{'Result-Code',2001}},4}] %% STA (send[1-4]) + = L; + +stats(S, L) + when S == ?SERVER1; + S == ?SERVER2; + S == ?SERVER3; + S == ?SERVER4 -> + [{{{0,257,0},send},1}, %% CEA + {{{0,257,1},recv},1}, %% CER + {{{0,275,0},send},1}, %% STA (send[1-4]) + {{{0,275,1},recv},1}, %% STR (send[1-4]) + {{{0,257,0},send,{'Result-Code',2001}},1}, %% CEA + {{{0,275,0},send,{'Result-Code',2001}},1}] %% STA (send[1-4]) + = L; + +stats(?RELAY1, L) -> + [{{{relay,0},recv},3}, %% STA x 2 (send[12]) + %% ASA (send_loop) + {{{relay,0},send},6}, %% STA x 2 (send[12]) + %% ASA x 2 (send_loop) + %% RAA x 2 (send_timeout_[12]) + {{{relay,1},recv},6}, %% STR x 2 (send[12]) + %% ASR x 2 (send_loop) + %% RAR x 2 (send_timeout_[12]) + {{{relay,1},send},5}, %% STR x 2 (send[12]) + %% ASR (send_loop) + %% RAR x 2 (send_timeout_[12]) + {{{0,257,0},recv},3}, %% CEA + {{{0,257,0},send},1}, %% " + {{{0,257,1},recv},1}, %% CER + {{{0,257,1},send},3}, %% " + {{{relay,0},recv,{'Result-Code',2001}},2}, %% STA x 2 (send[34]) + {{{relay,0},recv,{'Result-Code',3005}},1}, %% ASA (send_loop) + {{{relay,0},send,{'Result-Code',2001}},2}, %% STA x 2 (send[34]) + {{{relay,0},send,{'Result-Code',3002}},2}, %% RAA (send_timeout_[12]) + {{{relay,0},send,{'Result-Code',3005}},2}, %% ASA (send_loop) + {{{0,257,0},recv,{'Result-Code',2001}},3}, %% CEA + {{{0,257,0},send,{'Result-Code',2001}},1}] %% " + = L; + +stats(?RELAY2, L) -> + [{{{relay,0},recv},3}, %% STA x 2 (send[34]) + %% ASA (send_loop) + {{{relay,0},send},3}, %% STA x 2 (send[34]) + %% ASA (send_loop) + {{{relay,1},recv},5}, %% STR x 2 (send[34]) + %% RAR x 2 (send_timeout_[12]) + %% ASR (send_loop) + {{{relay,1},send},3}, %% STR x 2 (send[34]) + %% ASR (send_loop) + {{{0,257,0},recv},2}, %% CEA + {{{0,257,0},send},2}, %% " + {{{0,257,1},recv},2}, %% CER + {{{0,257,1},send},2}, %% " + {{{relay,0},recv,{'Result-Code',2001}},2}, %% STA x 2 (send[34]) + {{{relay,0},recv,{'Result-Code',3005}},1}, %% ASA (send_loop) + {{{relay,0},send,{'Result-Code',2001}},2}, %% STA x 2 (send[34]) + {{{relay,0},send,{'Result-Code',3005}},1}, %% ASA (send_loop) + {{{0,257,0},recv,{'Result-Code',2001}},2}, %% CEA + {{{0,257,0},send,{'Result-Code',2001}},2}] %% " + = L. + %% =========================================================================== realm(Host) -> @@ -303,18 +400,24 @@ handle_request(Pkt, OH, {_Ref, #diameter_caps{origin_host = {OH,_}} = Caps}) when OH /= ?CLIENT -> request(Pkt, Caps). -%% RELAY1 routes any ASR or RAR to RELAY2 ... +%% RELAY1 answers ACR after it's timed out at the client. +request(#diameter_packet{header = #diameter_header{cmd_code = 271}}, + #diameter_caps{origin_host = {?RELAY1, _}}) -> + receive after 1000 -> {answer_message, 3004} end; %% TOO_BUSY + +%% RELAY1 routes any ASR or RAR to RELAY2. request(#diameter_packet{header = #diameter_header{cmd_code = C}}, #diameter_caps{origin_host = {?RELAY1, _}}) when C == 274; %% ASR C == 258 -> %% RAR {relay, [{filter, {realm, realm(?RELAY2)}}]}; -%% ... which in turn routes it back. Expect diameter to either answer -%% either with DIAMETER_LOOP_DETECTED/DIAMETER_UNABLE_TO_COMPLY. +%% RELAY2 routes ASR back to RELAY1 to induce DIAMETER_LOOP_DETECTED. request(#diameter_packet{header = #diameter_header{cmd_code = 274}}, #diameter_caps{origin_host = {?RELAY2, _}}) -> {relay, [{filter, {host, ?RELAY1}}]}; + +%% RELAY2 discards RAR to induce DIAMETER_UNABLE_TO_DELIVER. request(#diameter_packet{header = #diameter_header{cmd_code = 258}}, #diameter_caps{origin_host = {?RELAY2, _}}) -> discard; diff --git a/lib/diameter/test/diameter_tls_SUITE.erl b/lib/diameter/test/diameter_tls_SUITE.erl index 55565692ec..e5bbda9c91 100644 --- a/lib/diameter/test/diameter_tls_SUITE.erl +++ b/lib/diameter/test/diameter_tls_SUITE.erl @@ -319,19 +319,19 @@ make_cert(Dir, Base) -> make_cert(Dir, Base ++ "_key.pem", Base ++ "_ca.pem"). make_cert(Dir, Keyfile, Certfile) -> - [K,C] = Paths = [filename:join([Dir, F]) || F <- [Keyfile, Certfile]], + [KP,CP] = [filename:join([Dir, F]) || F <- [Keyfile, Certfile]], - KCmd = join(["openssl genrsa -out", K, "2048"]), - CCmd = join(["openssl req -new -x509 -key", K, "-out", C, "-days 7", - "-subj /C=SE/ST=./L=Stockholm/CN=www.erlang.org"]), + KC = join(["openssl genrsa -out", KP, "2048"]), + CC = join(["openssl req -new -x509 -key", KP, "-out", CP, "-days 7", + "-subj /C=SE/ST=./L=Stockholm/CN=www.erlang.org"]), %% Hope for the best and only check that files are written. - os:cmd(KCmd), - os:cmd(CCmd), + [{_, _, {ok,_}},{_, _, {ok,_}}] + = [{P,O,T} || {P,C} <- [{KP,KC}, {CP,CC}], + O <- [os:cmd(C)], + T <- [file:read_file_info(P)]], - [_,_] = [T || P <- Paths, {ok, T} <- [file:read_file_info(P)]], - - {K,C}. + {KP,CP}. join(Strs) -> string:join(Strs, " "). diff --git a/lib/diameter/test/diameter_traffic_SUITE.erl b/lib/diameter/test/diameter_traffic_SUITE.erl index 7ff6ba7ab9..17faf30a9b 100644 --- a/lib/diameter/test/diameter_traffic_SUITE.erl +++ b/lib/diameter/test/diameter_traffic_SUITE.erl @@ -145,8 +145,12 @@ %% them as binary. -define(STRING_DECODES, [true, false]). +%% Which transport protocol to use. +-define(TRANSPORTS, [tcp, sctp]). + -record(group, - {client_service, + {transport, + client_service, client_encoding, client_dict0, client_strings, @@ -234,19 +238,20 @@ %% =========================================================================== suite() -> - [{timetrap, {seconds, 60}}]. + [{timetrap, {seconds, 10}}]. all() -> [start, result_codes, {group, traffic}, outstanding, empty, stop]. groups() -> Ts = tc(), + Sctp = ?util:have_sctp(), [{?util:name([R,D,A,C]), [parallel], Ts} || R <- ?ENCODINGS, D <- ?RFCS, A <- ?ENCODINGS, C <- ?CONTAINERS] ++ - [{?util:name([R,D,A,C,SD,CD]), + [{?util:name([T,R,D,A,C,SD,CD]), [], [start_services, add_transports, @@ -254,15 +259,19 @@ groups() -> {group, ?util:name([R,D,A,C])}, remove_transports, stop_services]} - || R <- ?ENCODINGS, + || T <- ?TRANSPORTS, + T /= sctp orelse Sctp, + R <- ?ENCODINGS, D <- ?RFCS, A <- ?ENCODINGS, C <- ?CONTAINERS, SD <- ?STRING_DECODES, CD <- ?STRING_DECODES] ++ - [{traffic, [parallel], [{group, ?util:name([R,D,A,C,SD,CD])} - || R <- ?ENCODINGS, + [{traffic, [parallel], [{group, ?util:name([T,R,D,A,C,SD,CD])} + || T <- ?TRANSPORTS, + T /= sctp orelse Sctp, + R <- ?ENCODINGS, D <- ?RFCS, A <- ?ENCODINGS, C <- ?CONTAINERS, @@ -271,8 +280,9 @@ groups() -> init_per_group(Name, Config) -> case ?util:name(Name) of - [R,D,A,C,SD,CD] -> - G = #group{client_service = [$C|?util:unique_string()], + [T,R,D,A,C,SD,CD] -> + G = #group{transport = T, + client_service = [$C|?util:unique_string()], client_encoding = R, client_dict0 = dict0(D), client_strings = CD, @@ -288,8 +298,18 @@ init_per_group(Name, Config) -> end_per_group(_, _) -> ok. +%% Skip testcases that can reasonably fail under SCTP. init_per_testcase(Name, Config) -> - [{testcase, Name} | Config]. + case [skip || #group{transport = sctp} + <- [proplists:get_value(group, Config)], + send_maxlen == Name + orelse send_long == Name] + of + [skip] -> + {skip, sctp}; + [] -> + [{testcase, Name} | Config] + end. end_per_testcase(_, _) -> ok. @@ -367,16 +387,18 @@ start_services(Config) -> | ?SERVICE(CN, CD)]). add_transports(Config) -> - #group{client_service = CN, + #group{transport = T, + client_service = CN, server_service = SN} = group(Config), LRef = ?util:listen(SN, - tcp, + T, [{capabilities_cb, fun capx/2}, + {pool_size, 8}, {spawn_opt, [{min_heap_size, 8096}]}, {applications, apps(rfc3588)}]), Cs = [?util:connect(CN, - tcp, + T, LRef, [{id, Id}, {capabilities, [{'Origin-State-Id', origin(Id)}]}, diff --git a/lib/diameter/test/diameter_transport_SUITE.erl b/lib/diameter/test/diameter_transport_SUITE.erl index f098851bea..78bddbd1cf 100644 --- a/lib/diameter/test/diameter_transport_SUITE.erl +++ b/lib/diameter/test/diameter_transport_SUITE.erl @@ -64,7 +64,7 @@ = #diameter_caps{host_ip_address = Addrs}}). -%% The term we register after open a listening port with gen_tcp. +%% The term we register after open a listening port with gen_{tcp,sctp}. -define(TEST_LISTENER(Ref, PortNr), {?MODULE, listen, Ref, PortNr}). @@ -85,7 +85,7 @@ %% =========================================================================== suite() -> - [{timetrap, {minutes, 2}}]. + [{timetrap, {seconds, 15}}]. all() -> [start, @@ -401,12 +401,13 @@ gen_listen(tcp) -> %% gen_accept/2 gen_accept(sctp, Sock) -> - Assoc = ?RECV(?SCTP(Sock, {_, #sctp_assoc_change{state = comm_up, - outbound_streams = O, - inbound_streams = I, - assoc_id = A}}), - {O, I, A}), - putr(assoc, Assoc), + #sctp_assoc_change{state = comm_up, + outbound_streams = OS, + inbound_streams = IS, + assoc_id = Id} + = ?RECV(?SCTP(Sock, {_, #sctp_assoc_change{} = S}), S), + + putr(assoc, {OS, IS, Id}), {ok, Sock}; gen_accept(tcp, LSock) -> gen_tcp:accept(LSock). diff --git a/lib/diameter/test/diameter_util.erl b/lib/diameter/test/diameter_util.erl index c496876ee1..df7d268429 100644 --- a/lib/diameter/test/diameter_util.erl +++ b/lib/diameter/test/diameter_util.erl @@ -204,13 +204,14 @@ seed() -> %% unique_string/0 unique_string() -> - us(diameter_lib:now()). - -us({M,S,U}) -> - tl(lists:append(["-" ++ integer_to_list(N) || N <- [M,S,U]])); - -us(MonoT) -> - integer_to_list(MonoT). + try erlang:unique_integer() of + N -> + integer_to_list(N) + catch + error: undef -> %% OTP < 18 + {M,S,U} = timestamp(), + tl(lists:append(["-" ++ integer_to_list(N) || N <- [M,S,U]])) + end. %% --------------------------------------------------------------------------- %% have_sctp/0 diff --git a/lib/diameter/test/diameter_watchdog_SUITE.erl b/lib/diameter/test/diameter_watchdog_SUITE.erl index 5a3ff2c92f..f39e12686e 100644 --- a/lib/diameter/test/diameter_watchdog_SUITE.erl +++ b/lib/diameter/test/diameter_watchdog_SUITE.erl @@ -673,8 +673,7 @@ jitter(T,D) -> %% Generate a unique hostname for the faked peer. hostname() -> - {M,S,U} = diameter_util:timestamp(), - lists:flatten(io_lib:format("~p-~p-~p", [M,S,U])). + ?util:unique_string(). putr(Key, Val) -> put({?MODULE, Key}, Val). diff --git a/lib/diameter/vsn.mk b/lib/diameter/vsn.mk index db7f72c44e..c278e74dca 100644 --- a/lib/diameter/vsn.mk +++ b/lib/diameter/vsn.mk @@ -16,5 +16,5 @@ # %CopyrightEnd% APPLICATION = diameter -DIAMETER_VSN = 1.9.1 +DIAMETER_VSN = 1.9.2 APP_VSN = $(APPLICATION)-$(DIAMETER_VSN)$(PRE_VSN) diff --git a/lib/edoc/doc/src/notes.xml b/lib/edoc/doc/src/notes.xml index 48bb415ab3..e350adb540 100644 --- a/lib/edoc/doc/src/notes.xml +++ b/lib/edoc/doc/src/notes.xml @@ -31,21 +31,6 @@ <p>This document describes the changes made to the EDoc application.</p> -<section><title>Edoc 0.7.17</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Remove functionality related to packages</p> - <p> - Own Id: OTP-12431</p> - </item> - </list> - </section> - -</section> - <section><title>Edoc 0.7.16</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/edoc/src/edoc.app.src b/lib/edoc/src/edoc.app.src index 9e1155d3e8..e4b9040c78 100644 --- a/lib/edoc/src/edoc.app.src +++ b/lib/edoc/src/edoc.app.src @@ -23,5 +23,5 @@ {registered,[]}, {applications, [compiler,kernel,stdlib,syntax_tools]}, {env, []}, - {runtime_dependencies, ["xmerl-1.3.7","syntax_tools-1.6.14","stdlib-2.0", + {runtime_dependencies, ["xmerl-1.3.7","syntax_tools-1.6.14","stdlib-2.5", "kernel-3.0","inets-5.10","erts-6.0"]}]}. diff --git a/lib/edoc/src/edoc_layout.erl b/lib/edoc/src/edoc_layout.erl index 62d5eb9a18..b67ec31ae3 100644 --- a/lib/edoc/src/edoc_layout.erl +++ b/lib/edoc/src/edoc_layout.erl @@ -520,7 +520,7 @@ format_spec(Name, Type, Defs, #opts{pretty_printer = erl_pp}=Opts) -> {R, ".\n"} = etypef(L, O), [{pre, R}] catch _:_ -> - %% Example: "@spec ... -> record(a)" + %% Should not happen. format_spec(Name, Type, Defs, Opts#opts{pretty_printer=''}) end; format_spec(Sep, Type, Defs, _Opts) -> diff --git a/lib/edoc/src/edoc_specs.erl b/lib/edoc/src/edoc_specs.erl index 59f6cb8ddf..eb69058148 100644 --- a/lib/edoc/src/edoc_specs.erl +++ b/lib/edoc/src/edoc_specs.erl @@ -295,47 +295,54 @@ arg_name([A | As], Default) -> is_name(A) -> is_atom(A). -d2e({ann_type,_,[V, T0]}) -> +d2e(T) -> + d2e(T, 0). + +d2e({ann_type,_,[V, T0]}, Prec) -> %% Note: the -spec/-type syntax allows annotations everywhere, but %% EDoc does not. The fact that the annotation is added to the %% type here does not necessarily mean that it will be used by the %% layout module. - T = d2e(T0), - ?add_t_ann(T, element(3, V)); -d2e({remote_type,_,[{atom,_,M},{atom,_,F},Ts0]}) -> + {_L,P,R} = erl_parse:type_inop_prec('::'), + T1 = d2e(T0, R), + T = ?add_t_ann(T1, element(3, V)), + maybe_paren(P, Prec, T); % the only necessary call to maybe_paren() +d2e({remote_type,_,[{atom,_,M},{atom,_,F},Ts0]}, _Prec) -> Ts = d2e(Ts0), typevar_anno(#t_type{name = #t_name{module = M, name = F}, args = Ts}, Ts); -d2e({type,_,'fun',[{type,_,product,As0},Ran0]}) -> +d2e({type,_,'fun',[{type,_,product,As0},Ran0]}, _Prec) -> Ts = [Ran|As] = d2e([Ran0|As0]), %% Assume that the linter has checked type variables. typevar_anno(#t_fun{args = As, range = Ran}, Ts); -d2e({type,_,'fun',[A0={type,_,any},Ran0]}) -> +d2e({type,_,'fun',[A0={type,_,any},Ran0]}, _Prec) -> Ts = [A, Ran] = d2e([A0, Ran0]), typevar_anno(#t_fun{args = [A], range = Ran}, Ts); -d2e({type,_,'fun',[]}) -> +d2e({type,_,'fun',[]}, _Prec) -> #t_type{name = #t_name{name = function}, args = []}; -d2e({type,_,any}) -> +d2e({type,_,any}, _Prec) -> #t_var{name = '...'}; % Kludge... not a type variable! -d2e({type,_,nil,[]}) -> +d2e({type,_,nil,[]}, _Prec) -> #t_nil{}; -d2e({paren_type,_,[T]}) -> - #t_paren{type = d2e(T)}; -d2e({type,_,list,[T0]}) -> +d2e({paren_type,_,[T]}, Prec) -> + d2e(T, Prec); +d2e({type,_,list,[T0]}, _Prec) -> T = d2e(T0), typevar_anno(#t_list{type = T}, [T]); -d2e({type,_,nonempty_list,[T0]}) -> +d2e({type,_,nonempty_list,[T0]}, _Prec) -> T = d2e(T0), typevar_anno(#t_nonempty_list{type = T}, [T]); -d2e({type,_,bounded_fun,[T,Gs]}) -> +d2e({type,_,bounded_fun,[T,Gs]}, _Prec) -> [F0|Defs] = d2e([T|Gs]), F = ?set_t_ann(F0, lists:keydelete(type_variables, 1, ?t_ann(F0))), %% Assume that the linter has checked type variables. #t_spec{type = typevar_anno(F, [F0]), defs = Defs}; -d2e({type,_,range,[V1,V2]}) -> +d2e({type,_,range,[V1,V2]}, Prec) -> + {_L,P,_R} = erl_parse:type_inop_prec('..'), {integer,_,I1} = erl_eval:partial_eval(V1), {integer,_,I2} = erl_eval:partial_eval(V2), - #t_integer_range{from = I1, to = I2}; -d2e({type,_,constraint,[Sub,Ts0]}) -> + T0 = #t_integer_range{from = I1, to = I2}, + maybe_paren(P, Prec, T0); +d2e({type,_,constraint,[Sub,Ts0]}, _Prec) -> case {Sub,Ts0} of {{atom,_,is_subtype},[{var,_,N},T0]} -> Ts = [T] = d2e([T0]), @@ -347,50 +354,60 @@ d2e({type,_,constraint,[Sub,Ts0]}) -> _ -> throw_error(get_line(element(2, Sub)), "cannot handle guard", []) end; -d2e({type,_,union,Ts0}) -> - Ts = d2e(Ts0), - typevar_anno(#t_union{types = Ts}, Ts); -d2e({type,_,tuple,any}) -> +d2e({type,_,union,Ts0}, Prec) -> + {_L,P,R} = erl_parse:type_inop_prec('|'), + Ts = d2e(Ts0, R), + T = maybe_paren(P, Prec, #t_union{types = Ts}), + typevar_anno(T, Ts); +d2e({type,_,tuple,any}, _Prec) -> #t_type{name = #t_name{name = tuple}, args = []}; -d2e({type,_,binary,[Base,Unit]}) -> - #t_binary{base_size = element(3, Base), - unit_size = element(3, Unit)}; -d2e({type,_,map,any}) -> - #t_map{ types = []}; -d2e({type,_,map,Es}) -> - #t_map{ types = d2e(Es) }; -d2e({type,_,map_field_assoc,[K,V]}) -> - #t_map_field{ k_type = d2e(K), v_type=d2e(V) }; -d2e({type,_,map_field_exact,K,V}) -> - #t_map_field{ k_type = d2e(K), v_type=d2e(V) }; -d2e({type,_,tuple,Ts0}) -> +d2e({type,_,binary,[Base,Unit]}, _Prec) -> + {integer,_,B} = erl_eval:partial_eval(Base), + {integer,_,U} = erl_eval:partial_eval(Unit), + #t_binary{base_size = B, unit_size = U}; +d2e({type,_,map,any}, _Prec) -> + #t_map{types = []}; +d2e({type,_,map,Es}, _Prec) -> + #t_map{types = d2e(Es) }; +d2e({type,_,map_field_assoc,[K,V]}, Prec) -> + T = #t_map_field{k_type = d2e(K), v_type=d2e(V) }, + {P,_R} = erl_parse:type_preop_prec('#'), + maybe_paren(P, Prec, T); +d2e({type,_,map_field_exact,K,V}, Prec) -> + T = #t_map_field{k_type = d2e(K), v_type=d2e(V) }, + {P,_R} = erl_parse:type_preop_prec('#'), + maybe_paren(P, Prec, T); +d2e({type,_,tuple,Ts0}, _Prec) -> Ts = d2e(Ts0), typevar_anno(#t_tuple{types = Ts}, Ts); -d2e({type,_,record,[Name|Fs0]}) -> +d2e({type,_,record,[Name|Fs0]}, Prec) -> Atom = #t_atom{val = element(3, Name)}, Fs = d2e(Fs0), - typevar_anno(#t_record{name = Atom, fields = Fs}, Fs); -d2e({type,_,field_type,[Name,Type0]}) -> - Type = d2e(Type0), - typevar_anno(#t_field{name = #t_atom{val = element(3, Name)}, type = Type}, - [Type]); -d2e({typed_record_field,{record_field,L,Name},Type}) -> - d2e({type,L,field_type,[Name,Type]}); -d2e({typed_record_field,{record_field,L,Name,_E},Type}) -> - d2e({type,L,field_type,[Name,Type]}); -d2e({record_field,L,_Name,_E}=F) -> - d2e({typed_record_field,F,{type,L,any,[]}}); % Maybe skip... -d2e({record_field,L,_Name}=F) -> - d2e({typed_record_field,F,{type,L,any,[]}}); % Maybe skip... -d2e({type,_,Name,Types0}) -> + {P,_R} = erl_parse:type_preop_prec('#'), + T = maybe_paren(P, Prec, #t_record{name = Atom, fields = Fs}), + typevar_anno(T, Fs); +d2e({type,_,field_type,[Name,Type0]}, Prec) -> + {_L,P,R} = erl_parse:type_inop_prec('::'), + Type = maybe_paren(P, Prec, d2e(Type0, R)), + T = #t_field{name = #t_atom{val = element(3, Name)}, type = Type}, + typevar_anno(T, [Type]); +d2e({typed_record_field,{record_field,L,Name},Type}, Prec) -> + d2e({type,L,field_type,[Name,Type]}, Prec); +d2e({typed_record_field,{record_field,L,Name,_E},Type}, Prec) -> + d2e({type,L,field_type,[Name,Type]}, Prec); +d2e({record_field,L,_Name,_E}=F, Prec) -> + d2e({typed_record_field,F,{type,L,any,[]}}, Prec); % Maybe skip... +d2e({record_field,L,_Name}=F, Prec) -> + d2e({typed_record_field,F,{type,L,any,[]}}, Prec); % Maybe skip... +d2e({type,_,Name,Types0}, _Prec) -> Types = d2e(Types0), typevar_anno(#t_type{name = #t_name{name = Name}, args = Types}, Types); -d2e({user_type,_,Name,Types0}) -> +d2e({user_type,_,Name,Types0}, _Prec) -> Types = d2e(Types0), typevar_anno(#t_type{name = #t_name{name = Name}, args = Types}, Types); -d2e({var,_,'_'}) -> +d2e({var,_,'_'}, _Prec) -> #t_type{name = #t_name{name = ?TOP_TYPE}}; -d2e({var,_,TypeName}) -> +d2e({var,_,TypeName}, _Prec) -> TypeVar = ordsets:from_list([TypeName]), T = #t_var{name = TypeName}, %% Annotate type variables with the name of the variable. @@ -398,13 +415,13 @@ d2e({var,_,TypeName}) -> %% from using the argument name from the source or to invent a new name. T1 = ?add_t_ann(T, {type_variables, TypeVar}), ?add_t_ann(T1, TypeName); -d2e(L) when is_list(L) -> - [d2e(T) || T <- L]; -d2e({atom,_,A}) -> +d2e(L, Prec) when is_list(L) -> + [d2e(T, Prec) || T <- L]; +d2e({atom,_,A}, _Prec) -> #t_atom{val = A}; -d2e(undefined = U) -> % opaque +d2e(undefined = U, _Prec) -> % opaque U; -d2e(Expr) -> +d2e(Expr, _Prec) -> {integer,_,I} = erl_eval:partial_eval(Expr), #t_integer{val = I}. @@ -422,6 +439,11 @@ typevars(Ts) -> get_typevars(Ts) -> [Vs || T <- Ts, T =/= undefined, {type_variables, Vs} <- ?t_ann(T)]. +maybe_paren(P, Prec, T) when P < Prec -> + #t_paren{type = T}; +maybe_paren(_P, _Prec, T) -> + T. + -record(parms, {tab, warn, file, line}). %% Expands record references. Explicitly given record fields are kept, @@ -484,11 +506,11 @@ xrecs(#t_fun{args = Args0, range = Range0}=T, P) -> Args = xrecs(Args0, P), Range = xrecs(Range0, P), T#t_fun{args = Args, range = Range}; -xrecs(#t_map{ types = Ts0 }=T,P) -> +xrecs(#t_map{types = Ts0 }=T,P) -> Ts = xrecs(Ts0, P), - T#t_map{ types = Ts }; -xrecs(#t_map_field{ k_type=Kt, v_type=Vt}=T, P) -> - T#t_map_field{ k_type=xrecs(Kt,P), v_type=xrecs(Vt,P)}; + T#t_map{types = Ts }; +xrecs(#t_map_field{k_type=Kt, v_type=Vt}=T, P) -> + T#t_map_field{k_type=xrecs(Kt,P), v_type=xrecs(Vt,P)}; xrecs(#t_tuple{types = Types0}=T, P) -> Types = xrecs(Types0, P), T#t_tuple{types = Types}; diff --git a/lib/eldap/doc/src/eldap.xml b/lib/eldap/doc/src/eldap.xml index a6fad8a857..253ba7c2ff 100644 --- a/lib/eldap/doc/src/eldap.xml +++ b/lib/eldap/doc/src/eldap.xml @@ -121,7 +121,7 @@ filter() See present/1, substrings/2, <item>Any error responded from ssl:connect/3</item> </taglist> <p>The <c>Timeout</c> parameter is for the actual tls upgrade (phase 2) while the timeout in - <seealso marker="#open/2">erl_tar:open/2</seealso> is used for the initial negotiation about + <seealso marker="#open/2">eldap:open/2</seealso> is used for the initial negotiation about upgrade (phase 1). </p> </desc> @@ -298,7 +298,7 @@ filter() See present/1, substrings/2, search(Handle, [{base, "dc=example, dc=com"}, {filter, Filter}, {attributes, ["cn"]}]), </pre> <p>The <c>timeout</c> option in the <c>SearchOptions</c> is for the ldap server, while - the timeout in <seealso marker="#open/2">erl_tar:open/2</seealso> is used for each + the timeout in <seealso marker="#open/2">eldap:open/2</seealso> is used for each individual request in the search operation. </p> </desc> diff --git a/lib/eldap/doc/src/notes.xml b/lib/eldap/doc/src/notes.xml index a7aefefb4a..e76101c30e 100644 --- a/lib/eldap/doc/src/notes.xml +++ b/lib/eldap/doc/src/notes.xml @@ -30,22 +30,6 @@ </header> <p>This document describes the changes made to the Eldap application.</p> -<section><title>Eldap 1.2</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Support added for LDAP Password Modify Extended Operation - (RFC 3062). Thanks to danielwhite.</p> - <p> - Own Id: OTP-12282</p> - </item> - </list> - </section> - -</section> - <section><title>Eldap 1.1.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/erl_docgen/src/erl_docgen.app.src b/lib/erl_docgen/src/erl_docgen.app.src index e2830b2692..d63d880d89 100644 --- a/lib/erl_docgen/src/erl_docgen.app.src +++ b/lib/erl_docgen/src/erl_docgen.app.src @@ -9,6 +9,6 @@ {registered,[]}, {applications, [kernel,stdlib]}, {env, []}, - {runtime_dependencies, ["xmerl-1.3.7","stdlib-2.0","edoc-0.7.13","erts-6.0"]} + {runtime_dependencies, ["xmerl-1.3.7","stdlib-2.5","edoc-0.7.13","erts-6.0"]} ] }. diff --git a/lib/erl_docgen/vsn.mk b/lib/erl_docgen/vsn.mk index 5823c96253..2abd3d2b7e 100644 --- a/lib/erl_docgen/vsn.mk +++ b/lib/erl_docgen/vsn.mk @@ -1 +1 @@ -ERL_DOCGEN_VSN = 0.3.8 +ERL_DOCGEN_VSN = 0.4 diff --git a/lib/erl_interface/doc/src/ei.xml b/lib/erl_interface/doc/src/ei.xml index 90495eebd6..32e0e0e2d8 100644 --- a/lib/erl_interface/doc/src/ei.xml +++ b/lib/erl_interface/doc/src/ei.xml @@ -202,6 +202,9 @@ typedef enum { <desc> <p>Encodes a double-precision (64 bit) floating point number in the binary format.</p> + <p> + The function returns <c><![CDATA[-1]]></c> if the floating point number is not finite. + </p> </desc> </func> <func> diff --git a/lib/erl_interface/doc/src/erl_eterm.xml b/lib/erl_interface/doc/src/erl_eterm.xml index 429f77501c..2152192696 100644 --- a/lib/erl_interface/doc/src/erl_eterm.xml +++ b/lib/erl_interface/doc/src/erl_eterm.xml @@ -371,9 +371,11 @@ iohead ::= Binary <p><c><![CDATA[f]]></c> is a value to be converted to an Erlang float.</p> <p></p> <p>The function returns an Erlang float object with the value - specified in <c><![CDATA[f]]></c>.</p> + specified in <c><![CDATA[f]]></c> or <c><![CDATA[NULL]]></c> if + <c><![CDATA[f]]></c> is not finite. + </p> <p><c><![CDATA[ERL_FLOAT_VALUE(t)]]></c> can be used to retrieve the - value from an Erlang float.</p> + value from an Erlang float.</p> </desc> </func> <func> diff --git a/lib/erl_interface/src/decode/decode_big.c b/lib/erl_interface/src/decode/decode_big.c index 477880b331..016ed2eac2 100644 --- a/lib/erl_interface/src/decode/decode_big.c +++ b/lib/erl_interface/src/decode/decode_big.c @@ -150,27 +150,6 @@ int ei_big_comp(erlang_big *x, erlang_big *y) #define INLINED_FP_CONVERSION 1 #endif -#ifdef USE_ISINF_ISNAN /* simulate finite() */ -# define isfinite(f) (!isinf(f) && !isnan(f)) -# define HAVE_ISFINITE -#elif defined(__GNUC__) && defined(HAVE_FINITE) -/* We use finite in gcc as it emits assembler instead of - the function call that isfinite emits. The assembler is - significantly faster. */ -# ifdef isfinite -# undef isfinite -# endif -# define isfinite finite -# ifndef HAVE_ISFINITE -# define HAVE_ISFINITE -# endif -#elif defined(isfinite) && !defined(HAVE_ISFINITE) -# define HAVE_ISFINITE -#elif !defined(HAVE_ISFINITE) && defined(HAVE_FINITE) -# define isfinite finite -# define HAVE_ISFINITE -#endif - #ifdef NO_FPE_SIGNALS # define ERTS_FP_CHECK_INIT() do {} while (0) # define ERTS_FP_ERROR(f, Action) if (!isfinite(f)) { Action; } else {} diff --git a/lib/erl_interface/src/encode/encode_double.c b/lib/erl_interface/src/encode/encode_double.c index 148a49f73a..72a1c60808 100644 --- a/lib/erl_interface/src/encode/encode_double.c +++ b/lib/erl_interface/src/encode/encode_double.c @@ -21,12 +21,24 @@ #include "eidef.h" #include "eiext.h" #include "putget.h" +#if defined(HAVE_ISFINITE) +#include <math.h> +#endif int ei_encode_double(char *buf, int *index, double p) { char *s = buf + *index; char *s0 = s; + /* Erlang does not handle Inf and NaN, so we return an error rather + * than letting the Erlang VM complain about a bad external + * term. */ +#if defined(HAVE_ISFINITE) + if(!isfinite(p)) { + return -1; + } +#endif + if (!buf) s += 9; else { diff --git a/lib/erl_interface/src/legacy/erl_eterm.c b/lib/erl_interface/src/legacy/erl_eterm.c index 636d26b24b..66cca7decf 100644 --- a/lib/erl_interface/src/legacy/erl_eterm.c +++ b/lib/erl_interface/src/legacy/erl_eterm.c @@ -26,6 +26,9 @@ #include <stdlib.h> #include <string.h> #include <ctype.h> +#if defined(HAVE_ISFINITE) +#include <math.h> +#endif #include "ei_locking.h" #include "ei_resolve.h" @@ -125,6 +128,15 @@ ETERM *erl_mk_float (double d) { ETERM *ep; +#if defined(HAVE_ISFINITE) + /* Erlang does not handle Inf and NaN, so we return an error + * rather than letting the Erlang VM complain about a bad external + * term. */ + if(!isfinite(d)) { + return NULL; + } +#endif + ep = erl_alloc_eterm(ERL_FLOAT); ERL_COUNT(ep) = 1; ERL_FLOAT_VALUE(ep) = d; diff --git a/lib/erl_interface/src/misc/eidef.h b/lib/erl_interface/src/misc/eidef.h index bd3d0bf631..e0dc325b48 100644 --- a/lib/erl_interface/src/misc/eidef.h +++ b/lib/erl_interface/src/misc/eidef.h @@ -41,6 +41,27 @@ typedef int socklen_t; #endif +#ifdef USE_ISINF_ISNAN /* simulate finite() */ +# define isfinite(f) (!isinf(f) && !isnan(f)) +# define HAVE_ISFINITE +#elif defined(__GNUC__) && defined(HAVE_FINITE) +/* We use finite in gcc as it emits assembler instead of + the function call that isfinite emits. The assembler is + significantly faster. */ +# ifdef isfinite +# undef isfinite +# endif +# define isfinite finite +# ifndef HAVE_ISFINITE +# define HAVE_ISFINITE +# endif +#elif defined(isfinite) && !defined(HAVE_ISFINITE) +# define HAVE_ISFINITE +#elif !defined(HAVE_ISFINITE) && defined(HAVE_FINITE) +# define isfinite finite +# define HAVE_ISFINITE +#endif + typedef unsigned char uint8; /* FIXME use configure */ typedef unsigned short uint16; typedef unsigned int uint32; diff --git a/lib/erl_interface/test/ei_encode_SUITE.erl b/lib/erl_interface/test/ei_encode_SUITE.erl index 50dc8b6a3c..86e0d8cd08 100644 --- a/lib/erl_interface/test/ei_encode_SUITE.erl +++ b/lib/erl_interface/test/ei_encode_SUITE.erl @@ -174,7 +174,7 @@ test_ei_encode_ulonglong(Config) when is_list(Config) -> %% ######################################################################## %% -%% A "character" for us is an 8 bit integer, alwasy positive, i.e. +%% A "character" for us is an 8 bit integer, always positive, i.e. %% it is unsigned. %% FIXME maybe the API should change to use "unsigned char" to be clear?! diff --git a/lib/et/src/Makefile b/lib/et/src/Makefile index 377e593712..b6873371ed 100644 --- a/lib/et/src/Makefile +++ b/lib/et/src/Makefile @@ -65,7 +65,7 @@ APPUP_TARGET = $(EBIN)/$(APPUP_FILE) # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- -ERL_COMPILE_FLAGS += -pa $(ERL_TOP)/lib/et/ebin -I../include +ERL_COMPILE_FLAGS += -pa $(ERL_TOP)/lib/et/ebin -I../include -Werror # ---------------------------------------------------- # Special Build Targets diff --git a/lib/et/src/et_collector.erl b/lib/et/src/et_collector.erl index e05c67be60..1f60dee8ca 100644 --- a/lib/et/src/et_collector.erl +++ b/lib/et/src/et_collector.erl @@ -64,6 +64,8 @@ -export([init/1,terminate/2, code_change/3, handle_call/3, handle_cast/2, handle_info/2]). +-compile([{nowarn_deprecated_function,[{erlang,now,0}]}]). + -include("et_internal.hrl"). -include("../include/et.hrl"). diff --git a/lib/et/src/et_selector.erl b/lib/et/src/et_selector.erl index c8e9c907b2..5497096377 100644 --- a/lib/et/src/et_selector.erl +++ b/lib/et/src/et_selector.erl @@ -28,6 +28,8 @@ parse_event/2 ]). +-compile([{nowarn_deprecated_function,[{erlang,now,0}]}]). + -include("../include/et.hrl"). %%---------------------------------------------------------------------- diff --git a/lib/eunit/doc/src/notes.xml b/lib/eunit/doc/src/notes.xml index 3c6daf142e..6b76e097b6 100644 --- a/lib/eunit/doc/src/notes.xml +++ b/lib/eunit/doc/src/notes.xml @@ -32,20 +32,6 @@ </header> <p>This document describes the changes made to the EUnit application.</p> -<section><title>Eunit 2.2.10</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p>The <c>eunit</c> application is now unicode safe.</p> - <p> - Own Id: OTP-11660</p> - </item> - </list> - </section> - -</section> - <section><title>Eunit 2.2.9</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/eunit/include/eunit.hrl b/lib/eunit/include/eunit.hrl index 53d291430d..88e9d6c19b 100644 --- a/lib/eunit/include/eunit.hrl +++ b/lib/eunit/include/eunit.hrl @@ -15,11 +15,14 @@ %% %% Copyright (C) 2004-2006 Mickaël Rémond, Richard Carlsson +-ifndef(EUNIT_HRL). +-define(EUNIT_HRL, true). + %% Including this file turns on testing and defines TEST, unless NOTEST %% is defined before the file is included. If both NOTEST and TEST are %% already defined, then TEST takes precedence, and NOTEST will become %% undefined. -%% +%% %% If NODEBUG is defined before this file is included, the debug macros %% are disabled, unless DEBUG is also defined, in which case NODEBUG %% will become undefined. NODEBUG also implies NOASSERT, unless testing @@ -31,14 +34,10 @@ %% even if NODEBUG is defined. If both ASSERT and NOASSERT are defined %% before the file is included, then ASSERT takes precedence, and NOASSERT %% will become undefined regardless of TEST. -%% +%% %% After including this file, EUNIT will be defined if and only if TEST %% is defined. --ifndef(EUNIT_HRL). --define(EUNIT_HRL, true). - - %% allow defining TEST to override NOTEST -ifdef(TEST). -undef(NOTEST). @@ -49,13 +48,6 @@ -undef(NODEBUG). -endif. -%% allow NODEBUG to imply NOASSERT, unless overridden below --ifdef(NODEBUG). --ifndef(NOASSERT). --define(NOASSERT, true). --endif. --endif. - %% note that the main switch used within this file is NOTEST; however, %% both TEST and EUNIT may be used to check whether testing is enabled -ifndef(NOTEST). @@ -70,10 +62,8 @@ -undef(EUNIT). -endif. -%% allow ASSERT to override NOASSERT (regardless of TEST/NOTEST) --ifdef(ASSERT). --undef(NOASSERT). --endif. +%% include the assert macros; ASSERT overrides NOASSERT if defined +-include_lib("stdlib/include/assert.hrl"). %% Parse transforms for automatic exporting/stripping of test functions. %% (Note that although automatic stripping is convenient, it will make @@ -91,7 +81,7 @@ %% All macros should be available even if testing is turned off, and %% should preferably not require EUnit to be present at runtime. -%% +%% %% We must use fun-call wrappers ((fun () -> ... end)()) to avoid %% exporting local variables, and furthermore we only use variable names %% prefixed with "__", that hopefully will not be bound outside the fun. @@ -128,211 +118,24 @@ current_function)))). -endif. -%% The plain assert macro should be defined to do nothing if this file -%% is included when debugging/testing is turned off. --ifdef(NOASSERT). --ifndef(assert). --define(assert(BoolExpr),ok). --endif. --else. -%% The assert macro is written the way it is so as not to cause warnings -%% for clauses that cannot match, even if the expression is a constant. --undef(assert). --define(assert(BoolExpr), - begin - ((fun () -> - case (BoolExpr) of - true -> ok; - __V -> erlang:error({assertion_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??BoolExpr)}, - {expected, true}, - {value, case __V of false -> __V; - _ -> {not_a_boolean,__V} - end}]}) - end - end)()) - end). --endif. --define(assertNot(BoolExpr), ?assert(not (BoolExpr))). +%% General test macros -define(_test(Expr), {?LINE, fun () -> (Expr) end}). - -define(_assert(BoolExpr), ?_test(?assert(BoolExpr))). - -define(_assertNot(BoolExpr), ?_assert(not (BoolExpr))). - -%% This is mostly a convenience which gives more detailed reports. -%% Note: Guard is a guarded pattern, and can not be used for value. --ifdef(NOASSERT). --define(assertMatch(Guard, Expr), ok). --else. --define(assertMatch(Guard, Expr), - begin - ((fun () -> - case (Expr) of - Guard -> ok; - __V -> erlang:error({assertMatch_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??Expr)}, - {pattern, (??Guard)}, - {value, __V}]}) - end - end)()) - end). --endif. -define(_assertMatch(Guard, Expr), ?_test(?assertMatch(Guard, Expr))). - -%% This is the inverse case of assertMatch, for convenience. --ifdef(NOASSERT). --define(assertNotMatch(Guard, Expr), ok). --else. --define(assertNotMatch(Guard, Expr), - begin - ((fun () -> - __V = (Expr), - case __V of - Guard -> erlang:error({assertNotMatch_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??Expr)}, - {pattern, (??Guard)}, - {value, __V}]}); - _ -> ok - end - end)()) - end). --endif. -define(_assertNotMatch(Guard, Expr), ?_test(?assertNotMatch(Guard, Expr))). - -%% This is a convenience macro which gives more detailed reports when -%% the expected LHS value is not a pattern, but a computed value --ifdef(NOASSERT). --define(assertEqual(Expect, Expr), ok). --else. --define(assertEqual(Expect, Expr), - begin - ((fun (__X) -> - case (Expr) of - __X -> ok; - __V -> erlang:error({assertEqual_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??Expr)}, - {expected, __X}, - {value, __V}]}) - end - end)(Expect)) - end). --endif. -define(_assertEqual(Expect, Expr), ?_test(?assertEqual(Expect, Expr))). - -%% This is the inverse case of assertEqual, for convenience. --ifdef(NOASSERT). --define(assertNotEqual(Unexpected, Expr), ok). --else. --define(assertNotEqual(Unexpected, Expr), - begin - ((fun (__X) -> - case (Expr) of - __X -> erlang:error({assertNotEqual_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??Expr)}, - {value, __X}]}); - _ -> ok - end - end)(Unexpected)) - end). --endif. -define(_assertNotEqual(Unexpected, Expr), ?_test(?assertNotEqual(Unexpected, Expr))). - -%% Note: Class and Term are patterns, and can not be used for value. -%% Term can be a guarded pattern, but Class cannot. --ifdef(NOASSERT). --define(assertException(Class, Term, Expr), ok). --else. --define(assertException(Class, Term, Expr), - begin - ((fun () -> - try (Expr) of - __V -> erlang:error({assertException_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??Expr)}, - {pattern, - "{ "++(??Class)++" , "++(??Term) - ++" , [...] }"}, - {unexpected_success, __V}]}) - catch - Class:Term -> ok; - __C:__T -> - erlang:error({assertException_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??Expr)}, - {pattern, - "{ "++(??Class)++" , "++(??Term) - ++" , [...] }"}, - {unexpected_exception, - {__C, __T, - erlang:get_stacktrace()}}]}) - end - end)()) - end). --endif. - --define(assertError(Term, Expr), ?assertException(error, Term, Expr)). --define(assertExit(Term, Expr), ?assertException(exit, Term, Expr)). --define(assertThrow(Term, Expr), ?assertException(throw, Term, Expr)). - -define(_assertException(Class, Term, Expr), ?_test(?assertException(Class, Term, Expr))). -define(_assertError(Term, Expr), ?_assertException(error, Term, Expr)). -define(_assertExit(Term, Expr), ?_assertException(exit, Term, Expr)). -define(_assertThrow(Term, Expr), ?_assertException(throw, Term, Expr)). - -%% This is the inverse case of assertException, for convenience. -%% Note: Class and Term are patterns, and can not be used for value. -%% Both Class and Term can be guarded patterns. --ifdef(NOASSERT). --define(assertNotException(Class, Term, Expr), ok). --else. --define(assertNotException(Class, Term, Expr), - begin - ((fun () -> - try (Expr) of - _ -> ok - catch - __C:__T -> - case __C of - Class -> - case __T of - Term -> - erlang:error({assertNotException_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {expression, (??Expr)}, - {pattern, - "{ "++(??Class)++" , " - ++(??Term)++" , [...] }"}, - {unexpected_exception, - {__C, __T, - erlang:get_stacktrace() - }}]}); - _ -> ok - end; - _ -> ok - end - end - end)()) - end). --endif. -define(_assertNotException(Class, Term, Expr), ?_test(?assertNotException(Class, Term, Expr))). +-define(_assertReceive(Guard, Expr), ?_test(?assertReceive(Guard, Expr))). %% Macros for running operating system commands. (Note that these %% require EUnit to be present at runtime, or at least eunit_lib.) @@ -364,18 +167,18 @@ -else. -define(assertCmdStatus(N, Cmd), begin - ((fun () -> - case ?_cmd_(Cmd) of - {(N), _} -> ok; - {__N, _} -> erlang:error({assertCmd_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {command, (Cmd)}, - {expected_status,(N)}, - {status,__N}]}) - end - end)()) - end). + ((fun () -> + case ?_cmd_(Cmd) of + {(N), _} -> ok; + {__N, _} -> erlang:error({assertCmd_failed, + [{module, ?MODULE}, + {line, ?LINE}, + {command, (Cmd)}, + {expected_status,(N)}, + {status,__N}]}) + end + end)()) + end). -endif. -define(assertCmd(Cmd), ?assertCmdStatus(0, Cmd)). @@ -384,17 +187,17 @@ -else. -define(assertCmdOutput(T, Cmd), begin - ((fun () -> - case ?_cmd_(Cmd) of - {_, (T)} -> ok; - {_, __T} -> erlang:error({assertCmdOutput_failed, - [{module, ?MODULE}, - {line, ?LINE}, - {command,(Cmd)}, - {expected_output,(T)}, - {output,__T}]}) - end - end)()) + ((fun () -> + case ?_cmd_(Cmd) of + {_, (T)} -> ok; + {_, __T} -> erlang:error({assertCmdOutput_failed, + [{module, ?MODULE}, + {line, ?LINE}, + {command,(Cmd)}, + {expected_output,(T)}, + {output,__T}]}) + end + end)()) end). -endif. @@ -439,5 +242,4 @@ end). -endif. - -endif. % EUNIT_HRL diff --git a/lib/eunit/src/Makefile b/lib/eunit/src/Makefile index 47aef104ff..86a6d8831e 100644 --- a/lib/eunit/src/Makefile +++ b/lib/eunit/src/Makefile @@ -24,7 +24,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/eunit-$(VSN) EBIN = ../ebin INCLUDE=../include -ERL_COMPILE_FLAGS += -pa $(EBIN) -I$(INCLUDE) +warn_unused_vars +nowarn_shadow_vars +warn_unused_import +warn_obsolete_guard +ERL_COMPILE_FLAGS += -pa $(EBIN) -pa ../../stdlib/ebin -I$(INCLUDE) +warn_unused_vars +nowarn_shadow_vars +warn_unused_import +warn_obsolete_guard PARSE_TRANSFORM = eunit_autoexport.erl diff --git a/lib/eunit/src/eunit.app.src b/lib/eunit/src/eunit.app.src index 7a3978e200..b4ff6c9242 100644 --- a/lib/eunit/src/eunit.app.src +++ b/lib/eunit/src/eunit.app.src @@ -19,4 +19,4 @@ {registered,[]}, {applications, [kernel,stdlib]}, {env, []}, - {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]}]}. + {runtime_dependencies, ["stdlib-2.5","kernel-3.0","erts-6.0"]}]}. diff --git a/lib/eunit/src/eunit_server.erl b/lib/eunit/src/eunit_server.erl index 2002930abb..387976eba1 100644 --- a/lib/eunit/src/eunit_server.erl +++ b/lib/eunit/src/eunit_server.erl @@ -200,7 +200,7 @@ server_command(From, stop, St) -> server(St#state{stopped = true}); server_command(From, {watch, Target, _Opts}, St) -> %% the code watcher is only started on demand - %% FIXME: this is disabled for now in the OTP distribution + %% TODO: this is disabled for now %%code_monitor:monitor(self()), %% TODO: propagate options to testing stage St1 = add_watch(Target, St), diff --git a/lib/eunit/src/eunit_surefire.erl b/lib/eunit/src/eunit_surefire.erl index d6684f33cb..f3e58a3d1c 100644 --- a/lib/eunit/src/eunit_surefire.erl +++ b/lib/eunit/src/eunit_surefire.erl @@ -203,10 +203,9 @@ handle_cancel(test, Data, St) -> testcases=[TestCase|TestSuite#testsuite.testcases] }, St#state{testsuites=store_suite(NewTestSuite, TestSuites)}. -format_name({Module, Function, Arity}, Line) -> - lists:flatten([atom_to_list(Module), ":", atom_to_list(Function), "/", - integer_to_list(Arity), "_", integer_to_list(Line)]). - +format_name({Module, Function, _Arity}, Line) -> + lists:flatten([atom_to_list(Module), ":", integer_to_list(Line), " ", + atom_to_list(Function)]). format_desc(undefined) -> ""; format_desc(Desc) when is_binary(Desc) -> @@ -335,12 +334,11 @@ write_testcase( FileDescriptor) -> DescriptionAttr = case Description of [] -> []; - _ -> [<<" description=\"">>, escape_attr(Description), <<"\"">>] + _ -> [<<" (">>, escape_attr(Description), <<")">>] end, StartTag = [ ?INDENT, <<"<testcase time=\"">>, format_time(Time), - <<"\" name=\"">>, escape_attr(Name), <<"\"">>, - DescriptionAttr], + <<"\" name=\"">>, escape_attr(Name), DescriptionAttr, <<"\"">>], ContentAndEndTag = case {Result, Output} of {ok, <<>>} -> [<<"/>">>, ?NEWLINE]; _ -> [<<">">>, ?NEWLINE, format_testcase_result(Result), format_testcase_output(Output), ?INDENT, <<"</testcase>">>, ?NEWLINE] diff --git a/lib/eunit/vsn.mk b/lib/eunit/vsn.mk index 8b489bdc04..b551ee6eb6 100644 --- a/lib/eunit/vsn.mk +++ b/lib/eunit/vsn.mk @@ -1 +1 @@ -EUNIT_VSN = 2.2.10 +EUNIT_VSN = 2.3 diff --git a/lib/hipe/cerl/cerl_pmatch.erl b/lib/hipe/cerl/cerl_pmatch.erl index 3bc93e80dd..4f04b0a7ed 100644 --- a/lib/hipe/cerl/cerl_pmatch.erl +++ b/lib/hipe/cerl/cerl_pmatch.erl @@ -31,7 +31,7 @@ -module(cerl_pmatch). --define(NO_UNUSED, true). +%%-define(NO_UNUSED, true). -export([clauses/2]). -ifndef(NO_UNUSED). @@ -59,6 +59,8 @@ %% @see transform/2 -ifndef(NO_UNUSED). +-spec core_transform(cerl:c_module(), [_]) -> cerl:c_module(). + core_transform(M, Opts) -> cerl:to_records(transform(cerl:from_records(M), Opts)). -endif. % NO_UNUSED @@ -76,6 +78,8 @@ core_transform(M, Opts) -> %% @see core_transform/2 -ifndef(NO_UNUSED). +-spec transform(cerl:cerl(), [_]) -> cerl:cerl(). + transform(M, _Opts) -> expr(M, env__empty()). -endif. % NO_UNUSED @@ -109,7 +113,7 @@ transform(M, _Opts) -> %% @see expr/2 %% @see transform/2 --spec clauses([cerl:cerl()], rec_env:environment()) -> +-spec clauses([cerl:cerl(),...], rec_env:environment()) -> {cerl:cerl(), [cerl:cerl()]}. clauses(Cs, Env) -> @@ -406,6 +410,8 @@ make_let(Vs, A, B) -> %% @see rec_env -ifndef(NO_UNUSED). +-spec expr(cerl:cerl(), rec_env:environment()) -> cerl:cerl(). + expr(E, Env) -> case cerl:type(E) of literal -> diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl index 5b1401b34a..ee77d65932 100644 --- a/lib/hipe/cerl/erl_bif_types.erl +++ b/lib/hipe/cerl/erl_bif_types.erl @@ -1113,8 +1113,8 @@ type(hipe_bifs, set_native_address, 3, Xs, Opaques) -> type(hipe_bifs, set_native_address_in_fe, 2, Xs, Opaques) -> strict(hipe_bifs, set_native_address_in_fe, 2, Xs, fun (_) -> t_atom('true') end, Opaques); -type(hipe_bifs, system_crc, 1, Xs, Opaques) -> - strict(hipe_bifs, system_crc, 1, Xs, fun (_) -> t_crc32() end, Opaques); +type(hipe_bifs, system_crc, 0, _, _Opaques) -> + t_crc32(); type(hipe_bifs, term_to_word, 1, Xs, Opaques) -> strict(hipe_bifs, term_to_word, 1, Xs, fun (_) -> t_integer() end, Opaques); @@ -2490,8 +2490,8 @@ arg_types(hipe_bifs, set_native_address, 3) -> [t_mfa(), t_integer(), t_boolean()]; arg_types(hipe_bifs, set_native_address_in_fe, 2) -> [t_integer(), t_integer()]; -arg_types(hipe_bifs, system_crc, 1) -> - [t_crc32()]; +arg_types(hipe_bifs, system_crc, 0) -> + []; arg_types(hipe_bifs, term_to_word, 1) -> [t_any()]; arg_types(hipe_bifs, update_code_size, 3) -> diff --git a/lib/hipe/cerl/erl_types.erl b/lib/hipe/cerl/erl_types.erl index 14335cf635..d39f350c0f 100644 --- a/lib/hipe/cerl/erl_types.erl +++ b/lib/hipe/cerl/erl_types.erl @@ -82,6 +82,8 @@ t_from_form/4, t_from_form/5, t_from_form_without_remote/2, + t_check_record_fields/4, + t_check_record_fields/5, t_from_range/2, t_from_range_unsafe/2, t_from_term/1, @@ -581,13 +583,11 @@ t_find_opaque_mismatch_list([H|T]) -> %% calling t_contains_opaque/2 is that the traversal stops when %% there is a mismatch which means that unknown opaque types "below" %% the mismatch are not found. -%% XXX. Returns one element even if both oparands contain opaque types. -%% XXX. Slow since t_inf() is called but the results are ignored. t_find_unknown_opaque(_T1, _T2, 'universe') -> []; t_find_unknown_opaque(T1, T2, Opaques) -> try t_inf(T1, T2, {match, Opaques}) of _ -> [] - catch throw:N when is_integer(N) -> [N] + catch throw:{pos, Ns} -> Ns end. -spec t_decorate_with_opaque(erl_type(), erl_type(), [erl_type()]) -> erl_type(). @@ -747,14 +747,15 @@ t_opaque_from_records(RecDict) -> end end, RecDict), OpaqueTypeDict = - dict:map(fun({opaque, Name, _Arity}, {{Module, _Form, ArgNames}, _Type}) -> + dict:map(fun({opaque, Name, _Arity}, + {{Module, _FileLine, _Form, ArgNames}, _Type}) -> %% Args = args_to_types(ArgNames), %% List = lists:zip(ArgNames, Args), %% TmpVarDict = dict:from_list(List), %% Rep = t_from_form(Type, RecDict, TmpVarDict), - Rep = t_none(), % not used for anything right now + Rep = t_any(), % not used for anything right now Args = [t_any() || _ <- ArgNames], - skip_opaque_alias(Rep, Module, Name, Args) + t_opaque(Module, Name, Args, Rep) end, OpaqueRecDict), [OpaqueType || {_Key, OpaqueType} <- dict:to_list(OpaqueTypeDict)]. @@ -2605,7 +2606,7 @@ inf_opaque1(T1, ?opaque(Set2)=T2, Pos, Opaques) -> end. inf_is_opaque_type(T, Pos, {match, Opaques}) -> - is_opaque_type(T, Opaques) orelse throw(Pos); + is_opaque_type(T, Opaques) orelse throw({pos, [Pos]}); inf_is_opaque_type(T, _Pos, Opaques) -> is_opaque_type(T, Opaques). @@ -2624,13 +2625,13 @@ combine(S, T1, T2) -> #opaque{mod = Mod1, name = Name1, args = Args1} = T1, #opaque{mod = Mod2, name = Name2, args = Args2} = T2, Comb1 = comb(Mod1, Name1, Args1, S, T1), - case is_same_type_name({Mod1, Name1, Args1}, {Mod2, Name2, Args2}) of + case is_compat_opaque_names({Mod1, Name1, Args1}, {Mod2, Name2, Args2}) of true -> Comb1; false -> Comb1 ++ comb(Mod2, Name2, Args2, S, T2) end. comb(Mod, Name, Args, S, T) -> - case is_same_name(Mod, Name, Args, S) of + case can_combine_opaque_names(Mod, Name, Args, S) of true -> ?opaque(Set) = S, Set; @@ -2638,17 +2639,17 @@ comb(Mod, Name, Args, S, T) -> [T#opaque{struct = S}] end. -is_same_name(Mod1, Name1, Args1, - ?opaque([#opaque{mod = Mod2, name = Name2, args = Args2}])) -> - is_same_type_name({Mod1, Name1, Args1}, {Mod2, Name2, Args2}); -is_same_name(_, _, _, _) -> false. +can_combine_opaque_names(Mod1, Name1, Args1, + ?opaque([#opaque{mod = Mod2, name = Name2, args = Args2}])) -> + is_compat_opaque_names({Mod1, Name1, Args1}, {Mod2, Name2, Args2}); +can_combine_opaque_names(_, _, _, _) -> false. %% Combining two lists this way can be very time consuming... %% Note: two parameterized opaque types are not the same if their %% actual parameters differ inf_opaque(Set1, Set2, Opaques) -> - List1 = inf_look_up(Set1, 1, Opaques), - List2 = inf_look_up(Set2, 2, Opaques), + List1 = inf_look_up(Set1, Opaques), + List2 = inf_look_up(Set2, Opaques), List0 = [combine(Inf, T1, T2) || {Is1, ModNameArgs1, T1} <- List1, {Is2, ModNameArgs2, T2} <- List2, @@ -2659,14 +2660,14 @@ inf_opaque(Set1, Set2, Opaques) -> sup_opaque(List). %% Optimization: do just one lookup. -inf_look_up(Set, Pos, Opaques) -> - [{Opaques =:= 'universe' orelse inf_is_opaque_type2(T, Pos, Opaques), +inf_look_up(Set, Opaques) -> + [{Opaques =:= 'universe' orelse inf_is_opaque_type2(T, Opaques), {M, N, Args}, T} || #opaque{mod = M, name = N, args = Args} = T <- set_to_list(Set)]. -inf_is_opaque_type2(T, Pos, {match, Opaques}) -> - is_opaque_type2(T, Opaques) orelse throw(Pos); -inf_is_opaque_type2(T, _Pos, Opaques) -> +inf_is_opaque_type2(T, {match, Opaques}) -> + is_opaque_type2(T, Opaques); +inf_is_opaque_type2(T, Opaques) -> is_opaque_type2(T, Opaques). inf_opaque_types(IsOpaque1, ModNameArgs1, T1, @@ -2675,18 +2676,33 @@ inf_opaque_types(IsOpaque1, ModNameArgs1, T1, #opaque{struct = S2}=T2, case Opaques =:= 'universe' orelse - is_same_type_name(ModNameArgs1, ModNameArgs2) + is_compat_opaque_names(ModNameArgs1, ModNameArgs2) of true -> t_inf(S1, S2, Opaques); false -> case {IsOpaque1, IsOpaque2} of - {true, true} -> t_inf(S1, S2, Opaques); - {true, false} -> t_inf(S1, ?opaque(set_singleton(T2)), Opaques); - {false, true} -> t_inf(?opaque(set_singleton(T1)), S2, Opaques); + {true, true} -> t_inf(S1, S2, Opaques); + {true, false} -> t_inf(S1, ?opaque(set_singleton(T2)), Opaques); + {false, true} -> t_inf(?opaque(set_singleton(T1)), S2, Opaques); + {false, false} when element(1, Opaques) =:= match -> + throw({pos, [1, 2]}); {false, false} -> t_none() end end. +is_compat_opaque_names(ModNameArgs, ModNameArgs) -> true; +is_compat_opaque_names({Mod,Name,Args1}, {Mod,Name,Args2}) -> + is_compat_args(Args1, Args2); +is_compat_opaque_names(_, _) -> false. + +is_compat_args([A1|Args1], [A2|Args2]) -> + is_compat_arg(A1, A2) andalso is_compat_args(Args1, Args2); +is_compat_args([], []) -> true; +is_compat_args(_, _) -> false. + +is_compat_arg(A, A) -> true; +is_compat_arg(A1, A2) -> t_is_any(A1) orelse t_is_any(A2). + -spec t_inf_lists([erl_type()], [erl_type()]) -> [erl_type()]. t_inf_lists(L1, L2) -> @@ -2785,7 +2801,7 @@ inf_union(U1, U2, Opaques) -> {Union, ThrowList3} = inf_union(U1, U2, 0, [], [], Opaques), ThrowList = lists:merge3(ThrowList1, ThrowList2, ThrowList3), case t_sup([O1, O2, Union]) of - ?none when ThrowList =/= [] -> throw(hd(ThrowList)); + ?none when ThrowList =/= [] -> throw({pos, lists:usort(ThrowList)}); Sup -> Sup end. @@ -2797,8 +2813,8 @@ inf_union_collect([E|L], Opaque, InfFun, InfList, ThrowList) -> try InfFun(E, Opaque)of Inf -> inf_union_collect(L, Opaque, InfFun, [Inf|InfList], ThrowList) - catch throw:N when is_integer(N) -> - inf_union_collect(L, Opaque, InfFun, InfList, [N|ThrowList]) + catch throw:{pos, Ns} -> + inf_union_collect(L, Opaque, InfFun, InfList, Ns ++ ThrowList) end. inf_union([?none|Left1], [?none|Left2], N, Acc, ThrowList, Opaques) -> @@ -2807,8 +2823,8 @@ inf_union([T1|Left1], [T2|Left2], N, Acc, ThrowList, Opaques) -> try t_inf(T1, T2, Opaques) of ?none -> inf_union(Left1, Left2, N, [?none|Acc], ThrowList, Opaques); T -> inf_union(Left1, Left2, N+1, [T|Acc], ThrowList, Opaques) - catch throw:N when is_integer(N) -> - inf_union(Left1, Left2, N, [?none|Acc], [N|ThrowList], Opaques) + catch throw:{pos, Ns} -> + inf_union(Left1, Left2, N, [?none|Acc], Ns ++ ThrowList, Opaques) end; inf_union([], [], N, Acc, ThrowList, _Opaques) -> if N =:= 0 -> {?none, ThrowList}; @@ -4189,7 +4205,7 @@ builtin_type(Name, Type, TypeNames, ET, M, MR, V, D, L) -> case dict:find(M, MR) of {ok, R} -> case lookup_type(Name, 0, R) of - {_, {{_M, _F, _A}, _T}} -> + {_, {{_M, _FL, _F, _A}, _T}} -> type_from_form(Name, [], TypeNames, ET, M, MR, V, D, L); error -> {Type, L} @@ -4203,7 +4219,7 @@ type_from_form(Name, Args, TypeNames, ET, M, MR, V, D, L) -> {ArgTypes, L1} = list_from_form(Args, TypeNames, ET, M, MR, V, D, L), {ok, R} = dict:find(M, MR), case lookup_type(Name, ArgsLen, R) of - {type, {{Module, Form, ArgNames}, _Type}} -> + {type, {{Module, _FileName, Form, ArgNames}, _Type}} -> TypeName = {type, Module, Name, ArgsLen}, case can_unfold_more(TypeName, TypeNames) of true -> @@ -4213,7 +4229,7 @@ type_from_form(Name, Args, TypeNames, ET, M, MR, V, D, L) -> false -> {t_any(), L1} end; - {opaque, {{Module, Form, ArgNames}, Type}} -> + {opaque, {{Module, _FileName, Form, ArgNames}, Type}} -> TypeName = {opaque, Module, Name, ArgsLen}, {Rep, L2} = case can_unfold_more(TypeName, TypeNames) of @@ -4224,17 +4240,19 @@ type_from_form(Name, Args, TypeNames, ET, M, MR, V, D, L) -> false -> {t_any(), L1} end, Rep1 = choose_opaque_type(Rep, Type), - Args2 = [subst_all_vars_to_any(ArgType) || ArgType <- ArgTypes], - {skip_opaque_alias(Rep1, Module, Name, Args2), L2}; + Rep2 = case t_is_none(Rep1) of + true -> Rep1; + false -> + Args2 = [subst_all_vars_to_any(ArgType) || + ArgType <- ArgTypes], + t_opaque(Module, Name, Args2, Rep1) + end, + {Rep2, L2}; error -> Msg = io_lib:format("Unable to find type ~w/~w\n", [Name, ArgsLen]), throw({error, Msg}) end. -skip_opaque_alias(?opaque(_) = T, _Mod, _Name, _Args) -> T; -skip_opaque_alias(T, Module, Name, Args) -> - t_opaque(Module, Name, Args, T). - remote_from_form(RemMod, Name, Args, TypeNames, ET, M, MR, V, D, L) -> {ArgTypes, L1} = list_from_form(Args, TypeNames, ET, M, MR, V, D, L), if @@ -4251,7 +4269,7 @@ remote_from_form(RemMod, Name, Args, TypeNames, ET, M, MR, V, D, L) -> case sets:is_element(MFA, ET) of true -> case lookup_type(Name, ArgsLen, RemDict) of - {type, {{_Mod, Form, ArgNames}, _Type}} -> + {type, {{_Mod, _FileLine, Form, ArgNames}, _Type}} -> RemType = {type, RemMod, Name, ArgsLen}, case can_unfold_more(RemType, TypeNames) of true -> @@ -4263,7 +4281,7 @@ remote_from_form(RemMod, Name, Args, TypeNames, ET, M, MR, V, D, L) -> false -> {t_any(), L1} end; - {opaque, {{Mod, Form, ArgNames}, Type}} -> + {opaque, {{Mod, _FileLine, Form, ArgNames}, Type}} -> RemType = {opaque, RemMod, Name, ArgsLen}, List = lists:zip(ArgNames, ArgTypes), TmpVarDict = dict:from_list(List), @@ -4277,7 +4295,11 @@ remote_from_form(RemMod, Name, Args, TypeNames, ET, M, MR, V, D, L) -> {t_any(), L1} end, NewRep1 = choose_opaque_type(NewRep, Type), - {skip_opaque_alias(NewRep1, Mod, Name, ArgTypes), L2}; + NewRep2 = case t_is_none(NewRep1) of + true -> NewRep1; + false -> t_opaque(Mod, Name, ArgTypes, NewRep1) + end, + {NewRep2, L2}; error -> Msg = io_lib:format("Unable to find remote type ~w:~w()\n", [RemMod, Name]), @@ -4358,34 +4380,24 @@ build_field_dict(FieldTypes, TypeNames, ET, M, MR, V, D, L) -> build_field_dict([{type, _, field_type, [{atom, _, Name}, Type]}|Left], TypeNames, ET, M, MR, V, D, L, Acc) -> {T, L1} = t_from_form(Type, TypeNames, ET, M, MR, V, D, L - 1), - %% The cached record field type (DeclType) in - %% get_mod_record_types()), was created with a similar call as TT. - %% Using T for the subtype test does not work since any() is not - %% always a subset of the field type. - TT = t_from_form(Type, ET, M, MR, V), - NewAcc = [{Name, Type, T, TT}|Acc], + NewAcc = [{Name, Type, T}|Acc], {Dict, L2} = build_field_dict(Left, TypeNames, ET, M, MR, V, D, L1, NewAcc), {Dict, L2}; build_field_dict([], _TypeNames, _ET, _M, _MR, _V, _D, L, Acc) -> {lists:keysort(1, Acc), L}. -get_mod_record_types([{FieldName, _Abstr, DeclType}|Left1], - [{FieldName, TypeForm, ModType, ModTypeTest}|Left2], +get_mod_record_types([{FieldName, _Abstr, _DeclType}|Left1], + [{FieldName, TypeForm, ModType}|Left2], Acc) -> - ModTypeNoVars = subst_all_vars_to_any(ModTypeTest), - case t_is_subtype(ModTypeNoVars, DeclType) of - false -> {error, FieldName}; - true -> get_mod_record_types(Left1, Left2, - [{FieldName, TypeForm, ModType}|Acc]) - end; + get_mod_record_types(Left1, Left2, [{FieldName, TypeForm, ModType}|Acc]); get_mod_record_types([{FieldName1, _Abstr, _DeclType} = DT|Left1], - [{FieldName2, _FormType, _ModType, _TT}|_] = List2, + [{FieldName2, _FormType, _ModType}|_] = List2, Acc) when FieldName1 < FieldName2 -> get_mod_record_types(Left1, List2, [DT|Acc]); get_mod_record_types(Left1, [], Acc) -> {ok, lists:keysort(1, Left1++Acc)}; -get_mod_record_types(_, [{FieldName2, _FormType, _ModType, _TT}|_], _Acc) -> +get_mod_record_types(_, [{FieldName2, _FormType, _ModType}|_], _Acc) -> {error, FieldName2}. %% It is important to create a limited version of the record type @@ -4406,6 +4418,74 @@ list_from_form([H|Tail], TypeNames, ET, M, MR, V, D, L) -> {T1, L2} = list_from_form(Tail, TypeNames, ET, M, MR, V, D, L1), {[H1|T1], L2}. +-spec t_check_record_fields(parse_form(), sets:set(mfa()), module(), + mod_records()) -> ok. + +t_check_record_fields(Form, ExpTypes, Module, RecDict) -> + t_check_record_fields(Form, ExpTypes, Module, RecDict, dict:new()). + +-spec t_check_record_fields(parse_form(), sets:set(mfa()), module(), + mod_records(), var_table()) -> ok. + +%% If there is something wrong with parse_form() +%% throw({error, io_lib:chars()} is called. + +t_check_record_fields({var, _L, _}, _ET, _M, _MR, _V) -> ok; +t_check_record_fields({ann_type, _L, [_Var, Type]}, ET, M, MR, V) -> + t_check_record_fields(Type, ET, M, MR, V); +t_check_record_fields({paren_type, _L, [Type]}, ET, M, MR, V) -> + t_check_record_fields(Type, ET, M, MR, V); +t_check_record_fields({remote_type, _L, [{atom, _, _}, {atom, _, _}, Args]}, + ET, M, MR, V) -> + list_check_record_fields(Args, ET, M, MR, V); +t_check_record_fields({atom, _L, _}, _ET, _M, _MR, _V) -> ok; +t_check_record_fields({integer, _L, _}, _ET, _M, _MR, _V) -> ok; +t_check_record_fields({op, _L, _Op, _Arg}, _ET, _M, _MR, _V) -> ok; +t_check_record_fields({op, _L, _Op, _Arg1, _Arg2}, _ET, _M, _MR, _V) -> ok; +t_check_record_fields({type, _L, tuple, any}, _ET, _M, _MR, _V) -> ok; +t_check_record_fields({type, _L, map, any}, _ET, _M, _MR, _V) -> ok; +t_check_record_fields({type, _L, binary, [_Base, _Unit]}, _ET, _M, _MR, _V) -> + ok; +t_check_record_fields({type, _L, 'fun', [{type, _, any}, Range]}, + ET, M, MR, V) -> + t_check_record_fields(Range, ET, M, MR, V); +t_check_record_fields({type, _L, range, [_From, _To]}, _ET, _M, _MR, _V) -> + ok; +t_check_record_fields({type, _L, record, [Name|Fields]}, ET, M, MR, V) -> + check_record(Name, Fields, ET, M, MR, V); +t_check_record_fields({type, _L, _, Args}, ET, M, MR, V) -> + list_check_record_fields(Args, ET, M, MR, V); +t_check_record_fields({user_type, _L, _Name, Args}, ET, M, MR, V) -> + list_check_record_fields(Args, ET, M, MR, V). + +check_record({atom, _, Name}, ModFields, ET, M, MR, V) -> + {ok, R} = dict:find(M, MR), + {ok, DeclFields} = lookup_record(Name, R), + case check_fields(ModFields, DeclFields, ET, M, MR, V) of + {error, FieldName} -> + throw({error, io_lib:format("Illegal declaration of #~w{~w}\n", + [Name, FieldName])}); + ok -> ok + end. + +check_fields([{type, _, field_type, [{atom, _, Name}, Abstr]}|Left], + DeclFields, ET, M, MR, V) -> + Type = t_from_form(Abstr, ET, M, MR, V), + {Name, _, DeclType} = lists:keyfind(Name, 1, DeclFields), + TypeNoVars = subst_all_vars_to_any(Type), + case t_is_subtype(TypeNoVars, DeclType) of + false -> {error, Name}; + true -> check_fields(Left, DeclFields, ET, M, MR, V) + end; +check_fields([], _Decl, _ET, _M, _MR, _V) -> + ok. + +list_check_record_fields([], _ET, _M, _MR, _V) -> + ok; +list_check_record_fields([H|Tail], ET, M, MR, V) -> + ok = t_check_record_fields(H, ET, M, MR, V), + list_check_record_fields(Tail, ET, M, MR, V). + -spec t_var_names([erl_type()]) -> [atom()]. t_var_names([{var, _, Name}|L]) when L =/= '_' -> @@ -4556,9 +4636,9 @@ is_erl_type(_) -> false. lookup_record(Tag, RecDict) when is_atom(Tag) -> case dict:find({record, Tag}, RecDict) of - {ok, [{_Arity, Fields}]} -> + {ok, {_FileLine, [{_Arity, Fields}]}} -> {ok, Fields}; - {ok, List} when is_list(List) -> + {ok, {_FileLine, List}} when is_list(List) -> %% This will have to do, since we do not know which record we %% are looking for. error; @@ -4571,8 +4651,8 @@ lookup_record(Tag, RecDict) when is_atom(Tag) -> lookup_record(Tag, Arity, RecDict) when is_atom(Tag) -> case dict:find({record, Tag}, RecDict) of - {ok, [{Arity, Fields}]} -> {ok, Fields}; - {ok, OrdDict} -> orddict:find(Arity, OrdDict); + {ok, {_FileLine, [{Arity, Fields}]}} -> {ok, Fields}; + {ok, {_FileLine, OrdDict}} -> orddict:find(Arity, OrdDict); error -> error end. @@ -4619,17 +4699,6 @@ do_opaque(?union(List) = Type, Opaques, Pred) -> do_opaque(Type, _Opaques, Pred) -> Pred(Type). -is_same_type_name(ModNameArgs, ModNameArgs) -> true; -is_same_type_name({Mod, Name, Args1}, {Mod, Name, Args2}) -> - all_any(Args1) orelse all_any(Args2); -is_same_type_name(_ModNameArgs1, _ModNameArgs2) -> - false. - -all_any([]) -> true; -all_any([T|L]) -> - t_is_any(T) andalso all_any(L); -all_any(_) -> false. - map_keys(?map(Pairs)) -> [K || {K, _} <- Pairs]. diff --git a/lib/hipe/doc/src/notes.xml b/lib/hipe/doc/src/notes.xml index 1cc8558fe8..8d3358533b 100644 --- a/lib/hipe/doc/src/notes.xml +++ b/lib/hipe/doc/src/notes.xml @@ -30,42 +30,6 @@ </header> <p>This document describes the changes made to HiPE.</p> -<section><title>Hipe 3.12</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> Fix a minor bug in the handling of opaque types. </p> - <p> - Own Id: OTP-12666</p> - </item> - <item> - <p> - Fix hipe bug when matching a "writable" binary. The bug - has been seen to sometimes cause a failed binary matching - of a correct utf8 character, but other symptoms are also - possible.</p> - <p> - Own Id: OTP-12667</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Improved error handling when memory allocation for HiPE - code fails.</p> - <p> - Own Id: OTP-12448</p> - </item> - </list> - </section> - -</section> - <section><title>Hipe 3.11.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/hipe/llvm/hipe_llvm_main.erl b/lib/hipe/llvm/hipe_llvm_main.erl index 0e50c9539b..3c24425828 100644 --- a/lib/hipe/llvm/hipe_llvm_main.erl +++ b/lib/hipe/llvm/hipe_llvm_main.erl @@ -465,7 +465,7 @@ remove_temp_folder(Dir, Options) -> end. unique_id(FunName, Arity) -> - integer_to_list(erlang:phash2({FunName, Arity, now()})). + integer_to_list(erlang:phash2({FunName, Arity, erlang:unique_integer()})). unique_folder(FunName, Arity, Options) -> DirName = "llvm_" ++ unique_id(FunName, Arity) ++ "/", diff --git a/lib/hipe/main/hipe.app.src b/lib/hipe/main/hipe.app.src index 22ea71b4e6..7b6d9e30e3 100644 --- a/lib/hipe/main/hipe.app.src +++ b/lib/hipe/main/hipe.app.src @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2012. All Rights Reserved. +%% Copyright Ericsson AB 2002-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -223,5 +223,5 @@ {registered,[]}, {applications, [kernel,stdlib]}, {env, []}, - {runtime_dependencies, ["syntax_tools-1.6.14","stdlib-2.0","kernel-3.0", + {runtime_dependencies, ["syntax_tools-1.6.14","stdlib-2.5","kernel-3.0", "erts-7.0","compiler-5.0"]}]}. diff --git a/lib/hipe/main/hipe.erl b/lib/hipe/main/hipe.erl index 539ce883c0..b614f5f1ab 100644 --- a/lib/hipe/main/hipe.erl +++ b/lib/hipe/main/hipe.erl @@ -649,8 +649,9 @@ run_compiler_1(DisasmFun, IcodeFun, Options) -> %% The full option expansion is not done %% until the DisasmFun returns. {Code, CompOpts} = DisasmFun(Options), - Opts0 = expand_options(Options ++ CompOpts), - Opts = + Opts0 = expand_options(Options ++ CompOpts, + get(hipe_target_arch)), + Opts = case proplists:get_bool(to_llvm, Opts0) andalso not llvm_support_available() of true -> @@ -895,8 +896,7 @@ do_load(Mod, Bin, BeamBinOrPath) when is_binary(BeamBinOrPath); code:load_native_sticky(Mod, Bin, Beam); false -> %% Normal loading of a whole module - Architecture = erlang:system_info(hipe_architecture), - ChunkName = hipe_unified_loader:chunk_name(Architecture), + ChunkName = hipe_unified_loader:chunk_name(HostArch), {ok, _, Chunks0} = beam_lib:all_chunks(BeamBinOrPath), Chunks = [{ChunkName, Bin}|lists:keydelete(ChunkName, 1, Chunks0)], {ok, BeamPlusNative} = beam_lib:build_module(Chunks), @@ -933,9 +933,9 @@ assemble(CompiledCode, Closures, Exports, Options) -> %% but can be overridden by passing an option {target, Target}. set_architecture(Options) -> - put(hipe_host_arch, erlang:system_info(hipe_architecture)), - put(hipe_target_arch, - proplists:get_value(target, Options, get(hipe_host_arch))), + HostArch = erlang:system_info(hipe_architecture), + put(hipe_host_arch, HostArch), + put(hipe_target_arch, proplists:get_value(target, Options, HostArch)), ok. %% This sets up some globally accessed stuff that are needed by the @@ -943,7 +943,7 @@ set_architecture(Options) -> %% Therefore, this expands the current set of options for local use. pre_init(Opts) -> - Options = expand_options(Opts), + Options = expand_options(Opts, get(hipe_target_arch)), %% Initialise some counters used for measurements and benchmarking. If %% the option 'measure_regalloc' is given the compilation will return %% a keylist with the counter values. @@ -1105,10 +1105,10 @@ help_hiper() -> -spec help_options() -> 'ok'. help_options() -> - set_architecture([]), %% needed for target-specific option expansion - O1 = expand_options([o1]), - O2 = expand_options([o2]), - O3 = expand_options([o3]), + HostArch = erlang:system_info(hipe_architecture), + O1 = expand_options([o1], HostArch), + O2 = expand_options([o2], HostArch), + O3 = expand_options([o3], HostArch), io:format("HiPE Compiler Options\n" ++ " Boolean-valued options generally have corresponding " ++ "aliases `no_...',\n" ++ @@ -1134,7 +1134,7 @@ help_options() -> [ordsets:from_list([verbose, debug, time, load, pp_beam, pp_icode, pp_rtl, pp_native, pp_asm, timeout]), - expand_options([pp_all]), + expand_options([pp_all], HostArch), O1 -- [o1], (O2 -- O1) -- [o2], (O3 -- O2) -- [o3]]), @@ -1232,8 +1232,8 @@ option_text(Opt) when is_atom(Opt) -> -spec help_option(comp_option()) -> 'ok'. help_option(Opt) -> - set_architecture([]), %% needed for target-specific option expansion - case expand_options([Opt]) of + HostArch = erlang:system_info(hipe_architecture), + case expand_options([Opt], HostArch) of [Opt] -> Name = if is_atom(Opt) -> Opt; tuple_size(Opt) =:= 2 -> element(1, Opt) @@ -1364,11 +1364,11 @@ opt_keys() -> %% verbose_spills, x87]. -%% Definitions: +%% Definitions: -o1_opts() -> +o1_opts(TargetArch) -> Common = [inline_fp, pmatch, peephole], - case get(hipe_target_arch) of + case TargetArch of ultrasparc -> Common; powerpc -> @@ -1385,13 +1385,13 @@ o1_opts() -> ?EXIT({executing_on_an_unsupported_architecture,Arch}) end. -o2_opts() -> +o2_opts(TargetArch) -> Common = [icode_ssa_const_prop, icode_ssa_copy_prop, % icode_ssa_struct_reuse, icode_type, icode_inline_bifs, rtl_lcm, rtl_ssa, rtl_ssa_const_prop, - spillmin_color, use_indexing, remove_comments, - concurrent_comp, binary_opt | o1_opts()], - case get(hipe_target_arch) of + spillmin_color, use_indexing, remove_comments, + concurrent_comp, binary_opt | o1_opts(TargetArch)], + case TargetArch of ultrasparc -> Common; powerpc -> @@ -1409,9 +1409,9 @@ o2_opts() -> ?EXIT({executing_on_an_unsupported_architecture,Arch}) end. -o3_opts() -> - Common = [icode_range, {regalloc,coalescing} | o2_opts()], - case get(hipe_target_arch) of +o3_opts(TargetArch) -> + Common = [icode_range, {regalloc,coalescing} | o2_opts(TargetArch)], + case TargetArch of ultrasparc -> Common; powerpc -> @@ -1489,18 +1489,18 @@ opt_aliases() -> opt_basic_expansions() -> [{pp_all, [pp_beam, pp_icode, pp_rtl, pp_native]}]. -opt_expansions() -> - [{o1, o1_opts()}, - {o2, o2_opts()}, - {o3, o3_opts()}, +opt_expansions(TargetArch) -> + [{o1, o1_opts(TargetArch)}, + {o2, o2_opts(TargetArch)}, + {o3, o3_opts(TargetArch)}, {to_llvm, llvm_opts(o3)}, {{to_llvm, o0}, llvm_opts(o0)}, {{to_llvm, o1}, llvm_opts(o1)}, {{to_llvm, o2}, llvm_opts(o2)}, {{to_llvm, o3}, llvm_opts(o3)}, {x87, [x87, inline_fp]}, - {inline_fp, case get(hipe_target_arch) of %% XXX: Temporary until x86 - x86 -> [x87, inline_fp]; %% has sse2 + {inline_fp, case TargetArch of %% XXX: Temporary until x86 has sse2 + x86 -> [x87, inline_fp]; _ -> [inline_fp] end}]. llvm_opts(O) -> @@ -1523,18 +1523,18 @@ expand_kt2(Opts) -> [{use_callgraph, fixpoint}, core, {core_transform, cerl_typean}]}]}]). -%% Note that set_architecture/1 must be called first, and that the given +%% Note that the given %% list should contain the total set of options, since things like 'o2' %% are expanded here. Basic expansions are processed here also, since %% this function is called from the help functions. --spec expand_options(comp_options()) -> comp_options(). +-spec expand_options(comp_options(), hipe_architecture()) -> comp_options(). -expand_options(Opts) -> +expand_options(Opts, TargetArch) -> proplists:normalize(Opts, [{negations, opt_negations()}, {aliases, opt_aliases()}, {expand, opt_basic_expansions()}, - {expand, opt_expansions()}]). + {expand, opt_expansions(TargetArch)}]). -spec check_options(comp_options()) -> 'ok'. diff --git a/lib/inets/doc/src/Makefile b/lib/inets/doc/src/Makefile index 1a8e1c7ca8..961bfa838d 100644 --- a/lib/inets/doc/src/Makefile +++ b/lib/inets/doc/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2012. All Rights Reserved. +# Copyright Ericsson AB 1997-2015. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -52,6 +52,7 @@ XML_REF3_FILES = \ httpc.xml\ httpd.xml \ httpd_conf.xml \ + httpd_custom_api.xml \ httpd_socket.xml \ httpd_util.xml \ mod_alias.xml \ diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index 4178cb7d4c..6984408932 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -366,7 +366,7 @@ filename() = string() <tag><c><![CDATA[receiver]]></c></tag> <item> <p>Defines how the client will deliver the result of an - asynchroneous request (<c>sync</c> has the value + asynchronous request (<c>sync</c> has the value <c>false</c>). </p> <taglist> diff --git a/lib/inets/doc/src/httpd.xml b/lib/inets/doc/src/httpd.xml index e40660ab39..e6aa8d5e07 100644 --- a/lib/inets/doc/src/httpd.xml +++ b/lib/inets/doc/src/httpd.xml @@ -162,6 +162,20 @@ in the apache like configuration file. </p> </item> + <marker id="profile"></marker> + <tag>{profile, atom()}</tag> + <item> + <p>Used together with <seealso marker="prop_bind_address"><c>bind_address</c></seealso> + and <seealso marker="prop_port"><c>port</c></seealso> to uniquely identify + a HTTP server. This can be useful in a virtualized environment, + where there can + be more that one server that has the same bind_address and port. + If this property is not explicitly set, it is assumed that the + <seealso marker="prop_bind_address"><c>bind_address</c></seealso> and + <seealso marker="prop_port"><c>port</c></seealso>uniquely identifies the HTTP server. + </p> + </item> + <marker id="prop_socket_type"></marker> <tag>{socket_type, ip_comm | {essl, Config::proplist()}}</tag> <item> @@ -176,6 +190,8 @@ <p>Note that this option is only used when the option <c>socket_type</c> has the value <c>ip_comm</c>. </p> </item> + + <marker id="prop_minimum_bytes_per_second"></marker> <tag>{minimum_bytes_per_second, integer()}</tag> <item> @@ -204,7 +220,15 @@ <marker id="props_limit"></marker> <p><em>Limit properties</em> </p> - <taglist> + <taglist> + + <marker id="prop_customize"></marker> + <tag>{customize, atom()}</tag> + <item> + <p>A callback module to customize the inets HTTP servers behaviour + see <seealso marker="http_custom_api"> httpd_custom_api</seealso> </p> + </item> + <marker id="prop_disable_chunked_encoding"></marker> <tag>{disable_chunked_transfer_encoding_send, boolean()}</tag> <item> @@ -927,19 +951,22 @@ bytes <func> <marker id="info2"></marker> <name>info(Address, Port) -> </name> + <name>info(Address, Port, Profile) -> </name> + <name>info(Address, Port, Profile, Properties) -> [{Option, Value}] </name> <name>info(Address, Port, Properties) -> [{Option, Value}] </name> <fsummary>Fetches information about the HTTP server</fsummary> <type> <v>Address = ip_address()</v> <v>Port = integer()</v> + <v>Profile = atom()</v> <v>Properties = [property()]</v> <v>Option = property()</v> <v>Value = term()</v> </type> <desc> <p>Fetches information about the HTTP server. When called with - only the Address and Port all properties are fetched, when - called with a list of specific properties they are fetched. + only the Address, Port and Profile, if relevant, all properties are fetched. + When called with a list of specific properties they are fetched. Available properties are the same as the server's start options. </p> diff --git a/lib/inets/doc/src/httpd_custom_api.xml b/lib/inets/doc/src/httpd_custom_api.xml new file mode 100644 index 0000000000..faf1d277df --- /dev/null +++ b/lib/inets/doc/src/httpd_custom_api.xml @@ -0,0 +1,63 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2015</year><year>2015</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>httpd_custom_api</title> + <file>httpd_custom_api.xml</file> + </header> + <module>httpd_custom_api</module> + <modulesummary>Behaviour with optional callbacks to customize the inets HTTP server.</modulesummary> + <description> + <p> The module implementing this behaviour shall be supplied to to the servers + configuration with the option <seealso marker="httpd:prop_customize"> customize</seealso></p> + + </description> + <funcs> + <func> + <name>response_header({HeaderName, HeaderValue}) -> {true, Header} | false </name> + <fsummary>Filter and possible alter HTTP response headers.</fsummary> + <type> + <v>Header = {HeaderName :: string(), HeaderValue::string()}</v> + <d>The header name will be in lower case and should not be altered.</d> + </type> + <desc> + <p> Filter and possible alter HTTP response headers before they are sent to the client. + </p> + </desc> + </func> + + <func> + <name>request_header({HeaderName, HeaderValue}) -> {true, Header} | false </name> + <fsummary>Filter and possible alter HTTP request headers.</fsummary> + <type> + <v>Header = {HeaderName :: string(), HeaderValue::string()}</v> + <d>The header name will be in lower case and should not be altered.</d> + </type> + <desc> + <p> Filter and possible alter HTTP request headers before they are processed by the server. + </p> + </desc> + </func> + </funcs> +</erlref> + + diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 7939787601..f563a8c4b0 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -32,19 +32,31 @@ <file>notes.xml</file> </header> - <section><title>Inets 6.0</title> + <section><title>Inets 5.10.9</title> <section><title>Improvements and New Features</title> <list> <item> <p> - Remove Server Side Include support from inets, as this is - an old technic that has security issues and was not well - tested.</p> + Add behaviour with optional callbacks to customize the + inets HTTP server.</p> <p> - *** POTENTIAL INCOMPATIBILITY ***</p> + Own Id: OTP-12776</p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 5.10.8</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Reject messages with a Content-Length less than 0</p> <p> - Own Id: OTP-12156</p> + Own Id: OTP-12739 Aux Id: seq12860 </p> </item> </list> </section> diff --git a/lib/inets/doc/src/ref_man.xml b/lib/inets/doc/src/ref_man.xml index aaedf330b4..3afb020431 100644 --- a/lib/inets/doc/src/ref_man.xml +++ b/lib/inets/doc/src/ref_man.xml @@ -4,7 +4,7 @@ <application xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2015</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -39,6 +39,7 @@ <xi:include href="httpc.xml"/> <xi:include href="httpd.xml"/> <xi:include href="httpd_conf.xml"/> + <xi:include href="httpd_custom_api.xml"/> <xi:include href="httpd_socket.xml"/> <xi:include href="httpd_util.xml"/> <xi:include href="mod_alias.xml"/> diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index 8f2f11ce8e..a4971cec0f 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -87,7 +87,7 @@ %% block the httpc manager process in odd cases such as trying to call %% a server that does not exist. (See OTP-6735) The only API function %% sending messages to the handler process that can be called before -%% init has compleated is cancel and that is not a problem! (Send and +%% init has completed is cancel and that is not a problem! (Send and %% stream will not be called before the first request has been sent and %% the reply or part of it has arrived.) %%-------------------------------------------------------------------- @@ -392,7 +392,7 @@ handle_call(info, _, State) -> %% When the request in process has been canceled the handler process is %% stopped and the pipelined requests will be reissued or remaining %% requests will be sent on a new connection. This is is -%% based on the assumption that it is proably cheaper to reissue the +%% based on the assumption that it is probably cheaper to reissue the %% requests than to wait for a potentiall large response that we then %% only throw away. This of course is not always true maybe we could %% do something smarter here?! If the request canceled is not @@ -1302,7 +1302,8 @@ handle_pipeline(#state{status = pipeline, handle_keep_alive_queue(#state{status = keep_alive, session = Session, profile_name = ProfileName, - options = #options{keep_alive_timeout = TimeOut}} = State, + options = #options{keep_alive_timeout = TimeOut, + proxy = Proxy}} = State, Data) -> ?hcrd("handle keep_alive", [{profile, ProfileName}, @@ -1323,14 +1324,15 @@ handle_keep_alive_queue(#state{status = keep_alive, State#state{keep_alive = KeepAlive}, Data); false -> ?hcrv("next request", [{request, NextRequest}]), - #request{address = Address} = NextRequest, + #request{address = Addr} = NextRequest, + Address = handle_proxy(Addr, Proxy), case httpc_request:send(Address, Session, NextRequest) of ok -> receive_response(NextRequest, Session, <<>>, State#state{keep_alive = KeepAlive}); {error, Reason} -> - {stop, shutdown, {keepalive_failed, Reason}, State} + {stop, {shutdown, {keepalive_failed, Reason}}, State} end end end. @@ -1345,7 +1347,7 @@ handle_empty_queue(Session, ProfileName, TimeOut, State) -> %% closed by the server, the client may want to close it. NewState = activate_queue_timeout(TimeOut, State), update_session(ProfileName, Session, #session.queue_length, 0), - %% Note mfa will be initilized when a new request + %% Note mfa will be initialized when a new request %% arrives. {noreply, NewState#state{request = undefined, diff --git a/lib/inets/src/http_server/Makefile b/lib/inets/src/http_server/Makefile index 51e3dd9212..00bad51ff9 100644 --- a/lib/inets/src/http_server/Makefile +++ b/lib/inets/src/http_server/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2005-2013. All Rights Reserved. +# Copyright Ericsson AB 2005-2015. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -46,6 +46,7 @@ MODULES = \ httpd_connection_sup\ httpd_cgi \ httpd_conf \ + httpd_custom \ httpd_example \ httpd_esi \ httpd_file\ diff --git a/lib/inets/src/http_server/httpd.erl b/lib/inets/src/http_server/httpd.erl index e8148ea362..71be6dde00 100644 --- a/lib/inets/src/http_server/httpd.erl +++ b/lib/inets/src/http_server/httpd.erl @@ -23,6 +23,7 @@ -behaviour(inets_service). -include("httpd.hrl"). +-include("httpd_internal.hrl"). %% Behavior callbacks -export([ @@ -61,18 +62,27 @@ info(Pid, Properties) when is_pid(Pid) andalso is_list(Properties) -> {ok, ServiceInfo} = service_info(Pid), Address = proplists:get_value(bind_address, ServiceInfo), Port = proplists:get_value(port, ServiceInfo), + Profile = proplists:get_value(profile, ServiceInfo, default), case Properties of [] -> - info(Address, Port); + info(Address, Port, Profile); _ -> - info(Address, Port, Properties) + info(Address, Port, Profile, Properties) end; + info(Address, Port) when is_integer(Port) -> - httpd_conf:get_config(Address, Port). + info(Address, Port, default). + +info(Address, Port, Profile) when is_integer(Port), is_atom(Profile) -> + httpd_conf:get_config(Address, Port, Profile); info(Address, Port, Properties) when is_integer(Port) andalso is_list(Properties) -> - httpd_conf:get_config(Address, Port, Properties). + httpd_conf:get_config(Address, Port, default, Properties). + +info(Address, Port, Profile, Properties) when is_integer(Port) andalso + is_atom(Profile) andalso is_list(Properties) -> + httpd_conf:get_config(Address, Port, Profile, Properties). %%%======================================================================== @@ -86,14 +96,16 @@ start_service(Conf) -> httpd_sup:start_child(Conf). stop_service({Address, Port}) -> - httpd_sup:stop_child(Address, Port); - + stop_service({Address, Port, ?DEFAULT_PROFILE}); +stop_service({Address, Port, Profile}) -> + httpd_sup:stop_child(Address, Port, Profile); stop_service(Pid) when is_pid(Pid) -> case service_info(Pid) of {ok, Info} -> Address = proplists:get_value(bind_address, Info), Port = proplists:get_value(port, Info), - stop_service({Address, Port}); + Profile = proplists:get_value(profile, Info, ?DEFAULT_PROFILE), + stop_service({Address, Port, Profile}); Error -> Error end. @@ -101,7 +113,6 @@ stop_service(Pid) when is_pid(Pid) -> services() -> [{httpd, ChildPid} || {_, ChildPid, _, _} <- supervisor:which_children(httpd_sup)]. - service_info(Pid) -> try [{ChildName, ChildPid} || @@ -114,7 +125,6 @@ service_info(Pid) -> {error, service_not_available} end. - %%%-------------------------------------------------------------- %%% Internal functions %%%-------------------------------------------------------------------- @@ -128,12 +138,12 @@ child_name(Pid, [_ | Children]) -> child_name2info(undefined) -> {error, no_such_service}; -child_name2info({httpd_instance_sup, any, Port}) -> +child_name2info({httpd_instance_sup, any, Port, Profile}) -> {ok, Host} = inet:gethostname(), - Info = info(any, Port, [server_name]), + Info = info(any, Port, Profile, [server_name]), {ok, [{bind_address, any}, {host, Host}, {port, Port} | Info]}; -child_name2info({httpd_instance_sup, Address, Port}) -> - Info = info(Address, Port, [server_name]), +child_name2info({httpd_instance_sup, Address, Port, Profile}) -> + Info = info(Address, Port, Profile, [server_name]), case inet:gethostbyaddr(Address) of {ok, {_, Host, _, _,_, _}} -> {ok, [{bind_address, Address}, @@ -143,8 +153,8 @@ child_name2info({httpd_instance_sup, Address, Port}) -> end. -reload(Config, Address, Port) -> - Name = make_name(Address,Port), +reload(Config, Address, Port, Profile) -> + Name = make_name(Address,Port, Profile), case whereis(Name) of Pid when is_pid(Pid) -> httpd_manager:reload(Pid, Config); @@ -191,51 +201,19 @@ reload(Config, Address, Port) -> %%% Timeout -> integer() %%% -block(Addr, Port, disturbing) when is_integer(Port) -> - do_block(Addr, Port, disturbing); -block(Addr, Port, non_disturbing) when is_integer(Port) -> - do_block(Addr, Port, non_disturbing); - -block(ConfigFile, Mode, Timeout) - when is_list(ConfigFile) andalso - is_atom(Mode) andalso - is_integer(Timeout) -> - case get_addr_and_port(ConfigFile) of - {ok, Addr, Port} -> - block(Addr, Port, Mode, Timeout); - Error -> - Error - end. - - -block(Addr, Port, non_disturbing, Timeout) - when is_integer(Port) andalso is_integer(Timeout) -> - do_block(Addr, Port, non_disturbing, Timeout); -block(Addr,Port,disturbing,Timeout) - when is_integer(Port) andalso is_integer(Timeout) -> - do_block(Addr, Port, disturbing, Timeout). - -do_block(Addr, Port, Mode) when is_integer(Port) andalso is_atom(Mode) -> - Name = make_name(Addr,Port), +block(Addr, Port, Profile, disturbing) when is_integer(Port) -> + do_block(Addr, Port, Profile, disturbing); +block(Addr, Port, Profile, non_disturbing) when is_integer(Port) -> + do_block(Addr, Port, Profile, non_disturbing). +do_block(Addr, Port, Profile, Mode) when is_integer(Port) andalso is_atom(Mode) -> + Name = make_name(Addr, Port, Profile), case whereis(Name) of Pid when is_pid(Pid) -> - httpd_manager:block(Pid,Mode); + httpd_manager:block(Pid, Mode); _ -> {error,not_started} end. - -do_block(Addr, Port, Mode, Timeout) - when is_integer(Port) andalso is_atom(Mode) -> - Name = make_name(Addr,Port), - case whereis(Name) of - Pid when is_pid(Pid) -> - httpd_manager:block(Pid,Mode,Timeout); - _ -> - {error,not_started} - end. - - %%% ========================================================= %%% Function: unblock/2 %%% unblock(Addr, Port) @@ -248,8 +226,8 @@ do_block(Addr, Port, Mode, Timeout) %%% ConfigFile -> string() %%% -unblock(Addr, Port) when is_integer(Port) -> - Name = make_name(Addr,Port), +unblock(Addr, Port, Profile) when is_integer(Port) -> + Name = make_name(Addr,Port, Profile), case whereis(Name) of Pid when is_pid(Pid) -> httpd_manager:unblock(Pid); @@ -269,24 +247,9 @@ foreach([KeyValue|Rest]) -> foreach(Rest) end. -get_addr_and_port(ConfigFile) -> - case httpd_conf:load(ConfigFile) of - {ok, ConfigList} -> - case (catch httpd_conf:validate_properties(ConfigList)) of - {ok, Config} -> - Address = proplists:get_value(bind_address, Config, any), - Port = proplists:get_value(port, Config, 80), - {ok, Address, Port}; - Error -> - Error - end; - Error -> - Error - end. - -make_name(Addr, Port) -> - httpd_util:make_name("httpd", Addr, Port). +make_name(Addr, Port, Profile) -> + httpd_util:make_name("httpd", Addr, Port, Profile). do_reload_config(ConfigList, Mode) -> @@ -294,10 +257,11 @@ do_reload_config(ConfigList, Mode) -> {ok, Config} -> Address = proplists:get_value(bind_address, Config, any), Port = proplists:get_value(port, Config, 80), - case block(Address, Port, Mode) of + Profile = proplists:get_value(profile, Config, default), + case block(Address, Port, Profile, Mode) of ok -> - reload(Config, Address, Port), - unblock(Address, Port); + reload(Config, Address, Port, Profile), + unblock(Address, Port, Profile); Error -> Error end; diff --git a/lib/inets/src/http_server/httpd_acceptor_sup.erl b/lib/inets/src/http_server/httpd_acceptor_sup.erl index cc2b582b52..a6a0fe2eea 100644 --- a/lib/inets/src/http_server/httpd_acceptor_sup.erl +++ b/lib/inets/src/http_server/httpd_acceptor_sup.erl @@ -26,6 +26,8 @@ -behaviour(supervisor). +-include("httpd_internal.hrl"). + %% API -export([start_link/1]). %%, start_acceptor/6, start_acceptor/7, stop_acceptor/2]). @@ -36,8 +38,9 @@ %%%========================================================================= %%% API %%%========================================================================= -start_link([Addr, Port| _] = Args) -> - SupName = make_name(Addr, Port), +start_link([Addr, Port, Config| _] = Args) -> + Profile = proplists:get_value(profile, Config, ?DEFAULT_PROFILE), + SupName = make_name(Addr, Port, Profile), supervisor:start_link({local, SupName}, ?MODULE, [Args]). %%%========================================================================= @@ -54,20 +57,23 @@ init([Args]) -> %%% Internal functions %%%========================================================================= child_spec([Address, Port, ConfigList, AcceptTimeout, ListenInfo]) -> - Name = id(Address, Port), - Manager = httpd_util:make_name("httpd", Address, Port), + Profile = proplists:get_value(profile, ConfigList, ?DEFAULT_PROFILE), + Name = id(Address, Port, Profile), + Manager = httpd_util:make_name("httpd", Address, Port, Profile), SockType = proplists:get_value(socket_type, ConfigList, ip_comm), IpFamily = proplists:get_value(ipfamily, ConfigList, inet), StartFunc = case ListenInfo of undefined -> - {httpd_acceptor, start_link, [Manager, SockType, Address, Port, IpFamily, - httpd_util:make_name("httpd_conf", Address, Port), - AcceptTimeout]}; + {httpd_acceptor, start_link, + [Manager, SockType, Address, Port, IpFamily, + httpd_util:make_name("httpd_conf", Address, Port, Profile), + AcceptTimeout]}; _ -> - {httpd_acceptor, start_link, [Manager, SockType, Address, Port, ListenInfo, - IpFamily, - httpd_util:make_name("httpd_conf", Address, Port), - AcceptTimeout]} + {httpd_acceptor, start_link, + [Manager, SockType, Address, Port, ListenInfo, + IpFamily, + httpd_util:make_name("httpd_conf", Address, Port, Profile), + AcceptTimeout]} end, Restart = transient, Shutdown = brutal_kill, @@ -75,9 +81,9 @@ child_spec([Address, Port, ConfigList, AcceptTimeout, ListenInfo]) -> Type = worker, {Name, StartFunc, Restart, Shutdown, Type, Modules}. -id(Address, Port) -> - {httpd_acceptor_sup, Address, Port}. +id(Address, Port, Profile) -> + {httpd_acceptor_sup, Address, Port, Profile}. -make_name(Addr,Port) -> - httpd_util:make_name("httpd_acceptor_sup", Addr, Port). +make_name(Addr, Port, Profile) -> + httpd_util:make_name("httpd_acceptor_sup", Addr, Port, Profile). diff --git a/lib/inets/src/http_server/httpd_conf.erl b/lib/inets/src/http_server/httpd_conf.erl index a21eb915d4..9c70f8d1b8 100644 --- a/lib/inets/src/http_server/httpd_conf.erl +++ b/lib/inets/src/http_server/httpd_conf.erl @@ -25,7 +25,7 @@ %% Application internal API -export([load/1, load/2, load_mime_types/1, store/1, store/2, - remove/1, remove_all/1, get_config/2, get_config/3, + remove/1, remove_all/1, get_config/3, get_config/4, lookup_socket_type/1, lookup/2, lookup/3, lookup/4, validate_properties/1]). @@ -757,8 +757,9 @@ store(ConfigList0) -> ?hdrt("store", [{modules, Modules}]), Port = proplists:get_value(port, ConfigList0), Addr = proplists:get_value(bind_address, ConfigList0, any), + Profile = proplists:get_value(profile, ConfigList0, default), ConfigList = fix_mime_types(ConfigList0), - Name = httpd_util:make_name("httpd_conf", Addr, Port), + Name = httpd_util:make_name("httpd_conf", Addr, Port, Profile), ConfigDB = ets:new(Name, [named_table, bag, protected]), store(ConfigDB, ConfigList, lists:append(Modules, [?MODULE]), @@ -909,15 +910,15 @@ remove(ConfigDB) -> %% end. -get_config(Address, Port) -> - Tab = httpd_util:make_name("httpd_conf", Address, Port), +get_config(Address, Port, Profile) -> + Tab = httpd_util:make_name("httpd_conf", Address, Port, Profile), Properties = ets:tab2list(Tab), MimeTab = proplists:get_value(mime_types, Properties), NewProperties = proplists:delete(mime_types, Properties), [{mime_types, ets:tab2list(MimeTab)} | NewProperties]. -get_config(Address, Port, Properties) -> - Tab = httpd_util:make_name("httpd_conf", Address, Port), +get_config(Address, Port, Profile, Properties) -> + Tab = httpd_util:make_name("httpd_conf", Address, Port, Profile), Config = lists:map(fun(Prop) -> {Prop, httpd_util:lookup(Tab, Prop)} end, Properties), diff --git a/lib/inets/src/http_server/httpd_custom.erl b/lib/inets/src/http_server/httpd_custom.erl new file mode 100644 index 0000000000..342469a579 --- /dev/null +++ b/lib/inets/src/http_server/httpd_custom.erl @@ -0,0 +1,69 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015-2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% +-module(httpd_custom). + +-export([response_header/1, request_header/1]). +-export([customize_headers/3]). + +-include_lib("inets/src/inets_app/inets_internal.hrl"). + +response_header(Header) -> + {true, httpify(Header)}. +request_header(Header) -> + {true, Header}. + +customize_headers(?MODULE, Function, Arg) -> + ?MODULE:Function(Arg); +customize_headers(Module, Function, Arg) -> + try Module:Function(Arg) of + {true, Value} -> + ?MODULE:Function(Value); + false -> + false + catch + _:_ -> + ?MODULE:Function(Arg) + end. + +httpify({Key0, Value}) -> + %% make sure first letter is capital (defacto standard) + Words1 = string:tokens(Key0, "-"), + Words2 = upify(Words1, []), + Key = new_key(Words2), + Key ++ ": " ++ Value ++ ?CRLF . + +new_key([]) -> + ""; +new_key([W]) -> + W; +new_key([W1,W2]) -> + W1 ++ "-" ++ W2; +new_key([W|R]) -> + W ++ "-" ++ new_key(R). + +upify([], Acc) -> + lists:reverse(Acc); +upify([Key|Rest], Acc) -> + upify(Rest, [upify2(Key)|Acc]). + +upify2([C|Rest]) when (C >= $a) andalso (C =< $z) -> + [C-($a-$A)|Rest]; +upify2(Str) -> + Str. diff --git a/lib/inets/src/http_server/httpd_instance_sup.erl b/lib/inets/src/http_server/httpd_instance_sup.erl index b95be44b2a..90800f2724 100644 --- a/lib/inets/src/http_server/httpd_instance_sup.erl +++ b/lib/inets/src/http_server/httpd_instance_sup.erl @@ -27,6 +27,8 @@ -behaviour(supervisor). +-include("httpd_internal.hrl"). + %% Internal application API -export([start_link/3, start_link/4]). @@ -41,7 +43,8 @@ start_link([{_, _}| _] = Config, AcceptTimeout, Debug) -> {ok, Config2} -> Address = proplists:get_value(bind_address, Config2), Port = proplists:get_value(port, Config2), - Name = make_name(Address, Port), + Profile = proplists:get_value(profile, Config2, ?DEFAULT_PROFILE), + Name = make_name(Address, Port, Profile), SupName = {local, Name}, supervisor:start_link(SupName, ?MODULE, [undefined, Config2, AcceptTimeout, @@ -54,7 +57,8 @@ start_link([{_, _}| _] = Config, AcceptTimeout, Debug) -> start_link(ConfigFile, AcceptTimeout, Debug) -> case file_2_config(ConfigFile) of {ok, ConfigList, Address, Port} -> - Name = make_name(Address, Port), + Profile = proplists:get_value(profile, ConfigList, ?DEFAULT_PROFILE), + Name = make_name(Address, Port, Profile), SupName = {local, Name}, supervisor:start_link(SupName, ?MODULE, [ConfigFile, ConfigList, AcceptTimeout, @@ -70,7 +74,8 @@ start_link([{_, _}| _] = Config, AcceptTimeout, ListenInfo, Debug) -> {ok, Config2} -> Address = proplists:get_value(bind_address, Config2), Port = proplists:get_value(port, Config2), - Name = make_name(Address, Port), + Profile = proplists:get_value(profile, Config2, ?DEFAULT_PROFILE), + Name = make_name(Address, Port, Profile), SupName = {local, Name}, supervisor:start_link(SupName, ?MODULE, [undefined, Config2, AcceptTimeout, @@ -83,7 +88,8 @@ start_link([{_, _}| _] = Config, AcceptTimeout, ListenInfo, Debug) -> start_link(ConfigFile, AcceptTimeout, ListenInfo, Debug) -> case file_2_config(ConfigFile) of {ok, ConfigList, Address, Port} -> - Name = make_name(Address, Port), + Profile = proplists:get_value(profile, ConfigList, ?DEFAULT_PROFILE), + Name = make_name(Address, Port, Profile), SupName = {local, Name}, supervisor:start_link(SupName, ?MODULE, [ConfigFile, ConfigList, AcceptTimeout, @@ -99,22 +105,24 @@ start_link(ConfigFile, AcceptTimeout, ListenInfo, Debug) -> %%%========================================================================= init([ConfigFile, ConfigList, AcceptTimeout, Debug, Address, Port]) -> httpd_util:enable_debug(Debug), + Profile = proplists:get_value(profile, ConfigList, ?DEFAULT_PROFILE), Flags = {one_for_one, 0, 1}, - Children = [httpd_connection_sup_spec(Address, Port), - httpd_acceptor_sup_spec(Address, Port, ConfigList, AcceptTimeout, + Children = [httpd_connection_sup_spec(Address, Port, Profile), + httpd_acceptor_sup_spec(Address, Port, Profile, ConfigList, AcceptTimeout, undefined), - sup_spec(httpd_misc_sup, Address, Port), - worker_spec(httpd_manager, Address, Port, + sup_spec(httpd_misc_sup, Address, Port, Profile), + worker_spec(httpd_manager, Address, Port, Profile, ConfigFile, ConfigList,AcceptTimeout)], {ok, {Flags, Children}}; init([ConfigFile, ConfigList, AcceptTimeout, Debug, Address, Port, ListenInfo]) -> httpd_util:enable_debug(Debug), + Profile = proplists:get_value(profile, ConfigList, ?DEFAULT_PROFILE), Flags = {one_for_one, 0, 1}, - Children = [httpd_connection_sup_spec(Address, Port), - httpd_acceptor_sup_spec(Address, Port, ConfigList, AcceptTimeout, - ListenInfo), - sup_spec(httpd_misc_sup, Address, Port), - worker_spec(httpd_manager, Address, Port, ListenInfo, + Children = [httpd_connection_sup_spec(Address, Port, Profile), + httpd_acceptor_sup_spec(Address, Port, Profile, ConfigList, AcceptTimeout, + ListenInfo), + sup_spec(httpd_misc_sup, Address, Port, Profile), + worker_spec(httpd_manager, Address, Port, Profile, ListenInfo, ConfigFile, ConfigList, AcceptTimeout)], {ok, {Flags, Children}}. @@ -122,8 +130,8 @@ init([ConfigFile, ConfigList, AcceptTimeout, Debug, Address, Port, ListenInfo]) %%%========================================================================= %%% Internal functions %%%========================================================================= -httpd_connection_sup_spec(Address, Port) -> - Name = {httpd_connection_sup, Address, Port}, +httpd_connection_sup_spec(Address, Port, Profile) -> + Name = {httpd_connection_sup, Address, Port, Profile}, StartFunc = {httpd_connection_sup, start_link, [[Address, Port]]}, Restart = permanent, Shutdown = 5000, @@ -131,8 +139,8 @@ httpd_connection_sup_spec(Address, Port) -> Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. -httpd_acceptor_sup_spec(Address, Port, ConfigList, AcceptTimeout, ListenInfo) -> - Name = {httpd_acceptor_sup, Address, Port}, +httpd_acceptor_sup_spec(Address, Port, Profile, ConfigList, AcceptTimeout, ListenInfo) -> + Name = {httpd_acceptor_sup, Address, Port, Profile}, StartFunc = {httpd_acceptor_sup, start_link, [[Address, Port, ConfigList, AcceptTimeout, ListenInfo]]}, Restart = permanent, Shutdown = infinity, @@ -140,18 +148,18 @@ httpd_acceptor_sup_spec(Address, Port, ConfigList, AcceptTimeout, ListenInfo) -> Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. -sup_spec(SupModule, Address, Port) -> - Name = {SupModule, Address, Port}, - StartFunc = {SupModule, start_link, [Address, Port]}, +sup_spec(SupModule, Address, Port, Profile) -> + Name = {SupModule, Address, Port, Profile}, + StartFunc = {SupModule, start_link, [Address, Port, Profile]}, Restart = permanent, Shutdown = infinity, Modules = [SupModule], Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. -worker_spec(WorkerModule, Address, Port, ConfigFile, +worker_spec(WorkerModule, Address, Port, Profile, ConfigFile, ConfigList, AcceptTimeout) -> - Name = {WorkerModule, Address, Port}, + Name = {WorkerModule, Address, Port, Profile}, StartFunc = {WorkerModule, start_link, [ConfigFile, ConfigList, AcceptTimeout]}, Restart = permanent, @@ -160,9 +168,9 @@ worker_spec(WorkerModule, Address, Port, ConfigFile, Type = worker, {Name, StartFunc, Restart, Shutdown, Type, Modules}. -worker_spec(WorkerModule, Address, Port, ListenInfo, ConfigFile, +worker_spec(WorkerModule, Address, Port, Profile, ListenInfo, ConfigFile, ConfigList, AcceptTimeout) -> - Name = {WorkerModule, Address, Port}, + Name = {WorkerModule, Address, Port, Profile}, StartFunc = {WorkerModule, start_link, [ConfigFile, ConfigList, AcceptTimeout, ListenInfo]}, Restart = permanent, @@ -171,8 +179,8 @@ worker_spec(WorkerModule, Address, Port, ListenInfo, ConfigFile, Type = worker, {Name, StartFunc, Restart, Shutdown, Type, Modules}. -make_name(Address,Port) -> - httpd_util:make_name("httpd_instance_sup", Address, Port). +make_name(Address, Port, Profile) -> + httpd_util:make_name("httpd_instance_sup", Address, Port, Profile). file_2_config(ConfigFile) -> diff --git a/lib/inets/src/http_server/httpd_internal.hrl b/lib/inets/src/http_server/httpd_internal.hrl index 108469ea0a..9829ca255c 100644 --- a/lib/inets/src/http_server/httpd_internal.hrl +++ b/lib/inets/src/http_server/httpd_internal.hrl @@ -31,6 +31,8 @@ -define(SOCKET_MAX_POLL,25). -define(FILE_CHUNK_SIZE,64*1024). -define(GATEWAY_INTERFACE,"CGI/1.1"). +-define(DEFAULT_PROFILE, default). + -define(NICE(Reason),lists:flatten(atom_to_list(?MODULE)++": "++Reason)). -define(DEFAULT_CONTEXT, [{errmsg,"[an error occurred while processing this directive]"}, diff --git a/lib/inets/src/http_server/httpd_manager.erl b/lib/inets/src/http_server/httpd_manager.erl index 3da0343401..995316d5e8 100644 --- a/lib/inets/src/http_server/httpd_manager.erl +++ b/lib/inets/src/http_server/httpd_manager.erl @@ -28,7 +28,7 @@ -export([start/2, start_link/2, start_link/3, start_link/4, stop/1, reload/2]). -export([new_connection/1]). --export([config_match/2, config_match/3]). +-export([config_match/3, config_match/4]). -export([block/2, block/3, unblock/1]). %% gen_server exports @@ -54,7 +54,8 @@ start(ConfigFile, ConfigList) -> Port = proplists:get_value(port,ConfigList,80), Addr = proplists:get_value(bind_address, ConfigList), - Name = make_name(Addr,Port), + Profile = proplists:get_value(profile, ConfigList, default), + Name = make_name(Addr, Port, Profile), gen_server:start({local,Name},?MODULE, [ConfigFile, ConfigList, 15000, Addr, Port],[]). @@ -65,7 +66,8 @@ start_link(ConfigFile, ConfigList) -> start_link(ConfigFile, ConfigList, AcceptTimeout) -> Port = proplists:get_value(port, ConfigList, 80), Addr = proplists:get_value(bind_address, ConfigList), - Name = make_name(Addr, Port), + Profile = proplists:get_value(profile, ConfigList, default), + Name = make_name(Addr, Port, Profile), gen_server:start_link({local, Name},?MODULE, [ConfigFile, ConfigList, @@ -74,7 +76,8 @@ start_link(ConfigFile, ConfigList, AcceptTimeout) -> start_link(ConfigFile, ConfigList, AcceptTimeout, ListenSocket) -> Port = proplists:get_value(port, ConfigList, 80), Addr = proplists:get_value(bind_address, ConfigList), - Name = make_name(Addr, Port), + Profile = proplists:get_value(profile, ConfigList, default), + Name = make_name(Addr, Port, Profile), gen_server:start_link({local, Name},?MODULE, [ConfigFile, ConfigList, AcceptTimeout, Addr, @@ -97,10 +100,10 @@ unblock(ServerRef) -> new_connection(Manager) -> call(Manager, {new_connection, self()}). -config_match(Port, Pattern) -> - config_match(undefined,Port,Pattern). -config_match(Addr, Port, Pattern) -> - Name = httpd_util:make_name("httpd",Addr,Port), +config_match(Port, Profile, Pattern) -> + config_match(undefined,Port, Profile, Pattern). +config_match(Addr, Port, Profile, Pattern) -> + Name = httpd_util:make_name("httpd",Addr,Port, Profile), call(whereis(Name), {config_match, Pattern}). %%%-------------------------------------------------------------------- @@ -446,8 +449,8 @@ get_ustate(ConnectionCnt,State) -> active end. -make_name(Addr,Port) -> - httpd_util:make_name("httpd",Addr,Port). +make_name(Addr, Port, Profile) -> + httpd_util:make_name("httpd", Addr, Port, Profile). report_error(State,String) -> diff --git a/lib/inets/src/http_server/httpd_misc_sup.erl b/lib/inets/src/http_server/httpd_misc_sup.erl index fd7c28bd7d..e5de66d773 100644 --- a/lib/inets/src/http_server/httpd_misc_sup.erl +++ b/lib/inets/src/http_server/httpd_misc_sup.erl @@ -27,8 +27,8 @@ -behaviour(supervisor). %% API --export([start_link/2, start_auth_server/2, stop_auth_server/2, - start_sec_server/2, stop_sec_server/2]). +-export([start_link/3, start_auth_server/3, stop_auth_server/3, + start_sec_server/3, stop_sec_server/3]). %% Supervisor callback -export([init/1]). @@ -37,26 +37,26 @@ %%% API %%%========================================================================= -start_link(Addr, Port) -> - SupName = make_name(Addr, Port), +start_link(Addr, Port, Profile) -> + SupName = make_name(Addr, Port, Profile), supervisor:start_link({local, SupName}, ?MODULE, []). %%---------------------------------------------------------------------- %% Function: [start|stop]_[auth|sec]_server/3 %% Description: Starts a [auth | security] worker (child) process %%---------------------------------------------------------------------- -start_auth_server(Addr, Port) -> - start_permanent_worker(mod_auth_server, Addr, Port, [gen_server]). +start_auth_server(Addr, Port, Profile) -> + start_permanent_worker(mod_auth_server, Addr, Port, Profile, [gen_server]). -stop_auth_server(Addr, Port) -> - stop_permanent_worker(mod_auth_server, Addr, Port). +stop_auth_server(Addr, Port, Profile) -> + stop_permanent_worker(mod_auth_server, Addr, Port, Profile). -start_sec_server(Addr, Port) -> - start_permanent_worker(mod_security_server, Addr, Port, [gen_server]). +start_sec_server(Addr, Port, Profile) -> + start_permanent_worker(mod_security_server, Addr, Port, Profile, [gen_server]). -stop_sec_server(Addr, Port) -> - stop_permanent_worker(mod_security_server, Addr, Port). +stop_sec_server(Addr, Port, Profile) -> + stop_permanent_worker(mod_security_server, Addr, Port, Profile). %%%========================================================================= @@ -70,15 +70,15 @@ init(_) -> %%%========================================================================= %%% Internal functions %%%========================================================================= -start_permanent_worker(Mod, Addr, Port, Modules) -> - SupName = make_name(Addr, Port), +start_permanent_worker(Mod, Addr, Port, Profile, Modules) -> + SupName = make_name(Addr, Port, Profile), Spec = {{Mod, Addr, Port}, - {Mod, start_link, [Addr, Port]}, + {Mod, start_link, [Addr, Port, Profile]}, permanent, timer:seconds(1), worker, [Mod] ++ Modules}, supervisor:start_child(SupName, Spec). -stop_permanent_worker(Mod, Addr, Port) -> - SupName = make_name(Addr, Port), +stop_permanent_worker(Mod, Addr, Port, Profile) -> + SupName = make_name(Addr, Port, Profile), Name = {Mod, Addr, Port}, case supervisor:terminate_child(SupName, Name) of ok -> @@ -87,5 +87,5 @@ stop_permanent_worker(Mod, Addr, Port) -> Error end. -make_name(Addr,Port) -> - httpd_util:make_name("httpd_misc_sup",Addr,Port). +make_name(Addr,Port, Profile) -> + httpd_util:make_name("httpd_misc_sup",Addr,Port, Profile). diff --git a/lib/inets/src/http_server/httpd_request.erl b/lib/inets/src/http_server/httpd_request.erl index 6985065c3e..782120c284 100644 --- a/lib/inets/src/http_server/httpd_request.erl +++ b/lib/inets/src/http_server/httpd_request.erl @@ -42,28 +42,28 @@ %%%========================================================================= %%% Internal application API %%%========================================================================= -parse([Bin, MaxSizes]) -> - ?hdrt("parse", [{bin, Bin}, {max_sizes, MaxSizes}]), - parse_method(Bin, [], 0, proplists:get_value(max_method, MaxSizes), MaxSizes, []); +parse([Bin, Options]) -> + ?hdrt("parse", [{bin, Bin}, {max_sizes, Options}]), + parse_method(Bin, [], 0, proplists:get_value(max_method, Options), Options, []); parse(Unknown) -> ?hdrt("parse", [{unknown, Unknown}]), exit({bad_args, Unknown}). %% Functions that may be returned during the decoding process %% if the input data is incompleate. -parse_method([Bin, Method, Current, Max, MaxSizes, Result]) -> - parse_method(Bin, Method, Current, Max, MaxSizes, Result). +parse_method([Bin, Method, Current, Max, Options, Result]) -> + parse_method(Bin, Method, Current, Max, Options, Result). -parse_uri([Bin, URI, Current, Max, MaxSizes, Result]) -> - parse_uri(Bin, URI, Current, Max, MaxSizes, Result). +parse_uri([Bin, URI, Current, Max, Options, Result]) -> + parse_uri(Bin, URI, Current, Max, Options, Result). -parse_version([Bin, Rest, Version, Current, Max, MaxSizes, Result]) -> - parse_version(<<Rest/binary, Bin/binary>>, Version, Current, Max, MaxSizes, +parse_version([Bin, Rest, Version, Current, Max, Options, Result]) -> + parse_version(<<Rest/binary, Bin/binary>>, Version, Current, Max, Options, Result). -parse_headers([Bin, Rest, Header, Headers, Current, Max, MaxSizes, Result]) -> +parse_headers([Bin, Rest, Header, Headers, Current, Max, Options, Result]) -> parse_headers(<<Rest/binary, Bin/binary>>, - Header, Headers, Current, Max, MaxSizes, Result). + Header, Headers, Current, Max, Options, Result). whole_body([Bin, Body, Length]) -> whole_body(<<Body/binary, Bin/binary>>, Length). @@ -134,13 +134,13 @@ update_mod_data(ModData, Method, RequestURI, HTTPVersion, Headers)-> %%%======================================================================== %%% Internal functions %%%======================================================================== -parse_method(<<>>, Method, Current, Max, MaxSizes, Result) -> - {?MODULE, parse_method, [Method, Current, Max, MaxSizes, Result]}; -parse_method(<<?SP, Rest/binary>>, Method, _Current, _Max, MaxSizes, Result) -> - parse_uri(Rest, [], 0, proplists:get_value(max_uri, MaxSizes), MaxSizes, +parse_method(<<>>, Method, Current, Max, Options, Result) -> + {?MODULE, parse_method, [Method, Current, Max, Options, Result]}; +parse_method(<<?SP, Rest/binary>>, Method, _Current, _Max, Options, Result) -> + parse_uri(Rest, [], 0, proplists:get_value(max_uri, Options), Options, [string:strip(lists:reverse(Method)) | Result]); -parse_method(<<Octet, Rest/binary>>, Method, Current, Max, MaxSizes, Result) when Current =< Max -> - parse_method(Rest, [Octet | Method], Current + 1, Max, MaxSizes, Result); +parse_method(<<Octet, Rest/binary>>, Method, Current, Max, Options, Result) when Current =< Max -> + parse_method(Rest, [Octet | Method], Current + 1, Max, Options, Result); parse_method(_, _, _, Max, _, _) -> %% We do not know the version of the client as it comes after the %% method send the lowest version in the response so that the client @@ -153,30 +153,30 @@ parse_uri(_, _, Current, MaxURI, _, _) %% uri send the lowest version in the response so that the client %% will be able to handle it. {error, {size_error, MaxURI, 414, "URI unreasonably long"},lowest_version()}; -parse_uri(<<>>, URI, Current, Max, MaxSizes, Result) -> - {?MODULE, parse_uri, [URI, Current, Max, MaxSizes, Result]}; -parse_uri(<<?SP, Rest/binary>>, URI, _, _, MaxSizes, Result) -> - parse_version(Rest, [], 0, proplists:get_value(max_version, MaxSizes), MaxSizes, +parse_uri(<<>>, URI, Current, Max, Options, Result) -> + {?MODULE, parse_uri, [URI, Current, Max, Options, Result]}; +parse_uri(<<?SP, Rest/binary>>, URI, _, _, Options, Result) -> + parse_version(Rest, [], 0, proplists:get_value(max_version, Options), Options, [string:strip(lists:reverse(URI)) | Result]); %% Can happen if it is a simple HTTP/0.9 request e.i "GET /\r\n\r\n" -parse_uri(<<?CR, _Rest/binary>> = Data, URI, _, _, MaxSizes, Result) -> - parse_version(Data, [], 0, proplists:get_value(max_version, MaxSizes), MaxSizes, +parse_uri(<<?CR, _Rest/binary>> = Data, URI, _, _, Options, Result) -> + parse_version(Data, [], 0, proplists:get_value(max_version, Options), Options, [string:strip(lists:reverse(URI)) | Result]); -parse_uri(<<Octet, Rest/binary>>, URI, Current, Max, MaxSizes, Result) -> - parse_uri(Rest, [Octet | URI], Current + 1, Max, MaxSizes, Result). +parse_uri(<<Octet, Rest/binary>>, URI, Current, Max, Options, Result) -> + parse_uri(Rest, [Octet | URI], Current + 1, Max, Options, Result). -parse_version(<<>>, Version, Current, Max, MaxSizes, Result) -> - {?MODULE, parse_version, [<<>>, Version, Current, Max, MaxSizes, Result]}; -parse_version(<<?LF, Rest/binary>>, Version, Current, Max, MaxSizes, Result) -> +parse_version(<<>>, Version, Current, Max, Options, Result) -> + {?MODULE, parse_version, [<<>>, Version, Current, Max, Options, Result]}; +parse_version(<<?LF, Rest/binary>>, Version, Current, Max, Options, Result) -> %% If ?CR is is missing RFC2616 section-19.3 - parse_version(<<?CR, ?LF, Rest/binary>>, Version, Current, Max, MaxSizes, Result); -parse_version(<<?CR, ?LF, Rest/binary>>, Version, _, _, MaxSizes, Result) -> - parse_headers(Rest, [], [], 0, proplists:get_value(max_header, MaxSizes), MaxSizes, + parse_version(<<?CR, ?LF, Rest/binary>>, Version, Current, Max, Options, Result); +parse_version(<<?CR, ?LF, Rest/binary>>, Version, _, _, Options, Result) -> + parse_headers(Rest, [], [], 0, proplists:get_value(max_header, Options), Options, [string:strip(lists:reverse(Version)) | Result]); -parse_version(<<?CR>> = Data, Version, Current, Max, MaxSizes, Result) -> - {?MODULE, parse_version, [Data, Version, Current, Max, MaxSizes, Result]}; -parse_version(<<Octet, Rest/binary>>, Version, Current, Max, MaxSizes, Result) when Current =< Max -> - parse_version(Rest, [Octet | Version], Current + 1, Max, MaxSizes, Result); +parse_version(<<?CR>> = Data, Version, Current, Max, Options, Result) -> + {?MODULE, parse_version, [Data, Version, Current, Max, Options, Result]}; +parse_version(<<Octet, Rest/binary>>, Version, Current, Max, Options, Result) when Current =< Max -> + parse_version(Rest, [Octet | Version], Current + 1, Max, Options, Result); parse_version(_, _, _, Max,_,_) -> {error, {size_error, Max, 413, "Version string unreasonably long"}, lowest_version()}. @@ -185,34 +185,42 @@ parse_headers(_, _, _, Current, Max, _, Result) HttpVersion = lists:nth(3, lists:reverse(Result)), {error, {size_error, Max, 413, "Headers unreasonably long"}, HttpVersion}; -parse_headers(<<>>, Header, Headers, Current, Max, MaxSizes, Result) -> +parse_headers(<<>>, Header, Headers, Current, Max, Options, Result) -> {?MODULE, parse_headers, [<<>>, Header, Headers, Current, Max, - MaxSizes, Result]}; -parse_headers(<<?CR,?LF,?LF,Body/binary>>, [], [], Current, Max, MaxSizes, Result) -> + Options, Result]}; +parse_headers(<<?CR,?LF,?LF,Body/binary>>, [], [], Current, Max, Options, Result) -> %% If ?CR is is missing RFC2616 section-19.3 parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, [], [], Current, Max, - MaxSizes, Result); + Options, Result); -parse_headers(<<?LF,?LF,Body/binary>>, [], [], Current, Max, MaxSizes, Result) -> +parse_headers(<<?LF,?LF,Body/binary>>, [], [], Current, Max, Options, Result) -> %% If ?CR is is missing RFC2616 section-19.3 parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, [], [], Current, Max, - MaxSizes, Result); + Options, Result); parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, [], [], _, _, _, Result) -> NewResult = list_to_tuple(lists:reverse([Body, {#http_request_h{}, []} | Result])), {ok, NewResult}; parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, Header, Headers, _, _, - MaxSizes, Result) -> + Options, Result) -> + Customize = proplists:get_value(customize, Options), case http_request:key_value(lists:reverse(Header)) of undefined -> %% Skip headers with missing : - {ok, list_to_tuple(lists:reverse([Body, {http_request:headers(Headers, #http_request_h{}), Headers} | Result]))}; + FinalHeaders = lists:filtermap(fun(H) -> + httpd_custom:customize_headers(Customize, request_header, H) + end, + Headers), + {ok, list_to_tuple(lists:reverse([Body, {http_request:headers(FinalHeaders, #http_request_h{}), FinalHeaders} | Result]))}; NewHeader -> - case check_header(NewHeader, MaxSizes) of + case check_header(NewHeader, Options) of ok -> - {ok, list_to_tuple(lists:reverse([Body, {http_request:headers([NewHeader | Headers], + FinalHeaders = lists:filtermap(fun(H) -> + httpd_custom:customize_headers(Customize, request_header, H) + end, [NewHeader | Headers]), + {ok, list_to_tuple(lists:reverse([Body, {http_request:headers(FinalHeaders, #http_request_h{}), - [NewHeader | Headers]} | Result]))}; + FinalHeaders} | Result]))}; {error, Reason} -> HttpVersion = lists:nth(3, lists:reverse(Result)), @@ -221,12 +229,12 @@ parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, Header, Headers, _, _, end; parse_headers(<<?CR,?LF,?CR>> = Data, Header, Headers, Current, Max, - MaxSizes, Result) -> + Options, Result) -> {?MODULE, parse_headers, [Data, Header, Headers, Current, Max, - MaxSizes, Result]}; -parse_headers(<<?LF>>, [], [], Current, Max, MaxSizes, Result) -> + Options, Result]}; +parse_headers(<<?LF>>, [], [], Current, Max, Options, Result) -> %% If ?CR is is missing RFC2616 section-19.3 - parse_headers(<<?CR,?LF>>, [], [], Current, Max, MaxSizes, Result); + parse_headers(<<?CR,?LF>>, [], [], Current, Max, Options, Result); %% There where no headers, which is unlikely to happen. parse_headers(<<?CR,?LF>>, [], [], _, _, _, Result) -> @@ -235,30 +243,30 @@ parse_headers(<<?CR,?LF>>, [], [], _, _, _, Result) -> {ok, NewResult}; parse_headers(<<?LF>>, Header, Headers, Current, Max, - MaxSizes, Result) -> + Options, Result) -> %% If ?CR is is missing RFC2616 section-19.3 - parse_headers(<<?CR,?LF>>, Header, Headers, Current, Max, MaxSizes, Result); + parse_headers(<<?CR,?LF>>, Header, Headers, Current, Max, Options, Result); parse_headers(<<?CR,?LF>> = Data, Header, Headers, Current, Max, - MaxSizes, Result) -> + Options, Result) -> {?MODULE, parse_headers, [Data, Header, Headers, Current, Max, - MaxSizes, Result]}; + Options, Result]}; parse_headers(<<?LF, Octet, Rest/binary>>, Header, Headers, Current, Max, - MaxSizes, Result) -> + Options, Result) -> %% If ?CR is is missing RFC2616 section-19.3 parse_headers(<<?CR,?LF, Octet, Rest/binary>>, Header, Headers, Current, Max, - MaxSizes, Result); + Options, Result); parse_headers(<<?CR,?LF, Octet, Rest/binary>>, Header, Headers, _, Max, - MaxSizes, Result) -> + Options, Result) -> case http_request:key_value(lists:reverse(Header)) of undefined -> %% Skip headers with missing : parse_headers(Rest, [Octet], Headers, - 0, Max, MaxSizes, Result); + 0, Max, Options, Result); NewHeader -> - case check_header(NewHeader, MaxSizes) of + case check_header(NewHeader, Options) of ok -> parse_headers(Rest, [Octet], [NewHeader | Headers], - 0, Max, MaxSizes, Result); + 0, Max, Options, Result); {error, Reason} -> HttpVersion = lists:nth(3, lists:reverse(Result)), {error, Reason, HttpVersion} @@ -266,19 +274,19 @@ parse_headers(<<?CR,?LF, Octet, Rest/binary>>, Header, Headers, _, Max, end; parse_headers(<<?CR>> = Data, Header, Headers, Current, Max, - MaxSizes, Result) -> + Options, Result) -> {?MODULE, parse_headers, [Data, Header, Headers, Current, Max, - MaxSizes, Result]}; + Options, Result]}; parse_headers(<<?LF>>, Header, Headers, Current, Max, - MaxSizes, Result) -> + Options, Result) -> %% If ?CR is is missing RFC2616 section-19.3 parse_headers(<<?CR, ?LF>>, Header, Headers, Current, Max, - MaxSizes, Result); + Options, Result); parse_headers(<<Octet, Rest/binary>>, Header, Headers, Current, - Max, MaxSizes, Result) -> + Max, Options, Result) -> parse_headers(Rest, [Octet | Header], Headers, Current + 1, Max, - MaxSizes, Result). + Options, Result). whole_body(Body, Length) -> case size(Body) of @@ -417,8 +425,12 @@ check_header({"content-length", Value}, Maxsizes) -> case length(Value) =< MaxLen of true -> try - _ = list_to_integer(Value), - ok + list_to_integer(Value) + of + I when I>= 0 -> + ok; + _ -> + {error, {size_error, Max, 411, "negative content-length"}} catch _:_ -> {error, {size_error, Max, 411, "content-length not an integer"}} end; diff --git a/lib/inets/src/http_server/httpd_request_handler.erl b/lib/inets/src/http_server/httpd_request_handler.erl index f7a9fe5d49..9947e17b47 100644 --- a/lib/inets/src/http_server/httpd_request_handler.erl +++ b/lib/inets/src/http_server/httpd_request_handler.erl @@ -121,13 +121,15 @@ continue_init(Manager, ConfigDB, SocketType, Socket, TimeOut) -> MaxURISize = max_uri_size(ConfigDB), NrOfRequest = max_keep_alive_request(ConfigDB), MaxContentLen = max_content_length(ConfigDB), + Customize = customize(ConfigDB), {_, Status} = httpd_manager:new_connection(Manager), MFA = {httpd_request, parse, [[{max_uri, MaxURISize}, {max_header, MaxHeaderSize}, {max_version, ?HTTP_MAX_VERSION_STRING}, {max_method, ?HTTP_MAX_METHOD_STRING}, - {max_content_length, MaxContentLen} + {max_content_length, MaxContentLen}, + {customize, Customize} ]]}, State = #state{mod = Mod, @@ -550,11 +552,13 @@ handle_next_request(#state{mod = #mod{connection = true} = ModData, MaxHeaderSize = max_header_size(ModData#mod.config_db), MaxURISize = max_uri_size(ModData#mod.config_db), MaxContentLen = max_content_length(ModData#mod.config_db), + Customize = customize(ModData#mod.config_db), MFA = {httpd_request, parse, [[{max_uri, MaxURISize}, {max_header, MaxHeaderSize}, {max_version, ?HTTP_MAX_VERSION_STRING}, {max_method, ?HTTP_MAX_METHOD_STRING}, - {max_content_length, MaxContentLen} + {max_content_length, MaxContentLen}, + {customize, Customize} ]]}, TmpState = State#state{mod = NewModData, mfa = MFA, @@ -640,3 +644,6 @@ max_keep_alive_request(ConfigDB) -> max_content_length(ConfigDB) -> httpd_util:lookup(ConfigDB, max_content_length, ?HTTP_MAX_CONTENT_LENGTH). + +customize(ConfigDB) -> + httpd_util:lookup(ConfigDB, customize, httpd_custom). diff --git a/lib/inets/src/http_server/httpd_response.erl b/lib/inets/src/http_server/httpd_response.erl index 2fa91d47a0..71dc05e46d 100644 --- a/lib/inets/src/http_server/httpd_response.erl +++ b/lib/inets/src/http_server/httpd_response.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -176,7 +176,7 @@ send_header(#mod{socket_type = Type, StatusLine = [NewVer, " ", io_lib:write(NewStatusCode), " ", httpd_util:reason_phrase(NewStatusCode), ?CRLF], ConnectionHeader = get_connection(Conn, NewVer), - Head = list_to_binary([StatusLine, Headers, ConnectionHeader , ?CRLF]), + Head = [StatusLine, Headers, ConnectionHeader , ?CRLF], httpd_socket:deliver(Type, Sock, Head). map_status_code("HTTP/1.0", Code) @@ -286,45 +286,21 @@ create_header(ConfigDb, KeyValueTupleHeaders) -> Date = httpd_util:rfc1123_date(), ContentType = "text/html", Server = server(ConfigDb), - NewHeaders = add_default_headers([{"date", Date}, - {"content-type", ContentType} - | if Server=="" -> []; - true -> [{"server", Server}] - end - ], - KeyValueTupleHeaders), - lists:map(fun fix_header/1, NewHeaders). - - + Headers0 = add_default_headers([{"date", Date}, + {"content-type", ContentType} + | if Server=="" -> []; + true -> [{"server", Server}] + end + ], + KeyValueTupleHeaders), + CustomizeCB = httpd_util:lookup(ConfigDb, customize, httpd_custom), + lists:filtermap(fun(H) -> + httpd_custom:customize_headers(CustomizeCB, response_header, H) + end, + [Header || Header <- Headers0]). server(ConfigDb) -> httpd_util:lookup(ConfigDb, server, ?SERVER_SOFTWARE). -fix_header({Key0, Value}) -> - %% make sure first letter is capital - Words1 = string:tokens(Key0, "-"), - Words2 = upify(Words1, []), - Key = new_key(Words2), - Key ++ ": " ++ Value ++ ?CRLF . - -new_key([]) -> - ""; -new_key([W]) -> - W; -new_key([W1,W2]) -> - W1 ++ "-" ++ W2; -new_key([W|R]) -> - W ++ "-" ++ new_key(R). - -upify([], Acc) -> - lists:reverse(Acc); -upify([Key|Rest], Acc) -> - upify(Rest, [upify2(Key)|Acc]). - -upify2([C|Rest]) when (C >= $a) andalso (C =< $z) -> - [C-($a-$A)|Rest]; -upify2(Str) -> - Str. - add_default_headers([], Headers) -> Headers; diff --git a/lib/inets/src/http_server/httpd_sup.erl b/lib/inets/src/http_server/httpd_sup.erl index 3b1e16cf78..b45742136a 100644 --- a/lib/inets/src/http_server/httpd_sup.erl +++ b/lib/inets/src/http_server/httpd_sup.erl @@ -28,7 +28,7 @@ %% Internal application API -export([start_link/1, start_link/2]). --export([start_child/1, restart_child/2, stop_child/2]). +-export([start_child/1, restart_child/3, stop_child/3]). %% Supervisor callback -export([init/1]). @@ -37,7 +37,6 @@ -define(TIMEOUT, 15000). -include("httpd_internal.hrl"). --include("inets_internal.hrl"). %%%========================================================================= %%% API @@ -64,33 +63,32 @@ start_child(Config) -> end. -restart_child(Address, Port) -> - Name = id(Address, Port), +restart_child(Address, Port, Profile) -> + Name = id(Address, Port, Profile), case supervisor:terminate_child(?MODULE, Name) of - ok -> - supervisor:restart_child(?MODULE, Name); - Error -> - Error - end. - -stop_child(Address, Port) -> - Name = id(Address, Port), + ok -> + supervisor:restart_child(?MODULE, Name); + Error -> + Error + end. + +stop_child(Address, Port, Profile) -> + Name = id(Address, Port, Profile), case supervisor:terminate_child(?MODULE, Name) of - ok -> - supervisor:delete_child(?MODULE, Name); - Error -> + ok -> + supervisor:delete_child(?MODULE, Name); + Error -> Error end. - -id(Address, Port) -> - {httpd_instance_sup, Address, Port}. + +id(Address, Port, Profile) -> + {httpd_instance_sup, Address, Port, Profile}. %%%========================================================================= %%% Supervisor callback %%%========================================================================= init([HttpdServices]) -> - ?hdrd("starting", [{httpd_service, HttpdServices}]), RestartStrategy = one_for_one, MaxR = 10, MaxT = 3600, @@ -118,23 +116,18 @@ init([HttpdServices]) -> child_specs([], Acc) -> Acc; child_specs([{httpd, HttpdService} | Rest], Acc) -> - ?hdrd("child specs", [{httpd, HttpdService}]), NewHttpdService = (catch mk_tuple_list(HttpdService)), - ?hdrd("child specs", [{new_httpd, NewHttpdService}]), case catch child_spec(NewHttpdService) of {error, Reason} -> - ?hdri("failed generating child spec", [{reason, Reason}]), error_msg("Failed to start service: ~n~p ~n due to: ~p~n", [HttpdService, Reason]), child_specs(Rest, Acc); Spec -> - ?hdrt("child spec", [{child_spec, Spec}]), child_specs(Rest, [Spec | Acc]) end. child_spec(HttpdService) -> {ok, Config} = httpd_config(HttpdService), - ?hdrt("child spec", [{config, Config}]), Debug = proplists:get_value(debug, Config, []), AcceptTimeout = proplists:get_value(accept_timeout, Config, 15000), httpd_util:valid_options(Debug, AcceptTimeout, Config), @@ -162,32 +155,27 @@ httpd_config([Value| _] = Config) when is_tuple(Value) -> httpd_child_spec([Value| _] = Config, AcceptTimeout, Debug) when is_tuple(Value) -> - ?hdrt("httpd_child_spec - entry", [{accept_timeout, AcceptTimeout}, - {debug, Debug}]), Address = proplists:get_value(bind_address, Config, any), Port = proplists:get_value(port, Config, 80), - httpd_child_spec(Config, AcceptTimeout, Debug, Address, Port); + Profile = proplists:get_value(profile, Config, ?DEFAULT_PROFILE), + httpd_child_spec(Config, AcceptTimeout, Debug, Address, Port, Profile); %% In this case the AcceptTimeout and Debug will only have default values... httpd_child_spec(ConfigFile, AcceptTimeoutDef, DebugDef) -> - ?hdrt("httpd_child_spec - entry", [{config_file, ConfigFile}, - {accept_timeout_def, AcceptTimeoutDef}, - {debug_def, DebugDef}]), case httpd_conf:load(ConfigFile) of {ok, ConfigList} -> - ?hdrt("httpd_child_spec - loaded", [{config_list, ConfigList}]), case (catch httpd_conf:validate_properties(ConfigList)) of {ok, Config} -> - ?hdrt("httpd_child_spec - validated", [{config, Config}]), Address = proplists:get_value(bind_address, Config, any), Port = proplists:get_value(port, Config, 80), + Profile = proplists:get_value(profile, Config, ?DEFAULT_PROFILE), AcceptTimeout = proplists:get_value(accept_timeout, Config, AcceptTimeoutDef), Debug = proplists:get_value(debug, Config, DebugDef), httpd_child_spec([{file, ConfigFile} | Config], - AcceptTimeout, Debug, Address, Port); + AcceptTimeout, Debug, Address, Port, Profile); Error -> Error end; @@ -195,19 +183,19 @@ httpd_child_spec(ConfigFile, AcceptTimeoutDef, DebugDef) -> Error end. -httpd_child_spec(Config, AcceptTimeout, Debug, Addr, Port) -> +httpd_child_spec(Config, AcceptTimeout, Debug, Addr, Port, Profile) -> Fd = proplists:get_value(fd, Config, undefined), case Port == 0 orelse Fd =/= undefined of true -> - httpd_child_spec_listen(Config, AcceptTimeout, Debug, Addr, Port); + httpd_child_spec_listen(Config, AcceptTimeout, Debug, Addr, Port, Profile); false -> - httpd_child_spec_nolisten(Config, AcceptTimeout, Debug, Addr, Port) + httpd_child_spec_nolisten(Config, AcceptTimeout, Debug, Addr, Port, Profile) end. -httpd_child_spec_listen(Config, AcceptTimeout, Debug, Addr, Port) -> +httpd_child_spec_listen(Config, AcceptTimeout, Debug, Addr, Port, Profile) -> case start_listen(Addr, Port, Config) of {Pid, {NewPort, NewConfig, ListenSocket}} -> - Name = {httpd_instance_sup, Addr, NewPort}, + Name = {httpd_instance_sup, Addr, NewPort, Profile}, StartFunc = {httpd_instance_sup, start_link, [NewConfig, AcceptTimeout, {Pid, ListenSocket}, Debug]}, @@ -221,8 +209,8 @@ httpd_child_spec_listen(Config, AcceptTimeout, Debug, Addr, Port) -> {error, Reason} end. -httpd_child_spec_nolisten(Config, AcceptTimeout, Debug, Addr, Port) -> - Name = {httpd_instance_sup, Addr, Port}, +httpd_child_spec_nolisten(Config, AcceptTimeout, Debug, Addr, Port, Profile) -> + Name = {httpd_instance_sup, Addr, Port, Profile}, StartFunc = {httpd_instance_sup, start_link, [Config, AcceptTimeout, Debug]}, Restart = permanent, diff --git a/lib/inets/src/http_server/httpd_util.erl b/lib/inets/src/http_server/httpd_util.erl index 0d04a75205..b1ddc1abbb 100644 --- a/lib/inets/src/http_server/httpd_util.erl +++ b/lib/inets/src/http_server/httpd_util.erl @@ -572,7 +572,10 @@ make_name(Prefix,Port) -> make_name(Prefix,Addr,Port) -> make_name(Prefix,Addr,Port,""). - + +make_name(Prefix, Addr,Port,Postfix) when is_atom(Postfix)-> + make_name(Prefix, Addr,Port, atom_to_list(Postfix)); + make_name(Prefix,"*",Port,Postfix) -> make_name(Prefix,undefined,Port,Postfix); @@ -595,15 +598,7 @@ make_name2({A,B,C,D}) -> io_lib:format("~w_~w_~w_~w", [A,B,C,D]); make_name2({A, B, C, D, E, F, G, H}) -> - io_lib:format("~s_~s_~s_~s_~s_~s_~s_~s", [integer_to_hexlist(A), - integer_to_hexlist(B), - integer_to_hexlist(C), - integer_to_hexlist(D), - integer_to_hexlist(E), - integer_to_hexlist(F), - integer_to_hexlist(G), - integer_to_hexlist(H) - ]); + io_lib:format("~w_~w_~w_~w_~w_~w_~w_~w", [A,B,C,D,E,F,G,H]); make_name2(Addr) -> search_and_replace(Addr,$.,$_). diff --git a/lib/inets/src/http_server/mod_auth.erl b/lib/inets/src/http_server/mod_auth.erl index 85a87ab884..1f4470622d 100644 --- a/lib/inets/src/http_server/mod_auth.erl +++ b/lib/inets/src/http_server/mod_auth.erl @@ -38,15 +38,16 @@ -include("httpd.hrl"). -include("mod_auth.hrl"). -include("httpd_internal.hrl"). --include("inets_internal.hrl"). -define(VMODULE,"AUTH"). -define(NOPASSWORD,"NoPassword"). -%% do +%%==================================================================== +%% Internal application API +%%==================================================================== + do(Info) -> - ?hdrt("do", [{info, Info}]), case proplists:get_value(status,Info#mod.data) of %% A status code has been generated! {_StatusCode, _PhraseArgs, _Reason} -> @@ -61,22 +62,15 @@ do(Info) -> %% Is it a secret area? case secretp(Path,Info#mod.config_db) of {yes, {Directory, DirectoryData}} -> - ?hdrt("secret area", - [{directory, Directory}, - {directory_data, DirectoryData}]), - - %% Authenticate (allow) case allow((Info#mod.init_data)#init_data.peername, Info#mod.socket_type,Info#mod.socket, DirectoryData) of allowed -> - ?hdrt("allowed", []), case deny((Info#mod.init_data)#init_data.peername, Info#mod.socket_type, Info#mod.socket, DirectoryData) of not_denied -> - ?hdrt("not denied", []), case proplists:get_value(auth_type, DirectoryData) of undefined -> @@ -90,15 +84,13 @@ do(Info) -> AuthType) end; {denied, Reason} -> - ?hdrt("denied", [{reason, Reason}]), {proceed, [{status, {403, - Info#mod.request_uri, - Reason}}| + Info#mod.request_uri, + Reason}}| Info#mod.data]} end; {not_allowed, Reason} -> - ?hdrt("not allowed", [{reason, Reason}]), {proceed,[{status,{403, Info#mod.request_uri, Reason}} | @@ -114,18 +106,299 @@ do(Info) -> end. -do_auth(Info, Directory, DirectoryData, AuthType) -> +%% mod_auth recognizes the following Configuration Directives: +%% <Directory /path/to/directory> +%% AuthDBType +%% AuthName +%% AuthUserFile +%% AuthGroupFile +%% AuthAccessPassword +%% require +%% allow +%% </Directory> + +%% When a <Directory> directive is found, a new context is set to +%% [{directory, Directory, DirData}|OtherContext] +%% DirData in this case is a key-value list of data belonging to the +%% directory in question. +%% +%% When the </Directory> statement is found, the Context created earlier +%% will be returned as a ConfigList and the context will return to the +%% state it was previously. + +load("<Directory " ++ Directory,[]) -> + Dir = httpd_conf:custom_clean(Directory,"",">"), + {ok,[{directory, {Dir, [{path, Dir}]}}]}; +load(eof,[{directory, {Directory, _DirData}}|_]) -> + {error, ?NICE("Premature end-of-file in "++ Directory)}; + +load("AuthName " ++ AuthName, [{directory, {Directory, DirData}}|Rest]) -> + {ok, [{directory, {Directory, + [{auth_name, httpd_conf:clean(AuthName)} | DirData]}} + | Rest ]}; +load("AuthUserFile " ++ AuthUserFile0, + [{directory, {Directory, DirData}}|Rest]) -> + AuthUserFile = httpd_conf:clean(AuthUserFile0), + {ok, [{directory, {Directory, + [{auth_user_file, AuthUserFile}|DirData]}} | Rest ]}; +load("AuthGroupFile " ++ AuthGroupFile0, + [{directory, {Directory, DirData}}|Rest]) -> + AuthGroupFile = httpd_conf:clean(AuthGroupFile0), + {ok,[{directory, {Directory, + [{auth_group_file, AuthGroupFile}|DirData]}} | Rest]}; + +load("AuthAccessPassword " ++ AuthAccessPassword0, + [{directory, {Directory, DirData}}|Rest]) -> + AuthAccessPassword = httpd_conf:clean(AuthAccessPassword0), + {ok,[{directory, {Directory, + [{auth_access_password, AuthAccessPassword}|DirData]}} | Rest]}; + +load("AuthDBType " ++ Type, + [{directory, {Dir, DirData}}|Rest]) -> + case httpd_conf:clean(Type) of + "plain" -> + {ok, [{directory, {Dir, [{auth_type, plain}|DirData]}} | Rest ]}; + "mnesia" -> + {ok, [{directory, {Dir, [{auth_type, mnesia}|DirData]}} | Rest ]}; + "dets" -> + {ok, [{directory, {Dir, [{auth_type, dets}|DirData]}} | Rest ]}; + _ -> + {error, ?NICE(httpd_conf:clean(Type)++" is an invalid AuthDBType")} + end; + +load("require " ++ Require,[{directory, {Directory, DirData}}|Rest]) -> + case inets_regexp:split(Require," ") of + {ok,["user"|Users]} -> + {ok,[{directory, {Directory, + [{require_user,Users}|DirData]}} | Rest]}; + {ok,["group"|Groups]} -> + {ok,[{directory, {Directory, + [{require_group,Groups}|DirData]}} | Rest]}; + {ok,_} -> + {error,?NICE(httpd_conf:clean(Require) ++" is an invalid require")} + end; + +load("allow " ++ Allow,[{directory, {Directory, DirData}}|Rest]) -> + case inets_regexp:split(Allow," ") of + {ok,["from","all"]} -> + {ok,[{directory, {Directory, + [{allow_from,all}|DirData]}} | Rest]}; + {ok,["from"|Hosts]} -> + {ok,[{directory, {Directory, + [{allow_from,Hosts}|DirData]}} | Rest]}; + {ok,_} -> + {error,?NICE(httpd_conf:clean(Allow) ++" is an invalid allow")} + end; + +load("deny " ++ Deny,[{directory, {Directory, DirData}}|Rest]) -> + case inets_regexp:split(Deny," ") of + {ok, ["from", "all"]} -> + {ok,[{{directory, Directory, + [{deny_from, all}|DirData]}} | Rest]}; + {ok, ["from"|Hosts]} -> + {ok,[{{directory, Directory, + [{deny_from, Hosts}|DirData]}} | Rest]}; + {ok, _} -> + {error,?NICE(httpd_conf:clean(Deny) ++" is an invalid deny")} + end; + +load("</Directory>",[{directory, {Directory, DirData}}|Rest]) -> + {ok, Rest, {directory, {Directory, DirData}}}; + +load("AuthMnesiaDB " ++ AuthMnesiaDB, + [{directory, {Dir, DirData}}|Rest]) -> + case httpd_conf:clean(AuthMnesiaDB) of + "On" -> + {ok,[{directory, {Dir,[{auth_type,mnesia}|DirData]}}|Rest]}; + "Off" -> + {ok,[{directory, {Dir,[{auth_type,plain}|DirData]}}|Rest]}; + _ -> + {error, ?NICE(httpd_conf:clean(AuthMnesiaDB) ++ + " is an invalid AuthMnesiaDB")} + end. + +store({directory, {Directory, DirData}}, ConfigList) + when is_list(Directory) andalso is_list(DirData) -> + try directory_config_check(Directory, DirData) of + ok -> + store_directory(Directory, DirData, ConfigList) + catch + throw:Error -> + {error, Error, {directory, Directory, DirData}} + end; +store({directory, {Directory, DirData}}, _) -> + {error, {wrong_type, {directory, {Directory, DirData}}}}. + +remove(ConfigDB) -> + lists:foreach(fun({directory, {_Dir, DirData}}) -> + AuthMod = auth_mod_name(DirData), + (catch apply(AuthMod, remove, [DirData])) + end, + ets:match_object(ConfigDB,{directory,{'_','_'}})), + + Addr = httpd_util:lookup(ConfigDB, bind_address, undefined), + Port = httpd_util:lookup(ConfigDB, port), + Profile = httpd_util:lookup(ConfigDB, profile, ?DEFAULT_PROFILE), + mod_auth_server:stop(Addr, Port, Profile), + ok. + +add_user(UserName, Opt) -> + case get_options(Opt, mandatory) of + {Addr, Port, Dir, AuthPwd}-> + case get_options(Opt, userData) of + {error, Reason}-> + {error, Reason}; + {UserData, Password}-> + User = [#httpd_user{username = UserName, + password = Password, + user_data = UserData}], + mod_auth_server:add_user(Addr, Port, Dir, User, AuthPwd) + end + end. + + +add_user(UserName, Password, UserData, Port, Dir) -> + add_user(UserName, Password, UserData, undefined, Port, Dir). +add_user(UserName, Password, UserData, Addr, Port, Dir) -> + User = [#httpd_user{username = UserName, + password = Password, + user_data = UserData}], + mod_auth_server:add_user(Addr, Port, Dir, User, ?NOPASSWORD). + +get_user(UserName, Opt) -> + case get_options(Opt, mandatory) of + {Addr, Port, Dir, AuthPwd} -> + mod_auth_server:get_user(Addr, Port, Dir, UserName, AuthPwd); + {error, Reason} -> + {error, Reason} + end. + +get_user(UserName, Port, Dir) -> + get_user(UserName, undefined, Port, Dir). +get_user(UserName, Addr, Port, Dir) -> + mod_auth_server:get_user(Addr, Port, Dir, UserName, ?NOPASSWORD). + +add_group_member(GroupName, UserName, Opt)-> + case get_options(Opt, mandatory) of + {Addr, Port, Dir, AuthPwd}-> + mod_auth_server:add_group_member(Addr, Port, Dir, + GroupName, UserName, AuthPwd); + {error, Reason} -> + {error, Reason} + end. + +add_group_member(GroupName, UserName, Port, Dir) -> + add_group_member(GroupName, UserName, undefined, Port, Dir). + +add_group_member(GroupName, UserName, Addr, Port, Dir) -> + mod_auth_server:add_group_member(Addr, Port, Dir, + GroupName, UserName, ?NOPASSWORD). + +delete_group_member(GroupName, UserName, Opt) -> + case get_options(Opt, mandatory) of + {Addr, Port, Dir, AuthPwd} -> + mod_auth_server:delete_group_member(Addr, Port, Dir, + GroupName, UserName, AuthPwd); + {error, Reason} -> + {error, Reason} + end. + +delete_group_member(GroupName, UserName, Port, Dir) -> + delete_group_member(GroupName, UserName, undefined, Port, Dir). +delete_group_member(GroupName, UserName, Addr, Port, Dir) -> + mod_auth_server:delete_group_member(Addr, Port, Dir, + GroupName, UserName, ?NOPASSWORD). + +list_users(Opt) -> + case get_options(Opt, mandatory) of + {Addr, Port, Dir, AuthPwd} -> + mod_auth_server:list_users(Addr, Port, Dir, AuthPwd); + {error, Reason} -> + {error, Reason} + end. + +list_users(Port, Dir) -> + list_users(undefined, Port, Dir). +list_users(Addr, Port, Dir) -> + mod_auth_server:list_users(Addr, Port, Dir, ?NOPASSWORD). + +delete_user(UserName, Opt) -> + case get_options(Opt, mandatory) of + {Addr, Port, Dir, AuthPwd} -> + mod_auth_server:delete_user(Addr, Port, Dir, UserName, AuthPwd); + {error, Reason} -> + {error, Reason} + end. + +delete_user(UserName, Port, Dir) -> + delete_user(UserName, undefined, Port, Dir). +delete_user(UserName, Addr, Port, Dir) -> + mod_auth_server:delete_user(Addr, Port, Dir, UserName, ?NOPASSWORD). + +delete_group(GroupName, Opt) -> + case get_options(Opt, mandatory) of + {Addr, Port, Dir, AuthPwd} -> + mod_auth_server:delete_group(Addr, Port, Dir, GroupName, AuthPwd); + {error, Reason} -> + {error, Reason} + end. + +delete_group(GroupName, Port, Dir) -> + delete_group(GroupName, undefined, Port, Dir). +delete_group(GroupName, Addr, Port, Dir) -> + mod_auth_server:delete_group(Addr, Port, Dir, GroupName, ?NOPASSWORD). + +list_groups(Opt) -> + case get_options(Opt, mandatory) of + {Addr, Port, Dir, AuthPwd} -> + mod_auth_server:list_groups(Addr, Port, Dir, AuthPwd); + {error, Reason} -> + {error, Reason} + end. + +list_groups(Port, Dir) -> + list_groups(undefined, Port, Dir). +list_groups(Addr, Port, Dir) -> + mod_auth_server:list_groups(Addr, Port, Dir, ?NOPASSWORD). + +list_group_members(GroupName, Opt) -> + case get_options(Opt, mandatory) of + {Addr, Port, Dir, AuthPwd} -> + mod_auth_server:list_group_members(Addr, Port, Dir, GroupName, + AuthPwd); + {error, Reason} -> + {error, Reason} + end. + +list_group_members(GroupName, Port, Dir) -> + list_group_members(GroupName, undefined, Port, Dir). +list_group_members(GroupName, Addr, Port, Dir) -> + mod_auth_server:list_group_members(Addr, Port, Dir, + GroupName, ?NOPASSWORD). + +update_password(Port, Dir, Old, New, New)-> + update_password(undefined, Port, Dir, Old, New, New). + +update_password(Addr, Port, Dir, Old, New, New) when is_list(New) -> + mod_auth_server:update_password(Addr, Port, Dir, Old, New); + +update_password(_Addr, _Port, _Dir, _Old, _New, _New) -> + {error, badtype}; +update_password(_Addr, _Port, _Dir, _Old, _New, _New1) -> + {error, notqeual}. + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- + +do_auth(Info, Directory, DirectoryData, _AuthType) -> %% Authenticate (require) - ?hdrt("authenticate", [{auth_type, AuthType}]), case require(Info, Directory, DirectoryData) of authorized -> - ?hdrt("authorized", []), {proceed,Info#mod.data}; {authorized, User} -> - ?hdrt("authorized", [{user, User}]), {proceed, [{remote_user,User}|Info#mod.data]}; {authorization_required, Realm} -> - ?hdrt("authorization required", [{realm, Realm}]), ReasonPhrase = httpd_util:reason_phrase(401), Message = httpd_util:message(401,none,Info#mod.config_db), {proceed, @@ -142,8 +415,6 @@ do_auth(Info, Directory, DirectoryData, AuthType) -> Info#mod.data]} end. -%% require - require(Info, Directory, DirectoryData) -> ParsedHeader = Info#mod.parsed_header, ValidUsers = proplists:get_value(require_user, DirectoryData), @@ -270,13 +541,6 @@ auth_mod_name(DirData) -> dets -> mod_auth_dets end. - -%% -%% Is it a secret area? -%% - -%% secretp - secretp(Path,ConfigDB) -> Directories = ets:match(ConfigDB,{directory, {'$1','_'}}), case secret_path(Path, Directories) of @@ -307,12 +571,6 @@ secret_path(Path, [[NewDirectory] | Rest], Directory) -> secret_path(Path, Rest, Directory) end. -%% -%% Authenticate -%% - -%% allow - allow({_,RemoteAddr}, _SocketType, _Socket, DirectoryData) -> Hosts = proplists:get_value(allow_from, DirectoryData, all), case validate_addr(RemoteAddr, Hosts) of @@ -336,8 +594,6 @@ validate_addr(RemoteAddr, [HostRegExp | Rest]) -> validate_addr(RemoteAddr,Rest) end. -%% deny - deny({_,RemoteAddr}, _SocketType, _Socket,DirectoryData) -> Hosts = proplists:get_value(deny_from, DirectoryData, none), case validate_addr(RemoteAddr,Hosts) of @@ -347,124 +603,6 @@ deny({_,RemoteAddr}, _SocketType, _Socket,DirectoryData) -> not_denied end. -%% -%% Configuration -%% - -%% load/2 -%% - -%% mod_auth recognizes the following Configuration Directives: -%% <Directory /path/to/directory> -%% AuthDBType -%% AuthName -%% AuthUserFile -%% AuthGroupFile -%% AuthAccessPassword -%% require -%% allow -%% </Directory> - -%% When a <Directory> directive is found, a new context is set to -%% [{directory, Directory, DirData}|OtherContext] -%% DirData in this case is a key-value list of data belonging to the -%% directory in question. -%% -%% When the </Directory> statement is found, the Context created earlier -%% will be returned as a ConfigList and the context will return to the -%% state it was previously. - -load("<Directory " ++ Directory,[]) -> - Dir = httpd_conf:custom_clean(Directory,"",">"), - {ok,[{directory, {Dir, [{path, Dir}]}}]}; -load(eof,[{directory, {Directory, _DirData}}|_]) -> - {error, ?NICE("Premature end-of-file in "++ Directory)}; - -load("AuthName " ++ AuthName, [{directory, {Directory, DirData}}|Rest]) -> - {ok, [{directory, {Directory, - [{auth_name, httpd_conf:clean(AuthName)} | DirData]}} - | Rest ]}; -load("AuthUserFile " ++ AuthUserFile0, - [{directory, {Directory, DirData}}|Rest]) -> - AuthUserFile = httpd_conf:clean(AuthUserFile0), - {ok, [{directory, {Directory, - [{auth_user_file, AuthUserFile}|DirData]}} | Rest ]}; -load("AuthGroupFile " ++ AuthGroupFile0, - [{directory, {Directory, DirData}}|Rest]) -> - AuthGroupFile = httpd_conf:clean(AuthGroupFile0), - {ok,[{directory, {Directory, - [{auth_group_file, AuthGroupFile}|DirData]}} | Rest]}; - -%AuthAccessPassword -load("AuthAccessPassword " ++ AuthAccessPassword0, - [{directory, {Directory, DirData}}|Rest]) -> - AuthAccessPassword = httpd_conf:clean(AuthAccessPassword0), - {ok,[{directory, {Directory, - [{auth_access_password, AuthAccessPassword}|DirData]}} | Rest]}; - -load("AuthDBType " ++ Type, - [{directory, {Dir, DirData}}|Rest]) -> - case httpd_conf:clean(Type) of - "plain" -> - {ok, [{directory, {Dir, [{auth_type, plain}|DirData]}} | Rest ]}; - "mnesia" -> - {ok, [{directory, {Dir, [{auth_type, mnesia}|DirData]}} | Rest ]}; - "dets" -> - {ok, [{directory, {Dir, [{auth_type, dets}|DirData]}} | Rest ]}; - _ -> - {error, ?NICE(httpd_conf:clean(Type)++" is an invalid AuthDBType")} - end; - -load("require " ++ Require,[{directory, {Directory, DirData}}|Rest]) -> - case inets_regexp:split(Require," ") of - {ok,["user"|Users]} -> - {ok,[{directory, {Directory, - [{require_user,Users}|DirData]}} | Rest]}; - {ok,["group"|Groups]} -> - {ok,[{directory, {Directory, - [{require_group,Groups}|DirData]}} | Rest]}; - {ok,_} -> - {error,?NICE(httpd_conf:clean(Require) ++" is an invalid require")} - end; - -load("allow " ++ Allow,[{directory, {Directory, DirData}}|Rest]) -> - case inets_regexp:split(Allow," ") of - {ok,["from","all"]} -> - {ok,[{directory, {Directory, - [{allow_from,all}|DirData]}} | Rest]}; - {ok,["from"|Hosts]} -> - {ok,[{directory, {Directory, - [{allow_from,Hosts}|DirData]}} | Rest]}; - {ok,_} -> - {error,?NICE(httpd_conf:clean(Allow) ++" is an invalid allow")} - end; - -load("deny " ++ Deny,[{directory, {Directory, DirData}}|Rest]) -> - case inets_regexp:split(Deny," ") of - {ok, ["from", "all"]} -> - {ok,[{{directory, Directory, - [{deny_from, all}|DirData]}} | Rest]}; - {ok, ["from"|Hosts]} -> - {ok,[{{directory, Directory, - [{deny_from, Hosts}|DirData]}} | Rest]}; - {ok, _} -> - {error,?NICE(httpd_conf:clean(Deny) ++" is an invalid deny")} - end; - -load("</Directory>",[{directory, {Directory, DirData}}|Rest]) -> - {ok, Rest, {directory, {Directory, DirData}}}; - -load("AuthMnesiaDB " ++ AuthMnesiaDB, - [{directory, {Dir, DirData}}|Rest]) -> - case httpd_conf:clean(AuthMnesiaDB) of - "On" -> - {ok,[{directory, {Dir,[{auth_type,mnesia}|DirData]}}|Rest]}; - "Off" -> - {ok,[{directory, {Dir,[{auth_type,plain}|DirData]}}|Rest]}; - _ -> - {error, ?NICE(httpd_conf:clean(AuthMnesiaDB) ++ - " is an invalid AuthMnesiaDB")} - end. directory_config_check(Directory, DirData) -> case proplists:get_value(auth_type, DirData) of @@ -482,25 +620,7 @@ check_filename_present(Dir,AuthFile,DirData) -> throw({missing_auth_file, AuthFile, {directory, {Dir, DirData}}}) end. -%% store - -store({directory, {Directory, DirData}}, ConfigList) - when is_list(Directory) andalso is_list(DirData) -> - ?hdrt("store", - [{directory, Directory}, {dir_data, DirData}]), - try directory_config_check(Directory, DirData) of - ok -> - store_directory(Directory, DirData, ConfigList) - catch - throw:Error -> - {error, Error, {directory, Directory, DirData}} - end; -store({directory, {Directory, DirData}}, _) -> - {error, {wrong_type, {directory, {Directory, DirData}}}}. - store_directory(Directory0, DirData0, ConfigList) -> - ?hdrt("store directory - entry", - [{directory, Directory0}, {dir_data, DirData0}]), Port = proplists:get_value(port, ConfigList), DirData = case proplists:get_value(bind_address, ConfigList) of undefined -> @@ -522,9 +642,7 @@ store_directory(Directory0, DirData0, ConfigList) -> dets -> mod_auth_dets; plain -> mod_auth_plain; _ -> no_module_at_all - end, - ?hdrt("store directory", - [{directory, Directory}, {dir_data, DirData}, {auth_mod, AuthMod}]), + end, case AuthMod of no_module_at_all -> {ok, {directory, {Directory, DirData}}}; @@ -560,204 +678,10 @@ store_directory(Directory0, DirData0, ConfigList) -> add_auth_password(Dir, Pwd0, ConfigList) -> Addr = proplists:get_value(bind_address, ConfigList), Port = proplists:get_value(port, ConfigList), - mod_auth_server:start(Addr, Port), + Profile = proplists:get_value(profile, ConfigList, ?DEFAULT_PROFILE), + mod_auth_server:start(Addr, Port, Profile), mod_auth_server:add_password(Addr, Port, Dir, Pwd0). -%% remove - - -remove(ConfigDB) -> - lists:foreach(fun({directory, {_Dir, DirData}}) -> - AuthMod = auth_mod_name(DirData), - (catch apply(AuthMod, remove, [DirData])) - end, - ets:match_object(ConfigDB,{directory,{'_','_'}})), - Addr = case lookup(ConfigDB, bind_address) of - [] -> - undefined; - [{bind_address, Address}] -> - Address - end, - [{port, Port}] = lookup(ConfigDB, port), - mod_auth_server:stop(Addr, Port), - ok. - -%% -------------------------------------------------------------------- - -%% update_password - -update_password(Port, Dir, Old, New, New)-> - update_password(undefined, Port, Dir, Old, New, New). - -update_password(Addr, Port, Dir, Old, New, New) when is_list(New) -> - mod_auth_server:update_password(Addr, Port, Dir, Old, New); - -update_password(_Addr, _Port, _Dir, _Old, _New, _New) -> - {error, badtype}; -update_password(_Addr, _Port, _Dir, _Old, _New, _New1) -> - {error, notqeual}. - - -%% add_user - -add_user(UserName, Opt) -> - case get_options(Opt, mandatory) of - {Addr, Port, Dir, AuthPwd}-> - case get_options(Opt, userData) of - {error, Reason}-> - {error, Reason}; - {UserData, Password}-> - User = [#httpd_user{username = UserName, - password = Password, - user_data = UserData}], - mod_auth_server:add_user(Addr, Port, Dir, User, AuthPwd) - end - end. - - -add_user(UserName, Password, UserData, Port, Dir) -> - add_user(UserName, Password, UserData, undefined, Port, Dir). -add_user(UserName, Password, UserData, Addr, Port, Dir) -> - User = [#httpd_user{username = UserName, - password = Password, - user_data = UserData}], - mod_auth_server:add_user(Addr, Port, Dir, User, ?NOPASSWORD). - - -%% get_user - -get_user(UserName, Opt) -> - case get_options(Opt, mandatory) of - {Addr, Port, Dir, AuthPwd} -> - mod_auth_server:get_user(Addr, Port, Dir, UserName, AuthPwd); - {error, Reason} -> - {error, Reason} - end. - -get_user(UserName, Port, Dir) -> - get_user(UserName, undefined, Port, Dir). -get_user(UserName, Addr, Port, Dir) -> - mod_auth_server:get_user(Addr, Port, Dir, UserName, ?NOPASSWORD). - - -%% add_group_member - -add_group_member(GroupName, UserName, Opt)-> - case get_options(Opt, mandatory) of - {Addr, Port, Dir, AuthPwd}-> - mod_auth_server:add_group_member(Addr, Port, Dir, - GroupName, UserName, AuthPwd); - {error, Reason} -> - {error, Reason} - end. - -add_group_member(GroupName, UserName, Port, Dir) -> - add_group_member(GroupName, UserName, undefined, Port, Dir). - -add_group_member(GroupName, UserName, Addr, Port, Dir) -> - mod_auth_server:add_group_member(Addr, Port, Dir, - GroupName, UserName, ?NOPASSWORD). - - -%% delete_group_member - -delete_group_member(GroupName, UserName, Opt) -> - case get_options(Opt, mandatory) of - {Addr, Port, Dir, AuthPwd} -> - mod_auth_server:delete_group_member(Addr, Port, Dir, - GroupName, UserName, AuthPwd); - {error, Reason} -> - {error, Reason} - end. - -delete_group_member(GroupName, UserName, Port, Dir) -> - delete_group_member(GroupName, UserName, undefined, Port, Dir). -delete_group_member(GroupName, UserName, Addr, Port, Dir) -> - mod_auth_server:delete_group_member(Addr, Port, Dir, - GroupName, UserName, ?NOPASSWORD). - - -%% list_users - -list_users(Opt) -> - case get_options(Opt, mandatory) of - {Addr, Port, Dir, AuthPwd} -> - mod_auth_server:list_users(Addr, Port, Dir, AuthPwd); - {error, Reason} -> - {error, Reason} - end. - -list_users(Port, Dir) -> - list_users(undefined, Port, Dir). -list_users(Addr, Port, Dir) -> - mod_auth_server:list_users(Addr, Port, Dir, ?NOPASSWORD). - - -%% delete_user - -delete_user(UserName, Opt) -> - case get_options(Opt, mandatory) of - {Addr, Port, Dir, AuthPwd} -> - mod_auth_server:delete_user(Addr, Port, Dir, UserName, AuthPwd); - {error, Reason} -> - {error, Reason} - end. - -delete_user(UserName, Port, Dir) -> - delete_user(UserName, undefined, Port, Dir). -delete_user(UserName, Addr, Port, Dir) -> - mod_auth_server:delete_user(Addr, Port, Dir, UserName, ?NOPASSWORD). - - -%% delete_group - -delete_group(GroupName, Opt) -> - case get_options(Opt, mandatory) of - {Addr, Port, Dir, AuthPwd} -> - mod_auth_server:delete_group(Addr, Port, Dir, GroupName, AuthPwd); - {error, Reason} -> - {error, Reason} - end. - -delete_group(GroupName, Port, Dir) -> - delete_group(GroupName, undefined, Port, Dir). -delete_group(GroupName, Addr, Port, Dir) -> - mod_auth_server:delete_group(Addr, Port, Dir, GroupName, ?NOPASSWORD). - - -%% list_groups - -list_groups(Opt) -> - case get_options(Opt, mandatory) of - {Addr, Port, Dir, AuthPwd} -> - mod_auth_server:list_groups(Addr, Port, Dir, AuthPwd); - {error, Reason} -> - {error, Reason} - end. - -list_groups(Port, Dir) -> - list_groups(undefined, Port, Dir). -list_groups(Addr, Port, Dir) -> - mod_auth_server:list_groups(Addr, Port, Dir, ?NOPASSWORD). - - -%% list_group_members - -list_group_members(GroupName, Opt) -> - case get_options(Opt, mandatory) of - {Addr, Port, Dir, AuthPwd} -> - mod_auth_server:list_group_members(Addr, Port, Dir, GroupName, - AuthPwd); - {error, Reason} -> - {error, Reason} - end. - -list_group_members(GroupName, Port, Dir) -> - list_group_members(GroupName, undefined, Port, Dir). -list_group_members(GroupName, Addr, Port, Dir) -> - mod_auth_server:list_group_members(Addr, Port, Dir, - GroupName, ?NOPASSWORD). - %% Opt = [{port, Port}, %% {addr, Addr}, %% {dir, Dir}, @@ -792,7 +716,3 @@ get_options(Opt, userData)-> {UserData, Pwd} end end. - - -lookup(Db, Key) -> - ets:lookup(Db, Key). diff --git a/lib/inets/src/http_server/mod_auth_dets.erl b/lib/inets/src/http_server/mod_auth_dets.erl index a48725d5d9..4220f46166 100644 --- a/lib/inets/src/http_server/mod_auth_dets.erl +++ b/lib/inets/src/http_server/mod_auth_dets.erl @@ -38,23 +38,23 @@ -include("httpd_internal.hrl"). -include("mod_auth.hrl"). -store_directory_data(_Directory, DirData, Server_root) -> - ?CDEBUG("store_directory_data -> ~n" - " Directory: ~p~n" - " DirData: ~p", - [_Directory, DirData]), +%%==================================================================== +%% Internal application API +%%==================================================================== +store_directory_data(_Directory, DirData, Server_root) -> {PWFile, Absolute_pwdfile} = absolute_file_name(auth_user_file, DirData, Server_root), {GroupFile, Absolute_groupfile} = absolute_file_name(auth_group_file, DirData, Server_root), Addr = proplists:get_value(bind_address, DirData), Port = proplists:get_value(port, DirData), + Profile = proplists:get_value(profile, DirData, ?DEFAULT_PROFILE), - PWName = httpd_util:make_name("httpd_dets_pwdb",Addr,Port), + PWName = httpd_util:make_name("httpd_dets_pwdb", Addr, Port, Profile), case dets:open_file(PWName,[{type,set},{file,Absolute_pwdfile},{repair,true}]) of {ok, PWDB} -> - GDBName = httpd_util:make_name("httpd_dets_groupdb",Addr,Port), + GDBName = httpd_util:make_name("httpd_dets_groupdb", Addr, Port, Profile), case dets:open_file(GDBName,[{type,set},{file,Absolute_groupfile},{repair,true}]) of {ok, GDB} -> NDD1 = lists:keyreplace(auth_user_file, 1, DirData, @@ -69,11 +69,8 @@ store_directory_data(_Directory, DirData, Server_root) -> {error, {{file, PWFile},Err2}} end. -%% %% Storage format of users in the dets table: %% {{UserName, Addr, Port, Dir}, Password, UserData} -%% - add_user(DirData, UStruct) -> {Addr, Port, Dir} = lookup_common(DirData), PWDB = proplists:get_value(auth_user_file, DirData), @@ -99,21 +96,15 @@ get_user(DirData, UserName) -> end. list_users(DirData) -> - ?DEBUG("list_users -> ~n" - " DirData: ~p", [DirData]), {Addr, Port, Dir} = lookup_common(DirData), PWDB = proplists:get_value(auth_user_file, DirData), - case dets:traverse(PWDB, fun(X) -> {continue, X} end) of %% SOOOO Ugly ! + case dets:traverse(PWDB, fun(X) -> {continue, X} end) of Records when is_list(Records) -> - ?DEBUG("list_users -> ~n" - " Records: ~p", [Records]), {ok, [UserName || {{UserName, AnyAddr, AnyPort, AnyDir}, _Password, _Data} <- Records, AnyAddr == Addr, AnyPort == Port, AnyDir == Dir]}; _O -> - ?DEBUG("list_users -> ~n" - " O: ~p", [_O]), {ok, []} end. @@ -134,10 +125,8 @@ delete_user(DirData, UserName) -> {error, no_such_user} end. -%% %% Storage of groups in the dets table: %% {Group, UserList} where UserList is a list of strings. -%% add_group_member(DirData, GroupName, UserName) -> {Addr, Port, Dir} = lookup_common(DirData), GDB = proplists:get_value(auth_group_file, DirData), @@ -215,16 +204,7 @@ delete_group(DirData, GroupName) -> {error, no_such_group} end. -lookup_common(DirData) -> - Dir = proplists:get_value(path, DirData), - Port = proplists:get_value(port, DirData), - Addr = proplists:get_value(bind_address, DirData), - {Addr, Port, Dir}. - -%% remove/1 -%% %% Closes dets tables used by this auth mod. -%% remove(DirData) -> PWDB = proplists:get_value(auth_user_file, DirData), GDB = proplists:get_value(auth_group_file, DirData), @@ -232,8 +212,9 @@ remove(DirData) -> dets:close(PWDB), ok. -%% absolute_file_name/2 -%% +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- %% Return the absolute path name of File_type. absolute_file_name(File_type, DirData, Server_root) -> Path = proplists:get_value(File_type, DirData), @@ -253,3 +234,8 @@ absolute_file_name(File_type, DirData, Server_root) -> end, {Path, Absolute_path}. +lookup_common(DirData) -> + Dir = proplists:get_value(path, DirData), + Port = proplists:get_value(port, DirData), + Addr = proplists:get_value(bind_address, DirData), + {Addr, Port, Dir}. diff --git a/lib/inets/src/http_server/mod_auth_plain.erl b/lib/inets/src/http_server/mod_auth_plain.erl index c0a83711ba..7bb86fc812 100644 --- a/lib/inets/src/http_server/mod_auth_plain.erl +++ b/lib/inets/src/http_server/mod_auth_plain.erl @@ -22,15 +22,11 @@ -include("httpd.hrl"). -include("mod_auth.hrl"). -include("httpd_internal.hrl"). --include("inets_internal.hrl"). - -define(VMODULE,"AUTH_PLAIN"). %% Internal API -export([store_directory_data/3]). - - -export([get_user/2, list_group_members/2, add_user/2, @@ -42,17 +38,13 @@ delete_group/2, remove/1]). -%% -%% API -%% +%%==================================================================== +%% Internal application API +%%==================================================================== -%% %% Storage format of users in the ets table: %% {UserName, Password, UserData} -%% - add_user(DirData, #httpd_user{username = User} = UStruct) -> - ?hdrt("add user", [{user, UStruct}]), PWDB = proplists:get_value(auth_user_file, DirData), Record = {User, UStruct#httpd_user.password, @@ -66,7 +58,6 @@ add_user(DirData, #httpd_user{username = User} = UStruct) -> end. get_user(DirData, User) -> - ?hdrt("get user", [{dir_data, DirData}, {user, User}]), PWDB = proplists:get_value(auth_user_file, DirData), case ets:lookup(PWDB, User) of [{User, PassWd, Data}] -> @@ -84,7 +75,6 @@ list_users(DirData) -> [], lists:flatten(Records))}. delete_user(DirData, UserName) -> - ?hdrt("delete user", [{dir_data, DirData}, {user, UserName}]), PWDB = proplists:get_value(auth_user_file, DirData), case ets:lookup(PWDB, UserName) of [{UserName, _SomePassword, _SomeData}] -> @@ -98,11 +88,8 @@ delete_user(DirData, UserName) -> {error, no_such_user} end. -%% %% Storage of groups in the ets table: %% {Group, UserList} where UserList is a list of strings. -%% - add_group_member(DirData, Group, UserName) -> GDB = proplists:get_value(auth_group_file, DirData), case ets:lookup(GDB, Group) of @@ -163,17 +150,12 @@ delete_group(DirData, Group) -> end. store_directory_data(_Directory, DirData, Server_root) -> - ?hdrt("store directory data", - [{dir_data, DirData}, {server_root, Server_root}]), PWFile = absolute_file_name(auth_user_file, DirData, Server_root), GroupFile = absolute_file_name(auth_group_file, DirData, Server_root), case load_passwd(PWFile) of {ok, PWDB} -> - ?hdrt("password file loaded", [{file, PWFile}, {pwdb, PWDB}]), case load_group(GroupFile) of {ok, GRDB} -> - ?hdrt("group file loaded", - [{file, GroupFile}, {grdb, GRDB}]), %% Address and port is included in the file names... Addr = proplists:get_value(bind_address, DirData), Port = proplists:get_value(port, DirData), @@ -191,9 +173,83 @@ store_directory_data(_Directory, DirData, Server_root) -> {error, Err2} end. +%% Deletes ets tables used by this auth mod. +remove(DirData) -> + PWDB = proplists:get_value(auth_user_file, DirData), + GDB = proplists:get_value(auth_group_file, DirData), + ets:delete(PWDB), + ets:delete(GDB). +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +%% Return the absolute path name of File_type. +absolute_file_name(File_type, DirData, Server_root) -> + Path = proplists:get_value(File_type, DirData), + case filename:pathtype(Path) of + relative -> + case Server_root of + undefined -> + {error, + ?NICE(Path++ + " is an invalid file name because " + "ServerRoot is not defined")}; + _ -> + filename:join(Server_root,Path) + end; + _ -> + Path + end. -%% load_passwd +store_group(Addr,Port,GroupList) -> + %% Not a named table so not importante to add Profile to name + Name = httpd_util:make_name("httpd_group",Addr,Port), + GroupDB = ets:new(Name, [set, public]), + store_group(GroupDB, GroupList). + +store_group(GroupDB,[]) -> + {ok, GroupDB}; +store_group(GroupDB, [User|Rest]) -> + ets:insert(GroupDB, User), + store_group(GroupDB, Rest). + +store_passwd(Addr,Port,PasswdList) -> + %% Not a named table so not importante to add Profile to name + Name = httpd_util:make_name("httpd_passwd",Addr,Port), + PasswdDB = ets:new(Name, [set, public]), + store_passwd(PasswdDB, PasswdList). + +store_passwd(PasswdDB, []) -> + {ok, PasswdDB}; +store_passwd(PasswdDB, [User|Rest]) -> + ets:insert(PasswdDB, User), + store_passwd(PasswdDB, Rest). + +parse_group(Stream, GroupList) -> + Line = + case io:get_line(Stream,'') of + eof -> + eof; + String -> + httpd_conf:clean(String) + end, + parse_group(Stream, GroupList, Line). + +parse_group(Stream, GroupList, eof) -> + file:close(Stream), + {ok, GroupList}; +parse_group(Stream, GroupList, "") -> + parse_group(Stream, GroupList); +parse_group(Stream, GroupList, [$#|_]) -> + parse_group(Stream, GroupList); +parse_group(Stream, GroupList, Line) -> + case inets_regexp:split(Line, ":") of + {ok, [Group,Users]} -> + {ok, UserList} = inets_regexp:split(Users," "), + parse_group(Stream, [{Group,UserList}|GroupList]); + {ok, _} -> + {error, ?NICE(Line)} + end. load_passwd(AuthUserFile) -> case file:open(AuthUserFile, [read]) of @@ -228,8 +284,6 @@ parse_passwd(Stream, PasswdList, Line) -> {error, ?NICE(Line)} end. -%% load_group - load_group(AuthGroupFile) -> case file:open(AuthGroupFile, [read]) of {ok, Stream} -> @@ -237,91 +291,3 @@ load_group(AuthGroupFile) -> {error, _} -> {error, ?NICE("Can't open " ++ AuthGroupFile)} end. - -parse_group(Stream, GroupList) -> - Line = - case io:get_line(Stream,'') of - eof -> - eof; - String -> - httpd_conf:clean(String) - end, - parse_group(Stream, GroupList, Line). - -parse_group(Stream, GroupList, eof) -> - file:close(Stream), - {ok, GroupList}; -parse_group(Stream, GroupList, "") -> - parse_group(Stream, GroupList); -parse_group(Stream, GroupList, [$#|_]) -> - parse_group(Stream, GroupList); -parse_group(Stream, GroupList, Line) -> - case inets_regexp:split(Line, ":") of - {ok, [Group,Users]} -> - {ok, UserList} = inets_regexp:split(Users," "), - parse_group(Stream, [{Group,UserList}|GroupList]); - {ok, _} -> - {error, ?NICE(Line)} - end. - - -%% store_passwd - -store_passwd(Addr,Port,PasswdList) -> - Name = httpd_util:make_name("httpd_passwd",Addr,Port), - PasswdDB = ets:new(Name, [set, public]), - store_passwd(PasswdDB, PasswdList). - -store_passwd(PasswdDB, []) -> - {ok, PasswdDB}; -store_passwd(PasswdDB, [User|Rest]) -> - ets:insert(PasswdDB, User), - store_passwd(PasswdDB, Rest). - -%% store_group - -store_group(Addr,Port,GroupList) -> - Name = httpd_util:make_name("httpd_group",Addr,Port), - GroupDB = ets:new(Name, [set, public]), - store_group(GroupDB, GroupList). - - -store_group(GroupDB,[]) -> - {ok, GroupDB}; -store_group(GroupDB, [User|Rest]) -> - ets:insert(GroupDB, User), - store_group(GroupDB, Rest). - - -%% remove/1 -%% -%% Deletes ets tables used by this auth mod. -%% -remove(DirData) -> - PWDB = proplists:get_value(auth_user_file, DirData), - GDB = proplists:get_value(auth_group_file, DirData), - ets:delete(PWDB), - ets:delete(GDB). - - - -%% absolute_file_name/2 -%% -%% Return the absolute path name of File_type. -absolute_file_name(File_type, DirData, Server_root) -> - Path = proplists:get_value(File_type, DirData), - case filename:pathtype(Path) of - relative -> - case Server_root of - undefined -> - {error, - ?NICE(Path++ - " is an invalid file name because " - "ServerRoot is not defined")}; - _ -> - filename:join(Server_root,Path) - end; - _ -> - Path - end. - diff --git a/lib/inets/src/http_server/mod_auth_server.erl b/lib/inets/src/http_server/mod_auth_server.erl index 947273bd9e..2a45f402d7 100644 --- a/lib/inets/src/http_server/mod_auth_server.erl +++ b/lib/inets/src/http_server/mod_auth_server.erl @@ -22,246 +22,184 @@ -include("httpd.hrl"). -include("httpd_internal.hrl"). --include("inets_internal.hrl"). -behaviour(gen_server). - %% mod_auth exports --export([start/2, stop/2, +-export([start/3, stop/3, add_password/4, update_password/5, add_user/5, delete_user/5, get_user/5, list_users/4, add_group_member/6, delete_group_member/6, list_group_members/5, delete_group/5, list_groups/4]). %% gen_server exports --export([start_link/2, init/1, +-export([start_link/3, init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -record(state, {tab}). +%%==================================================================== +%% Internal application API +%%==================================================================== -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% %% -%% External API %% -%% %% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -%% start_link/3 -%% %% NOTE: This is called by httpd_misc_sup when the process is started %% -start_link(Addr, Port) -> - ?hdrt("start_link", [{address, Addr}, {port, Port}]), - Name = make_name(Addr, Port), +start_link(Addr, Port, Profile) -> + Name = make_name(Addr, Port, Profile), gen_server:start_link({local, Name}, ?MODULE, [], [{timeout, infinity}]). - -%% start/2 - -start(Addr, Port) -> - ?hdrd("start", [{address, Addr}, {port, Port}]), - Name = make_name(Addr, Port), +start(Addr, Port, Profile) -> + Name = make_name(Addr, Port, Profile), case whereis(Name) of undefined -> - httpd_misc_sup:start_auth_server(Addr, Port); + httpd_misc_sup:start_auth_server(Addr, Port, Profile); _ -> %% Already started... ok end. - -%% stop/2 - -stop(Addr, Port) -> - ?hdrd("stop", [{address, Addr}, {port, Port}]), - Name = make_name(Addr, Port), +stop(Addr, Port, Profile) -> + Name = make_name(Addr, Port, Profile), case whereis(Name) of undefined -> %% Already stopped ok; _ -> - (catch httpd_misc_sup:stop_auth_server(Addr, Port)) + (catch httpd_misc_sup:stop_auth_server(Addr, Port, Profile)) end. -%% add_password/4 - add_password(Addr, Port, Dir, Password) -> - ?hdrt("add password", [{address, Addr}, {port, Port}]), - Name = make_name(Addr, Port), + add_password(Addr, Port, ?DEFAULT_PROFILE, Dir, Password). +add_password(Addr, Port, Profile, Dir, Password) -> + Name = make_name(Addr, Port, Profile), Req = {add_password, Dir, Password}, call(Name, Req). - -%% update_password/6 - -update_password(Addr, Port, Dir, Old, New) when is_list(New) -> - ?hdrt("update password", - [{address, Addr}, {port, Port}, {dir, Dir}, {old, Old}, {new, New}]), - Name = make_name(Addr, Port), +update_password(Addr, Port, Dir, Old, New) -> + update_password(Addr, Port, ?DEFAULT_PROFILE, Dir, Old, New). +update_password(Addr, Port, Profile, Dir, Old, New) when is_list(New) -> + Name = make_name(Addr, Port, Profile), Req = {update_password, Dir, Old, New}, call(Name, Req). - - -%% add_user/5 add_user(Addr, Port, Dir, User, Password) -> - ?hdrt("add user", - [{address, Addr}, {port, Port}, - {dir, Dir}, {user, User}, {passwd, Password}]), - Name = make_name(Addr, Port), - Req = {add_user, Addr, Port, Dir, User, Password}, + add_user(Addr, Port, ?DEFAULT_PROFILE, Dir, User, Password). +add_user(Addr, Port, Profile, Dir, User, Password) -> + Name = make_name(Addr, Port, Profile), + Req = {add_user, Addr, Port, Profile, Dir, User, Password}, call(Name, Req). - -%% delete_user/5 - delete_user(Addr, Port, Dir, UserName, Password) -> - ?hdrt("delete user", - [{address, Addr}, {port, Port}, - {dir, Dir}, {user, UserName}, {passwd, Password}]), - Name = make_name(Addr, Port), - Req = {delete_user, Addr, Port, Dir, UserName, Password}, + delete_user(Addr, Port, ?DEFAULT_PROFILE, Dir, UserName, Password). +delete_user(Addr, Port, Profile, Dir, UserName, Password) -> + Name = make_name(Addr, Port, Profile), + Req = {delete_user, Addr, Port, Profile, Dir, UserName, Password}, call(Name, Req). - -%% get_user/5 - get_user(Addr, Port, Dir, UserName, Password) -> - ?hdrt("get user", - [{address, Addr}, {port, Port}, - {dir, Dir}, {user, UserName}, {passwd, Password}]), - Name = make_name(Addr, Port), - Req = {get_user, Addr, Port, Dir, UserName, Password}, + get_user(Addr, Port, ?DEFAULT_PROFILE, Dir, UserName, Password). +get_user(Addr, Port, Profile,Dir, UserName, Password) -> + Name = make_name(Addr, Port, Profile), + Req = {get_user, Addr, Port, Profile, Dir, UserName, Password}, call(Name, Req). - -%% list_users/4 - list_users(Addr, Port, Dir, Password) -> - ?hdrt("list users", - [{address, Addr}, {port, Port}, {dir, Dir}, {passwd, Password}]), - Name = make_name(Addr,Port), - Req = {list_users, Addr, Port, Dir, Password}, + list_users(Addr, Port, ?DEFAULT_PROFILE, Dir, Password). +list_users(Addr, Port, Profile, Dir, Password) -> + Name = make_name(Addr,Port, Profile), + Req = {list_users, Addr, Port, Profile, Dir, Password}, call(Name, Req). - -%% add_group_member/6 - add_group_member(Addr, Port, Dir, GroupName, UserName, Password) -> - ?hdrt("add group member", - [{address, Addr}, {port, Port}, {dir, Dir}, - {group, GroupName}, {user, UserName}, {passwd, Password}]), - Name = make_name(Addr,Port), - Req = {add_group_member, Addr, Port, Dir, GroupName, UserName, Password}, + add_group_member(Addr, Port, ?DEFAULT_PROFILE, Dir, GroupName, UserName, Password). +add_group_member(Addr, Port, Profile, Dir, GroupName, UserName, Password) -> + Name = make_name(Addr,Port, Profile), + Req = {add_group_member, Addr, Port, Profile, Dir, GroupName, UserName, Password}, call(Name, Req). - -%% delete_group_member/6 - delete_group_member(Addr, Port, Dir, GroupName, UserName, Password) -> - ?hdrt("delete group member", - [{address, Addr}, {port, Port}, {dir, Dir}, - {group, GroupName}, {user, UserName}, {passwd, Password}]), - Name = make_name(Addr,Port), - Req = {delete_group_member, Addr, Port, Dir, GroupName, UserName, Password}, + delete_group_member(Addr, Port, ?DEFAULT_PROFILE, Dir, GroupName, UserName, Password). +delete_group_member(Addr, Port, Profile, Dir, GroupName, UserName, Password) -> + Name = make_name(Addr,Port,Profile), + Req = {delete_group_member, Addr, Port, Profile, Dir, GroupName, UserName, Password}, call(Name, Req). - -%% list_group_members/4 - list_group_members(Addr, Port, Dir, Group, Password) -> - ?hdrt("list group members", - [{address, Addr}, {port, Port}, {dir, Dir}, - {group, Group}, {passwd, Password}]), - Name = make_name(Addr, Port), + list_group_members(Addr, Port, ?DEFAULT_PROFILE, Dir, Group, Password). +list_group_members(Addr, Port, Profile, Dir, Group, Password) -> + Name = make_name(Addr, Port, Profile), Req = {list_group_members, Addr, Port, Dir, Group, Password}, call(Name, Req). - -%% delete_group/5 - delete_group(Addr, Port, Dir, GroupName, Password) -> - ?hdrt("delete group", - [{address, Addr}, {port, Port}, {dir, Dir}, - {group, GroupName}, {passwd, Password}]), - Name = make_name(Addr, Port), - Req = {delete_group, Addr, Port, Dir, GroupName, Password}, + delete_group(Addr, Port, ?DEFAULT_PROFILE, Dir, GroupName, Password). +delete_group(Addr, Port, Profile, Dir, GroupName, Password) -> + Name = make_name(Addr, Port, Profile), + Req = {delete_group, Addr, Port, Profile, Dir, GroupName, Password}, call(Name, Req). - -%% list_groups/4 - list_groups(Addr, Port, Dir, Password) -> - ?hdrt("list groups", - [{address, Addr}, {port, Port}, {dir, Dir}, {passwd, Password}]), - Name = make_name(Addr, Port), - Req = {list_groups, Addr, Port, Dir, Password}, + list_groups(Addr, Port, ?DEFAULT_PROFILE, Dir, Password). +list_groups(Addr, Port, Profile, Dir, Password) -> + Name = make_name(Addr, Port, Profile), + Req = {list_groups, Addr, Port,Profile, Dir, Password}, call(Name, Req). - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% %% -%% Server call-back functions %% -%% %% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -%% init - +%%==================================================================== +%% Behavior call backs +%%==================================================================== init(_) -> - ?hdrv("initiating", []), {ok,#state{tab = ets:new(auth_pwd,[set,protected])}}. %% handle_call %% Add a user -handle_call({add_user, Addr, Port, Dir, User, AuthPwd}, _From, State) -> - Reply = api_call(Addr, Port, Dir, add_user, User, AuthPwd, State), - ?hdrt("add user", [{reply, Reply}]), +handle_call({add_user, Addr, Port, Profile, Dir, User, AuthPwd}, _From, State) -> + Reply = api_call(Addr, Port, Profile, Dir, add_user, User, AuthPwd, State), {reply, Reply, State}; %% Get data about a user -handle_call({get_user, Addr, Port, Dir, User, AuthPwd}, _From, State) -> - Reply = api_call(Addr, Port, Dir, get_user, [User], AuthPwd, State), +handle_call({get_user, Addr, Port, Profile, Dir, User, AuthPwd}, _From, State) -> + Reply = api_call(Addr, Port, Profile, Dir, get_user, [User], AuthPwd, State), {reply, Reply, State}; %% Add a group member -handle_call({add_group_member, Addr, Port, Dir, Group, User, AuthPwd}, +handle_call({add_group_member, Addr, Port, Profile, Dir, Group, User, AuthPwd}, _From, State) -> - Reply = api_call(Addr, Port, Dir, add_group_member, [Group, User], + Reply = api_call(Addr, Port, Profile, Dir, add_group_member, [Group, User], AuthPwd, State), {reply, Reply, State}; %% delete a group -handle_call({delete_group_member, Addr, Port, Dir, Group, User, AuthPwd}, +handle_call({delete_group_member, Addr, Port, Profile, Dir, Group, User, AuthPwd}, _From, State) -> - Reply = api_call(Addr, Port, Dir, delete_group_member, [Group, User], + Reply = api_call(Addr, Port, Profile, Dir, delete_group_member, [Group, User], AuthPwd, State), {reply, Reply, State}; %% List all users thats standalone users -handle_call({list_users, Addr, Port, Dir, AuthPwd}, _From, State) -> - Reply = api_call(Addr, Port, Dir, list_users, [], AuthPwd, State), +handle_call({list_users, Addr, Port, Profile, Dir, AuthPwd}, _From, State) -> + Reply = api_call(Addr, Port, Profile, Dir, list_users, [], AuthPwd, State), {reply, Reply, State}; %% Delete a user -handle_call({delete_user, Addr, Port, Dir, User, AuthPwd}, _From, State) -> - Reply = api_call(Addr, Port, Dir, delete_user, [User], AuthPwd, State), +handle_call({delete_user, Addr, Port, Profile, Dir, User, AuthPwd}, _From, State) -> + Reply = api_call(Addr, Port, Profile, Dir, delete_user, [User], AuthPwd, State), {reply, Reply, State}; %% Delete a group -handle_call({delete_group, Addr, Port, Dir, Group, AuthPwd}, _From, State) -> - Reply = api_call(Addr, Port, Dir, delete_group, [Group], AuthPwd, State), +handle_call({delete_group, Addr, Port, Profile, Dir, Group, AuthPwd}, _From, State) -> + Reply = api_call(Addr, Port, Profile, Dir, delete_group, [Group], AuthPwd, State), {reply, Reply, State}; %% List the current groups -handle_call({list_groups, Addr, Port, Dir, AuthPwd}, _From, State) -> - Reply = api_call(Addr, Port, Dir, list_groups, [], AuthPwd, State), +handle_call({list_groups, Addr, Port, Profile, Dir, AuthPwd}, _From, State) -> + Reply = api_call(Addr, Port, Profile, Dir, list_groups, [], AuthPwd, State), {reply, Reply, State}; %% List the members of the given group -handle_call({list_group_members, Addr, Port, Dir, Group, AuthPwd}, +handle_call({list_group_members, Addr, Port, Profile, Dir, Group, AuthPwd}, _From, State) -> - Reply = api_call(Addr, Port, Dir, list_group_members, [Group], + Reply = api_call(Addr, Port, Profile, Dir, list_group_members, [Group], AuthPwd, State), {reply, Reply, State}; @@ -306,26 +244,16 @@ terminate(_Reason,State) -> ets:delete(State#state.tab), ok. - -%% code_change(Vsn, State, Extra) -%% code_change(_Vsn, State, _Extra) -> {ok, State}. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% %% -%% The functions that really changes the data in the database %% -%% of users to different directories %% -%% %% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -%% API gateway - -api_call(Addr, Port, Dir, Func, Args,Password,State) -> +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +api_call(Addr, Port, Profile, Dir, Func, Args,Password,State) -> case controlPassword(Password, State, Dir) of ok-> - ConfigName = httpd_util:make_name("httpd_conf", Addr, Port), + ConfigName = httpd_util:make_name("httpd_conf", Addr, Port, Profile), case ets:match_object(ConfigName, {directory, {Dir, '$1'}}) of [{directory, {Dir, DirData}}] -> AuthMod = auth_mod_name(DirData), @@ -386,8 +314,8 @@ lookup(Db, Key) -> ets:lookup(Db, Key). -make_name(Addr,Port) -> - httpd_util:make_name("httpd_auth",Addr,Port). +make_name(Addr, Port, Profile) -> + httpd_util:make_name(?MODULE, Addr, Port, Profile). call(Name, Req) -> @@ -397,5 +325,3 @@ call(Name, Req) -> Reply -> Reply end. - - diff --git a/lib/inets/src/http_server/mod_security.erl b/lib/inets/src/http_server/mod_security.erl index 41988732ad..a85383a921 100644 --- a/lib/inets/src/http_server/mod_security.erl +++ b/lib/inets/src/http_server/mod_security.erl @@ -32,14 +32,13 @@ -include("httpd.hrl"). -include("httpd_internal.hrl"). --include("inets_internal.hrl"). -define(VMODULE,"SEC"). - -%% do/1 +%%==================================================================== +%% Internal application API +%%==================================================================== do(Info) -> - ?hdrt("do", [{info, Info}]), %% Check and see if any user has been authorized. case proplists:get_value(remote_user, Info#mod.data,not_defined_user) of not_defined_user -> @@ -84,151 +83,66 @@ do(Info) -> {_Dir, SDirData} = secretp(Path, Info#mod.config_db), Addr = httpd_util:lookup(Info#mod.config_db, bind_address), Port = httpd_util:lookup(Info#mod.config_db, port), + Profile = httpd_util:lookup(Info#mod.config_db, profile, ?DEFAULT_PROFILE), case mod_security_server:check_blocked_user(Info, User, SDirData, - Addr, Port) of + Addr, Port, Profile) of true -> report_failed(Info, User ,"User Blocked"), {proceed, [{status, {403, Info#mod.request_uri, ""}} | Info#mod.data]}; false -> report_failed(Info, User,"Authentication Succedded"), - mod_security_server:store_successful_auth(Addr, Port, + mod_security_server:store_successful_auth(Addr, Port, Profile, User, SDirData), {proceed, Info#mod.data} end end. -report_failed(Info, Auth, Event) -> - Request = Info#mod.request_line, - {_PortNumber,RemoteHost}=(Info#mod.init_data)#init_data.peername, - String = RemoteHost ++ " : " ++ Event ++ " : " ++ Request ++ - " : " ++ Auth, - mod_disk_log:security_log(Info,String), - mod_log:security_log(Info, String). - -take_failed_action(Info, Auth) -> - ?hdrd("take failed action", [{auth, Auth}]), - Path = mod_alias:path(Info#mod.data, Info#mod.config_db, - Info#mod.request_uri), - {_Dir, SDirData} = secretp(Path, Info#mod.config_db), - Addr = httpd_util:lookup(Info#mod.config_db, bind_address), - Port = httpd_util:lookup(Info#mod.config_db, port), - mod_security_server:store_failed_auth(Info, Addr, Port, - Auth, SDirData). - -secretp(Path, ConfigDB) -> - Directories = ets:match(ConfigDB,{directory,{'$1','_'}}), - case secret_path(Path, Directories) of - {yes, Directory} -> - ?hdrd("secretp - yes", [{dir, Directory}]), - SDirs0 = httpd_util:multi_lookup(ConfigDB, security_directory), - [SDir] = lists:filter(fun({Directory0, _}) - when Directory0 == Directory -> - true; - (_) -> - false - end, SDirs0), - SDir; - no -> - {[], []} - end. - -secret_path(Path,Directories) -> - secret_path(Path, httpd_util:uniq(lists:sort(Directories)), to_be_found). - -secret_path(_Path, [], to_be_found) -> - no; -secret_path(_Path, [], Dir) -> - {yes, Dir}; -secret_path(Path, [[NewDir]|Rest], Dir) -> - case inets_regexp:match(Path, NewDir) of - {match, _, _} when Dir =:= to_be_found -> - secret_path(Path, Rest, NewDir); - {match, _, Length} when Length > length(Dir) -> - secret_path(Path, Rest, NewDir); - {match, _, _} -> - secret_path(Path, Rest, Dir); - nomatch -> - secret_path(Path, Rest, Dir) - end. - - load("<Directory " ++ Directory, []) -> - ?hdrt("load security directory - begin", [{directory, Directory}]), Dir = httpd_conf:custom_clean(Directory,"",">"), {ok, [{security_directory, {Dir, [{path, Dir}]}}]}; load(eof,[{security_directory, {Directory, _DirData}}|_]) -> {error, ?NICE("Premature end-of-file in "++Directory)}; load("SecurityDataFile " ++ FileName, [{security_directory, {Dir, DirData}}]) -> - ?hdrt("load security directory", - [{file, FileName}, {dir, Dir}, {dir_data, DirData}]), File = httpd_conf:clean(FileName), {ok, [{security_directory, {Dir, [{data_file, File}|DirData]}}]}; load("SecurityCallbackModule " ++ ModuleName, [{security_directory, {Dir, DirData}}]) -> - ?hdrt("load security directory", - [{module, ModuleName}, {dir, Dir}, {dir_data, DirData}]), Mod = list_to_atom(httpd_conf:clean(ModuleName)), {ok, [{security_directory, {Dir, [{callback_module, Mod}|DirData]}}]}; load("SecurityMaxRetries " ++ Retries, [{security_directory, {Dir, DirData}}]) -> - ?hdrt("load security directory", - [{max_retries, Retries}, {dir, Dir}, {dir_data, DirData}]), load_return_int_tag("SecurityMaxRetries", max_retries, httpd_conf:clean(Retries), Dir, DirData); load("SecurityBlockTime " ++ Time, [{security_directory, {Dir, DirData}}]) -> - ?hdrt("load security directory", - [{block_time, Time}, {dir, Dir}, {dir_data, DirData}]), load_return_int_tag("SecurityBlockTime", block_time, httpd_conf:clean(Time), Dir, DirData); load("SecurityFailExpireTime " ++ Time, [{security_directory, {Dir, DirData}}]) -> - ?hdrt("load security directory", - [{expire_time, Time}, {dir, Dir}, {dir_data, DirData}]), load_return_int_tag("SecurityFailExpireTime", fail_expire_time, httpd_conf:clean(Time), Dir, DirData); load("SecurityAuthTimeout " ++ Time0, [{security_directory, {Dir, DirData}}]) -> - ?hdrt("load security directory", - [{auth_timeout, Time0}, {dir, Dir}, {dir_data, DirData}]), Time = httpd_conf:clean(Time0), load_return_int_tag("SecurityAuthTimeout", auth_timeout, httpd_conf:clean(Time), Dir, DirData); load("AuthName " ++ Name0, [{security_directory, {Dir, DirData}}]) -> - ?hdrt("load security directory", - [{name, Name0}, {dir, Dir}, {dir_data, DirData}]), Name = httpd_conf:clean(Name0), {ok, [{security_directory, {Dir, [{auth_name, Name}|DirData]}}]}; load("</Directory>",[{security_directory, {Dir, DirData}}]) -> - ?hdrt("load security directory - end", - [{dir, Dir}, {dir_data, DirData}]), {ok, [], {security_directory, {Dir, DirData}}}. -load_return_int_tag(Name, Atom, Time, Dir, DirData) -> - case Time of - "infinity" -> - {ok, [{security_directory, {Dir, - [{Atom, 99999999999999999999999999999} | DirData]}}]}; - _Int -> - case catch list_to_integer(Time) of - {'EXIT', _} -> - {error, Time++" is an invalid "++Name}; - Val -> - {ok, [{security_directory, {Dir, [{Atom, Val}|DirData]}}]} - end - end. - store({security_directory, {Dir, DirData}}, ConfigList) when is_list(Dir) andalso is_list(DirData) -> - ?hdrt("store security directory", [{dir, Dir}, {dir_data, DirData}]), Addr = proplists:get_value(bind_address, ConfigList), Port = proplists:get_value(port, ConfigList), - mod_security_server:start(Addr, Port), + Profile = proplists:get_value(profile, ConfigList, ?DEFAULT_PROFILE), + mod_security_server:start(Addr, Port, Profile), SR = proplists:get_value(server_root, ConfigList), case proplists:get_value(data_file, DirData, no_data_file) of no_data_file -> @@ -241,7 +155,7 @@ store({security_directory, {Dir, DirData}}, ConfigList) _ -> DataFile0 end, - case mod_security_server:new_table(Addr, Port, DataFile) of + case mod_security_server:new_table(Addr, Port, Profile, DataFile) of {ok, TwoTables} -> NewDirData0 = lists:keyreplace(data_file, 1, DirData, {data_file, TwoTables}), @@ -261,45 +175,35 @@ store({directory, {Directory, DirData}}, _) -> {error, {wrong_type, {security_directory, {Directory, DirData}}}}. remove(ConfigDB) -> - Addr = case ets:lookup(ConfigDB, bind_address) of - [] -> - undefined; - [{bind_address, Address}] -> - Address - end, - [{port, Port}] = ets:lookup(ConfigDB, port), - mod_security_server:delete_tables(Addr, Port), - mod_security_server:stop(Addr, Port). + Addr = httpd_util:lookup(ConfigDB, bind_address, undefined), + Port = httpd_util:lookup(ConfigDB, port), + Profile = httpd_util:lookup(ConfigDB, profile, ?DEFAULT_PROFILE), + mod_security_server:delete_tables(Addr, Port, Profile), + mod_security_server:stop(Addr, Port, Profile). -%% -%% User API -%% - -%% list_blocked_users - list_blocked_users(Port) -> list_blocked_users(undefined, Port). list_blocked_users(Port, Dir) when is_integer(Port) -> list_blocked_users(undefined,Port,Dir); list_blocked_users(Addr, Port) when is_integer(Port) -> - mod_security_server:list_blocked_users(Addr, Port). + lists:map(fun({User, Addr0, Port0, ?DEFAULT_PROFILE, Dir0, Time}) -> + {User, Addr0, Port0, Dir0,Time} + end, + mod_security_server:list_blocked_users(Addr, Port)). list_blocked_users(Addr, Port, Dir) -> - mod_security_server:list_blocked_users(Addr, Port, Dir). - - -%% block_user + lists:map(fun({User, Addr0, Port0, ?DEFAULT_PROFILE, Dir0, Time}) -> + {User, Addr0, Port0, Dir0,Time} + end, + mod_security_server:list_blocked_users(Addr, Port, Dir)). block_user(User, Port, Dir, Time) -> block_user(User, undefined, Port, Dir, Time). block_user(User, Addr, Port, Dir, Time) -> mod_security_server:block_user(User, Addr, Port, Dir, Time). - -%% unblock_user - unblock_user(User, Port) -> unblock_user(User, undefined, Port). @@ -311,9 +215,6 @@ unblock_user(User, Addr, Port) when is_integer(Port) -> unblock_user(User, Addr, Port, Dir) -> mod_security_server:unblock_user(User, Addr, Port, Dir). - -%% list_auth_users - list_auth_users(Port) -> list_auth_users(undefined,Port). @@ -324,3 +225,76 @@ list_auth_users(Addr, Port) when is_integer(Port) -> list_auth_users(Addr, Port, Dir) -> mod_security_server:list_auth_users(Addr, Port, Dir). + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- + +report_failed(Info, Auth, Event) -> + Request = Info#mod.request_line, + {_PortNumber,RemoteHost}=(Info#mod.init_data)#init_data.peername, + String = RemoteHost ++ " : " ++ Event ++ " : " ++ Request ++ + " : " ++ Auth, + mod_disk_log:security_log(Info,String), + mod_log:security_log(Info, String). + +take_failed_action(Info, Auth) -> + Path = mod_alias:path(Info#mod.data, Info#mod.config_db, + Info#mod.request_uri), + {_Dir, SDirData} = secretp(Path, Info#mod.config_db), + Addr = httpd_util:lookup(Info#mod.config_db, bind_address), + Port = httpd_util:lookup(Info#mod.config_db, port), + Profile = httpd_util:lookup(Info#mod.config_db, profile, ?DEFAULT_PROFILE), + mod_security_server:store_failed_auth(Info, Addr, Port, Profile, + Auth, SDirData). + +secretp(Path, ConfigDB) -> + Directories = ets:match(ConfigDB,{directory,{'$1','_'}}), + case secret_path(Path, Directories) of + {yes, Directory} -> + SDirs0 = httpd_util:multi_lookup(ConfigDB, security_directory), + [SDir] = lists:filter(fun({Directory0, _}) + when Directory0 == Directory -> + true; + (_) -> + false + end, SDirs0), + SDir; + no -> + {[], []} + end. + +secret_path(Path,Directories) -> + secret_path(Path, httpd_util:uniq(lists:sort(Directories)), to_be_found). + +secret_path(_Path, [], to_be_found) -> + no; +secret_path(_Path, [], Dir) -> + {yes, Dir}; +secret_path(Path, [[NewDir]|Rest], Dir) -> + case inets_regexp:match(Path, NewDir) of + {match, _, _} when Dir =:= to_be_found -> + secret_path(Path, Rest, NewDir); + {match, _, Length} when Length > length(Dir) -> + secret_path(Path, Rest, NewDir); + {match, _, _} -> + secret_path(Path, Rest, Dir); + nomatch -> + secret_path(Path, Rest, Dir) + end. + + + +load_return_int_tag(Name, Atom, Time, Dir, DirData) -> + case Time of + "infinity" -> + {ok, [{security_directory, {Dir, + [{Atom, 99999999999999999999999999999} | DirData]}}]}; + _Int -> + case catch list_to_integer(Time) of + {'EXIT', _} -> + {error, Time++" is an invalid "++Name}; + Val -> + {ok, [{security_directory, {Dir, [{Atom, Val}|DirData]}}]} + end + end. diff --git a/lib/inets/src/http_server/mod_security_server.erl b/lib/inets/src/http_server/mod_security_server.erl index 784b3eba70..4f37dff18c 100644 --- a/lib/inets/src/http_server/mod_security_server.erl +++ b/lib/inets/src/http_server/mod_security_server.erl @@ -45,7 +45,6 @@ -include("httpd.hrl"). -include("httpd_internal.hrl"). --include("inets_internal.hrl"). -behaviour(gen_server). @@ -57,129 +56,105 @@ list_auth_users/2, list_auth_users/3]). %% Internal exports (for mod_security only) --export([start/2, stop/1, stop/2, - new_table/3, delete_tables/2, - store_failed_auth/5, store_successful_auth/4, - check_blocked_user/5]). +-export([start/3, stop/2, stop/3, + new_table/4, delete_tables/3, + store_failed_auth/6, store_successful_auth/5, + check_blocked_user/6]). %% gen_server exports --export([start_link/2, init/1, +-export([start_link/3, init/1, handle_info/2, handle_call/3, handle_cast/2, terminate/2, code_change/3]). +%%==================================================================== +%% Internal application API +%%==================================================================== -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% %% -%% External API %% -%% %% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -%% start_link/3 -%% %% NOTE: This is called by httpd_misc_sup when the process is started -%% - -start_link(Addr, Port) -> - ?hdrt("start_link", [{address, Addr}, {port, Port}]), - Name = make_name(Addr, Port), +start_link(Addr, Port, Profile) -> + Name = make_name(Addr, Port, Profile), gen_server:start_link({local, Name}, ?MODULE, [], [{timeout, infinity}]). - -%% start/2 %% Called by the mod_security module. - -start(Addr, Port) -> - ?hdrt("start", [{address, Addr}, {port, Port}]), - Name = make_name(Addr, Port), +start(Addr, Port, Profile) -> + Name = make_name(Addr, Port, Profile), case whereis(Name) of undefined -> - httpd_misc_sup:start_sec_server(Addr, Port); + httpd_misc_sup:start_sec_server(Addr, Port, Profile); _ -> %% Already started... ok end. - -%% stop - -stop(Port) -> - stop(undefined, Port). -stop(Addr, Port) -> - ?hdrt("stop", [{address, Addr}, {port, Port}]), - Name = make_name(Addr, Port), +stop(Port, Profile) -> + stop(undefined, Port, Profile). +stop(Addr, Port, Profile) -> + Name = make_name(Addr, Port, Profile), case whereis(Name) of undefined -> ok; _ -> - httpd_misc_sup:stop_sec_server(Addr, Port) + httpd_misc_sup:stop_sec_server(Addr, Port, Profile) end. - addr(undefined) -> any; addr(Addr) -> Addr. - -%% list_blocked_users - list_blocked_users(Addr, Port) -> - Name = make_name(Addr, Port), - Req = {list_blocked_users, addr(Addr), Port, '_'}, - call(Name, Req). - + list_blocked_users(Addr, Port, ?DEFAULT_PROFILE). +list_blocked_users(Addr, Port, Profile) when is_atom(Profile)-> + Name = make_name(Addr, Port, Profile), + Req = {list_blocked_users, addr(Addr), Port, Profile,'_'}, + call(Name, Req); list_blocked_users(Addr, Port, Dir) -> - Name = make_name(Addr, Port), - Req = {list_blocked_users, addr(Addr), Port, Dir}, + list_blocked_users(Addr, Port, ?DEFAULT_PROFILE, Dir). +list_blocked_users(Addr, Port, Profile, Dir) -> + Name = make_name(Addr, Port, Profile), + Req = {list_blocked_users, addr(Addr), Port, Profile, Dir}, call(Name, Req). - -%% block_user - block_user(User, Addr, Port, Dir, Time) -> - Name = make_name(Addr, Port), - Req = {block_user, User, addr(Addr), Port, Dir, Time}, + block_user(User, Addr, Port, ?DEFAULT_PROFILE, Dir, Time). +block_user(User, Addr, Port, Profile, Dir, Time) -> + Name = make_name(Addr, Port, Profile), + Req = {block_user, User, addr(Addr), Port, Profile, Dir, Time}, call(Name, Req). - -%% unblock_user - unblock_user(User, Addr, Port) -> - Name = make_name(Addr, Port), - Req = {unblock_user, User, addr(Addr), Port, '_'}, - call(Name, Req). - + unblock_user(User, Addr, Port, ?DEFAULT_PROFILE). +unblock_user(User, Addr, Port, Profile) when is_atom(Profile)-> + Name = make_name(Addr, Port, Profile), + Req = {unblock_user, User, addr(Addr), Port, Profile, '_'}, + call(Name, Req); unblock_user(User, Addr, Port, Dir) -> - Name = make_name(Addr, Port), - Req = {unblock_user, User, addr(Addr), Port, Dir}, + unblock_user(User, Addr, Port, ?DEFAULT_PROFILE, Dir). +unblock_user(User, Addr, Port, Profile, Dir) -> + Name = make_name(Addr, Port, Profile), + Req = {unblock_user, User, addr(Addr), Port, Profile, Dir}, call(Name, Req). - -%% list_auth_users - list_auth_users(Addr, Port) -> - Name = make_name(Addr, Port), - Req = {list_auth_users, addr(Addr), Port, '_'}, - call(Name, Req). - + list_auth_users(Addr, Port, ?DEFAULT_PROFILE). +list_auth_users(Addr, Port, Profile) when is_atom(Profile) -> + Name = make_name(Addr, Port, Profile), + Req = {list_auth_users, addr(Addr), Port, Profile, '_'}, + call(Name, Req); list_auth_users(Addr, Port, Dir) -> - Name = make_name(Addr,Port), - Req = {list_auth_users, addr(Addr), Port, Dir}, + list_auth_users(Addr, Port, ?DEFAULT_PROFILE, Dir). +list_auth_users(Addr, Port, Profile, Dir) -> + Name = make_name(Addr,Port, Profile), + Req = {list_auth_users, addr(Addr), Port, Profile, Dir}, call(Name, Req). - -%% new_table - -new_table(Addr, Port, TabName) -> - Name = make_name(Addr,Port), - Req = {new_table, addr(Addr), Port, TabName}, +new_table(Addr, Port, Profile, TabName) -> + Name = make_name(Addr,Port, Profile), + Req = {new_table, addr(Addr), Port, Profile, TabName}, call(Name, Req). - -%% delete_tables - -delete_tables(Addr, Port) -> - Name = make_name(Addr, Port), +delete_tables(Addr, Port, Profile) -> + Name = make_name(Addr, Port, Profile), case whereis(Name) of undefined -> ok; @@ -187,79 +162,53 @@ delete_tables(Addr, Port) -> call(Name, delete_tables) end. - -%% store_failed_auth - -store_failed_auth(Info, Addr, Port, DecodedString, SDirData) -> - ?hdrv("store failed auth", - [{addr, Addr}, {port, Port}, - {decoded_string, DecodedString}, {sdir_data, SDirData}]), - Name = make_name(Addr,Port), - Msg = {store_failed_auth,[Info,DecodedString,SDirData]}, +store_failed_auth(Info, Addr, Port, Profile, DecodedString, SDirData) -> + Name = make_name(Addr, Port, Profile), + Msg = {store_failed_auth, Profile, [Info,DecodedString,SDirData]}, cast(Name, Msg). - -%% store_successful_auth - -store_successful_auth(Addr, Port, User, SDirData) -> - Name = make_name(Addr,Port), - Msg = {store_successful_auth, [User,Addr,Port,SDirData]}, +store_successful_auth(Addr, Port, Profile, User, SDirData) -> + Name = make_name(Addr,Port, Profile), + Msg = {store_successful_auth, [User,Addr,Port, Profile, SDirData]}, cast(Name, Msg). - - -%% check_blocked_user - -check_blocked_user(Info, User, SDirData, Addr, Port) -> - Name = make_name(Addr, Port), - Req = {check_blocked_user, [Info, User, SDirData]}, + +check_blocked_user(Info, User, SDirData, Addr, Port, Profile) -> + Name = make_name(Addr, Port, Profile), + Req = {check_blocked_user, Profile, [Info, User, SDirData]}, call(Name, Req). - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% %% -%% Server call-back functions %% -%% %% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - +%%==================================================================== +%% Behavior call backs +%%==================================================================== init(_) -> - ?hdrv("initiating", []), process_flag(trap_exit, true), {ok, []}. handle_call(stop, _From, _Tables) -> {stop, normal, ok, []}; -handle_call({block_user, User, Addr, Port, Dir, Time}, _From, Tables) -> - ?hdrv("block user", - [{user, User}, {addr, Addr}, {port, Port}, {dir, Dir}, - {time, Time}]), - Ret = block_user_int(User, Addr, Port, Dir, Time), +handle_call({block_user, User, Addr, Port, Profile, Dir, Time}, _From, Tables) -> + Ret = block_user_int(User, Addr, Port, Profile, Dir, Time), {reply, Ret, Tables}; -handle_call({list_blocked_users, Addr, Port, Dir}, _From, Tables) -> - ?hdrv("list blocked users", - [{addr, Addr}, {port, Port}, {dir, Dir}]), - Blocked = list_blocked(Tables, Addr, Port, Dir, []), +handle_call({list_blocked_users, Addr, Port, Profile, Dir}, _From, Tables) -> + Blocked = list_blocked(Tables, Addr, Port, Profile, Dir, []), {reply, Blocked, Tables}; -handle_call({unblock_user, User, Addr, Port, Dir}, _From, Tables) -> - ?hdrv("block user", - [{user, User}, {addr, Addr}, {port, Port}, {dir, Dir}]), - Ret = unblock_user_int(User, Addr, Port, Dir), +handle_call({unblock_user, User, Addr, Port, Profile, Dir}, _From, Tables) -> + Ret = unblock_user_int(User, Addr, Port, Profile,Dir), {reply, Ret, Tables}; -handle_call({list_auth_users, Addr, Port, Dir}, _From, Tables) -> - ?hdrv("list auth users", - [{addr, Addr}, {port, Port}, {dir, Dir}]), - Auth = list_auth(Tables, Addr, Port, Dir, []), +handle_call({list_auth_users, Addr, Port, Profile, Dir}, _From, Tables) -> + Auth = list_auth(Tables, Addr, Port, Profile, Dir, []), {reply, Auth, Tables}; -handle_call({new_table, Addr, Port, Name}, _From, Tables) -> +handle_call({new_table, Addr, Port, Profile, Name}, _From, Tables) -> case lists:keysearch(Name, 1, Tables) of {value, {Name, {Ets, Dets}}} -> {reply, {ok, {Ets, Dets}}, Tables}; false -> - TName = make_name(Addr,Port,length(Tables)), + TName = make_name(Addr,Port, Profile, length(Tables)), case dets:open_file(TName, [{type, bag}, {file, Name}, {repair, true}, {access, read_write}]) of @@ -280,7 +229,7 @@ handle_call(delete_tables, _From, Tables) -> end, Tables), {reply, ok, []}; -handle_call({check_blocked_user, [Info, User, SDirData]}, _From, Tables) -> +handle_call({check_blocked_user, Profile, [Info, User, SDirData]}, _From, Tables) -> {ETS, DETS} = proplists:get_value(data_file, SDirData), Dir = proplists:get_value(path, SDirData), Addr = proplists:get_value(bind_address, SDirData), @@ -288,27 +237,24 @@ handle_call({check_blocked_user, [Info, User, SDirData]}, _From, Tables) -> CBModule = proplists:get_value(callback_module, SDirData, no_module_at_all), Ret = - check_blocked_user(Info, User, Dir, Addr, Port, ETS, DETS, CBModule), + check_blocked_user(Info, User, Dir, Addr, Port, Profile, ETS, DETS, CBModule), {reply, Ret, Tables}; handle_call(_Request,_From,Tables) -> {reply,ok,Tables}. - -%% handle_cast - -handle_cast({store_failed_auth, [_, _, []]}, Tables) -> +handle_cast({store_failed_auth, _,[_, _, []]}, Tables) -> %% Some other authentication scheme than mod_auth (example mod_htacess) %% was the source for the authentication failure so we should ignor it! {noreply, Tables}; -handle_cast({store_failed_auth, [Info, DecodedString, SDirData]}, Tables) -> +handle_cast({store_failed_auth, Profile, [Info, DecodedString, SDirData]}, Tables) -> {ETS, DETS} = proplists:get_value(data_file, SDirData), Dir = proplists:get_value(path, SDirData), Addr = proplists:get_value(bind_address, SDirData), Port = proplists:get_value(port, SDirData), {ok, [User,Password]} = httpd_util:split(DecodedString,":",2), Seconds = universal_time(), - Key = {User, Dir, Addr, Port}, + Key = {User, Dir, Addr, Port, Profile}, %% Event CBModule = proplists:get_value(callback_module, SDirData, no_module_at_all), @@ -363,7 +309,7 @@ handle_cast({store_failed_auth, [Info, DecodedString, SDirData]}, Tables) -> dets:match_delete(DETS, {blocked_user, {User, Addr, Port, Dir, '$1'}}), BlockRecord = {blocked_user, - {User, Addr, Port, Dir, Future}}, + {User, Addr, Port, Profile, Dir, Future}}, ets:insert(ETS, BlockRecord), dets:insert(DETS, BlockRecord), %% Remove previous failed requests. @@ -374,11 +320,11 @@ handle_cast({store_failed_auth, [Info, DecodedString, SDirData]}, Tables) -> end, {noreply, Tables}; -handle_cast({store_successful_auth, [User, Addr, Port, SDirData]}, Tables) -> +handle_cast({store_successful_auth, [User, Addr, Port, Profile, SDirData]}, Tables) -> {ETS, DETS} = proplists:get_value(data_file, SDirData), AuthTimeOut = proplists:get_value(auth_timeout, SDirData, 30), Dir = proplists:get_value(path, SDirData), - Key = {User, Dir, Addr, Port}, + Key = {User, Dir, Addr, Port, Profile}, %% Remove failed entries for this Key dets:match_delete(DETS, {failed, {Key, '_', '_'}}), @@ -396,33 +342,22 @@ handle_cast(Req, Tables) -> error_msg("security server got unknown cast: ~p",[Req]), {noreply, Tables}. - -%% handle_info - handle_info(_Info, State) -> {noreply, State}. - -%% terminate - terminate(_Reason, _Tables) -> ok. - -%% code_change({down, ToVsn}, State, Extra) -%% -code_change({down, _}, State, _Extra) -> - {ok, State}; - - -%% code_change(FromVsn, State, Extra) -%% code_change(_, State, _Extra) -> {ok, State}. +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- + %% block_user_int/5 -block_user_int(User, Addr, Port, Dir, Time) -> - Dirs = httpd_manager:config_match(Addr, Port, +block_user_int(User, Addr, Port, Profile, Dir, Time) -> + Dirs = httpd_manager:config_match(Addr, Port, Profile, {security_directory, {'_', '_'}}), case find_dirdata(Dirs, Dir) of {ok, DirData, {ETS, DETS}} -> @@ -434,11 +369,11 @@ block_user_int(User, Addr, Port, Dir, Time) -> Time end, Future = universal_time()+Time1, - ets:match_delete(ETS, {blocked_user, {User,Addr,Port,Dir,'_'}}), + ets:match_delete(ETS, {blocked_user, {User,Addr,Port,Profile, Dir,'_'}}), dets:match_delete(DETS, {blocked_user, - {User,Addr,Port,Dir,'_'}}), - ets:insert(ETS, {blocked_user, {User,Addr,Port,Dir,Future}}), - dets:insert(DETS, {blocked_user, {User,Addr,Port,Dir,Future}}), + {User,Addr,Port,Profile, Dir,'_'}}), + ets:insert(ETS, {blocked_user, {User,Addr,Port, Profile, Dir,Future}}), + dets:insert(DETS, {blocked_user, {User,Addr,Port,Profile, Dir,Future}}), CBModule = proplists:get_value(callback_module, DirData, no_module_at_all), user_block_event(CBModule,Addr,Port,Dir,User), @@ -447,7 +382,6 @@ block_user_int(User, Addr, Port, Dir, Time) -> {error, no_such_directory} end. - find_dirdata([], _Dir) -> false; find_dirdata([{security_directory, {_, DirData}}|SDirs], Dir) -> @@ -460,21 +394,20 @@ find_dirdata([{security_directory, {_, DirData}}|SDirs], Dir) -> find_dirdata(SDirs, Dir) end. -%% unblock_user_int/4 -unblock_user_int(User, Addr, Port, Dir) -> - Dirs = httpd_manager:config_match(Addr, Port, +unblock_user_int(User, Addr, Port, Profile, Dir) -> + Dirs = httpd_manager:config_match(Addr, Port, Profile, {security_directory, {'_', '_'}}), case find_dirdata(Dirs, Dir) of {ok, DirData, {ETS, DETS}} -> case ets:match_object(ETS, - {blocked_user,{User,Addr,Port,Dir,'_'}}) of + {blocked_user,{User,Addr,Port,Profile,Dir,'_'}}) of [] -> {error, not_blocked}; _Objects -> ets:match_delete(ETS, {blocked_user, - {User, Addr, Port, Dir, '_'}}), + {User, Addr, Port, Profile, Dir, '_'}}), dets:match_delete(DETS, {blocked_user, - {User, Addr, Port, Dir, '_'}}), + {User, Addr, Port, Profile, Dir, '_'}}), CBModule = proplists:get_value(callback_module, DirData, no_module_at_all), @@ -485,63 +418,51 @@ unblock_user_int(User, Addr, Port, Dir) -> {error, no_such_directory} end. - - -%% list_auth/2 - -list_auth([], _Addr, _Port, _Dir, Acc) -> +list_auth([], _, _, _, _, Acc) -> Acc; -list_auth([{_Name, {ETS, DETS}}|Tables], Addr, Port, Dir, Acc) -> - case ets:match_object(ETS, {success, {{'_', Dir, Addr, Port}, '_'}}) of +list_auth([{_Name, {ETS, DETS}}|Tables], Addr, Port, Profile, Dir, Acc) -> + case ets:match_object(ETS, {success, {{'_', Dir, Addr, Port, Profile}, '_'}}) of [] -> - list_auth(Tables, Addr, Port, Dir, Acc); + list_auth(Tables, Addr, Port, Profile, Dir, Acc); List -> TN = universal_time(), - NewAcc = lists:foldr(fun({success,{{U,Ad,P,D},T}},Ac) -> + NewAcc = lists:foldr(fun({success,{{U,Ad,P, Pr,D},T}},Ac) -> if T-TN > 0 -> [U|Ac]; true -> Rec = {success, - {{U,Ad,P,D},T}}, + {{U,Ad,P,Pr,D},T}}, ets:match_delete(ETS,Rec), dets:match_delete(DETS,Rec), Ac end end, Acc, List), - list_auth(Tables, Addr, Port, Dir, NewAcc) + list_auth(Tables, Addr, Port, Profile, Dir, NewAcc) end. - -%% list_blocked/2 - -list_blocked([], _Addr, _Port, _Dir, Acc) -> - ?hdrv("list blocked", [{acc, Acc}]), +list_blocked([], _, _, _, _, Acc) -> TN = universal_time(), - lists:foldl(fun({U,Ad,P,D,T}, Ac) -> + lists:foldl(fun({U,Ad,P,Pr,D,T}, Ac) -> if T-TN > 0 -> - [{U,Ad,P,D,local_time(T)}|Ac]; + [{U,Ad,P, Pr,D,local_time(T)}|Ac]; true -> Ac end end, [], Acc); -list_blocked([{_Name, {ETS, _DETS}}|Tables], Addr, Port, Dir, Acc) -> - ?hdrv("list blocked", [{ets, ETS}, {tab2list, ets:tab2list(ETS)}]), +list_blocked([{_Name, {ETS, _DETS}}|Tables], Addr, Port, Profile, Dir, Acc) -> List = ets:match_object(ETS, {blocked_user, - {'_',Addr,Port,Dir,'_'}}), + {'_',Addr,Port,Profile, Dir,'_'}}), NewBlocked = lists:foldl(fun({blocked_user, X}, A) -> [X|A] end, Acc, List), - list_blocked(Tables, Addr, Port, Dir, NewBlocked). + list_blocked(Tables, Addr, Port, Profile, Dir, NewBlocked). -%% -%% sync_dets_to_ets/2 -%% %% Reads dets-table DETS and syncronizes it with the ets-table ETS. %% sync_dets_to_ets(DETS, ETS) -> @@ -550,68 +471,62 @@ sync_dets_to_ets(DETS, ETS) -> continue end). -%% -%% check_blocked_user/7 -> true | false -%% %% Check if a specific user is blocked from access. %% %% The sideeffect of this routine is that it unblocks also other users %% whos blocking time has expired. This to keep the tables as small %% as possible. %% -check_blocked_user(Info, User, Dir, Addr, Port, ETS, DETS, CBModule) -> +check_blocked_user(Info, User, Dir, Addr, Port, Profile, ETS, DETS, CBModule) -> TN = universal_time(), - BlockList = ets:match_object(ETS, {blocked_user, {User, '_', '_', '_', '_'}}), + BlockList = ets:match_object(ETS, {blocked_user, {User, '_', '_', '_', '_', '_'}}), Blocked = lists:foldl(fun({blocked_user, X}, A) -> [X|A] end, [], BlockList), check_blocked_user(Info,User,Dir, - Addr,Port,ETS,DETS,TN,Blocked,CBModule). + Addr,Port, Profile, ETS,DETS,TN,Blocked,CBModule). -check_blocked_user(_Info, _User, _Dir, _Addr, _Port, _ETS, _DETS, _TN, - [], _CBModule) -> +check_blocked_user(_Info, _, _, _, _, _, _, _, _,[], _CBModule) -> false; -check_blocked_user(Info, User, Dir, Addr, Port, ETS, DETS, TN, - [{User,Addr,Port,Dir,T}| _], CBModule) -> +check_blocked_user(Info, User, Dir, Addr, Port, Profile, ETS, DETS, TN, + [{User,Addr,Port,Profile, Dir,T}| _], CBModule) -> TD = T-TN, if TD =< 0 -> %% Blocking has expired, remove and grant access. - unblock_user(Info, User, Dir, Addr, Port, ETS, DETS, CBModule), + unblock_user(Info, User, Dir, Addr, Port, Profile, ETS, DETS, CBModule), false; true -> true end; -check_blocked_user(Info, User, Dir, Addr, Port, ETS, DETS, TN, - [{OUser,ODir,OAddr,OPort,T}|Ls], CBModule) -> +check_blocked_user(Info, User, Dir, Addr, Port, Profile, ETS, DETS, TN, + [{OUser,ODir,OAddr,OPort, OProfile, T}|Ls], CBModule) -> TD = T-TN, if TD =< 0 -> %% Blocking has expired, remove. - unblock_user(Info, OUser, ODir, OAddr, OPort, + unblock_user(Info, OUser, ODir, OAddr, OPort, OProfile, ETS, DETS, CBModule); true -> true end, - check_blocked_user(Info, User, Dir, Addr, Port, ETS, DETS, + check_blocked_user(Info, User, Dir, Addr, Port, Profile, ETS, DETS, TN, Ls, CBModule). -unblock_user(Info, User, Dir, Addr, Port, ETS, DETS, CBModule) -> +unblock_user(Info, User, Dir, Addr, Port, Profile, ETS, DETS, CBModule) -> Reason = io_lib:format("User ~s was removed from the block list for dir ~s", [User, Dir]), mod_log:security_log(Info, lists:flatten(Reason)), user_unblock_event(CBModule,Addr,Port,Dir,User), - dets:match_delete(DETS, {blocked_user, {User, Addr, Port, Dir, '_'}}), - ets:match_delete(ETS, {blocked_user, {User, Addr, Port, Dir, '_'}}). + dets:match_delete(DETS, {blocked_user, {User, Addr, Port, Profile, Dir, '_'}}), + ets:match_delete(ETS, {blocked_user, {User, Addr, Port, Profile, Dir, '_'}}). +make_name(Addr,Port, Profile) -> + httpd_util:make_name(?MODULE,Addr,Port, Profile). -make_name(Addr,Port) -> - httpd_util:make_name("httpd_security",Addr,Port). - -make_name(Addr,Port,Num) -> - httpd_util:make_name("httpd_security",Addr,Port, - "__" ++ integer_to_list(Num)). - +make_name(Addr,Port, Profile, Num) -> + httpd_util:make_name(?MODULE,Addr,Port, + atom_to_list(Profile) ++ "__" ++ integer_to_list(Num)). auth_fail_event(Mod,Addr,Port,Dir,User,Passwd) -> event(auth_fail,Mod,Addr,Port,Dir,[{user,User},{password,Passwd}]). @@ -623,17 +538,10 @@ user_unblock_event(Mod,Addr,Port,Dir,User) -> event(user_unblock,Mod,Addr,Port,Dir,[{user,User}]). event(Event, Mod, undefined, Port, Dir, Info) -> - ?hdrt("event", - [{event, Event}, {mod, Mod}, {port, Port}, {dir, Dir}]), (catch Mod:event(Event,Port,Dir,Info)); event(Event, Mod, any, Port, Dir, Info) -> - ?hdrt("event", - [{event, Event}, {mod, Mod}, {port, Port}, {dir, Dir}]), (catch Mod:event(Event,Port,Dir,Info)); event(Event, Mod, Addr, Port, Dir, Info) -> - ?hdrt("event", - [{event, Event}, {mod, Mod}, - {addr, Addr}, {port, Port}, {dir, Dir}]), (catch Mod:event(Event,Addr,Port,Dir,Info)). universal_time() -> @@ -643,11 +551,9 @@ local_time(T) -> calendar:universal_time_to_local_time( calendar:gregorian_seconds_to_datetime(T)). - error_msg(F, A) -> error_logger:error_msg(F, A). - call(Name, Req) -> case (catch gen_server:call(Name, Req)) of {'EXIT', Reason} -> @@ -656,7 +562,6 @@ call(Name, Req) -> Reply end. - cast(Name, Msg) -> case (catch gen_server:cast(Name, Msg)) of {'EXIT', Reason} -> diff --git a/lib/inets/src/inets_app/inets.app.src b/lib/inets/src/inets_app/inets.app.src index b7c3e341e8..6ba9795d9e 100644 --- a/lib/inets/src/inets_app/inets.app.src +++ b/lib/inets/src/inets_app/inets.app.src @@ -63,6 +63,7 @@ httpd_cgi, httpd_connection_sup, httpd_conf, + httpd_custom, httpd_esi, httpd_example, httpd_file, diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 0dfc65e8f7..ab7ffadf75 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -1289,7 +1289,9 @@ dummy_server_init(Caller, ip_comm, Inet, _) -> {max_header, ?HTTP_MAX_HEADER_SIZE}, {max_version,?HTTP_MAX_VERSION_STRING}, {max_method, ?HTTP_MAX_METHOD_STRING}, - {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}]]}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}, + {customize, httpd_custom} + ]]}, [], ListenSocket); dummy_server_init(Caller, ssl, Inet, SSLOptions) -> @@ -1305,7 +1307,8 @@ dummy_ssl_server_init(Caller, BaseOpts, Inet) -> {max_method, ?HTTP_MAX_METHOD_STRING}, {max_version,?HTTP_MAX_VERSION_STRING}, {max_method, ?HTTP_MAX_METHOD_STRING}, - {max_content_length, ?HTTP_MAX_CONTENT_LENGTH} + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}, + {customize, httpd_custom} ]]}, [], ListenSocket). @@ -1384,18 +1387,20 @@ handle_request(Module, Function, Args, Socket) -> stop; <<>> -> {httpd_request, parse, [[{max_uri,?HTTP_MAX_URI_SIZE}, - {max_header, ?HTTP_MAX_HEADER_SIZE}, - {max_version,?HTTP_MAX_VERSION_STRING}, - {max_method, ?HTTP_MAX_METHOD_STRING}, - {max_content_length, ?HTTP_MAX_CONTENT_LENGTH} - ]]}; + {max_header, ?HTTP_MAX_HEADER_SIZE}, + {max_version,?HTTP_MAX_VERSION_STRING}, + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}, + {customize, httpd_custom} + ]]}; Data -> handle_request(httpd_request, parse, [Data, [{max_uri, ?HTTP_MAX_URI_SIZE}, - {max_header, ?HTTP_MAX_HEADER_SIZE}, + {max_header, ?HTTP_MAX_HEADER_SIZE}, {max_version,?HTTP_MAX_VERSION_STRING}, {max_method, ?HTTP_MAX_METHOD_STRING}, - {max_content_length, ?HTTP_MAX_CONTENT_LENGTH} + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}, + {customize, httpd_custom} ]], Socket) end; NewMFA -> diff --git a/lib/inets/test/httpc_proxy_SUITE.erl b/lib/inets/test/httpc_proxy_SUITE.erl index ddd23d0c65..fbd85e9e42 100644 --- a/lib/inets/test/httpc_proxy_SUITE.erl +++ b/lib/inets/test/httpc_proxy_SUITE.erl @@ -79,7 +79,7 @@ local_proxy_cases() -> %%-------------------------------------------------------------------- init_per_suite(Config0) -> - case init_apps([crypto,public_key], Config0) of + case init_apps(suite_apps(), Config0) of Config when is_list(Config) -> make_cert_files(dsa, "server-", Config), Config; @@ -94,7 +94,7 @@ end_per_suite(_Config) -> %% internal functions suite_apps() -> - [crypto,public_key]. + [asn1,crypto,public_key]. %%-------------------------------------------------------------------- diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index 7670c2cc60..c90887bcf3 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -53,6 +53,8 @@ all() -> {group, https_basic}, {group, http_limit}, {group, https_limit}, + {group, http_custom}, + {group, https_custom}, {group, http_basic_auth}, {group, https_basic_auth}, {group, http_auth_api}, @@ -76,6 +78,8 @@ groups() -> {https_basic, [], basic_groups()}, {http_limit, [], [{group, limit}]}, {https_limit, [], [{group, limit}]}, + {http_custom, [], [{group, custom}]}, + {https_custom, [], [{group, custom}]}, {http_basic_auth, [], [{group, basic_auth}]}, {https_basic_auth, [], [{group, basic_auth}]}, {http_auth_api, [], [{group, auth_api}]}, @@ -92,6 +96,7 @@ groups() -> {https_reload, [], [{group, reload}]}, {http_mime_types, [], [alias_1_1, alias_1_0, alias_0_9]}, {limit, [], [max_clients_1_1, max_clients_1_0, max_clients_0_9]}, + {custom, [], [customize]}, {reload, [], [non_disturbing_reconfiger_dies, disturbing_reconfiger_dies, non_disturbing_1_1, @@ -178,6 +183,7 @@ end_per_suite(_Config) -> %%-------------------------------------------------------------------- init_per_group(Group, Config0) when Group == https_basic; Group == https_limit; + Group == https_custom; Group == https_basic_auth; Group == https_auth_api; Group == https_auth_api_dets; @@ -188,6 +194,7 @@ init_per_group(Group, Config0) when Group == https_basic; init_ssl(Group, Config0); init_per_group(Group, Config0) when Group == http_basic; Group == http_limit; + Group == http_custom; Group == http_basic_auth; Group == http_auth_api; Group == http_auth_api_dets; @@ -977,6 +984,30 @@ missing_CR(Config) -> {version, Version}]). %%------------------------------------------------------------------------- +customize() -> + [{doc, "Test filtering of headers with custom callback"}]. + +customize(Config) when is_list(Config) -> + Version = "HTTP/1.1", + Host = ?config(host, Config), + Type = ?config(type, Config), + ok = httpd_test_lib:verify_request(?config(type, Config), Host, + ?config(port, Config), + transport_opts(Type, Config), + ?config(node, Config), + http_request("GET /index.html ", Version, Host), + [{statuscode, 200}, + {header, "Content-Type", "text/html"}, + {header, "Date"}, + {no_header, "Server"}, + {version, Version}]). + +response_header({"server", _}) -> + false; +response_header(Header) -> + {true, Header}. + +%%------------------------------------------------------------------------- max_header() -> ["Denial Of Service (DOS) attack, prevented by max_header"]. max_header(Config) when is_list(Config) -> @@ -1320,24 +1351,26 @@ setup_server_dirs(ServerRoot, DocRoot, DataDir) -> start_apps(Group) when Group == https_basic; Group == https_limit; + Group == https_custom; Group == https_basic_auth; Group == https_auth_api; Group == https_auth_api_dets; Group == https_auth_api_mnesia; - Group == http_htaccess; - Group == http_security; - Group == http_reload + Group == https_htaccess; + Group == https_security; + Group == https_reload -> inets_test_lib:start_apps([inets, asn1, crypto, public_key, ssl]); start_apps(Group) when Group == http_basic; Group == http_limit; + Group == http_custom; Group == http_basic_auth; Group == http_auth_api; Group == http_auth_api_dets; Group == http_auth_api_mnesia; - Group == https_htaccess; - Group == https_security; - Group == https_reload; + Group == http_htaccess; + Group == http_security; + Group == http_reload; Group == http_mime_types-> inets_test_lib:start_apps([inets]). @@ -1390,6 +1423,10 @@ server_config(http_limit, Config) -> [{max_clients, 1}, %% Make sure option checking code is run {max_content_length, 100000002}] ++ server_config(http, Config); +server_config(http_custom, Config) -> + [{custom, ?MODULE}] ++ server_config(http, Config); +server_config(https_custom, Config) -> + [{custom, ?MODULE}] ++ server_config(https, Config); server_config(https_limit, Config) -> [{max_clients, 1}] ++ server_config(https, Config); server_config(http_basic_auth, Config) -> diff --git a/lib/inets/test/httpd_block.erl b/lib/inets/test/httpd_block.erl index 9790623b6f..a95a5ee62d 100644 --- a/lib/inets/test/httpd_block.erl +++ b/lib/inets/test/httpd_block.erl @@ -292,7 +292,7 @@ httpd_restart(Addr, Port) -> end. make_name(Addr, Port) -> - httpd_util:make_name("httpd", Addr, Port). + httpd_util:make_name("httpd", Addr, Port, default). get_admin_state(_, _Host, Port) -> Name = make_name(undefined, Port), diff --git a/lib/inets/test/inets_sup_SUITE.erl b/lib/inets/test/inets_sup_SUITE.erl index 60979278fc..1479681e30 100644 --- a/lib/inets/test/inets_sup_SUITE.erl +++ b/lib/inets/test/inets_sup_SUITE.erl @@ -22,14 +22,14 @@ -include_lib("common_test/include/ct.hrl"). - %% Note: This directive should only be used in test suites. -compile(export_all). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [default_tree, ftpc_worker, tftpd_worker, httpd_subtree, + [default_tree, ftpc_worker, tftpd_worker, + httpd_subtree, httpd_subtree_profile, httpc_subtree]. groups() -> @@ -41,54 +41,29 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - -%%-------------------------------------------------------------------- -%% Function: init_per_suite(Config) -> Config -%% Config - [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Description: Initiation before the whole suite -%% -%% Note: This function is free to add any key/value pairs to the Config -%% variable, but should NOT alter/remove any existing entries. -%%-------------------------------------------------------------------- init_per_suite(Config) -> Config. -%%-------------------------------------------------------------------- -%% Function: end_per_suite(Config) -> _ -%% Config - [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Description: Cleanup after the whole suite -%%-------------------------------------------------------------------- end_per_suite(_) -> inets:stop(), ok. -%%-------------------------------------------------------------------- -%% Function: init_per_testcase(Case, Config) -> Config -%% Case - atom() -%% Name of the test case that is about to be run. -%% Config - [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% -%% Description: Initiation before each test case -%% -%% Note: This function is free to add any key/value pairs to the Config -%% variable, but should NOT alter/remove any existing entries. -%%-------------------------------------------------------------------- init_per_testcase(httpd_subtree, Config) -> Dog = test_server:timetrap(?t:minutes(1)), NewConfig = lists:keydelete(watchdog, 1, Config), PrivDir = ?config(priv_dir, Config), - + Dir = filename:join(PrivDir, "root"), + ok = file:make_dir(Dir), + SimpleConfig = [{port, 0}, {server_name,"www.test"}, {modules, [mod_get]}, - {server_root, PrivDir}, - {document_root, PrivDir}, + {server_root, Dir}, + {document_root, Dir}, {bind_address, any}, {ipfamily, inet}], try + inets:stop(), inets:start(), inets:start(httpd, SimpleConfig), [{watchdog, Dog} | NewConfig] @@ -97,7 +72,33 @@ init_per_testcase(httpd_subtree, Config) -> inets:stop(), exit({failed_starting_inets, Reason}) end; - + +init_per_testcase(httpd_subtree_profile, Config) -> + Dog = test_server:timetrap(?t:minutes(1)), + NewConfig = lists:keydelete(watchdog, 1, Config), + PrivDir = ?config(priv_dir, Config), + Dir = filename:join(PrivDir, "root"), + ok = file:make_dir(Dir), + + SimpleConfig = [{port, 0}, + {server_name,"www.test"}, + {modules, [mod_get]}, + {server_root, Dir}, + {document_root, Dir}, + {bind_address, any}, + {profile, test_profile}, + {ipfamily, inet}], + try + inets:stop(), + inets:start(), + {ok, _} = inets:start(httpd, SimpleConfig), + [{watchdog, Dog} | NewConfig] + catch + _:Reason -> + inets:stop(), + exit({failed_starting_inets, Reason}) + end; + init_per_testcase(_Case, Config) -> Dog = test_server:timetrap(?t:minutes(5)), @@ -106,20 +107,13 @@ init_per_testcase(_Case, Config) -> ok = inets:start(), [{watchdog, Dog} | NewConfig]. - -%%-------------------------------------------------------------------- -%% Function: end_per_testcase(Case, Config) -> _ -%% Case - atom() -%% Name of the test case that is about to be run. -%% Config - [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Description: Cleanup after each test case -%%-------------------------------------------------------------------- -end_per_testcase(httpd_subtree, Config) -> +end_per_testcase(Case, Config) when Case == httpd_subtree; + Case == httpd_subtree_profile -> Dog = ?config(watchdog, Config), test_server:timetrap_cancel(Dog), - PrivDir = ?config(priv_dir, Config), - inets_test_lib:del_dirs(PrivDir), + PrivDir = ?config(priv_dir, Config), + Dir = filename:join(PrivDir, "root"), + inets_test_lib:del_dirs(Dir), ok; end_per_testcase(_, Config) -> @@ -131,16 +125,9 @@ end_per_testcase(_, Config) -> %%------------------------------------------------------------------------- %% Test cases starts here. %%------------------------------------------------------------------------- - - -%%------------------------------------------------------------------------- -%% default_tree -%%------------------------------------------------------------------------- -default_tree(doc) -> - ["Makes sure the correct processes are started and linked," - "in the default case."]; -default_tree(suite) -> - []; +default_tree() -> + [{doc, "Makes sure the correct processes are started and linked," + "in the default case."}]. default_tree(Config) when is_list(Config) -> TopSupChildren = supervisor:which_children(inets_sup), 4 = length(TopSupChildren), @@ -173,15 +160,9 @@ default_tree(Config) when is_list(Config) -> ok. - -%%------------------------------------------------------------------------- -%% ftpc_worker -%%------------------------------------------------------------------------- -ftpc_worker(doc) -> - ["Makes sure the ftp worker processes are added and removed " - "appropriatly to/from the supervison tree."]; -ftpc_worker(suite) -> - []; +ftpc_worker() -> + [{doc, "Makes sure the ftp worker processes are added and removed " + "appropriatly to/from the supervison tree."}]. ftpc_worker(Config) when is_list(Config) -> [] = supervisor:which_children(ftp_sup), try @@ -207,14 +188,8 @@ ftpc_worker(Config) when is_list(Config) -> {skip, "No available FTP servers"} end. - -%%------------------------------------------------------------------------- -%% tftpd_worker -%%------------------------------------------------------------------------- -tftpd_worker(doc) -> - ["Makes sure the tftp sub tree is correct."]; -tftpd_worker(suite) -> - []; +tftpd_worker() -> + [{doc, "Makes sure the tftp sub tree is correct."}]. tftpd_worker(Config) when is_list(Config) -> [] = supervisor:which_children(tftp_sup), {ok, Pid0} = inets:start(tftpd, [{host, inets_test_lib:hostname()}, @@ -228,22 +203,63 @@ tftpd_worker(Config) when is_list(Config) -> [] = supervisor:which_children(tftp_sup), ok. +httpd_subtree() -> + [{doc, "Makes sure the httpd sub tree is correct."}]. +httpd_subtree(Config) when is_list(Config) -> + do_httpd_subtree(Config, default). + +httpd_subtree_profile(doc) -> + ["Makes sure the httpd sub tree is correct when using a profile"]; +httpd_subtree_profile(Config) when is_list(Config) -> + do_httpd_subtree(Config, test_profile). + +httpc_subtree() -> + [{doc, "Makes sure the httpd sub tree is correct."}]. +httpc_subtree(Config) when is_list(Config) -> + {ok, Foo} = inets:start(httpc, [{profile, foo}]), + + {ok, Bar} = inets:start(httpc, [{profile, bar}], stand_alone), + + HttpcChildren = supervisor:which_children(httpc_profile_sup), + + {value, {httpc_manager, _, worker, [httpc_manager]}} = + lists:keysearch(httpc_manager, 1, HttpcChildren), + + {value,{{httpc,foo}, _Pid, worker, [httpc_manager]}} = + lists:keysearch({httpc, foo}, 1, HttpcChildren), + false = lists:keysearch({httpc, bar}, 1, HttpcChildren), + + inets:stop(httpc, Foo), + exit(Bar, normal). %%------------------------------------------------------------------------- -%% httpd_subtree +%% Internal functions %%------------------------------------------------------------------------- -httpd_subtree(doc) -> - ["Makes sure the httpd sub tree is correct."]; -httpd_subtree(suite) -> - []; -httpd_subtree(Config) when is_list(Config) -> - %% Check that we have the httpd top supervisor + +verify_child(Parent, Child, Type) -> + Children = supervisor:which_children(Parent), + verify_child(Children, Parent, Child, Type). + +verify_child([], Parent, Child, _Type) -> + {error, {child_not_found, Child, Parent}}; +verify_child([{Id, _Pid, Type2, Mods}|Children], Parent, Child, Type) -> + case lists:member(Child, Mods) of + true when (Type2 =:= Type) -> + {ok, Id}; + true when (Type2 =/= Type) -> + {error, {wrong_type, Type2, Child, Parent}}; + false -> + verify_child(Children, Parent, Child, Type) + end. + +do_httpd_subtree(_Config, Profile) -> + %% Check that we have the httpd top supervisor {ok, _} = verify_child(inets_sup, httpd_sup, supervisor), %% Check that we have the httpd instance supervisor {ok, Id} = verify_child(httpd_sup, httpd_instance_sup, supervisor), - {httpd_instance_sup, Addr, Port} = Id, - Instance = httpd_util:make_name("httpd_instance_sup", Addr, Port), + {httpd_instance_sup, Addr, Port, Profile} = Id, + Instance = httpd_util:make_name("httpd_instance_sup", Addr, Port, Profile), %% Check that we have the expected httpd instance children {ok, _} = verify_child(Instance, httpd_connection_sup, supervisor), @@ -252,7 +268,7 @@ httpd_subtree(Config) when is_list(Config) -> {ok, _} = verify_child(Instance, httpd_manager, worker), %% Check that the httpd instance acc supervisor has children - InstanceAcc = httpd_util:make_name("httpd_acceptor_sup", Addr, Port), + InstanceAcc = httpd_util:make_name("httpd_acceptor_sup", Addr, Port, Profile), case supervisor:which_children(InstanceAcc) of [_ | _] -> ok; @@ -263,7 +279,7 @@ httpd_subtree(Config) when is_list(Config) -> %% Check that the httpd instance misc supervisor has no children io:format("httpd_subtree -> verify misc~n", []), - InstanceMisc = httpd_util:make_name("httpd_misc_sup", Addr, Port), + InstanceMisc = httpd_util:make_name("httpd_misc_sup", Addr, Port, Profile), case supervisor:which_children(InstanceMisc) of [] -> ok; @@ -273,45 +289,3 @@ httpd_subtree(Config) when is_list(Config) -> end, io:format("httpd_subtree -> done~n", []), ok. - - -verify_child(Parent, Child, Type) -> - Children = supervisor:which_children(Parent), - verify_child(Children, Parent, Child, Type). - -verify_child([], Parent, Child, _Type) -> - {error, {child_not_found, Child, Parent}}; -verify_child([{Id, _Pid, Type2, Mods}|Children], Parent, Child, Type) -> - case lists:member(Child, Mods) of - true when (Type2 =:= Type) -> - {ok, Id}; - true when (Type2 =/= Type) -> - {error, {wrong_type, Type2, Child, Parent}}; - false -> - verify_child(Children, Parent, Child, Type) - end. - -%%------------------------------------------------------------------------- -%% httpc_subtree -%%------------------------------------------------------------------------- -httpc_subtree(doc) -> - ["Makes sure the httpc sub tree is correct."]; -httpc_subtree(suite) -> - []; -httpc_subtree(Config) when is_list(Config) -> - {ok, Foo} = inets:start(httpc, [{profile, foo}]), - - {ok, Bar} = inets:start(httpc, [{profile, bar}], stand_alone), - - HttpcChildren = supervisor:which_children(httpc_profile_sup), - - {value, {httpc_manager, _, worker, [httpc_manager]}} = - lists:keysearch(httpc_manager, 1, HttpcChildren), - - {value,{{httpc,foo}, _Pid, worker, [httpc_manager]}} = - lists:keysearch({httpc, foo}, 1, HttpcChildren), - false = lists:keysearch({httpc, bar}, 1, HttpcChildren), - - inets:stop(httpc, Foo), - exit(Bar, normal). - diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index d5cb460404..f52347e39e 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -2,7 +2,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2001-2014. All Rights Reserved. +# Copyright Ericsson AB 2001-2015. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/jinterface/doc/src/notes.xml b/lib/jinterface/doc/src/notes.xml index f854fa1f3a..fc5f8be53e 100644 --- a/lib/jinterface/doc/src/notes.xml +++ b/lib/jinterface/doc/src/notes.xml @@ -30,59 +30,6 @@ </header> <p>This document describes the changes made to the Jinterface application.</p> -<section><title>Jinterface 1.6</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Reformat the sources for JInterface uniformly and - according to the standard Java style guidelines. Provide - description of the rules applied in Eclipse format (for - other editors one can check the settings against these).</p> - <p> - In short, the formatting style is: * indentation uses - only spaces; each level is 4 positions * no trailing - whitespace * mostly default Java style formatting (any - difference is minor) * always use {} blocks * use 'final' - as much as possible</p> - <p> - Own Id: OTP-12333</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Add basic transport factory implementation. This makes - possible creating connections between nodes using ssh - channels for example.</p> - <p> - Own Id: OTP-12686</p> - </item> - <item> - <p> - Add Jinterface generic match and bind methods to provide - low level interface base methods sufficient for variety - of higher level pattern matching/variable binding - implementations.</p> - <p> - Own Id: OTP-12691</p> - </item> - <item> - <p> - Minimal Java version is now 1.6</p> - <p> - Own Id: OTP-12718</p> - </item> - </list> - </section> - -</section> - <section><title>Jinterface 1.5.12</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpSelf.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpSelf.java index 5b9d13ad81..74afbbcca6 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpSelf.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpSelf.java @@ -153,9 +153,6 @@ public class OtpSelf extends OtpLocalNode { * the port number you wish to use for incoming connections. * Specifying 0 lets the system choose an available port. * - * @param transportFactory - * the transport factory to use when creating connections. - * * @exception IOException * in case of server transport failure */ diff --git a/lib/jinterface/vsn.mk b/lib/jinterface/vsn.mk index 4df01d1151..72ad316333 100644 --- a/lib/jinterface/vsn.mk +++ b/lib/jinterface/vsn.mk @@ -1 +1 @@ -JINTERFACE_VSN = 1.6 +JINTERFACE_VSN = 1.5.12 diff --git a/lib/kernel/doc/src/error_logger.xml b/lib/kernel/doc/src/error_logger.xml index df2f0b01ee..f49d63b5a6 100644 --- a/lib/kernel/doc/src/error_logger.xml +++ b/lib/kernel/doc/src/error_logger.xml @@ -58,12 +58,11 @@ specific events. (<c>add_report_handler/1,2</c>). Also, there is a useful event handler in STDLIB for multi-file logging of events, see <c>log_mf_h(3)</c>.</p> - <p>Warning events were introduced in Erlang/OTP R9C. To retain - backwards compatibility, these are by default tagged as errors, - thus showing up as error reports in the logs. By using - the command line flag <c><![CDATA[+W <w | i>]]></c>, they can instead - be tagged as warnings or info. Tagging them as warnings may - require rewriting existing user defined event handlers.</p> + <p>Warning events were introduced in Erlang/OTP R9C and are enabled + by default as of 18.0. To retain backwards compatibility with existing + user defined event handlers, these may be tagged as errors or info + using the command line flag <c><![CDATA[+W <e | i | w>]]></c>, thus + showing up as error or info reports in the logs.</p> </description> <datatypes> <datatype> @@ -132,7 +131,7 @@ ok</pre> <desc> <p>Returns the current mapping for warning events. Events sent using <c>warning_msg/1,2</c> or <c>warning_report/1,2</c> - are tagged as errors (default), warnings or info, depending + are tagged as errors, warnings (default) or info, depending on the value of the command line flag <c>+W</c>.</p> <pre> os$ <input>erl</input> @@ -140,25 +139,25 @@ Erlang (BEAM) emulator version 5.4.8 [hipe] [threads:0] [kernel-poll] Eshell V5.4.8 (abort with ^G) 1> <input>error_logger:warning_map().</input> -error -2> <input>error_logger:warning_msg("Warnings tagged as: ~p~n", [error]).</input> +warning +2> <input>error_logger:warning_msg("Warnings tagged as: ~p~n", [warning]).</input> -=ERROR REPORT==== 11-Aug-2005::15:31:23 === -Warnings tagged as: error +=WARNING REPORT==== 11-Aug-2005::15:31:55 === +Warnings tagged as: warning ok 3> User switch command --> q -os$ <input>erl +W w</input> +os$ <input>erl +W e</input> Erlang (BEAM) emulator version 5.4.8 [hipe] [threads:0] [kernel-poll] Eshell V5.4.8 (abort with ^G) 1> <input>error_logger:warning_map().</input> -warning -2> <input>error_logger:warning_msg("Warnings tagged as: ~p~n", [warning]).</input> +error +2> <input>error_logger:warning_msg("Warnings tagged as: ~p~n", [error]).</input> -=WARNING REPORT==== 11-Aug-2005::15:31:55 === -Warnings tagged as: warning +=ERROR REPORT==== 11-Aug-2005::15:31:23 === +Warnings tagged as: error ok</pre> </desc> </func> diff --git a/lib/kernel/doc/src/gen_tcp.xml b/lib/kernel/doc/src/gen_tcp.xml index 820ecd1e30..71ef5cd48f 100644 --- a/lib/kernel/doc/src/gen_tcp.xml +++ b/lib/kernel/doc/src/gen_tcp.xml @@ -347,11 +347,22 @@ do_recv(Sock, Bs) -> </func> <func> <name name="shutdown" arity="2"/> - <fsummary>Immediately close a socket</fsummary> + <fsummary>Asynchronously close a socket</fsummary> <desc> - <p>Immediately close a socket in one or two directions.</p> + <p>Close a socket in one or two directions.</p> <p><c><anno>How</anno> == write</c> means closing the socket for writing, reading from it is still possible.</p> + <p>If <c><anno>How</anno> == read</c>, or there is no outgoing + data buffered in the <c><anno>Socket</anno></c> port, + then the socket is shutdown immediately and any error encountered + is returned in <c><anno>Reason</anno></c>.</p> + <p>If there is data buffered in the socket port, then the attempt + to shutdown the socket is postponed until that data is written to the + kernel socket send buffer. Any errors encountered will result + in the socket being closed and <c>{error, closed}</c> being returned + on the next + <seealso marker="gen_tcp#recv/2">recv/2</seealso> or + <seealso marker="gen_tcp#send/2">send/2</seealso>.</p> <p>To be able to handle that the peer has done a shutdown on the write side, the <c>{exit_on_close, false}</c> option is useful.</p> diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index 0bd9a067ca..6f7f18a8e7 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -30,79 +30,6 @@ </header> <p>This document describes the changes made to the Kernel application.</p> -<section><title>Kernel 4.0</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Fix error handling in <c>file:read_line/1</c> for Unicode - contents.</p> - <p> - Own Id: OTP-12144</p> - </item> - <item> - <p> - Introduce <c>os:getenv/2</c> which is similar to - <c>os:getenv/1</c> but returns the passed default value - if the required environment variable is undefined.</p> - <p> - Own Id: OTP-12342</p> - </item> - <item> - <p> - It is now possible to paste text in JCL mode (using - Ctrl-Y) that has been copied in the previous shell - session. Also a bug that caused the JCL mode to crash - when pasting text has been fixed.</p> - <p> - Own Id: OTP-12673</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - New BIF: <c>erlang:get_keys/0</c>, lists all keys - associated with the process dictionary. Note: - <c>erlang:get_keys/0</c> is auto-imported.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12151 Aux Id: seq12521 </p> - </item> - <item> - <p> - The internal group to user_drv protocol has been changed - to be synchronous in order to guarantee that output sent - to a process implementing the user_drv protocol is - printed before replying. This protocol is used by the - standard_output device and the ssh application when - acting as a client. </p> - <p> - This change changes the previous unlimited buffer when - printing to standard_io and other devices that end up in - user_drv to 1KB.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12240</p> - </item> - <item> - <p>The <c>inflateInit/2</c> and <c>deflateInit/6</c> - functions now accepts a WindowBits argument equal to 8 - and -8.</p> - <p> - Own Id: OTP-12564</p> - </item> - </list> - </section> - -</section> - <section><title>Kernel 3.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl index 6635885aaf..a1a99a4e18 100644 --- a/lib/kernel/src/application_controller.erl +++ b/lib/kernel/src/application_controller.erl @@ -490,7 +490,8 @@ init(Init, Kernel) -> %% called during start-up of any app. case check_conf_data(ConfData) of ok -> - _ = ets:new(ac_tab, [set, public, named_table]), + _ = ets:new(ac_tab, [set, public, named_table, + {read_concurrency,true}]), S = #state{conf_data = ConfData}, {ok, KAppl} = make_appl(Kernel), case catch load(S, KAppl) of diff --git a/lib/kernel/src/code.erl b/lib/kernel/src/code.erl index d73d1ff281..580c070389 100644 --- a/lib/kernel/src/code.erl +++ b/lib/kernel/src/code.erl @@ -107,7 +107,7 @@ is_module_native(_) -> -spec make_stub_module(Module, Beam, Info) -> Module when Module :: module(), Beam :: binary(), - Info :: {list(), list()}. + Info :: {list(), list(), binary()}. make_stub_module(_, _, _) -> erlang:nif_error(undef). @@ -339,7 +339,8 @@ do_start(Flags) -> ok end, %% Quietly load native code for all modules loaded so far - load_native_code_for_all_loaded(), + Architecture = erlang:system_info(hipe_architecture), + load_native_code_for_all_loaded(Architecture), Ok2; Other -> Other @@ -554,18 +555,18 @@ has_ext(Ext, Extlen, File) -> %%% Silently load native code for all modules loaded so far. %%% --spec load_native_code_for_all_loaded() -> ok. -load_native_code_for_all_loaded() -> - Architecture = erlang:system_info(hipe_architecture), +load_native_code_for_all_loaded(undefined) -> + ok; +load_native_code_for_all_loaded(Architecture) -> try hipe_unified_loader:chunk_name(Architecture) of ChunkTag -> Loaded = all_loaded(), - spawn(fun() -> load_all_native(Loaded, ChunkTag) end) + _ = spawn(fun() -> load_all_native(Loaded, ChunkTag) end), + ok catch _:_ -> ok - end, - ok. + end. load_all_native(Loaded, ChunkTag) -> catch load_all_native_1(Loaded, ChunkTag). @@ -582,7 +583,8 @@ load_all_native_1([{Mod,BeamFilename}|T], ChunkTag) -> undefined -> ok; NativeCode when is_binary(NativeCode) -> - load_native_partial(Mod, NativeCode) + _ = load_native_partial(Mod, NativeCode), + ok end; true -> ok end, diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl index 819554ce74..eecd26863a 100644 --- a/lib/kernel/src/code_server.erl +++ b/lib/kernel/src/code_server.erl @@ -324,12 +324,15 @@ handle_call({load_binary,Mod,File,Bin}, Caller, S) -> do_load_binary(Mod, File, Bin, Caller, S); handle_call({load_native_partial,Mod,Bin}, {_From,_Tag}, S) -> - Result = (catch hipe_unified_loader:load(Mod, Bin)), + Architecture = erlang:system_info(hipe_architecture), + Result = (catch hipe_unified_loader:load(Mod, Bin, Architecture)), Status = hipe_result_to_status(Result), {reply,Status,S}; handle_call({load_native_sticky,Mod,Bin,WholeModule}, {_From,_Tag}, S) -> - Result = (catch hipe_unified_loader:load_module(Mod, Bin, WholeModule)), + Architecture = erlang:system_info(hipe_architecture), + Result = (catch hipe_unified_loader:load_module(Mod, Bin, WholeModule, + Architecture)), Status = hipe_result_to_status(Result), {reply,Status,S}; @@ -1256,33 +1259,43 @@ try_load_module(File, Mod, Bin, {From,_}=Caller, St0) -> try_load_module_1(File, Mod, Bin, Caller, #state{moddb=Db}=St) -> case is_sticky(Mod, Db) of true -> %% Sticky file reject the load - error_msg("Can't load module that resides in sticky dir\n",[]), + error_msg("Can't load module '~w' that resides in sticky dir\n",[Mod]), {reply,{error,sticky_directory},St}; false -> - case catch load_native_code(Mod, Bin) of - {module,Mod} = Module -> - ets:insert(Db, {Mod,File}), - {reply,Module,St}; - no_native -> - case erlang:load_module(Mod, Bin) of - {module,Mod} = Module -> - ets:insert(Db, {Mod,File}), - post_beam_load(Mod), - {reply,Module,St}; - {error,on_load} -> - handle_on_load(Mod, File, Caller, St); - {error,What} = Error -> - error_msg("Loading of ~ts failed: ~p\n", [File, What]), - {reply,Error,St} - end; - Error -> - error_msg("Native loading of ~ts failed: ~p\n", - [File,Error]), - {reply,ok,St} - end + Architecture = erlang:system_info(hipe_architecture), + try_load_module_2(File, Mod, Bin, Caller, Architecture, St) + end. + +try_load_module_2(File, Mod, Bin, Caller, undefined, St) -> + try_load_module_3(File, Mod, Bin, Caller, undefined, St); +try_load_module_2(File, Mod, Bin, Caller, Architecture, + #state{moddb=Db}=St) -> + case catch load_native_code(Mod, Bin, Architecture) of + {module,Mod} = Module -> + ets:insert(Db, {Mod,File}), + {reply,Module,St}; + no_native -> + try_load_module_3(File, Mod, Bin, Caller, Architecture, St); + Error -> + error_msg("Native loading of ~ts failed: ~p\n", [File,Error]), + {reply,ok,St} + end. + +try_load_module_3(File, Mod, Bin, Caller, Architecture, + #state{moddb=Db}=St) -> + case erlang:load_module(Mod, Bin) of + {module,Mod} = Module -> + ets:insert(Db, {Mod,File}), + post_beam_load(Mod, Architecture), + {reply,Module,St}; + {error,on_load} -> + handle_on_load(Mod, File, Caller, St); + {error,What} = Error -> + error_msg("Loading of ~ts failed: ~p\n", [File, What]), + {reply,Error,St} end. -load_native_code(Mod, Bin) -> +load_native_code(Mod, Bin, Architecture) -> %% During bootstrapping of Open Source Erlang, we don't have any hipe %% loader modules, but the Erlang emulator might be hipe enabled. %% Therefore we must test for that the loader modules are available @@ -1291,7 +1304,8 @@ load_native_code(Mod, Bin) -> false -> no_native; true -> - Result = hipe_unified_loader:load_native_code(Mod, Bin), + Result = hipe_unified_loader:load_native_code(Mod, Bin, + Architecture), case Result of {module,_} -> put(?ANY_NATIVE_CODE_LOADED, true); @@ -1310,12 +1324,12 @@ hipe_result_to_status(Result) -> {error,Result} end. -post_beam_load(Mod) -> - %% post_beam_load/1 can potentially be very expensive because it +post_beam_load(Mod, Architecture) -> + %% post_beam_load/2 can potentially be very expensive because it %% blocks multi-scheduling; thus we want to avoid the call if we %% know that it is not needed. case get(?ANY_NATIVE_CODE_LOADED) of - true -> hipe_unified_loader:post_beam_load(Mod); + true -> hipe_unified_loader:post_beam_load(Mod, Architecture); false -> ok end. diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl index 49d4a8fe54..ddbbc548dd 100644 --- a/lib/kernel/src/hipe_unified_loader.erl +++ b/lib/kernel/src/hipe_unified_loader.erl @@ -43,10 +43,10 @@ -export([chunk_name/1, %% Only the code and code_server modules may call the entries below! - load_native_code/2, - post_beam_load/1, - load_module/3, - load/2]). + load_native_code/3, + post_beam_load/2, + load_module/4, + load/3]). %%-define(DEBUG,true). -define(DO_ASSERT,true). @@ -82,58 +82,57 @@ chunk_name(Architecture) -> %% HW32 %% HiPE, x86, Win32 end. +word_size(Architecture) -> + case Architecture of + amd64 -> 8; + ppc64 -> 8; + _ -> 4 + end. + %%======================================================================== --spec load_native_code(Mod, binary()) -> 'no_native' | {'module', Mod} - when Mod :: atom(). +-spec load_native_code(Mod, binary(), hipe_architecture()) -> + 'no_native' | {'module', Mod} when Mod :: atom(). %% @doc %% Loads the native code of a module Mod. %% Returns {module,Mod} on success (for compatibility with %% code:load_file/1) and the atom `no_native' on failure. -load_native_code(Mod, Bin) when is_atom(Mod), is_binary(Bin) -> - Architecture = erlang:system_info(hipe_architecture), - try chunk_name(Architecture) of - ChunkTag -> - %% patch_to_emu(Mod), - case code:get_chunk(Bin, ChunkTag) of - undefined -> no_native; - NativeCode when is_binary(NativeCode) -> - erlang:system_flag(multi_scheduling, block), - try - OldReferencesToPatch = patch_to_emu_step1(Mod), - case load_module(Mod, NativeCode, Bin, OldReferencesToPatch) of - bad_crc -> no_native; - Result -> Result - end - after - erlang:system_flag(multi_scheduling, unblock) - end +load_native_code(_Mod, _Bin, undefined) -> + no_native; +load_native_code(Mod, Bin, Architecture) when is_atom(Mod), is_binary(Bin) -> + %% patch_to_emu(Mod), + case code:get_chunk(Bin, chunk_name(Architecture)) of + undefined -> no_native; + NativeCode when is_binary(NativeCode) -> + erlang:system_flag(multi_scheduling, block), + try + OldReferencesToPatch = patch_to_emu_step1(Mod), + case load_module(Mod, NativeCode, Bin, OldReferencesToPatch, + Architecture) of + bad_crc -> no_native; + Result -> Result + end + after + erlang:system_flag(multi_scheduling, unblock) end - catch - _:_ -> - %% Unknown HiPE architecture. Can't happen (in principle). - no_native end. %%======================================================================== --spec post_beam_load(atom()) -> 'ok'. +-spec post_beam_load(atom(), hipe_architecture()) -> 'ok'. -post_beam_load(Mod) when is_atom(Mod) -> - Architecture = erlang:system_info(hipe_architecture), - try chunk_name(Architecture) of - _ChunkTag -> - erlang:system_flag(multi_scheduling, block), - try - patch_to_emu(Mod) - after - erlang:system_flag(multi_scheduling, unblock) - end - catch - _:_ -> - ok - end. +%% does nothing on a hipe-disabled system +post_beam_load(_Mod, undefined) -> + ok; +post_beam_load(Mod, _) when is_atom(Mod) -> + erlang:system_flag(multi_scheduling, block), + try + patch_to_emu(Mod) + after + erlang:system_flag(multi_scheduling, unblock) + end, + ok. %%======================================================================== @@ -148,46 +147,48 @@ version_check(Version, Mod) when is_atom(Mod) -> %%======================================================================== --spec load_module(Mod, binary(), _) -> 'bad_crc' | {'module', Mod} - when Mod :: atom(). -load_module(Mod, Bin, Beam) -> +-spec load_module(Mod, binary(), _, hipe_architecture()) -> + 'bad_crc' | {'module', Mod} when Mod :: atom(). + +load_module(Mod, Bin, Beam, Architecture) -> erlang:system_flag(multi_scheduling, block), try - load_module_nosmp(Mod, Bin, Beam) + load_module_nosmp(Mod, Bin, Beam, Architecture) after erlang:system_flag(multi_scheduling, unblock) end. -load_module_nosmp(Mod, Bin, Beam) -> - load_module(Mod, Bin, Beam, []). +load_module_nosmp(Mod, Bin, Beam, Architecture) -> + load_module(Mod, Bin, Beam, [], Architecture). -load_module(Mod, Bin, Beam, OldReferencesToPatch) -> +load_module(Mod, Bin, Beam, OldReferencesToPatch, Architecture) -> ?debug_msg("************ Loading Module ~w ************\n",[Mod]), %% Loading a whole module, let the BEAM loader patch closures. put(hipe_patch_closures, false), - load_common(Mod, Bin, Beam, OldReferencesToPatch). + load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture). %%======================================================================== --spec load(Mod, binary()) -> 'bad_crc' | {'module', Mod} when Mod :: atom(). +-spec load(Mod, binary(), hipe_architecture()) -> + 'bad_crc' | {'module', Mod} when Mod :: atom(). -load(Mod, Bin) -> +load(Mod, Bin, Architecture) -> erlang:system_flag(multi_scheduling, block), try - load_nosmp(Mod, Bin) + load_nosmp(Mod, Bin, Architecture) after erlang:system_flag(multi_scheduling, unblock) end. -load_nosmp(Mod, Bin) -> +load_nosmp(Mod, Bin, Architecture) -> ?debug_msg("********* Loading funs in module ~w *********\n",[Mod]), %% Loading just some functions in a module; patch closures separately. put(hipe_patch_closures, true), - load_common(Mod, Bin, [], []). + load_common(Mod, Bin, [], [], Architecture). %%------------------------------------------------------------------------ -load_common(Mod, Bin, Beam, OldReferencesToPatch) -> +load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) -> %% Unpack the binary. [{Version, CheckSum}, ConstAlign, ConstSize, ConstMap, LabelMap, ExportMap, @@ -212,18 +213,21 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch) -> bad_crc; true -> put(closures_to_patch, []), + WordSize = word_size(Architecture), + WriteWord = write_word_fun(WordSize), %% Create data segment {ConstAddr,ConstMap2} = - create_data_segment(ConstAlign, ConstSize, ConstMap), + create_data_segment(ConstAlign, ConstSize, ConstMap, WriteWord), %% Find callees for which we may need trampolines. - CalleeMFAs = find_callee_mfas(Refs), + CalleeMFAs = find_callee_mfas(Refs, Architecture), %% Write the code to memory. {CodeAddress,Trampolines} = enter_code(CodeSize, CodeBinary, CalleeMFAs, Mod, Beam), %% Construct CalleeMFA-to-trampoline mapping. - TrampolineMap = mk_trampoline_map(CalleeMFAs, Trampolines), + TrampolineMap = mk_trampoline_map(CalleeMFAs, Trampolines, + Architecture), %% Patch references to code labels in data seg. - ok = patch_consts(LabelMap, ConstAddr, CodeAddress), + ok = patch_consts(LabelMap, ConstAddr, CodeAddress, WriteWord), %% Find out which functions are being loaded (and where). %% Note: Addresses are sorted descending. {MFAs,Addresses} = exports(ExportMap, CodeAddress), @@ -275,14 +279,26 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch) -> %% Scan the list of patches and build a set (returned as a tuple) %% of the callees for which we may need trampolines. %% -find_callee_mfas(Patches) when is_list(Patches) -> - case erlang:system_info(hipe_architecture) of - amd64 -> []; - arm -> find_callee_mfas(Patches, gb_sets:empty(), false); - powerpc -> find_callee_mfas(Patches, gb_sets:empty(), true); - ppc64 -> find_callee_mfas(Patches, gb_sets:empty(), true); - ultrasparc -> []; - x86 -> [] +find_callee_mfas(Patches, Architecture) when is_list(Patches) -> + case needs_trampolines(Architecture) of + true -> find_callee_mfas(Patches, gb_sets:empty(), + no_erts_trampolines(Architecture)); + _ -> [] + end. + +needs_trampolines(Architecture) -> + case Architecture of + arm -> true; + powerpc -> true; + ppc64 -> true; + _ -> false + end. + +no_erts_trampolines(Architecture) -> + case Architecture of + powerpc -> true; + ppc64 -> true; + _ -> false end. find_callee_mfas([{Type,Data}|Patches], MFAs, SkipErtsSyms) -> @@ -318,14 +334,9 @@ add_callee_mfas([], MFAs, _SkipErtsSyms) -> MFAs. %%---------------------------------------------------------------- %% -mk_trampoline_map([], []) -> []; % archs not using trampolines -mk_trampoline_map(CalleeMFAs, Trampolines) -> - SizeofLong = - case erlang:system_info(hipe_architecture) of - amd64 -> 8; - ppc64 -> 8; - _ -> 4 - end, +mk_trampoline_map([], [], _) -> []; % archs not using trampolines +mk_trampoline_map(CalleeMFAs, Trampolines, Architecture) -> + SizeofLong = word_size(Architecture), mk_trampoline_map(tuple_size(CalleeMFAs), CalleeMFAs, Trampolines, SizeofLong, gb_trees:empty()). @@ -621,22 +632,24 @@ patch_load_mfa(CodeAddress, DestMFA, Addresses, RemoteOrLocal) -> %%---------------------------------------------------------------- %% Patch references to code labels in the data segment. %% -patch_consts(Labels, DataAddress, CodeAddress) -> +patch_consts(Labels, DataAddress, CodeAddress, WriteWord) -> lists:foreach(fun (L) -> - patch_label_or_labels(L, DataAddress, CodeAddress) + patch_label_or_labels(L, DataAddress, CodeAddress, + WriteWord) end, Labels). -patch_label_or_labels({Pos,Offset}, DataAddress, CodeAddress) -> +patch_label_or_labels({Pos,Offset}, DataAddress, CodeAddress, WriteWord) -> ?ASSERT(assert_local_patch(CodeAddress+Offset)), - write_word(DataAddress+Pos, CodeAddress+Offset); -patch_label_or_labels({sorted,Base,UnOrderdList}, DataAddress, CodeAddress) -> - sort_and_write(UnOrderdList, Base, DataAddress, CodeAddress). + WriteWord(DataAddress+Pos, CodeAddress+Offset); +patch_label_or_labels({sorted,Base,UnOrderdList}, DataAddress, CodeAddress, + WriteWord) -> + sort_and_write(UnOrderdList, Base, DataAddress, CodeAddress, WriteWord). -sort_and_write(UnOrderdList, Base, DataAddress, CodeAddress) -> +sort_and_write(UnOrderdList, Base, DataAddress, CodeAddress, WriteWord) -> WriteAndInc = fun ({_, Offset}, DataPos) -> ?ASSERT(assert_local_patch(CodeAddress+Offset)), - write_word(DataPos, CodeAddress+Offset) + WriteWord(DataPos, CodeAddress+Offset) end, lists:foldl(WriteAndInc, DataAddress+Base, sort_on_representation(UnOrderdList)). @@ -662,17 +675,18 @@ patch_instr(Address, Value, Type) -> %% XXX: It appears this is used for inserting both code addresses %% and other data. In HiPE, code addresses are still 32-bit on %% some 64-bit machines. -write_word(DataAddress, DataWord) -> - case erlang:system_info(hipe_architecture) of - amd64 -> - hipe_bifs:write_u64(DataAddress, DataWord), - DataAddress+8; - ppc64 -> - hipe_bifs:write_u64(DataAddress, DataWord), - DataAddress+8; - _ -> - hipe_bifs:write_u32(DataAddress, DataWord), - DataAddress+4 +write_word_fun(WordSize) -> + case WordSize of + 8 -> + fun (DataAddress, DataWord) -> + hipe_bifs:write_u64(DataAddress, DataWord), + DataAddress+8 + end; + 4 -> + fun (DataAddress, DataWord) -> + hipe_bifs:write_u32(DataAddress, DataWord), + DataAddress+4 + end end. %%-------------------------------------------------------------------- @@ -688,30 +702,31 @@ bif_address(Name) when is_atom(Name) -> %% memory, and produces a ConstMap2 mapping each constant's ConstNo to %% its runtime address, tagged if the constant is a term. %% -create_data_segment(DataAlign, DataSize, DataList) -> +create_data_segment(DataAlign, DataSize, DataList, WriteWord) -> %%io:format("create_data_segment: \nDataAlign: ~p\nDataSize: ~p\nDataList: ~p\n",[DataAlign,DataSize,DataList]), DataAddress = hipe_bifs:alloc_data(DataAlign, DataSize), - enter_data(DataList, [], DataAddress, DataSize). + enter_data(DataList, [], DataAddress, DataSize, WriteWord). -enter_data(List, ConstMap2, DataAddress, DataSize) -> +enter_data(List, ConstMap2, DataAddress, DataSize, WriteWord) -> case List of [ConstNo,Offset,Type,Data|Rest] when is_integer(Offset) -> %%?msg("Const ~w\n",[[ConstNo,Offset,Type,Data]]), ?ASSERT((Offset >= 0) and (Offset =< DataSize)), - Res = enter_datum(Type, Data, DataAddress+Offset), - enter_data(Rest, [{ConstNo,Res}|ConstMap2], DataAddress, DataSize); + Res = enter_datum(Type, Data, DataAddress+Offset, WriteWord), + enter_data(Rest, [{ConstNo,Res}|ConstMap2], DataAddress, DataSize, + WriteWord); [] -> {DataAddress, ConstMap2} end. -enter_datum(Type, Data, Address) -> +enter_datum(Type, Data, Address, WriteWord) -> case ?EXT2CONST_TYPE(Type) of term -> %% Address is unused for terms hipe_bifs:term_to_word(hipe_bifs:merge_term(Data)); sorted_block -> L = lists:sort([hipe_bifs:term_to_word(Term) || Term <- Data]), - write_words(L, Address), + write_words(L, Address, WriteWord), Address; block -> case Data of @@ -719,7 +734,7 @@ enter_datum(Type, Data, Address) -> write_bytes(Lbls, Address); {Lbls, SortOrder} -> SortedLbls = [Lbl || {_,Lbl} <- lists:sort(group(Lbls, SortOrder))], - write_words(SortedLbls, Address); + write_words(SortedLbls, Address, WriteWord); Lbls -> write_bytes(Lbls, Address) end, @@ -734,9 +749,9 @@ group([B1,B2,B3,B4|Ls], [O|Os]) -> bytes_to_32(B4,B3,B2,B1) -> (B4 bsl 24) bor (B3 bsl 16) bor (B2 bsl 8) bor B1. -write_words([W|Rest], Addr) -> - write_words(Rest, write_word(Addr, W)); -write_words([], Addr) when is_integer(Addr) -> true. +write_words([W|Rest], Addr, WriteWord) -> + write_words(Rest, WriteWord(Addr, W), WriteWord); +write_words([], Addr, _) when is_integer(Addr) -> true. write_bytes([B|Rest], Addr) -> hipe_bifs:write_u8(Addr, B), @@ -812,7 +827,7 @@ address_to_mfa_lth(_Address, [], Prev) -> %%---------------------------------------------------------------- %% Change callers of the given module to instead trap to BEAM. -%% load_native_code/2 calls this just before loading native code. +%% load_native_code/3 calls this just before loading native code. %% patch_to_emu(Mod) -> patch_to_emu_step2(patch_to_emu_step1(Mod)). diff --git a/lib/kernel/src/inet.erl b/lib/kernel/src/inet.erl index ec2c350931..d668738109 100644 --- a/lib/kernel/src/inet.erl +++ b/lib/kernel/src/inet.erl @@ -1527,26 +1527,28 @@ tcp_controlling_process(S, NewOwner) when is_port(S), is_pid(NewOwner) -> _ -> case prim_inet:getopt(S, active) of {ok, A0} -> - case A0 of - false -> ok; - _ -> ok = prim_inet:setopt(S, active, false) - end, - case tcp_sync_input(S, NewOwner, false) of - true -> %% socket already closed, + SetOptRes = + case A0 of + false -> ok; + _ -> prim_inet:setopt(S, active, false) + end, + case {tcp_sync_input(S, NewOwner, false), SetOptRes} of + {true, _} -> %% socket already closed ok; - false -> + {false, ok} -> try erlang:port_connect(S, NewOwner) of true -> unlink(S), %% unlink from port case A0 of false -> ok; - _ -> ok = prim_inet:setopt(S, active, A0) - end, - ok + _ -> prim_inet:setopt(S, active, A0) + end catch error:Reason -> {error, Reason} - end + end; + {false, Error} -> + Error end; Error -> Error diff --git a/lib/kernel/src/inet_db.erl b/lib/kernel/src/inet_db.erl index abe207295f..535c11271e 100644 --- a/lib/kernel/src/inet_db.erl +++ b/lib/kernel/src/inet_db.erl @@ -1372,7 +1372,7 @@ cache_rr(_Db, Cache, RR) -> ets:insert(Cache, RR). times() -> - erlang:monotonic_time(1). + erlang:convert_time_unit(erlang:monotonic_time() - erlang:system_info(start_time),native,seconds). %% lookup and remove old entries diff --git a/lib/kernel/src/inet_parse.erl b/lib/kernel/src/inet_parse.erl index a88c94a453..a694642b19 100644 --- a/lib/kernel/src/inet_parse.erl +++ b/lib/kernel/src/inet_parse.erl @@ -675,28 +675,22 @@ ipv6_addr_done(Ar, Br, N) -> ipv6_addr_done(Ar) -> list_to_tuple(lists:reverse(Ar)). -%% Collect Hex digits -hex(Cs) -> hex(Cs, []). -%% -hex([C|Cs], R) when C >= $0, C =< $9 -> - hex(Cs, [C|R]); -hex([C|Cs], R) when C >= $a, C =< $f -> - hex(Cs, [C|R]); -hex([C|Cs], R) when C >= $A, C =< $F -> - hex(Cs, [C|R]); -hex(Cs, [_|_]=R) when is_list(Cs) -> +%% Collect 1-4 Hex digits +hex(Cs) -> hex(Cs, [], 4). +%% +hex([C|Cs], R, N) when C >= $0, C =< $9, N > 0 -> + hex(Cs, [C|R], N-1); +hex([C|Cs], R, N) when C >= $a, C =< $f, N > 0 -> + hex(Cs, [C|R], N-1); +hex([C|Cs], R, N) when C >= $A, C =< $F, N > 0 -> + hex(Cs, [C|R], N-1); +hex(Cs, [_|_]=R, _) when is_list(Cs) -> {lists:reverse(R),Cs}; -hex(_, _) -> +hex(_, _, _) -> erlang:error(badarg). %% Hex string to integer -hex_to_int(Cs0) -> - case strip0(Cs0) of - Cs when length(Cs) =< 4 -> - erlang:list_to_integer("0"++Cs, 16); - _ -> - erlang:error(badarg) - end. +hex_to_int(Cs) -> erlang:list_to_integer(Cs, 16). %% Dup onto head of existing list dup(0, _, L) -> diff --git a/lib/kernel/src/inet_sctp.erl b/lib/kernel/src/inet_sctp.erl index 93528d305d..f0f13c8d4a 100644 --- a/lib/kernel/src/inet_sctp.erl +++ b/lib/kernel/src/inet_sctp.erl @@ -133,15 +133,18 @@ connect_get_assoc(S, Addr, Port, Active, Timer) -> Timeout = inet:timeout(Timer), receive {sctp,S,Addr,Port,{_,#sctp_assoc_change{state=St}=Ev}} -> - case Active of - once -> - ok = prim_inet:setopt(S, active, once); - _ -> ok - end, - if St =:= comm_up -> + SetOptRes = + case Active of + once -> prim_inet:setopt(S, active, once); + _ -> ok + end, + case {St, SetOptRes} of + {comm_up, ok} -> {ok,Ev}; - true -> - {error,Ev} + {_, ok} -> + {error,Ev}; + {_, Error} -> + Error end after Timeout -> {error,timeout} diff --git a/lib/kernel/src/inet_tcp_dist.erl b/lib/kernel/src/inet_tcp_dist.erl index 835dcf2705..fb60a14afb 100644 --- a/lib/kernel/src/inet_tcp_dist.erl +++ b/lib/kernel/src/inet_tcp_dist.erl @@ -112,7 +112,6 @@ listen_options(Opts0) -> end, case application:get_env(kernel, inet_dist_listen_options) of {ok,ListenOpts} -> - erlang:display({inet_dist_listen_options, ListenOpts}), ListenOpts ++ Opts1; _ -> Opts1 @@ -340,7 +339,6 @@ do_setup(Kernel, Node, Type, MyNode, LongOrShortNames,SetupTime) -> connect_options(Opts) -> case application:get_env(kernel, inet_dist_connect_options) of {ok,ConnectOpts} -> - erlang:display({inet_dist_listen_options, ConnectOpts}), ConnectOpts ++ Opts; _ -> Opts diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index 0cb10791d7..9787dca162 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -115,6 +115,6 @@ {applications, []}, {env, [{error_logger, tty}]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-7.0", "stdlib-2.0", "sasl-2.4"]} + {runtime_dependencies, ["erts-7.0", "stdlib-2.5", "sasl-2.4"]} ] }. diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index 1bae762bed..5d3836bad7 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -1,7 +1,7 @@ %% -*- erlang -*- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2014. All Rights Reserved. +%% Copyright Ericsson AB 1999-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -17,7 +17,7 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"3\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-17 + [{<<"3\\.[0-2](\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-17 %% Down to - max one major revision back - [{<<"3\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-17 + [{<<"3\\.[0-2](\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-17 }. diff --git a/lib/kernel/src/user_drv.erl b/lib/kernel/src/user_drv.erl index 380c685869..d3deca3a20 100644 --- a/lib/kernel/src/user_drv.erl +++ b/lib/kernel/src/user_drv.erl @@ -135,7 +135,7 @@ server1(Iport, Oport, Shell) -> Iport, Oport), %% Enter the server loop. - server_loop(Iport, Oport, Curr, User, Gr, queue:new()). + server_loop(Iport, Oport, Curr, User, Gr, {false, queue:new()}). rem_sh_opts(Node) -> [{expand_fun,fun(B)-> rpc:call(Node,edlin_expand,expand,[B]) end}]. @@ -165,7 +165,7 @@ server_loop(Iport, Oport, User, Gr, IOQueue) -> put(current_group, Curr), server_loop(Iport, Oport, Curr, User, Gr, IOQueue). -server_loop(Iport, Oport, Curr, User, Gr, IOQueue) -> +server_loop(Iport, Oport, Curr, User, Gr, {Resp, IOQ} = IOQueue) -> receive {Iport,{data,Bs}} -> BsBin = list_to_binary(Bs), @@ -182,9 +182,9 @@ server_loop(Iport, Oport, Curr, User, Gr, IOQueue) -> {Oport,ok} -> %% We get this ok from the port, in io_request we store %% info about where to send reply at head of queue - {{value,{Origin,Reply}},ReplyQ} = queue:out(IOQueue), + {Origin,Reply} = Resp, Origin ! {reply,Reply}, - NewQ = handle_req(next, Iport, Oport, ReplyQ), + NewQ = handle_req(next, Iport, Oport, {false, IOQ}), server_loop(Iport, Oport, Curr, User, Gr, NewQ); {'EXIT',Iport,_R} -> server_loop(Iport, Oport, Curr, User, Gr, IOQueue); @@ -238,28 +238,30 @@ handle_req({Curr,get_unicode_state},Iport,_Oport,IOQueue) -> handle_req({Curr,set_unicode_state, Bool},Iport,_Oport,IOQueue) -> Curr ! {self(),set_unicode_state,set_unicode_state(Iport,Bool)}, IOQueue; -handle_req(next,Iport,Oport,IOQueue) -> - case queue:out(IOQueue) of - {{value,Next},ExecQ} -> - NewQ = handle_req(Next,Iport,Oport,queue:new()), - queue:join(NewQ,ExecQ); +handle_req(next,Iport,Oport,{false,IOQ}=IOQueue) -> + case queue:out(IOQ) of {empty,_} -> - IOQueue - end; -handle_req(Msg,Iport,Oport,IOQueue) -> - case queue:peek(IOQueue) of - empty -> - {Origin,Req} = Msg, + IOQueue; + {{value,{Origin,Req}},ExecQ} -> case io_request(Req, Iport, Oport) of - ok -> IOQueue; + ok -> + handle_req(next,Iport,Oport,{false,ExecQ}); Reply -> - %% Push reply info to front of queue - queue:in_r({Origin,Reply},IOQueue) - end; - _Else -> - %% All requests are queued when we have outstanding sync put_chars - queue:in(Msg,IOQueue) - end. + {{Origin,Reply}, ExecQ} + end + end; +handle_req(Msg,Iport,Oport,{false,IOQ}=IOQueue) -> + empty = queue:peek(IOQ), + {Origin,Req} = Msg, + case io_request(Req, Iport, Oport) of + ok -> + IOQueue; + Reply -> + {{Origin,Reply}, IOQ} + end; +handle_req(Msg,_Iport,_Oport,{Resp, IOQ}) -> + %% All requests are queued when we have outstanding sync put_chars + {Resp, queue:in(Msg,IOQ)}. %% port_bytes(Bytes, InPort, OutPort, CurrentProcess, UserProcess, Group) %% Check the Bytes from the port to see if it contains a ^G. If so, diff --git a/lib/kernel/test/application_SUITE.erl b/lib/kernel/test/application_SUITE.erl index 4901206c8e..59efe85480 100644 --- a/lib/kernel/test/application_SUITE.erl +++ b/lib/kernel/test/application_SUITE.erl @@ -2699,10 +2699,7 @@ node_names(Names, Config) -> node_name(Name, Config) -> U = "_", - {{Y,M,D}, {H,Min,S}} = calendar:now_to_local_time(now()), - Date = io_lib:format("~4w_~2..0w_~2..0w__~2..0w_~2..0w_~2..0w", - [Y,M,D, H,Min,S]), - L = lists:flatten(Date), + L = integer_to_list(erlang:unique_integer([positive])), lists:concat([Name,U,?testcase,U,U,L]). stop_node_nice(Node) when is_atom(Node) -> diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 549c65d034..be55e25811 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -810,14 +810,6 @@ check_funs({'$M_EXPR','$F_EXPR',_}, {unicode,characters_to_binary,3}, {filename,filename_string_to_binary,1}|_]) -> 0; check_funs({'$M_EXPR','$F_EXPR',_}, - [{code_server,load_native_code,4}, - {code_server,load_native_code_1,2}, - {code_server,load_native_code,2}, - {code_server,try_load_module,4}, - {code_server,do_load_binary,4}, - {code_server,handle_call,3}, - {code_server,loop,1}|_]) -> 0; -check_funs({'$M_EXPR','$F_EXPR',_}, [{code_server,do_mod_call,4}, {code_server,handle_call,3}|_]) -> 0; check_funs({'$M_EXPR','$F_EXPR',_}, @@ -866,8 +858,14 @@ check_funs({'$M_EXPR','$F_EXPR',_}, check_funs({'$M_EXPR',module_info,1}, [{hipe_unified_loader,patch_to_emu_step1,1} | _]) -> 0; check_funs({'$M_EXPR','$F_EXPR',2}, + [{hipe_unified_loader,write_words,3} | _]) -> 0; +check_funs({'$M_EXPR','$F_EXPR',2}, + [{hipe_unified_loader,patch_label_or_labels,4} | _]) -> 0; +check_funs({'$M_EXPR','$F_EXPR',2}, + [{hipe_unified_loader,sort_and_write,5} | _]) -> 0; +check_funs({'$M_EXPR','$F_EXPR',2}, [{lists,foldl,3}, - {hipe_unified_loader,sort_and_write,4} | _]) -> 0; + {hipe_unified_loader,sort_and_write,5} | _]) -> 0; check_funs({'$M_EXPR','$F_EXPR',1}, [{lists,foreach,2}, {hipe_unified_loader,patch_consts,3} | _]) -> 0; @@ -1654,9 +1652,7 @@ get_mode(Config) when is_list(Config) -> init(Tester) -> {ok, Tester}. -handle_event({error, _GL, {emulator, _, _}}, Tester) -> - {ok, Tester}; -handle_event({error, _GL, Msg}, Tester) -> +handle_event({warning_msg, _GL, Msg}, Tester) -> Tester ! Msg, {ok, Tester}; handle_event(_Event, State) -> diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl index 15c2adc957..76564d4b0e 100644 --- a/lib/kernel/test/erl_distribution_SUITE.erl +++ b/lib/kernel/test/erl_distribution_SUITE.erl @@ -235,11 +235,10 @@ do_test_setuptime(Setuptime) when is_list(Setuptime) -> Res. time_ping(Node) -> - T0 = erlang:now(), + T0 = erlang:monotonic_time(), pang = net_adm:ping(Node), - T1 = erlang:now(), - time_diff(T0,T1). - + T1 = erlang:monotonic_time(), + erlang:convert_time_unit(T1 - T0, native, milli_seconds). %% Keep the connection with the client node up. %% This is neccessary as the client node runs with much shorter @@ -276,13 +275,15 @@ tick_cli_test1(Node) -> erlang:monitor_node(Node, true), sleep(2), rpc:call(Node, erlang, time, []), %% simulate action on the connection - T1 = now(), + T1 = erlang:monotonic_time(), receive {nodedown, Node} -> - T2 = now(), + T2 = erlang:monotonic_time(), receive {whats_the_result, From} -> - case time_diff(T1, T2) of + Diff = erlang:convert_time_unit(T2-T1, native, + milli_seconds), + case Diff of T when T > 8000, T < 16000 -> From ! {tick_test, T}; T -> @@ -1208,19 +1209,6 @@ print_my_messages() -> ?line ?t:format("Messages: ~p~n", [Messages]), ?line ok. -%% Time difference in milliseconds !! -time_diff({TimeM, TimeS, TimeU}, {CurM, CurS, CurU}) when CurM > TimeM -> - ((CurM - TimeM) * 1000000000) + sec_diff({TimeS, TimeU}, {CurS, CurU}); -time_diff({_, TimeS, TimeU}, {_, CurS, CurU}) -> - sec_diff({TimeS, TimeU}, {CurS, CurU}). - -sec_diff({TimeS, TimeU}, {CurS, CurU}) when CurS > TimeS -> - ((CurS - TimeS) * 1000) + micro_diff(TimeU, CurU); -sec_diff({_, TimeU}, {_, CurU}) -> - micro_diff(TimeU, CurU). - -micro_diff(TimeU, CurU) -> - trunc(CurU/1000) - trunc(TimeU/1000). sleep(T) -> receive after T * 1000 -> ok end. @@ -1267,16 +1255,12 @@ get_nodenames(N, T) -> get_nodenames(0, _, Acc) -> Acc; get_nodenames(N, T, Acc) -> - {A, B, C} = now(), + U = erlang:unique_integer([positive]), get_nodenames(N-1, T, [list_to_atom(atom_to_list(T) ++ "-" - ++ atom_to_list(?MODULE) - ++ "-" - ++ integer_to_list(A) + ++ ?MODULE_STRING ++ "-" - ++ integer_to_list(B) - ++ "-" - ++ integer_to_list(C)) | Acc]). + ++ integer_to_list(U)) | Acc]). get_numbered_nodenames(N, T) -> get_numbered_nodenames(N, T, []). @@ -1284,16 +1268,12 @@ get_numbered_nodenames(N, T) -> get_numbered_nodenames(0, _, Acc) -> Acc; get_numbered_nodenames(N, T, Acc) -> - {A, B, C} = now(), + U = erlang:unique_integer([positive]), NL = [list_to_atom(atom_to_list(T) ++ integer_to_list(N) ++ "-" - ++ atom_to_list(?MODULE) - ++ "-" - ++ integer_to_list(A) - ++ "-" - ++ integer_to_list(B) + ++ ?MODULE_STRING ++ "-" - ++ integer_to_list(C)) | Acc], + ++ integer_to_list(U)) | Acc], get_numbered_nodenames(N-1, T, NL). wait_until(Fun) -> diff --git a/lib/kernel/test/erl_distribution_wb_SUITE.erl b/lib/kernel/test/erl_distribution_wb_SUITE.erl index 3b8b2d9150..8e2bbf5b64 100644 --- a/lib/kernel/test/erl_distribution_wb_SUITE.erl +++ b/lib/kernel/test/erl_distribution_wb_SUITE.erl @@ -451,11 +451,8 @@ close_pair({Client, Server}) -> %% MD5 hashing %% -%% This is no proper random number, but that is not really important in -%% this test gen_challenge() -> - {_,_,N} = erlang:now(), - N. + rand:uniform(1000000). %% Generate a message digest from Challenge number and Cookie gen_digest(Challenge, Cookie) when is_integer(Challenge), is_atom(Cookie) -> @@ -712,13 +709,9 @@ get_nodenames(N, T) -> get_nodenames(0, _, Acc) -> Acc; get_nodenames(N, T, Acc) -> - {A, B, C} = now(), - get_nodenames(N-1, T, [list_to_atom(atom_to_list(?MODULE) + U = erlang:unique_integer([positive]), + get_nodenames(N-1, T, [list_to_atom(?MODULE_STRING ++ "-" ++ atom_to_list(T) ++ "-" - ++ integer_to_list(A) - ++ "-" - ++ integer_to_list(B) - ++ "-" - ++ integer_to_list(C)) | Acc]). + ++ integer_to_list(U)) | Acc]). diff --git a/lib/kernel/test/error_logger_warn_SUITE.erl b/lib/kernel/test/error_logger_warn_SUITE.erl index 2bf467610e..16e0286593 100644 --- a/lib/kernel/test/error_logger_warn_SUITE.erl +++ b/lib/kernel/test/error_logger_warn_SUITE.erl @@ -21,8 +21,8 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, - basic/1,warnings_info/1,warnings_warnings/1, - rb_basic/1,rb_warnings_info/1,rb_warnings_warnings/1, + basic/1,warnings_info/1,warnings_errors/1, + rb_basic/1,rb_warnings_info/1,rb_warnings_errors/1, rb_trunc/1,rb_utc/1,file_utc/1]). %% Internal exports. @@ -48,8 +48,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [basic, warnings_info, warnings_warnings, rb_basic, - rb_warnings_info, rb_warnings_warnings, rb_trunc, + [basic, warnings_info, warnings_errors, rb_basic, + rb_warnings_info, rb_warnings_errors, rb_trunc, rb_utc, file_utc]. groups() -> @@ -69,7 +69,7 @@ end_per_group(_GroupName, Config) -> init_per_testcase(_Case, Config) -> - ?line Dog = ?t:timetrap(?default_timeout), + Dog = ?t:timetrap(?default_timeout), [{watchdog, Dog} | Config]. end_per_testcase(_Case, Config) -> Dog = ?config(watchdog, Config), @@ -88,11 +88,11 @@ warnings_info(Config) when is_list(Config) -> put(elw_config,Config), warnings_info(). -warnings_warnings(doc) -> - ["Tests mapping warnings to warnings functionality"]; -warnings_warnings(Config) when is_list(Config) -> +warnings_errors(doc) -> + ["Tests mapping warnings to errors functionality"]; +warnings_errors(Config) when is_list(Config) -> put(elw_config,Config), - warnings_warnings(). + warnings_errors(). rb_basic(doc) -> ["Tests basic rb functionality"]; @@ -106,11 +106,11 @@ rb_warnings_info(Config) when is_list(Config) -> put(elw_config,Config), rb_warnings_info(). -rb_warnings_warnings(doc) -> - ["Tests warnings as warnings rb functionality"]; -rb_warnings_warnings(Config) when is_list(Config) -> +rb_warnings_errors(doc) -> + ["Tests warnings as errors rb functionality"]; +rb_warnings_errors(Config) when is_list(Config) -> put(elw_config,Config), - rb_warnings_warnings(). + rb_warnings_errors(). rb_trunc(doc) -> ["Tests rb functionality on truncated data"]; @@ -159,6 +159,9 @@ install_relay(Node) -> rpc:call(Node,error_logger,add_report_handler,[?MODULE,[self()]]). +warning_map(Node) -> + rpc:call(Node,error_logger,warning_map,[]). + format(Node,A,B) -> rpc:call(Node,error_logger,format,[A,B]). error_msg(Node,A,B) -> @@ -181,80 +184,81 @@ nn() -> basic() -> - ?line Node = start_node(nn(),[]), - ?line ok = install_relay(Node), - ?line Self = self(), - ?line GL = group_leader(), - ?line format(Node,"~p~n",[Self]), - ?line ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), - ?line error_msg(Node,"~p~n",[Self]), - ?line ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), - ?line warning_msg(Node,"~p~n",[Self]), - ?line ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), - ?line info_msg(Node,"~p~n",[Self]), - ?line ?EXPECT({handle_event,{info_msg,GL,{_,"~p~n",[Self]}}}), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line error_report(Node,Report), - ?line ?EXPECT({handle_event,{error_report,GL,{_,std_error,Report}}}), - ?line warning_report(Node,Report), - ?line ?EXPECT({handle_event,{error_report,GL,{_,std_error,Report}}}), - ?line info_report(Node,Report), - ?line ?EXPECT({handle_event,{info_report,GL,{_,std_info,Report}}}), - - ?line stop_node(Node), + Node = start_node(nn(),[]), + ok = install_relay(Node), + Self = self(), + GL = group_leader(), + warning = warning_map(Node), + format(Node,"~p~n",[Self]), + ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), + error_msg(Node,"~p~n",[Self]), + ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), + warning_msg(Node,"~p~n",[Self]), + ?EXPECT({handle_event,{warning_msg,GL,{_,"~p~n",[Self]}}}), + info_msg(Node,"~p~n",[Self]), + ?EXPECT({handle_event,{info_msg,GL,{_,"~p~n",[Self]}}}), + Report = [{self,Self},{gl,GL},make_ref()], + error_report(Node,Report), + ?EXPECT({handle_event,{error_report,GL,{_,std_error,Report}}}), + warning_report(Node,Report), + ?EXPECT({handle_event,{warning_report,GL,{_,std_warning,Report}}}), + info_report(Node,Report), + ?EXPECT({handle_event,{info_report,GL,{_,std_info,Report}}}), + stop_node(Node), ok. warnings_info() -> - ?line Node = start_node(nn(),"+Wi"), - ?line ok = install_relay(Node), - ?line Self = self(), - ?line GL = group_leader(), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line warning_msg(Node,"~p~n",[Self]), - ?line ?EXPECT({handle_event,{info_msg,GL,{_,"~p~n",[Self]}}}), - ?line warning_report(Node,Report), - ?line ?EXPECT({handle_event,{info_report,GL,{_,std_info,Report}}}), - ?line stop_node(Node), + Node = start_node(nn(),"+Wi"), + ok = install_relay(Node), + Self = self(), + GL = group_leader(), + info = warning_map(Node), + Report = [{self,Self},{gl,GL},make_ref()], + warning_msg(Node,"~p~n",[Self]), + ?EXPECT({handle_event,{info_msg,GL,{_,"~p~n",[Self]}}}), + warning_report(Node,Report), + ?EXPECT({handle_event,{info_report,GL,{_,std_info,Report}}}), + stop_node(Node), ok. -warnings_warnings() -> - ?line Node = start_node(nn(),"+Ww"), - ?line ok = install_relay(Node), - ?line Self = self(), - ?line GL = group_leader(), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line warning_msg(Node,"~p~n",[Self]), - ?line ?EXPECT({handle_event,{warning_msg,GL,{_,"~p~n",[Self]}}}), - ?line warning_report(Node,Report), - ?line ?EXPECT({handle_event,{warning_report,GL,{_,std_warning,Report}}}), - ?line stop_node(Node), +warnings_errors() -> + Node = start_node(nn(),"+We"), + ok = install_relay(Node), + Self = self(), + GL = group_leader(), + error = warning_map(Node), + Report = [{self,Self},{gl,GL},make_ref()], + warning_msg(Node,"~p~n",[Self]), + ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), + warning_report(Node,Report), + ?EXPECT({handle_event,{error_report,GL,{_,std_error,Report}}}), + stop_node(Node), ok. - - + % RB... quote(String) -> case os:type() of - {win32,_} -> - "\\\""++String++"\\\""; - _ -> - "'\""++String++"\"'" + {win32,_} -> + "\\\""++String++"\\\""; + _ -> + "'\""++String++"\"'" end. iquote(String) -> case os:type() of - {win32,_} -> - "\\\""++String++"\\\""; - _ -> - "\""++String++"\"" + {win32,_} -> + "\\\""++String++"\\\""; + _ -> + "\""++String++"\"" end. oquote(String) -> case os:type() of - {win32,_} -> - "\""++String++"\""; - _ -> - "'"++String++"'" + {win32,_} -> + "\""++String++"\""; + _ -> + "'"++String++"'" end. @@ -264,18 +268,18 @@ findstr(String,FileName) -> findstrc(String,File) -> case string:str(File,String) of - N when is_integer(N), - N > 0 -> - S2 = lists:sublist(File,N,length(File)), - case string:str(S2,"\n") of - 0 -> - 1; - M -> - S3 = lists:sublist(S2,M,length(S2)), - 1 + findstrc(String,S3) - end; - _ -> - 0 + N when is_integer(N), + N > 0 -> + S2 = lists:sublist(File,N,length(File)), + case string:str(S2,"\n") of + 0 -> + 1; + M -> + S3 = lists:sublist(S2,M,length(S2)), + 1 + findstrc(String,S3) + end; + _ -> + 0 end. % Doesn't count empty lines @@ -349,175 +353,182 @@ one_rb_findstr(Param,String) -> % Tests rb_basic() -> - ?line clean_rd(), + clean_rd(), % Behold, the magic parameters to activate rb logging... - ?line Node = start_node(nn(),"-boot start_sasl -sasl error_logger_mf_dir "++ - quote(rd())++" error_logger_mf_maxbytes 5000 " - "error_logger_mf_maxfiles 5"), - ?line Self = self(), - ?line GL = group_leader(), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line fake_gl(Node,warning_msg,"~p~n",[Self]), - ?line fake_gl(Node,warning_report,Report), - ?line nice_stop_node(Node), - ?line application:start(sasl), - ?line rb:start([{report_dir, rd()}]), - ?line rb:list(), - ?line true = (one_rb_lines([error]) > 1), - ?line true = (one_rb_lines([error_report]) > 1), - ?line 1 = one_rb_findstr([error],pid_to_list(Self)), - ?line 1 = one_rb_findstr([error_report],pid_to_list(Self)), - ?line 2 = one_rb_findstr([],pid_to_list(Self)), - ?line true = (one_rb_findstr([progress],"===") > 4), - ?line rb:stop(), - ?line application:stop(sasl), - ?line stop_node(Node), + Node = start_node(nn(),"-boot start_sasl -sasl error_logger_mf_dir "++ + quote(rd())++" error_logger_mf_maxbytes 5000 " + "error_logger_mf_maxfiles 5"), + Self = self(), + GL = group_leader(), + warning = warning_map(Node), + Report = [{self,Self},{gl,GL},make_ref()], + fake_gl(Node,warning_msg,"~p~n",[Self]), + fake_gl(Node,warning_report,Report), + nice_stop_node(Node), + application:start(sasl), + rb:start([{report_dir, rd()}]), + rb:list(), + true = (one_rb_lines([error]) =:= 0), + true = (one_rb_lines([error_report]) =:= 0), + 0 = one_rb_findstr([error],pid_to_list(Self)), + 0 = one_rb_findstr([error_report],pid_to_list(Self)), + 1 = one_rb_findstr([warning_msg],pid_to_list(Self)), + 1 = one_rb_findstr([warning_report],pid_to_list(Self)), + 0 = one_rb_findstr([info_msg],pid_to_list(Self)), + 0 = one_rb_findstr([info_report],pid_to_list(Self)), + 2 = one_rb_findstr([],pid_to_list(Self)), + true = (one_rb_findstr([progress],"===") > 4), + rb:stop(), + application:stop(sasl), + stop_node(Node), ok. rb_warnings_info() -> - ?line clean_rd(), - ?line Node = start_node(nn(),"+W i -boot start_sasl -sasl error_logger_mf_dir "++ - quote(rd())++" error_logger_mf_maxbytes 5000 " - "error_logger_mf_maxfiles 5"), - ?line Self = self(), - ?line GL = group_leader(), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line fake_gl(Node,warning_msg,"~p~n",[Self]), - ?line fake_gl(Node,warning_report,Report), - ?line nice_stop_node(Node), - ?line application:start(sasl), - ?line rb:start([{report_dir, rd()}]), - ?line rb:list(), - ?line true = (one_rb_lines([error]) =:= 0), - ?line true = (one_rb_lines([error_report]) =:= 0), - ?line 0 = one_rb_findstr([error],pid_to_list(Self)), - ?line 0 = one_rb_findstr([error_report],pid_to_list(Self)), - ?line 0 = one_rb_findstr([warning_msg],pid_to_list(Self)), - ?line 0 = one_rb_findstr([warning_report],pid_to_list(Self)), - ?line 1 = one_rb_findstr([info_msg],pid_to_list(Self)), - ?line 1 = one_rb_findstr([info_report],pid_to_list(Self)), - ?line 2 = one_rb_findstr([],pid_to_list(Self)), - ?line true = (one_rb_findstr([progress],"===") > 4), - ?line rb:stop(), - ?line application:stop(sasl), - ?line stop_node(Node), + clean_rd(), + Node = start_node(nn(),"+W i -boot start_sasl -sasl error_logger_mf_dir "++ + quote(rd())++" error_logger_mf_maxbytes 5000 " + "error_logger_mf_maxfiles 5"), + Self = self(), + GL = group_leader(), + info = warning_map(Node), + Report = [{self,Self},{gl,GL},make_ref()], + fake_gl(Node,warning_msg,"~p~n",[Self]), + fake_gl(Node,warning_report,Report), + nice_stop_node(Node), + application:start(sasl), + rb:start([{report_dir, rd()}]), + rb:list(), + true = (one_rb_lines([error]) =:= 0), + true = (one_rb_lines([error_report]) =:= 0), + 0 = one_rb_findstr([error],pid_to_list(Self)), + 0 = one_rb_findstr([error_report],pid_to_list(Self)), + 0 = one_rb_findstr([warning_msg],pid_to_list(Self)), + 0 = one_rb_findstr([warning_report],pid_to_list(Self)), + 1 = one_rb_findstr([info_msg],pid_to_list(Self)), + 1 = one_rb_findstr([info_report],pid_to_list(Self)), + 2 = one_rb_findstr([],pid_to_list(Self)), + true = (one_rb_findstr([progress],"===") > 4), + rb:stop(), + application:stop(sasl), + stop_node(Node), ok. -rb_warnings_warnings() -> - ?line clean_rd(), - ?line Node = start_node(nn(),"+W w -boot start_sasl -sasl error_logger_mf_dir "++ - quote(rd())++" error_logger_mf_maxbytes 5000 " - "error_logger_mf_maxfiles 5"), - ?line Self = self(), - ?line GL = group_leader(), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line fake_gl(Node,warning_msg,"~p~n",[Self]), - ?line fake_gl(Node,warning_report,Report), - ?line nice_stop_node(Node), - ?line application:start(sasl), - ?line rb:start([{report_dir, rd()}]), - ?line rb:list(), - ?line true = (one_rb_lines([error]) =:= 0), - ?line true = (one_rb_lines([error_report]) =:= 0), - ?line 0 = one_rb_findstr([error],pid_to_list(Self)), - ?line 0 = one_rb_findstr([error_report],pid_to_list(Self)), - ?line 1 = one_rb_findstr([warning_msg],pid_to_list(Self)), - ?line 1 = one_rb_findstr([warning_report],pid_to_list(Self)), - ?line 0 = one_rb_findstr([info_msg],pid_to_list(Self)), - ?line 0 = one_rb_findstr([info_report],pid_to_list(Self)), - ?line 2 = one_rb_findstr([],pid_to_list(Self)), - ?line true = (one_rb_findstr([progress],"===") > 4), - ?line rb:stop(), - ?line application:stop(sasl), - ?line stop_node(Node), +rb_warnings_errors() -> + clean_rd(), + Node = start_node(nn(),"+W e -boot start_sasl -sasl error_logger_mf_dir "++ + quote(rd())++" error_logger_mf_maxbytes 5000 " + "error_logger_mf_maxfiles 5"), + Self = self(), + GL = group_leader(), + error = warning_map(Node), + Report = [{self,Self},{gl,GL},make_ref()], + fake_gl(Node,warning_msg,"~p~n",[Self]), + fake_gl(Node,warning_report,Report), + nice_stop_node(Node), + application:start(sasl), + rb:start([{report_dir, rd()}]), + rb:list(), + true = (one_rb_lines([error]) > 1), + true = (one_rb_lines([error_report]) > 1), + 1 = one_rb_findstr([error],pid_to_list(Self)), + 1 = one_rb_findstr([error_report],pid_to_list(Self)), + 0 = one_rb_findstr([warning_msg],pid_to_list(Self)), + 0 = one_rb_findstr([warning_report],pid_to_list(Self)), + 0 = one_rb_findstr([info_msg],pid_to_list(Self)), + 0 = one_rb_findstr([info_report],pid_to_list(Self)), + 2 = one_rb_findstr([],pid_to_list(Self)), + true = (one_rb_findstr([progress],"===") > 4), + rb:stop(), + application:stop(sasl), + stop_node(Node), ok. rb_trunc() -> - ?line clean_rd(), - ?line Node = start_node(nn(),"+W w -boot start_sasl -sasl error_logger_mf_dir "++ - quote(rd())++" error_logger_mf_maxbytes 5000 " - "error_logger_mf_maxfiles 5"), - ?line Self = self(), - ?line GL = group_leader(), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line fake_gl(Node,warning_msg,"~p~n",[Self]), - ?line fake_gl(Node,warning_report,Report), - ?line nice_stop_node(Node), - ?line application:start(sasl), - ?line {ok,File} = file:read_file(rf()), - ?line S=byte_size(File)-2, - ?line <<TFile:S/binary,_/binary>>=File, - ?line file:write_file(rf(),TFile), - ?line rb:start([{report_dir, rd()}]), - ?line rb:list(), - ?line true = (one_rb_lines([error]) =:= 0), - ?line true = (one_rb_lines([error_report]) =:= 0), - ?line 0 = one_rb_findstr([error],pid_to_list(Self)), - ?line 0 = one_rb_findstr([error_report],pid_to_list(Self)), - ?line 1 = one_rb_findstr([warning_msg],pid_to_list(Self)), - ?line 0 = one_rb_findstr([warning_report],pid_to_list(Self)), - ?line 0 = one_rb_findstr([info_msg],pid_to_list(Self)), - ?line 0 = one_rb_findstr([info_report],pid_to_list(Self)), - ?line 1 = one_rb_findstr([],pid_to_list(Self)), - ?line true = (one_rb_findstr([progress],"===") > 4), - ?line rb:stop(), - ?line application:stop(sasl), - ?line stop_node(Node), + clean_rd(), + Node = start_node(nn(),"-boot start_sasl -sasl error_logger_mf_dir "++ + quote(rd())++" error_logger_mf_maxbytes 5000 " + "error_logger_mf_maxfiles 5"), + Self = self(), + GL = group_leader(), + Report = [{self,Self},{gl,GL},make_ref()], + fake_gl(Node,warning_msg,"~p~n",[Self]), + fake_gl(Node,warning_report,Report), + nice_stop_node(Node), + application:start(sasl), + {ok,File} = file:read_file(rf()), + S=byte_size(File)-2, + <<TFile:S/binary,_/binary>>=File, + file:write_file(rf(),TFile), + rb:start([{report_dir, rd()}]), + rb:list(), + true = (one_rb_lines([error]) =:= 0), + true = (one_rb_lines([error_report]) =:= 0), + 0 = one_rb_findstr([error],pid_to_list(Self)), + 0 = one_rb_findstr([error_report],pid_to_list(Self)), + 1 = one_rb_findstr([warning_msg],pid_to_list(Self)), + 0 = one_rb_findstr([warning_report],pid_to_list(Self)), + 0 = one_rb_findstr([info_msg],pid_to_list(Self)), + 0 = one_rb_findstr([info_report],pid_to_list(Self)), + 1 = one_rb_findstr([],pid_to_list(Self)), + true = (one_rb_findstr([progress],"===") > 4), + rb:stop(), + application:stop(sasl), + stop_node(Node), ok. rb_utc() -> - ?line clean_rd(), - ?line Node = start_node(nn(),"+W w -boot start_sasl -sasl error_logger_mf_dir "++ - quote(rd())++" error_logger_mf_maxbytes 5000 " - "error_logger_mf_maxfiles 5 -sasl utc_log true"), - ?line Self = self(), - ?line GL = group_leader(), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line fake_gl(Node,warning_msg,"~p~n",[Self]), - ?line fake_gl(Node,warning_report,Report), - ?line nice_stop_node(Node), - ?line application:stop(sasl), - ?line UtcLog=case application:get_env(sasl,utc_log) of - {ok,true} -> - true; - _AllOthers -> - application:set_env(sasl,utc_log,true), - false - end, - ?line application:start(sasl), - ?line rb:start([{report_dir, rd()}]), - ?line rb:list(), - ?line Pr=one_rb_findstr([progress],"==="), - ?line Wm=one_rb_findstr([warning_msg],"==="), - ?line Wr=one_rb_findstr([warning_report],"==="), - ?line Sum=Pr+Wm+Wr, - ?line Sum=one_rb_findstr([],"UTC"), - ?line rb:stop(), - ?line application:stop(sasl), - ?line application:set_env(sasl,utc_log,UtcLog), - ?line stop_node(Node), + clean_rd(), + Node = start_node(nn(),"-boot start_sasl -sasl error_logger_mf_dir "++ + quote(rd())++" error_logger_mf_maxbytes 5000 " + "error_logger_mf_maxfiles 5 -sasl utc_log true"), + Self = self(), + GL = group_leader(), + Report = [{self,Self},{gl,GL},make_ref()], + fake_gl(Node,warning_msg,"~p~n",[Self]), + fake_gl(Node,warning_report,Report), + nice_stop_node(Node), + application:stop(sasl), + UtcLog=case application:get_env(sasl,utc_log) of + {ok,true} -> + true; + _AllOthers -> + application:set_env(sasl,utc_log,true), + false + end, + application:start(sasl), + rb:start([{report_dir, rd()}]), + rb:list(), + Pr=one_rb_findstr([progress],"==="), + Wm=one_rb_findstr([warning_msg],"==="), + Wr=one_rb_findstr([warning_report],"==="), + Sum=Pr+Wm+Wr, + Sum=one_rb_findstr([],"UTC"), + rb:stop(), + application:stop(sasl), + application:set_env(sasl,utc_log,UtcLog), + stop_node(Node), ok. file_utc() -> - ?line file:delete(lf()), - ?line SS="+W w -stdlib utc_log true -kernel error_logger "++ oquote("{file,"++iquote(lf())++"}"), + file:delete(lf()), + SS="-stdlib utc_log true -kernel error_logger "++ oquote("{file,"++iquote(lf())++"}"), %erlang:display(SS), - ?line Node = start_node(nn(),SS), + Node = start_node(nn(),SS), %erlang:display(rpc:call(Node,application,get_env,[kernel,error_logger])), - ?line Self = self(), - ?line GL = group_leader(), - ?line fake_gl(Node,error_msg,"~p~n",[Self]), - ?line fake_gl(Node,warning_msg,"~p~n",[Self]), - ?line fake_gl(Node,info_msg,"~p~n",[Self]), - ?line Report = [{self,Self},{gl,GL},make_ref()], - ?line fake_gl(Node,error_report,Report), - ?line fake_gl(Node,warning_report,Report), - ?line fake_gl(Node,info_report,Report), - ?line nice_stop_node(Node), - ?line receive after 5000 -> ok end, % Let the node die, needed - ?line 6 = findstr("UTC",lf()), - ?line 2 = findstr("WARNING",lf()), - ?line 2 = findstr("ERROR",lf()), - ?line 2 = findstr("INFO",lf()), - ?line stop_node(Node), + Self = self(), + GL = group_leader(), + fake_gl(Node,error_msg,"~p~n",[Self]), + fake_gl(Node,warning_msg,"~p~n",[Self]), + fake_gl(Node,info_msg,"~p~n",[Self]), + Report = [{self,Self},{gl,GL},make_ref()], + fake_gl(Node,error_report,Report), + fake_gl(Node,warning_report,Report), + fake_gl(Node,info_report,Report), + nice_stop_node(Node), + receive after 5000 -> ok end, % Let the node die, needed + 6 = findstr("UTC",lf()), + 2 = findstr("WARNING",lf()), + 2 = findstr("ERROR",lf()), + 2 = findstr("INFO",lf()), + stop_node(Node), ok. diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index 1213d8e37e..48abc92e4c 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -3914,7 +3914,7 @@ response_analysis(Module, Function, Arguments) -> receive {Parent, start, Ts} -> ok end, Stat = iterate(response_stat(response_stat(init, Ts), - erlang:now()), + micro_ts()), done, fun (S) -> erlang:yield(), @@ -3922,12 +3922,12 @@ response_analysis(Module, Function, Arguments) -> {Parent, stop} -> done after 0 -> - response_stat(S, erlang:now()) + response_stat(S, micro_ts()) end end), - Parent ! {self(), stopped, response_stat(Stat, erlang:now())} + Parent ! {self(), stopped, response_stat(Stat, micro_ts())} end), - ?line Child ! {Parent, start, erlang:now()}, + Child ! {Parent, start, micro_ts()}, ?line Result = apply(Module, Function, Arguments), ?line Child ! {Parent, stop}, ?line {N, Sum, _, M, Max} = receive {Child, stopped, X} -> X end, @@ -3941,12 +3941,13 @@ response_analysis(Module, Function, Arguments) -> [Mean_ms, Max_ms, M, (N-1)])), ?line {Result, Comment}. - +micro_ts() -> + erlang:monotonic_time(micro_seconds). response_stat(init, Ts) -> {0, 0, Ts, 0, 0}; -response_stat({N, Sum, {A1, B1, C1}, M, Max}, {A2, B2, C2} = Ts) -> - D = C2-C1 + 1000000*((B2-B1) + 1000000*(A2-A1)), +response_stat({N, Sum, Ts0, M, Max}, Ts) -> + D = Ts - Ts0, if D > Max -> {N+1, Sum+D, Ts, N, D}; true -> diff --git a/lib/kernel/test/gen_tcp_api_SUITE.erl b/lib/kernel/test/gen_tcp_api_SUITE.erl index c27d265550..4a527e2f51 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE.erl +++ b/lib/kernel/test/gen_tcp_api_SUITE.erl @@ -32,6 +32,7 @@ t_connect_bad/1, t_recv_timeout/1, t_recv_eof/1, t_shutdown_write/1, t_shutdown_both/1, t_shutdown_error/1, + t_shutdown_async/1, t_fdopen/1, t_fdconnect/1, t_implicit_inet6/1]). -export([getsockfd/0,closesockfd/1]). @@ -41,7 +42,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [{group, t_accept}, {group, t_connect}, {group, t_recv}, t_shutdown_write, t_shutdown_both, t_shutdown_error, - t_fdopen, t_fdconnect, t_implicit_inet6]. + t_shutdown_async, t_fdopen, t_fdconnect, t_implicit_inet6]. groups() -> [{t_accept, [], [t_accept_timeout]}, @@ -155,7 +156,34 @@ t_shutdown_error(Config) when is_list(Config) -> ?line ok = gen_tcp:close(L), ?line {error, closed} = gen_tcp:shutdown(L, read_write), ok. - + +t_shutdown_async(Config) when is_list(Config) -> + ?line {OS, _} = os:type(), + ?line {ok, L} = gen_tcp:listen(0, [{sndbuf, 4096}]), + ?line {ok, Port} = inet:port(L), + ?line {ok, Client} = gen_tcp:connect(localhost, Port, + [{recbuf, 4096}, + {active, false}]), + ?line {ok, S} = gen_tcp:accept(L), + ?line PayloadSize = 1024 * 1024, + ?line Payload = lists:duplicate(PayloadSize, $.), + ?line ok = gen_tcp:send(S, Payload), + ?line case erlang:port_info(S, queue_size) of + {queue_size, N} when N > 0 -> ok; + {queue_size, 0} when OS =:= win32 -> ok; + {queue_size, 0} = T -> ?t:fail({unexpected, T}) + end, + + ?line ok = gen_tcp:shutdown(S, write), + ?line {ok, Buf} = gen_tcp:recv(Client, PayloadSize), + ?line {error, closed} = gen_tcp:recv(Client, 0), + ?line case length(Buf) of + PayloadSize -> ok; + Sz -> ?t:fail({payload_size, + {expected, PayloadSize}, + {received, Sz}}) + end. + %%% gen_tcp:fdopen/2 diff --git a/lib/kernel/test/gen_tcp_misc_SUITE.erl b/lib/kernel/test/gen_tcp_misc_SUITE.erl index 4e4aeb67e2..76a9708a58 100644 --- a/lib/kernel/test/gen_tcp_misc_SUITE.erl +++ b/lib/kernel/test/gen_tcp_misc_SUITE.erl @@ -38,7 +38,9 @@ % Accept tests primitive_accept/1,multi_accept_close_listen/1,accept_timeout/1, accept_timeouts_in_order/1,accept_timeouts_in_order2/1, - accept_timeouts_in_order3/1,accept_timeouts_mixed/1, + accept_timeouts_in_order3/1,accept_timeouts_in_order4/1, + accept_timeouts_in_order5/1,accept_timeouts_in_order6/1, + accept_timeouts_in_order7/1,accept_timeouts_mixed/1, killing_acceptor/1,killing_multi_acceptors/1,killing_multi_acceptors2/1, several_accepts_in_one_go/1, accept_system_limit/1, active_once_closed/1, send_timeout/1, send_timeout_active/1, @@ -60,19 +62,19 @@ init_per_testcase(wrapping_oct, Config) when is_list(Config) -> [{watchdog, Dog}|Config]; init_per_testcase(iter_max_socks, Config) when is_list(Config) -> Dog = case os:type() of - {win32,_} -> - test_server:timetrap(test_server:minutes(30)); - _Else -> - test_server:timetrap(test_server:seconds(240)) - end, + {win32,_} -> + test_server:timetrap(test_server:minutes(30)); + _Else -> + test_server:timetrap(test_server:seconds(240)) + end, [{watchdog, Dog}|Config]; init_per_testcase(accept_system_limit, Config) when is_list(Config) -> case os:type() of - {ose,_} -> - {skip,"Skip in OSE"}; - _ -> - Dog = test_server:timetrap(test_server:seconds(240)), - [{watchdog,Dog}|Config] + {ose,_} -> + {skip,"Skip in OSE"}; + _ -> + Dog = test_server:timetrap(test_server:seconds(240)), + [{watchdog,Dog}|Config] end; init_per_testcase(wrapping_oct, Config) when is_list(Config) -> Dog = test_server:timetrap(test_server:seconds(600)), @@ -99,7 +101,9 @@ all() -> so_priority, primitive_accept, multi_accept_close_listen, accept_timeout, accept_timeouts_in_order, accept_timeouts_in_order2, - accept_timeouts_in_order3, accept_timeouts_mixed, + accept_timeouts_in_order3, accept_timeouts_in_order4, + accept_timeouts_in_order5, accept_timeouts_in_order6, + accept_timeouts_in_order7, accept_timeouts_mixed, killing_acceptor, killing_multi_acceptors, killing_multi_acceptors2, several_accepts_in_one_go, accept_system_limit, active_once_closed, send_timeout, send_timeout_active, otp_7731, @@ -121,8 +125,6 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - - default_options(doc) -> ["Tests kernel application variables inet_default_listen_options and " "inet_default_connect_options"]; @@ -130,69 +132,68 @@ default_options(suite) -> []; default_options(Config) when is_list(Config) -> %% First check the delay_send option - ?line {true,true,true}=do_delay_send_1(), - ?line {false,false,false}=do_delay_send_2(), - ?line {true,false,false}=do_delay_send_3(), - ?line {false,false,false}=do_delay_send_4(), - ?line {false,false,false}=do_delay_send_5(), - ?line {false,true,true}=do_delay_send_6(), + {true,true,true}=do_delay_send_1(), + {false,false,false}=do_delay_send_2(), + {true,false,false}=do_delay_send_3(), + {false,false,false}=do_delay_send_4(), + {false,false,false}=do_delay_send_5(), + {false,true,true}=do_delay_send_6(), %% Now lets start some nodes with different combinations of options: - ?line {true,true,true} = do_delay_on_other_node("", - fun do_delay_send_1/0), - ?line {true,false,false} = + {true,true,true} = do_delay_on_other_node("", fun do_delay_send_1/0), + {true,false,false} = do_delay_on_other_node("-kernel inet_default_connect_options " "\"[{delay_send,true}]\"", fun do_delay_send_2/0), - ?line {false,true,true} = + {false,true,true} = do_delay_on_other_node("-kernel inet_default_listen_options " "\"[{delay_send,true}]\"", fun do_delay_send_2/0), - ?line {true,true,true} = + {true,true,true} = do_delay_on_other_node("-kernel inet_default_listen_options " "\"[{delay_send,true}]\"", fun do_delay_send_3/0), - ?line {true,true,true} = + {true,true,true} = do_delay_on_other_node("-kernel inet_default_connect_options " "\"[{delay_send,true}]\"", fun do_delay_send_6/0), - ?line {false,false,false} = + {false,false,false} = do_delay_on_other_node("-kernel inet_default_connect_options " "\"[{delay_send,true}]\"", fun do_delay_send_5/0), - ?line {false,true,true} = + {false,true,true} = do_delay_on_other_node("-kernel inet_default_connect_options " "\"[{delay_send,true}]\" " "-kernel inet_default_listen_options " "\"[{delay_send,true}]\"", fun do_delay_send_5/0), - ?line {true,false,false} = + {true,false,false} = do_delay_on_other_node("-kernel inet_default_connect_options " "\"[{delay_send,true}]\" " "-kernel inet_default_listen_options " "\"[{delay_send,true}]\"", fun do_delay_send_4/0), - ?line {true,true,true} = + {true,true,true} = do_delay_on_other_node("-kernel inet_default_connect_options " "\"{delay_send,true}\" " "-kernel inet_default_listen_options " "\"{delay_send,true}\"", fun do_delay_send_2/0), %% Active is to dangerous and is supressed - ?line {true,true,true} = + {true,true,true} = do_delay_on_other_node("-kernel inet_default_connect_options " "\"{active,false}\" " "-kernel inet_default_listen_options " "\"{active,false}\"", fun do_delay_send_7/0), - ?line {true,true,true} = + {true,true,true} = do_delay_on_other_node("-kernel inet_default_connect_options " "\"[{active,false},{delay_send,true}]\" " "-kernel inet_default_listen_options " "\"[{active,false},{delay_send,true}]\"", fun do_delay_send_7/0), - ?line {true,true,true} = + {true,true,true} = do_delay_on_other_node("-kernel inet_default_connect_options " "\"[{active,false},{delay_send,true}]\" " "-kernel inet_default_listen_options " @@ -204,12 +205,10 @@ default_options(Config) when is_list(Config) -> do_delay_on_other_node(XArgs, Function) -> Dir = filename:dirname(code:which(?MODULE)), {ok,Node} = test_server:start_node(test_default_options_slave,slave, - [{args,"-pa " ++ Dir ++ " " ++ - XArgs}]), + [{args,"-pa " ++ Dir ++ " " ++ XArgs}]), Res = rpc:call(Node,erlang,apply,[Function,[]]), test_server:stop_node(Node), Res. - do_delay_send_1() -> {ok,LS}=gen_tcp:listen(0,[{delay_send,true}]), @@ -301,8 +300,6 @@ do_delay_send_7() -> gen_tcp:close(S), gen_tcp:close(LS), {B1,B2,B3}. - - controlling_process(doc) -> ["Open a listen port and change controlling_process for it", @@ -313,18 +310,18 @@ controlling_process(Config) when is_list(Config) -> {ok,S} = gen_tcp:listen(0,[]), Pid2 = spawn(?MODULE,not_owner,[S]), Pid2 ! {self(),2,control}, - ?line {error, E} = receive {2,_E} -> + {error, E} = receive {2,_E} -> _E after 10000 -> timeout end, io:format("received ~p~n",[E]), Pid = spawn(?MODULE,not_owner,[S]), - ?line ok = gen_tcp:controlling_process(S,Pid), + ok = gen_tcp:controlling_process(S,Pid), Pid ! {self(),1,control}, - ?line ok = receive {1,ok} -> - ok - after 1000 -> timeout - end, + ok = receive {1,ok} -> + ok + after 1000 -> timeout + end, Pid ! close. not_owner(S) -> @@ -377,7 +374,7 @@ no_accept(Config) when is_list(Config) -> {tcp_closed, Client} -> ok after 5000 -> - ?line test_server:fail(never_closed) + test_server:fail(never_closed) end. @@ -386,30 +383,30 @@ close_with_pending_output(doc) -> "to the other end."]; close_with_pending_output(suite) -> []; close_with_pending_output(Config) when is_list(Config) -> - ?line {ok, L} = gen_tcp:listen(0, [binary, {active, false}]), - ?line {ok, {_, Port}} = inet:sockname(L), - ?line Packets = 16, - ?line Total = 2048*Packets, + {ok, L} = gen_tcp:listen(0, [binary, {active, false}]), + {ok, {_, Port}} = inet:sockname(L), + Packets = 16, + Total = 2048*Packets, case start_remote(close_pending) of {ok, Node} -> - ?line {ok, Host} = inet:gethostname(), - ?line spawn_link(Node, ?MODULE, sender, [Port, Packets, Host]), - ?line {ok, A} = gen_tcp:accept(L), - ?line case gen_tcp:recv(A, Total) of + {ok, Host} = inet:gethostname(), + spawn_link(Node, ?MODULE, sender, [Port, Packets, Host]), + {ok, A} = gen_tcp:accept(L), + case gen_tcp:recv(A, Total) of {ok, Bin} when byte_size(Bin) == Total -> gen_tcp:close(A), gen_tcp:close(L); {ok, Bin} -> - ?line test_server:fail({small_packet, + test_server:fail({small_packet, byte_size(Bin)}); Error -> - ?line test_server:fail({unexpected, Error}) + test_server:fail({unexpected, Error}) end, ok; {error, no_remote_hosts} -> {skipped,"No remote hosts"}; {error, Other} -> - ?line ?t:fail({failed_to_start_slave_node, Other}) + ?t:fail({failed_to_start_slave_node, Other}) end. sender(Port, Packets, Host) -> @@ -556,63 +553,62 @@ otp_3924(Config) when is_list(Config) -> otp_3924_1(MaxDelay). otp_3924_1(MaxDelay) -> - ?line {ok, Node} = start_node(otp_3924), - ?line DataLen = 100*1024, - ?line Data = otp_3924_data(DataLen), + {ok, Node} = start_node(otp_3924), + DataLen = 100*1024, + Data = otp_3924_data(DataLen), % Repeat the test a couple of times to prevent the test from passing % by chance. - repeat(10, - fun (N) -> - ?line ok = otp_3924(MaxDelay, Node, Data, DataLen, N) - end), - ?line test_server:stop_node(Node), + repeat(10, fun(N) -> + ok = otp_3924(MaxDelay, Node, Data, DataLen, N) + end), + test_server:stop_node(Node), ok. otp_3924(MaxDelay, Node, Data, DataLen, N) -> - ?line {ok, L} = gen_tcp:listen(0, [list, {active, false}]), - ?line {ok, {_, Port}} = inet:sockname(L), - ?line {ok, Host} = inet:gethostname(), - ?line Sender = spawn_link(Node, - ?MODULE, - otp_3924_sender, - [self(), Host, Port, Data]), - ?line Data = otp_3924_receive_data(L, Sender, MaxDelay, DataLen, N), - ?line ok = gen_tcp:close(L). + {ok, L} = gen_tcp:listen(0, [list, {active, false}]), + {ok, {_, Port}} = inet:sockname(L), + {ok, Host} = inet:gethostname(), + Sender = spawn_link(Node, + ?MODULE, + otp_3924_sender, + [self(), Host, Port, Data]), + Data = otp_3924_receive_data(L, Sender, MaxDelay, DataLen, N), + ok = gen_tcp:close(L). otp_3924_receive_data(LSock, Sender, MaxDelay, Len, N) -> - ?line OP = process_flag(priority, max), - ?line OTE = process_flag(trap_exit, true), - ?line TimeoutRef = make_ref(), - ?line Data = (catch begin - ?line Sender ! start, - ?line {ok, Sock} = gen_tcp:accept(LSock), - ?line D = otp_3924_receive_data(Sock, - TimeoutRef, - MaxDelay, - Len, - [], - 0), - ?line ok = gen_tcp:close(Sock), - D - end), - ?line unlink(Sender), - ?line process_flag(trap_exit, OTE), - ?line process_flag(priority, OP), + OP = process_flag(priority, max), + OTE = process_flag(trap_exit, true), + TimeoutRef = make_ref(), + Data = (catch begin + Sender ! start, + {ok, Sock} = gen_tcp:accept(LSock), + D = otp_3924_receive_data(Sock, + TimeoutRef, + MaxDelay, + Len, + [], + 0), + ok = gen_tcp:close(Sock), + D + end), + unlink(Sender), + process_flag(trap_exit, OTE), + process_flag(priority, OP), receive {'EXIT', _, TimeoutRef} -> - ?line test_server:fail({close_not_fast_enough,MaxDelay,N}); + test_server:fail({close_not_fast_enough,MaxDelay,N}); {'EXIT', Sender, Reason} -> - ?line test_server:fail({sender_exited, Reason}); + test_server:fail({sender_exited, Reason}); {'EXIT', _Other, Reason} -> - ?line test_server:fail({linked_process_exited, Reason}) + test_server:fail({linked_process_exited, Reason}) after 0 -> case Data of {'EXIT', {A,B}} -> - ?line test_server:fail({A,B,N}); + test_server:fail({A,B,N}); {'EXIT', Failure} -> - ?line test_server:fail(Failure); + test_server:fail(Failure); _ -> - ?line Data + Data end end. @@ -623,12 +619,12 @@ otp_3924_receive_data(Sock, TimeoutRef, MaxDelay, Len, Acc, AccLen) -> NewAccLen = AccLen + length(Data), if NewAccLen == Len -> - ?line {ok, TRef} = timer:exit_after(MaxDelay, + {ok, TRef} = timer:exit_after(MaxDelay, self(), TimeoutRef), - ?line {error, closed} = gen_tcp:recv(Sock, 0), - ?line timer:cancel(TRef), - ?line lists:flatten([Acc, Data]); + {error, closed} = gen_tcp:recv(Sock, 0), + timer:cancel(TRef), + lists:flatten([Acc, Data]); NewAccLen > Len -> exit({received_too_much, NewAccLen}); true -> @@ -713,8 +709,8 @@ get_status(doc) -> "is called."]; get_status(suite) -> []; get_status(Config) when is_list(Config) -> - ?line {ok,{socket,Pid,_,_}} = gen_tcp:listen(5678,[]), - ?line {status,Pid,_,_} = sys:get_status(Pid). + {ok,{socket,Pid,_,_}} = gen_tcp:listen(5678,[]), + {status,Pid,_,_} = sys:get_status(Pid). -define(RECOVER_SLEEP, 60000). -define(RETRY_SLEEP, 15000). @@ -744,19 +740,19 @@ do_iter_max_socks(N, failed) -> MS = max_socks(), [MS|do_iter_max_socks(N-1, failed)]; do_iter_max_socks(N, First) when is_integer(First) -> - ?line MS = max_socks(), + MS = max_socks(), if MS == First -> - ?line [MS|do_iter_max_socks(N-1, First)]; + [MS|do_iter_max_socks(N-1, First)]; true -> - ?line io:format("Sleeping for ~p seconds...~n", + io:format("Sleeping for ~p seconds...~n", [?RETRY_SLEEP/1000]), - ?line ?t:sleep(?RETRY_SLEEP), - ?line io:format("Trying again...~n", []), - ?line RetryMS = max_socks(), - ?line if RetryMS == First -> - ?line [RetryMS|do_iter_max_socks(N-1, First)]; + ?t:sleep(?RETRY_SLEEP), + io:format("Trying again...~n", []), + RetryMS = max_socks(), + if RetryMS == First -> + [RetryMS|do_iter_max_socks(N-1, First)]; true -> - ?line [RetryMS|do_iter_max_socks(N-1, failed)] + [RetryMS|do_iter_max_socks(N-1, failed)] end end. @@ -768,7 +764,7 @@ all_equal([Rule | T]) -> all_equal(Rule, [Rule | T]) -> all_equal(Rule, T); all_equal(_, [_ | _]) -> - ?line ?t:sleep(?RECOVER_SLEEP), % Wait a while and *hope* that we'll + ?t:sleep(?RECOVER_SLEEP), % Wait a while and *hope* that we'll % recover so other tests won't be % affected. ?t:fail(max_socket_mismatch); @@ -776,9 +772,9 @@ all_equal(_Rule, []) -> ok. max_socks() -> - ?line Socks = open_socks(), - ?line N = length(Socks), - ?line lists:foreach(fun(S) -> ok = gen_tcp:close(S) end, Socks), + Socks = open_socks(), + N = length(Socks), + lists:foreach(fun(S) -> ok = gen_tcp:close(S) end, Socks), io:format("Got ~p sockets", [N]), N. @@ -817,18 +813,18 @@ passive_sockets(doc) -> ["Tests that when 'the other side' on a passive socket closes, the connecting", "side still can read until the end of data."]; passive_sockets(Config) when is_list(Config) -> - ?line spawn_link(?MODULE, passive_sockets_server, - [[{active,false}],self()]), - ?line receive - {socket,Port} -> ok - end, + spawn_link(?MODULE, passive_sockets_server, + [[{active,false}],self()]), + receive + {socket,Port} -> ok + end, ?t:sleep(500), - ?line case gen_tcp:connect("localhost", Port, [{active, false}]) of - {ok, Sock} -> - passive_sockets_read(Sock); - Error -> - ?t:fail({"Could not connect to server", Error}) - end. + case gen_tcp:connect("localhost", Port, [{active, false}]) of + {ok, Sock} -> + passive_sockets_read(Sock); + Error -> + ?t:fail({"Could not connect to server", Error}) + end. %% %% Read until we get an {error, closed}. If we get another error, this test case @@ -847,58 +843,58 @@ passive_sockets_read(Sock) -> end. passive_sockets_server(Opts, Parent) -> - ?line case gen_tcp:listen(0, Opts) of - {ok, LSock} -> - {ok,{_,Port}} = inet:sockname(LSock), - Parent ! {socket,Port}, - passive_sockets_server_accept(LSock); - Error -> - ?t:fail({"Could not create listen socket", Error}) - end. + case gen_tcp:listen(0, Opts) of + {ok, LSock} -> + {ok,{_,Port}} = inet:sockname(LSock), + Parent ! {socket,Port}, + passive_sockets_server_accept(LSock); + Error -> + ?t:fail({"Could not create listen socket", Error}) + end. passive_sockets_server_accept(Sock) -> - ?line case gen_tcp:accept(Sock) of - {ok, Socket} -> - ?t:sleep(500), % Simulate latency - passive_sockets_server_send(Socket, 5), - passive_sockets_server_accept(Sock); - Error -> - ?t:fail({"Could not accept connection", Error}) - end. + case gen_tcp:accept(Sock) of + {ok, Socket} -> + ?t:sleep(500), % Simulate latency + passive_sockets_server_send(Socket, 5), + passive_sockets_server_accept(Sock); + Error -> + ?t:fail({"Could not accept connection", Error}) + end. passive_sockets_server_send(Socket, 0) -> io:format("Closing other end..~n", []), gen_tcp:close(Socket); passive_sockets_server_send(Socket, X) -> - ?line Data = lists:duplicate(1024*X, $a), - ?line case gen_tcp:send(Socket, Data) of - ok -> - ?t:sleep(50), % Simulate some processing. - passive_sockets_server_send(Socket, X-1); - {error, _Reason} -> - ?t:fail("Failed to send data") - end. + Data = lists:duplicate(1024*X, $a), + case gen_tcp:send(Socket, Data) of + ok -> + ?t:sleep(50), % Simulate some processing. + passive_sockets_server_send(Socket, X-1); + {error, _Reason} -> + ?t:fail("Failed to send data") + end. accept_closed_by_other_process(doc) -> ["Tests the return value from gen_tcp:accept when ", "the socket is closed from another process. (OTP-3817)"]; accept_closed_by_other_process(Config) when is_list(Config) -> - ?line Parent = self(), - ?line {ok, ListenSocket} = gen_tcp:listen(0, []), - ?line Child = + Parent = self(), + {ok, ListenSocket} = gen_tcp:listen(0, []), + Child = spawn_link( fun() -> Parent ! {self(), gen_tcp:accept(ListenSocket)} end), - ?line receive after 1000 -> ok end, - ?line ok = gen_tcp:close(ListenSocket), - ?line receive - {Child, {error, closed}} -> - ok; - {Child, Other} -> - ?t:fail({"Wrong result of gen_tcp:accept", Other}) - end. + receive after 1000 -> ok end, + ok = gen_tcp:close(ListenSocket), + receive + {Child, {error, closed}} -> + ok; + {Child, Other} -> + ?t:fail({"Wrong result of gen_tcp:accept", Other}) + end. repeat(N, Fun) -> repeat(N, N, Fun). @@ -915,9 +911,9 @@ closed_socket(suite) -> closed_socket(doc) -> ["Tests the response when using a closed socket as argument"]; closed_socket(Config) when is_list(Config) -> - ?line {ok, LS1} = gen_tcp:listen(0, []), - ?line erlang:yield(), - ?line ok = gen_tcp:close(LS1), + {ok, LS1} = gen_tcp:listen(0, []), + erlang:yield(), + ok = gen_tcp:close(LS1), %% If the following delay is uncommented, the result error values %% below will change from {error, einval} to {error, closed} since %% inet_db then will have noticed that the socket is closed. @@ -925,19 +921,18 @@ closed_socket(Config) when is_list(Config) -> %% in inet_db processes the 'EXIT' message from the port, %% the socket is unregistered. %% - %% ?line test_server:sleep(test_server:seconds(2)), + %% test_server:sleep(test_server:seconds(2)), %% - ?line {error, R_send} = gen_tcp:send(LS1, "data"), - ?line {error, R_recv} = gen_tcp:recv(LS1, 17), - ?line {error, R_accept} = gen_tcp:accept(LS1), - ?line {error, R_controlling_process} = + {error, R_send} = gen_tcp:send(LS1, "data"), + {error, R_recv} = gen_tcp:recv(LS1, 17), + {error, R_accept} = gen_tcp:accept(LS1), + {error, R_controlling_process} = gen_tcp:controlling_process(LS1, self()), %% - ?line ok = io:format("R_send = ~p~n", [R_send]), - ?line ok = io:format("R_recv = ~p~n", [R_recv]), - ?line ok = io:format("R_accept = ~p~n", [R_accept]), - ?line ok = io:format("R_controlling_process = ~p~n", - [R_controlling_process]), + ok = io:format("R_send = ~p~n", [R_send]), + ok = io:format("R_recv = ~p~n", [R_recv]), + ok = io:format("R_accept = ~p~n", [R_accept]), + ok = io:format("R_controlling_process = ~p~n", [R_controlling_process]), ok. %%% @@ -945,28 +940,27 @@ closed_socket(Config) when is_list(Config) -> %%% shutdown_active(Config) when is_list(Config) -> - ?line shutdown_common(true). + shutdown_common(true). shutdown_passive(Config) when is_list(Config) -> - ?line shutdown_common(false). + shutdown_common(false). shutdown_common(Active) -> - ?line P = sort_server(Active), + P = sort_server(Active), io:format("Sort server port: ~p\n", [P]), - ?line do_sort(P, []), - ?line do_sort(P, ["glurf"]), - ?line do_sort(P, ["abc","nisse","dum"]), - - ?line do_sort(P, [lists:reverse(integer_to_list(I)) || I <- lists:seq(25, 255)]), - ?line do_sort(P, [lists:reverse(integer_to_list(I)) || I <- lists:seq(77, 999)]), - ?line do_sort(P, [lists:reverse(integer_to_list(I)) || I <- lists:seq(25, 55)]), - ?line do_sort(P, []), - ?line do_sort(P, ["apa"]), - ?line do_sort(P, ["kluns","gorilla"]), - ?line do_sort(P, [lists:reverse(integer_to_list(I)) || I <- lists:seq(25, 1233)]), - ?line do_sort(P, []), - + do_sort(P, []), + do_sort(P, ["glurf"]), + do_sort(P, ["abc","nisse","dum"]), + + do_sort(P, [lists:reverse(integer_to_list(I)) || I <- lists:seq(25, 255)]), + do_sort(P, [lists:reverse(integer_to_list(I)) || I <- lists:seq(77, 999)]), + do_sort(P, [lists:reverse(integer_to_list(I)) || I <- lists:seq(25, 55)]), + do_sort(P, []), + do_sort(P, ["apa"]), + do_sort(P, ["kluns","gorilla"]), + do_sort(P, [lists:reverse(integer_to_list(I)) || I <- lists:seq(25, 1233)]), + do_sort(P, []), receive Any -> ?t:fail({unexpected_message,Any}) @@ -985,14 +979,14 @@ do_sort(P, List0) -> sort_server(Active) -> Opts = [{exit_on_close,false},{packet,line},{active,Active}], - ?line {ok,L} = gen_tcp:listen(0, Opts), + {ok,L} = gen_tcp:listen(0, Opts), Go = make_ref(), - ?line Pid = spawn_link(fun() -> - receive Go -> sort_server_1(L, Active) end - end), - ?line ok = gen_tcp:controlling_process(L, Pid), - ?line Pid ! Go, - ?line {ok,Port} = inet:port(L), + Pid = spawn_link(fun() -> + receive Go -> sort_server_1(L, Active) end + end), + ok = gen_tcp:controlling_process(L, Pid), + Pid ! Go, + {ok,Port} = inet:port(L), Port. sort_server_1(L, Active) -> @@ -1042,17 +1036,17 @@ shutdown_pending(Config) when is_list(Config) -> Data = [<<N:32>>,ones(N),42], P = a_server(), io:format("Server port: ~p\n", [P]), - ?line {ok,S} = gen_tcp:connect(localhost, P, []), - ?line gen_tcp:send(S, Data), - ?line gen_tcp:shutdown(S, write), - ?line receive - {tcp,S,Msg} -> - io:format("~p\n", [Msg]), - ?line N = list_to_integer(Msg) - 5; - Other -> - ?t:fail({unexpected,Other}) - end, - ok. + {ok,S} = gen_tcp:connect(localhost, P, []), + gen_tcp:send(S, Data), + gen_tcp:shutdown(S, write), + receive + {tcp,S,Msg} -> + io:format("~p\n", [Msg]), + N = list_to_integer(Msg) - 5; + Other -> + ?t:fail({unexpected,Other}) + end, + ok. ones(0) -> []; ones(1) -> [1]; @@ -1065,10 +1059,10 @@ shutdown_pending(Config) when is_list(Config) -> end. a_server() -> - ?line {ok,L} = gen_tcp:listen(0, [{exit_on_close,false},{active,false}]), - ?line Pid = spawn_link(fun() -> a_server(L) end), - ?line ok = gen_tcp:controlling_process(L, Pid), - ?line {ok,Port} = inet:port(L), + {ok,L} = gen_tcp:listen(0, [{exit_on_close,false},{active,false}]), + Pid = spawn_link(fun() -> a_server(L) end), + ok = gen_tcp:controlling_process(L, Pid), + {ok,Port} = inet:port(L), Port. a_server(L) -> @@ -1090,19 +1084,18 @@ shutdown_pending(Config) when is_list(Config) -> %% corrupt data. The testcase will be killed by the timetrap timeout %% if the bug is present. http_bad_packet(Config) when is_list(Config) -> - ?line {ok,L} = gen_tcp:listen(0, - [{active, false}, - binary, - {reuseaddr, true}, - {packet, http}]), - ?line {ok,Port} = inet:port(L), - ?line spawn_link(fun() -> erlang:yield(), http_bad_client(Port) end), - ?line case gen_tcp:accept(L) of - {ok,S} -> - http_worker(S); - Err -> - exit({accept,Err}) - end. + {ok,L} = gen_tcp:listen(0, [{active, false}, + binary, + {reuseaddr, true}, + {packet, http}]), + {ok,Port} = inet:port(L), + spawn_link(fun() -> erlang:yield(), http_bad_client(Port) end), + case gen_tcp:accept(L) of + {ok,S} -> + http_worker(S); + Err -> + exit({accept,Err}) + end. http_worker(S) -> case gen_tcp:recv(S, 0, 30000) of @@ -1122,9 +1115,9 @@ http_bad_client(Port) -> %% Fill send queue and then start receiving. %% busy_send(Config) when is_list(Config) -> - ?line Master = self(), - ?line Msg = <<"the quick brown fox jumps over a lazy dog~n">>, - ?line Server = + Master = self(), + Msg = <<"the quick brown fox jumps over a lazy dog~n">>, + Server = spawn_link(fun () -> {ok,L} = gen_tcp:listen (0, [{active,false},binary, @@ -1134,45 +1127,42 @@ busy_send(Config) when is_list(Config) -> busy_send_client(Port, Master, Msg)}, busy_send_srv(L, Master, Msg) end), - ?line io:format("~p Server~n", [Server]), - ?line receive - {Server,client,Client} -> - ?line io:format("~p Client~n", [Client]), - ?line busy_send_loop(Server, Client, 0) - end. + io:format("~p Server~n", [Server]), + receive + {Server,client,Client} -> + io:format("~p Client~n", [Client]), + busy_send_loop(Server, Client, 0) + end. busy_send_loop(Server, Client, N) -> %% Master %% - ?line receive {Server,send} -> + receive {Server,send} -> busy_send_loop(Server, Client, N+1) after 2000 -> %% Send queue full, sender blocked %% -> stop sender and release client - ?line io:format("Send timeout, time to receive...~n", []), - ?line Server ! {self(),close}, - ?line Client ! {self(),recv,N+1}, - ?line receive - {Server,send} -> - ?line busy_send_2(Server, Client, N+1) - after 10000 -> - %% If this happens, see busy_send_srv - ?t:fail({timeout,{server,not_send,flush([])}}) - end - end. + io:format("Send timeout, time to receive...~n", []), + Server ! {self(),close}, + Client ! {self(),recv,N+1}, + receive + {Server,send} -> + busy_send_2(Server, Client, N+1) + after 10000 -> + %% If this happens, see busy_send_srv + ?t:fail({timeout,{server,not_send,flush([])}}) + end + end. busy_send_2(Server, Client, _N) -> %% Master %% - ?line receive - {Server,[closed]} -> - ?line receive - {Client,[0,{error,closed}]} -> - ok - end - after 10000 -> - ?t:fail({timeout,{server,not_closed,flush([])}}) - end. + receive + {Server,[closed]} -> + receive {Client,[0,{error,closed}]} -> ok end + after 10000 -> + ?t:fail({timeout,{server,not_closed,flush([])}}) + end. busy_send_srv(L, Master, Msg) -> %% Server @@ -1228,7 +1218,7 @@ busy_send_client_loop(Socket, Master, Msg, N) -> busy_disconnect_passive(Config) when is_list(Config) -> MuchoData = list_to_binary(ones(64*1024)), - ?line [do_busy_disconnect_passive(MuchoData) || _ <- lists:seq(1, 10)], + [do_busy_disconnect_passive(MuchoData) || _ <- lists:seq(1, 10)], ok. do_busy_disconnect_passive(MuchoData) -> @@ -1236,8 +1226,8 @@ do_busy_disconnect_passive(MuchoData) -> busy_disconnect_passive_send(S, MuchoData). busy_disconnect_passive_send(S, Data) -> - ?line case gen_tcp:send(S, Data) of - ok -> ?line busy_disconnect_passive_send(S, Data); + case gen_tcp:send(S, Data) of + ok -> busy_disconnect_passive_send(S, Data); {error,closed} -> ok end. @@ -1248,7 +1238,7 @@ busy_disconnect_passive_send(S, Data) -> %%% busy_disconnect_active(Config) when is_list(Config) -> MuchoData = list_to_binary(ones(64*1024)), - ?line [do_busy_disconnect_active(MuchoData) || _ <- lists:seq(1, 10)], + [do_busy_disconnect_active(MuchoData) || _ <- lists:seq(1, 10)], ok. do_busy_disconnect_active(MuchoData) -> @@ -1256,21 +1246,21 @@ do_busy_disconnect_active(MuchoData) -> busy_disconnect_active_send(S, MuchoData). busy_disconnect_active_send(S, Data) -> - ?line case gen_tcp:send(S, Data) of - ok -> ?line busy_disconnect_active_send(S, Data); + case gen_tcp:send(S, Data) of + ok -> busy_disconnect_active_send(S, Data); {error,closed} -> receive {tcp_closed,S} -> ok; - _Other -> ?line ?t:fail() + _Other -> ?t:fail() end end. busy_disconnect_prepare_server(ConnectOpts) -> - ?line Sender = self(), - ?line Server = spawn_link(fun() -> busy_disconnect_server(Sender) end), + Sender = self(), + Server = spawn_link(fun() -> busy_disconnect_server(Sender) end), receive {port,Server,Port} -> ok end, - ?line {ok,S} = gen_tcp:connect(localhost, Port, ConnectOpts), + {ok,S} = gen_tcp:connect(localhost, Port, ConnectOpts), Server ! {Sender,sending}, S. @@ -1304,8 +1294,8 @@ busy_disconnect_server_wait_for_busy(Sender, S) -> %%% Fill send queue %%% fill_sendq(Config) when is_list(Config) -> - ?line Master = self(), - ?line Server = + Master = self(), + Server = spawn_link(fun () -> {ok,L} = gen_tcp:listen (0, [{active,false},binary, @@ -1315,12 +1305,12 @@ fill_sendq(Config) when is_list(Config) -> fill_sendq_client(Port, Master)}, fill_sendq_srv(L, Master) end), - ?line io:format("~p Server~n", [Server]), - ?line receive {Server,client,Client} -> - ?line io:format("~p Client~n", [Client]), - ?line receive {Server,reader,Reader} -> - ?line io:format("~p Reader~n", [Reader]), - ?line fill_sendq_loop(Server, Client, Reader) + io:format("~p Server~n", [Server]), + receive {Server,client,Client} -> + io:format("~p Client~n", [Client]), + receive {Server,reader,Reader} -> + io:format("~p Reader~n", [Reader]), + fill_sendq_loop(Server, Client, Reader) end end. @@ -1331,21 +1321,21 @@ fill_sendq_loop(Server, Client, Reader) -> fill_sendq_loop(Server, Client, Reader) after 2000 -> %% Send queue full, sender blocked -> close client. - ?line io:format("Send timeout, closing Client...~n", []), - ?line Client ! {self(),close}, - ?line receive {Server,[{error,closed}]} -> - ?line io:format("Got server closed.~n"), - ?line receive {Reader,[{error,closed}]} -> - ?line io:format + io:format("Send timeout, closing Client...~n", []), + Client ! {self(),close}, + receive {Server,[{error,closed}]} -> + io:format("Got server closed.~n"), + receive {Reader,[{error,closed}]} -> + io:format ("Got reader closed.~n"), ok after 3000 -> ?t:fail({timeout,{closed,reader}}) end; {Reader,[{error,closed}]} -> - ?line io:format("Got reader closed.~n"), - ?line receive {Server,[{error,closed}]} -> - ?line io:format("Got server closed~n"), + io:format("Got reader closed.~n"), + receive {Server,[{error,closed}]} -> + io:format("Got server closed~n"), ok after 3000 -> ?t:fail({timeout,{closed,server}}) @@ -1416,39 +1406,39 @@ fill_sendq_client(Port, Master) -> %%% a closed socket. %%% partial_recv_and_close(Config) when is_list(Config) -> - ?line Msg = "the quick brown fox jumps over a lazy dog 0123456789\n", - ?line Len = length(Msg), - ?line {ok,L} = gen_tcp:listen(0, [{active,false}]), - ?line {ok,P} = inet:port(L), - ?line {ok,S} = gen_tcp:connect("localhost", P, [{active,false}]), - ?line {ok,A} = gen_tcp:accept(L), - ?line ok = gen_tcp:send(S, Msg), - ?line ok = gen_tcp:close(S), - ?line {error,closed} = gen_tcp:recv(A, Len+1), + Msg = "the quick brown fox jumps over a lazy dog 0123456789\n", + Len = length(Msg), + {ok,L} = gen_tcp:listen(0, [{active,false}]), + {ok,P} = inet:port(L), + {ok,S} = gen_tcp:connect("localhost", P, [{active,false}]), + {ok,A} = gen_tcp:accept(L), + ok = gen_tcp:send(S, Msg), + ok = gen_tcp:close(S), + {error,closed} = gen_tcp:recv(A, Len+1), ok. %%% Try to receive more than available number of bytes from %%% a closed socket, this time waiting in the recv before closing. %%% partial_recv_and_close_2(Config) when is_list(Config) -> - ?line Msg = "the quick brown fox jumps over a lazy dog 0123456789\n", - ?line Len = length(Msg), - ?line {ok,L} = gen_tcp:listen(0, [{active,false}]), - ?line {ok,P} = inet:port(L), - ?line Server = self(), - ?line Client = + Msg = "the quick brown fox jumps over a lazy dog 0123456789\n", + Len = length(Msg), + {ok,L} = gen_tcp:listen(0, [{active,false}]), + {ok,P} = inet:port(L), + Server = self(), + Client = spawn_link( fun () -> receive after 2000 -> ok end, {ok,S} = gen_tcp:connect("localhost", P, [{active,false}]), - ?line ok = gen_tcp:send(S, Msg), + ok = gen_tcp:send(S, Msg), receive {Server,close} -> ok end, receive after 2000 -> ok end, - ?line ok = gen_tcp:close(S) + ok = gen_tcp:close(S) end), - ?line {ok,A} = gen_tcp:accept(L), - ?line Client ! {Server,close}, - ?line {error,closed} = gen_tcp:recv(A, Len+1), + {ok,A} = gen_tcp:accept(L), + Client ! {Server,close}, + {error,closed} = gen_tcp:recv(A, Len+1), ok. %%% Here we tests that gen_tcp:recv/2 will return {error,closed} following @@ -1471,151 +1461,151 @@ do_partial_recv_and_close_3() -> receive {port,Port} -> ok end, - ?line Much = ones(8*64*1024), - ?line {ok,S} = gen_tcp:connect(localhost, Port, [{active,false}]), + Much = ones(8*64*1024), + {ok,S} = gen_tcp:connect(localhost, Port, [{active,false}]), %% Send a lot of data (most of it will be queued). The receiver will read one byte %% and close the connection. The write operation will fail. - ?line gen_tcp:send(S, Much), + gen_tcp:send(S, Much), %% We should always get {error,closed} here. - ?line {error,closed} = gen_tcp:recv(S, 0). + {error,closed} = gen_tcp:recv(S, 0). test_prio_put_get() -> Tos = 3 bsl 5, - ?line {ok,L1} = gen_tcp:listen(0, [{active,false}]), - ?line ok = inet:setopts(L1,[{priority,3}]), - ?line ok = inet:setopts(L1,[{tos,Tos}]), - ?line {ok,[{priority,3},{tos,Tos}]} = inet:getopts(L1,[priority,tos]), - ?line ok = inet:setopts(L1,[{priority,3}]), % Dont destroy each other - ?line {ok,[{priority,3},{tos,Tos}]} = inet:getopts(L1,[priority,tos]), - ?line ok = inet:setopts(L1,[{reuseaddr,true}]), % Dont let others destroy - ?line {ok,[{priority,3},{tos,Tos}]} = inet:getopts(L1,[priority,tos]), - ?line gen_tcp:close(L1), + {ok,L1} = gen_tcp:listen(0, [{active,false}]), + ok = inet:setopts(L1,[{priority,3}]), + ok = inet:setopts(L1,[{tos,Tos}]), + {ok,[{priority,3},{tos,Tos}]} = inet:getopts(L1,[priority,tos]), + ok = inet:setopts(L1,[{priority,3}]), % Dont destroy each other + {ok,[{priority,3},{tos,Tos}]} = inet:getopts(L1,[priority,tos]), + ok = inet:setopts(L1,[{reuseaddr,true}]), % Dont let others destroy + {ok,[{priority,3},{tos,Tos}]} = inet:getopts(L1,[priority,tos]), + gen_tcp:close(L1), ok. test_prio_accept() -> - ?line {ok,Sock}=gen_tcp:listen(0,[binary,{packet,0},{active,false}, - {reuseaddr,true},{priority,4}]), - ?line {ok,Port} = inet:port(Sock), - ?line {ok,Sock2}=gen_tcp:connect("localhost",Port,[binary,{packet,0}, - {active,false}, - {reuseaddr,true}, - {priority,4}]), - ?line {ok,Sock3}=gen_tcp:accept(Sock), - ?line {ok,[{priority,4}]} = inet:getopts(Sock,[priority]), - ?line {ok,[{priority,4}]} = inet:getopts(Sock2,[priority]), - ?line {ok,[{priority,4}]} = inet:getopts(Sock3,[priority]), - ?line gen_tcp:close(Sock), - ?line gen_tcp:close(Sock2), - ?line gen_tcp:close(Sock3), + {ok,Sock}=gen_tcp:listen(0,[binary,{packet,0},{active,false}, + {reuseaddr,true},{priority,4}]), + {ok,Port} = inet:port(Sock), + {ok,Sock2}=gen_tcp:connect("localhost",Port,[binary,{packet,0}, + {active,false}, + {reuseaddr,true}, + {priority,4}]), + {ok,Sock3}=gen_tcp:accept(Sock), + {ok,[{priority,4}]} = inet:getopts(Sock,[priority]), + {ok,[{priority,4}]} = inet:getopts(Sock2,[priority]), + {ok,[{priority,4}]} = inet:getopts(Sock3,[priority]), + gen_tcp:close(Sock), + gen_tcp:close(Sock2), + gen_tcp:close(Sock3), ok. test_prio_accept2() -> Tos1 = 4 bsl 5, Tos2 = 3 bsl 5, - ?line {ok,Sock}=gen_tcp:listen(0,[binary,{packet,0},{active,false}, - {reuseaddr,true},{priority,4}, - {tos,Tos1}]), - ?line {ok,Port} = inet:port(Sock), - ?line {ok,Sock2}=gen_tcp:connect("localhost",Port,[binary,{packet,0}, - {active,false}, - {reuseaddr,true}, - {priority,4}, - {tos,Tos2}]), - ?line {ok,Sock3}=gen_tcp:accept(Sock), - ?line {ok,[{priority,4},{tos,Tos1}]} = inet:getopts(Sock,[priority,tos]), - ?line {ok,[{priority,4},{tos,Tos2}]} = inet:getopts(Sock2,[priority,tos]), - ?line {ok,[{priority,4},{tos,Tos1}]} = inet:getopts(Sock3,[priority,tos]), - ?line gen_tcp:close(Sock), - ?line gen_tcp:close(Sock2), - ?line gen_tcp:close(Sock3), + {ok,Sock}=gen_tcp:listen(0,[binary,{packet,0},{active,false}, + {reuseaddr,true},{priority,4}, + {tos,Tos1}]), + {ok,Port} = inet:port(Sock), + {ok,Sock2}=gen_tcp:connect("localhost",Port,[binary,{packet,0}, + {active,false}, + {reuseaddr,true}, + {priority,4}, + {tos,Tos2}]), + {ok,Sock3}=gen_tcp:accept(Sock), + {ok,[{priority,4},{tos,Tos1}]} = inet:getopts(Sock,[priority,tos]), + {ok,[{priority,4},{tos,Tos2}]} = inet:getopts(Sock2,[priority,tos]), + {ok,[{priority,4},{tos,Tos1}]} = inet:getopts(Sock3,[priority,tos]), + gen_tcp:close(Sock), + gen_tcp:close(Sock2), + gen_tcp:close(Sock3), ok. test_prio_accept3() -> Tos1 = 4 bsl 5, Tos2 = 3 bsl 5, - ?line {ok,Sock}=gen_tcp:listen(0,[binary,{packet,0},{active,false}, - {reuseaddr,true}, - {tos,Tos1}]), - ?line {ok,Port} = inet:port(Sock), - ?line {ok,Sock2}=gen_tcp:connect("localhost",Port,[binary,{packet,0}, - {active,false}, - {reuseaddr,true}, - {tos,Tos2}]), - ?line {ok,Sock3}=gen_tcp:accept(Sock), - ?line {ok,[{priority,0},{tos,Tos1}]} = inet:getopts(Sock,[priority,tos]), - ?line {ok,[{priority,0},{tos,Tos2}]} = inet:getopts(Sock2,[priority,tos]), - ?line {ok,[{priority,0},{tos,Tos1}]} = inet:getopts(Sock3,[priority,tos]), - ?line gen_tcp:close(Sock), - ?line gen_tcp:close(Sock2), - ?line gen_tcp:close(Sock3), + {ok,Sock}=gen_tcp:listen(0,[binary,{packet,0},{active,false}, + {reuseaddr,true}, + {tos,Tos1}]), + {ok,Port} = inet:port(Sock), + {ok,Sock2}=gen_tcp:connect("localhost",Port,[binary,{packet,0}, + {active,false}, + {reuseaddr,true}, + {tos,Tos2}]), + {ok,Sock3}=gen_tcp:accept(Sock), + {ok,[{priority,0},{tos,Tos1}]} = inet:getopts(Sock,[priority,tos]), + {ok,[{priority,0},{tos,Tos2}]} = inet:getopts(Sock2,[priority,tos]), + {ok,[{priority,0},{tos,Tos1}]} = inet:getopts(Sock3,[priority,tos]), + gen_tcp:close(Sock), + gen_tcp:close(Sock2), + gen_tcp:close(Sock3), ok. test_prio_accept_async() -> Tos1 = 4 bsl 5, Tos2 = 3 bsl 5, Ref = make_ref(), - ?line spawn(?MODULE,priority_server,[{self(),Ref}]), - ?line Port = receive - {Ref,P} -> P - after 5000 -> ?t:fail({error,"helper process timeout"}) - end, - ?line receive - after 3000 -> ok - end, - ?line {ok,Sock2}=gen_tcp:connect("localhost",Port,[binary,{packet,0}, - {active,false}, - {reuseaddr,true}, - {priority,4}, - {tos,Tos2}]), - ?line receive - {Ref,{ok,[{priority,4},{tos,Tos1}]}} -> - ok ; - {Ref,Error} -> - ?t:fail({missmatch,Error}) - after 5000 -> ?t:fail({error,"helper process timeout"}) - end, - ?line receive - {Ref,{ok,[{priority,4},{tos,Tos1}]}} -> - ok ; - {Ref,Error2} -> - ?t:fail({missmatch,Error2}) - after 5000 -> ?t:fail({error,"helper process timeout"}) - end, - - ?line {ok,[{priority,4},{tos,Tos2}]} = inet:getopts(Sock2,[priority,tos]), - ?line catch gen_tcp:close(Sock2), + spawn(?MODULE,priority_server,[{self(),Ref}]), + Port = receive + {Ref,P} -> P + after 5000 -> ?t:fail({error,"helper process timeout"}) + end, + receive + after 3000 -> ok + end, + {ok,Sock2}=gen_tcp:connect("localhost",Port,[binary,{packet,0}, + {active,false}, + {reuseaddr,true}, + {priority,4}, + {tos,Tos2}]), + receive + {Ref,{ok,[{priority,4},{tos,Tos1}]}} -> + ok; + {Ref,Error} -> + ?t:fail({missmatch,Error}) + after 5000 -> ?t:fail({error,"helper process timeout"}) + end, + receive + {Ref,{ok,[{priority,4},{tos,Tos1}]}} -> + ok; + {Ref,Error2} -> + ?t:fail({missmatch,Error2}) + after 5000 -> ?t:fail({error,"helper process timeout"}) + end, + + {ok,[{priority,4},{tos,Tos2}]} = inet:getopts(Sock2,[priority,tos]), + catch gen_tcp:close(Sock2), ok. priority_server({Parent,Ref}) -> Tos1 = 4 bsl 5, - ?line {ok,Sock}=gen_tcp:listen(0,[binary,{packet,0},{active,false}, - {reuseaddr,true},{priority,4}, - {tos,Tos1}]), - ?line {ok,Port} = inet:port(Sock), + {ok,Sock}=gen_tcp:listen(0,[binary,{packet,0},{active,false}, + {reuseaddr,true},{priority,4}, + {tos,Tos1}]), + {ok,Port} = inet:port(Sock), Parent ! {Ref,Port}, - ?line {ok,Sock3}=gen_tcp:accept(Sock), + {ok,Sock3}=gen_tcp:accept(Sock), Parent ! {Ref, inet:getopts(Sock,[priority,tos])}, Parent ! {Ref, inet:getopts(Sock3,[priority,tos])}, ok. test_prio_fail() -> - ?line {ok,L} = gen_tcp:listen(0, [{active,false}]), - ?line {error,_} = inet:setopts(L,[{priority,1000}]), + {ok,L} = gen_tcp:listen(0, [{active,false}]), + {error,_} = inet:setopts(L,[{priority,1000}]), % This error could only happen in linux kernels earlier than 2.6.24.4 % Privilege check is now disabled and IP_TOS can never fail (only silently % be masked). -% ?line {error,_} = inet:setopts(L,[{tos,6 bsl 5}]), - ?line gen_tcp:close(L), +% {error,_} = inet:setopts(L,[{tos,6 bsl 5}]), + gen_tcp:close(L), ok. test_prio_udp() -> Tos = 3 bsl 5, - ?line {ok,S} = gen_udp:open(0,[{active,false},binary,{tos, Tos}, - {priority,3}]), - ?line {ok,[{priority,3},{tos,Tos}]} = inet:getopts(S,[priority,tos]), - ?line gen_udp:close(S), + {ok,S} = gen_udp:open(0,[{active,false},binary,{tos, Tos}, + {priority,3}]), + {ok,[{priority,3},{tos,Tos}]} = inet:getopts(S,[priority,tos]), + gen_udp:close(S), ok. so_priority(doc) -> @@ -1623,9 +1613,9 @@ so_priority(doc) -> so_priority(suite) -> []; so_priority(Config) when is_list(Config) -> - ?line {ok,L} = gen_tcp:listen(0, [{active,false}]), - ?line ok = inet:setopts(L,[{priority,1}]), - ?line case inet:getopts(L,[priority]) of + {ok,L} = gen_tcp:listen(0, [{active,false}]), + ok = inet:setopts(L,[{priority,1}]), + case inet:getopts(L,[priority]) of {ok,[{priority,1}]} -> gen_tcp:close(L), test_prio_put_get(), @@ -1641,7 +1631,7 @@ so_priority(Config) when is_list(Config) -> {unix,linux} -> case os:version() of {X,Y,_} when (X > 2) or ((X =:= 2) and (Y >= 4)) -> - ?line ?t:fail({error, + ?t:fail({error, "so_priority should work on this " "OS, but does not"}); _ -> @@ -1655,21 +1645,21 @@ so_priority(Config) when is_list(Config) -> %% Accept test utilities (suites are below) millis() -> - {A,B,C}=erlang:now(), - (A*1000000*1000)+(B*1000)+(C div 1000). + erlang:monotonic_time(milli_seconds). -collect_accepts(Tmo) -> +collect_accepts(0,_) -> []; +collect_accepts(N,Tmo) -> A = millis(), receive {accepted,P,Msg} -> - [{P,Msg}] ++ collect_accepts(Tmo-(millis() - A)) + [{P,Msg}] ++ collect_accepts(N-1,Tmo-(millis() - A)) after Tmo -> [] end. --define(EXPECT_ACCEPTS(Pattern,Timeout), +-define(EXPECT_ACCEPTS(Pattern,N,Timeout), (fun() -> - case collect_accepts(Timeout) of + case collect_accepts(if N =:= infinity -> -1; true -> N end,Timeout) of Pattern -> ok; Other -> @@ -1705,20 +1695,20 @@ primitive_accept(suite) -> primitive_accept(doc) -> ["Test singular accept"]; primitive_accept(Config) when is_list(Config) -> - ?line {ok,LS}=gen_tcp:listen(0,[]), - ?line {ok,PortNo}=inet:port(LS), - ?line Parent = self(), - ?line F = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS)} end, - ?line P = spawn(F), - ?line gen_tcp:connect("localhost",PortNo,[]), - ?line receive - {accepted,P,{ok,P0}} when is_port(P0) -> - ok; - {accepted,P,Other0} -> - {error,Other0} - after 500 -> - {error,timeout} - end. + {ok,LS}=gen_tcp:listen(0,[]), + {ok,PortNo}=inet:port(LS), + Parent = self(), + F = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS)} end, + P = spawn(F), + gen_tcp:connect("localhost",PortNo,[]), + receive + {accepted,P,{ok,P0}} when is_port(P0) -> + ok; + {accepted,P,Other0} -> + {error,Other0} + after 500 -> + {error,timeout} + end. multi_accept_close_listen(suite) -> @@ -1726,111 +1716,175 @@ multi_accept_close_listen(suite) -> multi_accept_close_listen(doc) -> ["Closing listen socket when multi-accepting"]; multi_accept_close_listen(Config) when is_list(Config) -> - ?line {ok,LS}=gen_tcp:listen(0,[]), - ?line Parent = self(), - ?line F = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS)} end, - ?line spawn(F), - ?line spawn(F), - ?line spawn(F), - ?line spawn(F), - ?line gen_tcp:close(LS), - ?line ?EXPECT_ACCEPTS([{_,{error,closed}},{_,{error,closed}}, - {_,{error,closed}},{_,{error,closed}}], 500). + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + F = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS)} end, + spawn(F), + spawn(F), + spawn(F), + spawn(F), + gen_tcp:close(LS), + ok = ?EXPECT_ACCEPTS([{_,{error,closed}},{_,{error,closed}}, + {_,{error,closed}},{_,{error,closed}}],4,500). accept_timeout(suite) -> []; accept_timeout(doc) -> ["Single accept with timeout"]; accept_timeout(Config) when is_list(Config) -> - ?line {ok,LS}=gen_tcp:listen(0,[]), - ?line Parent = self(), - ?line F = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS,1000)} end, - ?line P = spawn(F), - ?line ?EXPECT_ACCEPTS([{P,{error,timeout}}],2000). + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + F = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS,1000)} end, + P = spawn(F), + ok = ?EXPECT_ACCEPTS([{P,{error,timeout}}],1,2000). accept_timeouts_in_order(suite) -> []; accept_timeouts_in_order(doc) -> ["Check that multi-accept timeouts happen in the correct order"]; accept_timeouts_in_order(Config) when is_list(Config) -> - ?line {ok,LS}=gen_tcp:listen(0,[]), - ?line Parent = self(), - ?line P1 = spawn(mktmofun(1000,Parent,LS)), - ?line P2 = spawn(mktmofun(1200,Parent,LS)), - ?line P3 = spawn(mktmofun(1300,Parent,LS)), - ?line P4 = spawn(mktmofun(1400,Parent,LS)), - ?line ?EXPECT_ACCEPTS([{P1,{error,timeout}},{P2,{error,timeout}}, - {P3,{error,timeout}},{P4,{error,timeout}}], 2000). + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + P1 = spawn(mktmofun(1000,Parent,LS)), + P2 = spawn(mktmofun(1200,Parent,LS)), + P3 = spawn(mktmofun(1300,Parent,LS)), + P4 = spawn(mktmofun(1400,Parent,LS)), + ok = ?EXPECT_ACCEPTS([{P1,{error,timeout}},{P2,{error,timeout}}, + {P3,{error,timeout}},{P4,{error,timeout}}],infinity,2000). accept_timeouts_in_order2(suite) -> []; accept_timeouts_in_order2(doc) -> ["Check that multi-accept timeouts happen in the correct order (more)"]; accept_timeouts_in_order2(Config) when is_list(Config) -> - ?line {ok,LS}=gen_tcp:listen(0,[]), - ?line Parent = self(), - ?line P1 = spawn(mktmofun(1400,Parent,LS)), - ?line P2 = spawn(mktmofun(1300,Parent,LS)), - ?line P3 = spawn(mktmofun(1200,Parent,LS)), - ?line P4 = spawn(mktmofun(1000,Parent,LS)), - ?line ?EXPECT_ACCEPTS([{P4,{error,timeout}},{P3,{error,timeout}}, - {P2,{error,timeout}},{P1,{error,timeout}}], 2000). + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + P1 = spawn(mktmofun(1400,Parent,LS)), + P2 = spawn(mktmofun(1300,Parent,LS)), + P3 = spawn(mktmofun(1200,Parent,LS)), + P4 = spawn(mktmofun(1000,Parent,LS)), + ok = ?EXPECT_ACCEPTS([{P4,{error,timeout}},{P3,{error,timeout}}, + {P2,{error,timeout}},{P1,{error,timeout}}],infinity,2000). accept_timeouts_in_order3(suite) -> []; accept_timeouts_in_order3(doc) -> ["Check that multi-accept timeouts happen in the correct order (even more)"]; accept_timeouts_in_order3(Config) when is_list(Config) -> - ?line {ok,LS}=gen_tcp:listen(0,[]), - ?line Parent = self(), - ?line P1 = spawn(mktmofun(1200,Parent,LS)), - ?line P2 = spawn(mktmofun(1400,Parent,LS)), - ?line P3 = spawn(mktmofun(1300,Parent,LS)), - ?line P4 = spawn(mktmofun(1000,Parent,LS)), - ?line ?EXPECT_ACCEPTS([{P4,{error,timeout}},{P1,{error,timeout}}, - {P3,{error,timeout}},{P2,{error,timeout}}], 2000). + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + P1 = spawn(mktmofun(1200,Parent,LS)), + P2 = spawn(mktmofun(1400,Parent,LS)), + P3 = spawn(mktmofun(1300,Parent,LS)), + P4 = spawn(mktmofun(1000,Parent,LS)), + ok = ?EXPECT_ACCEPTS([{P4,{error,timeout}},{P1,{error,timeout}}, + {P3,{error,timeout}},{P2,{error,timeout}}],infinity,2000). + +accept_timeouts_in_order4(suite) -> + []; +accept_timeouts_in_order4(doc) -> + ["Check that multi-accept timeouts happen in the correct order after " + "mixing millsec and sec timeouts"]; +accept_timeouts_in_order4(Config) when is_list(Config) -> + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + P1 = spawn(mktmofun(200,Parent,LS)), + P2 = spawn(mktmofun(400,Parent,LS)), + P3 = spawn(mktmofun(1000,Parent,LS)), + P4 = spawn(mktmofun(600,Parent,LS)), + ok = ?EXPECT_ACCEPTS([{P1,{error,timeout}},{P2,{error,timeout}}, + {P4,{error,timeout}},{P3,{error,timeout}}],infinity,2000). + +accept_timeouts_in_order5(suite) -> + []; +accept_timeouts_in_order5(doc) -> + ["Check that multi-accept timeouts happen in the correct order after " + "mixing millsec and sec timeouts (more)"]; +accept_timeouts_in_order5(Config) when is_list(Config) -> + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + P1 = spawn(mktmofun(400,Parent,LS)), + P2 = spawn(mktmofun(1000,Parent,LS)), + P3 = spawn(mktmofun(600,Parent,LS)), + P4 = spawn(mktmofun(200,Parent,LS)), + ok = ?EXPECT_ACCEPTS([{P4,{error,timeout}},{P1,{error,timeout}}, + {P3,{error,timeout}},{P2,{error,timeout}}],infinity,2000). + +accept_timeouts_in_order6(suite) -> + []; +accept_timeouts_in_order6(doc) -> + ["Check that multi-accept timeouts happen in the correct order after " + "mixing millsec and sec timeouts (even more)"]; +accept_timeouts_in_order6(Config) when is_list(Config) -> + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + P1 = spawn(mktmofun(1000,Parent,LS)), + P2 = spawn(mktmofun(400,Parent,LS)), + P3 = spawn(mktmofun(600,Parent,LS)), + P4 = spawn(mktmofun(200,Parent,LS)), + ok = ?EXPECT_ACCEPTS([{P4,{error,timeout}},{P2,{error,timeout}}, + {P3,{error,timeout}},{P1,{error,timeout}}],infinity,2000). + +accept_timeouts_in_order7(suite) -> + []; +accept_timeouts_in_order7(doc) -> + ["Check that multi-accept timeouts happen in the correct order after " + "mixing millsec and sec timeouts (even more++)"]; +accept_timeouts_in_order7(Config) when is_list(Config) -> + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + P1 = spawn(mktmofun(1000,Parent,LS)), + P2 = spawn(mktmofun(200,Parent,LS)), + P3 = spawn(mktmofun(1200,Parent,LS)), + P4 = spawn(mktmofun(600,Parent,LS)), + P5 = spawn(mktmofun(400,Parent,LS)), + P6 = spawn(mktmofun(800,Parent,LS)), + P7 = spawn(mktmofun(1600,Parent,LS)), + P8 = spawn(mktmofun(1400,Parent,LS)), + ok = ?EXPECT_ACCEPTS([{P2,{error,timeout}},{P5,{error,timeout}}, + {P4,{error,timeout}},{P6,{error,timeout}}, + {P1,{error,timeout}},{P3,{error,timeout}}, + {P8,{error,timeout}},{P7,{error,timeout}}],infinity,2000). accept_timeouts_mixed(suite) -> []; accept_timeouts_mixed(doc) -> ["Check that multi-accept timeouts behave correctly when mixed with successful timeouts"]; accept_timeouts_mixed(Config) when is_list(Config) -> - ?line {ok,LS}=gen_tcp:listen(0,[]), - ?line Parent = self(), - ?line {ok,PortNo}=inet:port(LS), - ?line P1 = spawn(mktmofun(1000,Parent,LS)), - ?line wait_until_accepting(P1,500), - ?line P2 = spawn(mktmofun(2000,Parent,LS)), - ?line wait_until_accepting(P2,500), - ?line P3 = spawn(mktmofun(3000,Parent,LS)), - ?line wait_until_accepting(P3,500), - ?line P4 = spawn(mktmofun(4000,Parent,LS)), - ?line wait_until_accepting(P4,500), - ?line ok = ?EXPECT_ACCEPTS([{P1,{error,timeout}}],1500), - ?line {ok,_}=gen_tcp:connect("localhost",PortNo,[]), - ?line ok = ?EXPECT_ACCEPTS([{P2,{ok,Port0}}] when is_port(Port0),100), - ?line ok = ?EXPECT_ACCEPTS([{P3,{error,timeout}}],2000), - ?line gen_tcp:connect("localhost",PortNo,[]), - ?line ?EXPECT_ACCEPTS([{P4,{ok,Port1}}] when is_port(Port1),100). + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + {ok,PortNo}=inet:port(LS), + P1 = spawn(mktmofun(1000,Parent,LS)), + wait_until_accepting(P1,500), + P2 = spawn(mktmofun(2000,Parent,LS)), + wait_until_accepting(P2,500), + P3 = spawn(mktmofun(3000,Parent,LS)), + wait_until_accepting(P3,500), + P4 = spawn(mktmofun(4000,Parent,LS)), + wait_until_accepting(P4,500), + ok = ?EXPECT_ACCEPTS([{P1,{error,timeout}}],infinity,1500), + {ok,_}=gen_tcp:connect("localhost",PortNo,[]), + ok = ?EXPECT_ACCEPTS([{P2,{ok,Port0}}] when is_port(Port0),infinity,100), + ok = ?EXPECT_ACCEPTS([{P3,{error,timeout}}],infinity,2000), + gen_tcp:connect("localhost",PortNo,[]), + ok = ?EXPECT_ACCEPTS([{P4,{ok,Port1}}] when is_port(Port1),infinity,100). killing_acceptor(suite) -> []; killing_acceptor(doc) -> ["Check that single acceptor behaves as expected when killed"]; killing_acceptor(Config) when is_list(Config) -> - ?line {ok,LS}=gen_tcp:listen(0,[]), - ?line Pid = spawn(fun() -> erlang:display({accepted,self(),gen_tcp:accept(LS)}) end), - ?line receive after 100 -> - ok - end, - ?line {ok,L1} = prim_inet:getstatus(LS), - ?line true = lists:member(accepting, L1), - ?line exit(Pid,kill), - ?line receive after 100 -> - ok - end, - ?line {ok,L2} = prim_inet:getstatus(LS), - ?line false = lists:member(accepting, L2), + {ok,LS}=gen_tcp:listen(0,[]), + Pid = spawn(fun() -> erlang:display({accepted,self(),gen_tcp:accept(LS)}) end), + receive after 100 -> ok + end, + {ok,L1} = prim_inet:getstatus(LS), + true = lists:member(accepting, L1), + exit(Pid,kill), + receive after 100 -> ok + end, + {ok,L2} = prim_inet:getstatus(LS), + false = lists:member(accepting, L2), ok. killing_multi_acceptors(suite) -> @@ -1838,26 +1892,24 @@ killing_multi_acceptors(suite) -> killing_multi_acceptors(doc) -> ["Check that multi acceptors behaves as expected when killed"]; killing_multi_acceptors(Config) when is_list(Config) -> - ?line {ok,LS}=gen_tcp:listen(0,[]), - ?line Parent = self(), - ?line F = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS)} end, - ?line F2 = mktmofun(1000,Parent,LS), - ?line Pid = spawn(F), - ?line Pid2 = spawn(F2), - ?line receive after 100 -> - ok - end, - ?line {ok,L1} = prim_inet:getstatus(LS), - ?line true = lists:member(accepting, L1), - ?line exit(Pid,kill), - ?line receive after 100 -> - ok - end, - ?line {ok,L2} = prim_inet:getstatus(LS), - ?line true = lists:member(accepting, L2), - ?line ok = ?EXPECT_ACCEPTS([{Pid2,{error,timeout}}],1000), - ?line {ok,L3} = prim_inet:getstatus(LS), - ?line false = lists:member(accepting, L3), + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + F = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS)} end, + F2 = mktmofun(1000,Parent,LS), + Pid = spawn(F), + Pid2 = spawn(F2), + receive after 100 -> ok + end, + {ok,L1} = prim_inet:getstatus(LS), + true = lists:member(accepting, L1), + exit(Pid,kill), + receive after 100 -> ok + end, + {ok,L2} = prim_inet:getstatus(LS), + true = lists:member(accepting, L2), + ok = ?EXPECT_ACCEPTS([{Pid2,{error,timeout}}],1,1000), + {ok,L3} = prim_inet:getstatus(LS), + false = lists:member(accepting, L3), ok. killing_multi_acceptors2(suite) -> @@ -1865,40 +1917,36 @@ killing_multi_acceptors2(suite) -> killing_multi_acceptors2(doc) -> ["Check that multi acceptors behaves as expected when killed (more)"]; killing_multi_acceptors2(Config) when is_list(Config) -> - ?line {ok,LS}=gen_tcp:listen(0,[]), - ?line Parent = self(), - ?line {ok,PortNo}=inet:port(LS), - ?line F = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS)} end, - ?line F2 = mktmofun(1000,Parent,LS), - ?line Pid = spawn(F), - ?line Pid2 = spawn(F), - ?line receive after 100 -> - ok - end, - ?line {ok,L1} = prim_inet:getstatus(LS), - ?line true = lists:member(accepting, L1), - ?line exit(Pid,kill), - ?line receive after 100 -> - ok - end, - ?line {ok,L2} = prim_inet:getstatus(LS), - ?line true = lists:member(accepting, L2), - ?line exit(Pid2,kill), - ?line receive after 100 -> - ok - end, - ?line {ok,L3} = prim_inet:getstatus(LS), - ?line false = lists:member(accepting, L3), - ?line Pid3 = spawn(F2), - ?line receive after 100 -> - ok - end, - ?line {ok,L4} = prim_inet:getstatus(LS), - ?line true = lists:member(accepting, L4), - ?line gen_tcp:connect("localhost",PortNo,[]), - ?line ok = ?EXPECT_ACCEPTS([{Pid3,{ok,Port}}] when is_port(Port),100), - ?line {ok,L5} = prim_inet:getstatus(LS), - ?line false = lists:member(accepting, L5), + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + {ok,PortNo}=inet:port(LS), + F = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS)} end, + F2 = mktmofun(1000,Parent,LS), + Pid = spawn(F), + Pid2 = spawn(F), + receive after 100 -> ok + end, + {ok,L1} = prim_inet:getstatus(LS), + true = lists:member(accepting, L1), + exit(Pid,kill), + receive after 100 -> ok + end, + {ok,L2} = prim_inet:getstatus(LS), + true = lists:member(accepting, L2), + exit(Pid2,kill), + receive after 100 -> ok + end, + {ok,L3} = prim_inet:getstatus(LS), + false = lists:member(accepting, L3), + Pid3 = spawn(F2), + receive after 100 -> ok + end, + {ok,L4} = prim_inet:getstatus(LS), + true = lists:member(accepting, L4), + gen_tcp:connect("localhost",PortNo,[]), + ok = ?EXPECT_ACCEPTS([{Pid3,{ok,Port}}] when is_port(Port),1,100), + {ok,L5} = prim_inet:getstatus(LS), + false = lists:member(accepting, L5), ok. several_accepts_in_one_go(suite) -> @@ -1907,33 +1955,19 @@ several_accepts_in_one_go(doc) -> ["checks that multi-accept works when more than one accept can be " "done at once (wb test of inet_driver)"]; several_accepts_in_one_go(Config) when is_list(Config) -> - ?line {ok,LS}=gen_tcp:listen(0,[]), - ?line Parent = self(), - ?line {ok,PortNo}=inet:port(LS), - ?line F1 = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS)} end, - ?line F2 = fun() -> Parent ! {connected,self(),gen_tcp:connect("localhost",PortNo,[])} end, - ?line spawn(F1), - ?line spawn(F1), - ?line spawn(F1), - ?line spawn(F1), - ?line spawn(F1), - ?line spawn(F1), - ?line spawn(F1), - ?line spawn(F1), - ?line ok = ?EXPECT_ACCEPTS([],500), - ?line spawn(F2), - ?line spawn(F2), - ?line spawn(F2), - ?line spawn(F2), - ?line spawn(F2), - ?line spawn(F2), - ?line spawn(F2), - ?line spawn(F2), - ?line ok = ?EXPECT_ACCEPTS([{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}}],15000), - ?line ok = ?EXPECT_CONNECTS([{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}}],1000), + {ok,LS}=gen_tcp:listen(0,[]), + Parent = self(), + {ok,PortNo}=inet:port(LS), + F1 = fun() -> Parent ! {accepted,self(),gen_tcp:accept(LS)} end, + F2 = fun() -> Parent ! {connected,self(),gen_tcp:connect("localhost",PortNo,[])} end, + Ns = lists:seq(1,8), + _ = [spawn(F1) || _ <- Ns], + ok = ?EXPECT_ACCEPTS([],1,500), % wait for tmo + _ = [spawn(F2) || _ <- Ns], + ok = ?EXPECT_ACCEPTS([{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}}],8,15000), + ok = ?EXPECT_CONNECTS([{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}},{_,{ok,_}}],1000), ok. - flush(Msgs) -> erlang:yield(), receive Msg -> flush([Msg|Msgs]) @@ -1968,13 +2002,13 @@ accept_system_limit(doc) -> ["Check that accept returns {error, system_limit} " "(and not {error, enfile}) when running out of ports"]; accept_system_limit(Config) when is_list(Config) -> - ?line {ok, LS} = gen_tcp:listen(0, []), - ?line {ok, TcpPort} = inet:port(LS), + {ok, LS} = gen_tcp:listen(0, []), + {ok, TcpPort} = inet:port(LS), Me = self(), - ?line Connector = spawn_link(fun () -> connector(TcpPort, Me) end), + Connector = spawn_link(fun () -> connector(TcpPort, Me) end), receive {Connector, sync} -> Connector ! {self(), continue} end, - ?line ok = acceptor(LS, false, []), - ?line Connector ! stop, + ok = acceptor(LS, false, []), + Connector ! stop, ok. acceptor(LS, GotSL, A) -> @@ -2021,49 +2055,49 @@ active_once_closed(doc) -> ["Check that active once and tcp_close messages behave as expected"]; active_once_closed(Config) when is_list(Config) -> (fun() -> - ?line {Loop,A} = setup_closed_ao(), - ?line Loop({{error,closed},{error,econnaborted}}, + {Loop,A} = setup_closed_ao(), + Loop({{error,closed},{error,econnaborted}}, fun() -> gen_tcp:send(A,"Hello") end), - ?line ok = inet:setopts(A,[{active,once}]), - ?line ok = receive {tcp_closed, A} -> ok after 1000 -> error end, - ?line {error,einval} = inet:setopts(A,[{active,once}]), - ?line ok = receive {tcp_closed, A} -> error after 1000 -> ok end + ok = inet:setopts(A,[{active,once}]), + ok = receive {tcp_closed, A} -> ok after 1000 -> error end, + {error,einval} = inet:setopts(A,[{active,once}]), + ok = receive {tcp_closed, A} -> error after 1000 -> ok end end)(), (fun() -> - ?line {Loop,A} = setup_closed_ao(), - ?line Loop({{error,closed},{error,econnaborted}}, + {Loop,A} = setup_closed_ao(), + Loop({{error,closed},{error,econnaborted}}, fun() -> gen_tcp:send(A,"Hello") end), - ?line ok = inet:setopts(A,[{active,true}]), - ?line ok = receive {tcp_closed, A} -> ok after 1000 -> error end, - ?line {error,einval} = inet:setopts(A,[{active,true}]), - ?line ok = receive {tcp_closed, A} -> error after 1000 -> ok end + ok = inet:setopts(A,[{active,true}]), + ok = receive {tcp_closed, A} -> ok after 1000 -> error end, + {error,einval} = inet:setopts(A,[{active,true}]), + ok = receive {tcp_closed, A} -> error after 1000 -> ok end end)(), (fun() -> - ?line {Loop,A} = setup_closed_ao(), - ?line Loop({{error,closed},{error,econnaborted}}, + {Loop,A} = setup_closed_ao(), + Loop({{error,closed},{error,econnaborted}}, fun() -> gen_tcp:send(A,"Hello") end), - ?line ok = inet:setopts(A,[{active,true}]), - ?line ok = receive {tcp_closed, A} -> ok after 1000 -> error end, - ?line {error,einval} = inet:setopts(A,[{active,once}]), - ?line ok = receive {tcp_closed, A} -> error after 1000 -> ok end + ok = inet:setopts(A,[{active,true}]), + ok = receive {tcp_closed, A} -> ok after 1000 -> error end, + {error,einval} = inet:setopts(A,[{active,once}]), + ok = receive {tcp_closed, A} -> error after 1000 -> ok end end)(), (fun() -> - ?line {Loop,A} = setup_closed_ao(), - ?line Loop({{error,closed},{error,econnaborted}}, + {Loop,A} = setup_closed_ao(), + Loop({{error,closed},{error,econnaborted}}, fun() -> gen_tcp:send(A,"Hello") end), - ?line ok = inet:setopts(A,[{active,once}]), - ?line ok = receive {tcp_closed, A} -> ok after 1000 -> error end, - ?line {error,einval} = inet:setopts(A,[{active,true}]), - ?line ok = receive {tcp_closed, A} -> error after 1000 -> ok end + ok = inet:setopts(A,[{active,once}]), + ok = receive {tcp_closed, A} -> ok after 1000 -> error end, + {error,einval} = inet:setopts(A,[{active,true}]), + ok = receive {tcp_closed, A} -> error after 1000 -> ok end end)(), (fun() -> - ?line {Loop,A} = setup_closed_ao(), - ?line Loop({{error,closed},{error,econnaborted}}, + {Loop,A} = setup_closed_ao(), + Loop({{error,closed},{error,econnaborted}}, fun() -> gen_tcp:send(A,"Hello") end), - ?line ok = inet:setopts(A,[{active,false}]), - ?line ok = receive {tcp_closed, A} -> error after 1000 -> ok end, - ?line ok = inet:setopts(A,[{active,once}]), - ?line ok = receive {tcp_closed, A} -> ok after 1000 -> error end + ok = inet:setopts(A,[{active,false}]), + ok = receive {tcp_closed, A} -> error after 1000 -> ok end, + ok = inet:setopts(A,[{active,once}]), + ok = receive {tcp_closed, A} -> ok after 1000 -> error end end)(). send_timeout(suite) -> @@ -2072,10 +2106,10 @@ send_timeout(doc) -> ["Test the send_timeout socket option"]; send_timeout(Config) when is_list(Config) -> %% Basic - BasicFun = + BasicFun = fun(AutoClose) -> - ?line {Loop,A,RNode} = setup_timeout_sink(1000, AutoClose), - ?line {error,timeout} = + {Loop,A,RNode} = setup_timeout_sink(1000, AutoClose), + {error,timeout} = Loop(fun() -> Res = gen_tcp:send(A,<<1:10000>>), %%erlang:display(Res), @@ -2083,64 +2117,63 @@ send_timeout(Config) when is_list(Config) -> end), %% Check that the socket is not busy/closed... Error = after_send_timeout(AutoClose), - ?line {error,Error} = gen_tcp:send(A,<<"Hej">>), - ?line test_server:stop_node(RNode) + {error,Error} = gen_tcp:send(A,<<"Hej">>), + test_server:stop_node(RNode) end, BasicFun(false), BasicFun(true), %% Check timeout length - ?line Self = self(), - ?line Pid = - spawn(fun() -> - {Loop,A,RNode} = setup_timeout_sink(1000, true), - {error,timeout} = - Loop(fun() -> - Res = gen_tcp:send(A,<<1:10000>>), - %%erlang:display(Res), - Self ! Res, - Res - end), - test_server:stop_node(RNode) - end), - ?line Diff = get_max_diff(), - ?line io:format("Max time for send: ~p~n",[Diff]), - ?line true = (Diff > 500) and (Diff < 1500), + Self = self(), + Pid = spawn(fun() -> + {Loop,A,RNode} = setup_timeout_sink(1000, true), + {error,timeout} = Loop(fun() -> + Res = gen_tcp:send(A,<<1:10000>>), + %%erlang:display(Res), + Self ! Res, + Res + end), + test_server:stop_node(RNode) + end), + Diff = get_max_diff(), + io:format("Max time for send: ~p~n",[Diff]), + true = (Diff > 500) and (Diff < 1500), %% Let test_server slave die... - ?line Mon = erlang:monitor(process, Pid), - ?line receive {'DOWN',Mon,process,Pid,_} -> ok end, + Mon = erlang:monitor(process, Pid), + receive {'DOWN',Mon,process,Pid,_} -> ok end, %% Check that parallell writers do not hang forever - ParaFun = + ParaFun = fun(AutoClose) -> - ?line {Loop,A,RNode} = setup_timeout_sink(1000, AutoClose), + {Loop,A,RNode} = setup_timeout_sink(1000, AutoClose), SenderFun = fun() -> - {error,Error} = + {error,Error} = Loop(fun() -> gen_tcp:send(A, <<1:10000>>) end), Self ! {error,Error} end, - ?line spawn_link(SenderFun), - ?line spawn_link(SenderFun), - ?line receive - {error,timeout} -> ok - after 10000 -> - ?line exit(timeout) - end, + spawn_link(SenderFun), + spawn_link(SenderFun), + receive + {error,timeout} -> ok + after 10000 -> + exit(timeout) + end, NextErr = after_send_timeout(AutoClose), - ?line receive - {error,NextErr} -> ok - after 10000 -> - ?line exit(timeout) - end, - ?line {error,NextErr} = gen_tcp:send(A,<<"Hej">>), - ?line test_server:stop_node(RNode) + receive + {error,NextErr} -> ok + after 10000 -> + exit(timeout) + end, + {error,NextErr} = gen_tcp:send(A,<<"Hej">>), + test_server:stop_node(RNode) end, ParaFun(false), ParaFun(true), ok. + mad_sender(S) -> - {_, _, USec} = now(), - case gen_tcp:send(S, integer_to_list(USec)) of + U = rand:uniform(1000000), + case gen_tcp:send(S, integer_to_list(U)) of ok -> mad_sender(S); Err -> @@ -2166,25 +2199,25 @@ send_timeout_active(Config) when is_list(Config) -> %% Basic BasicFun = fun(AutoClose) -> - ?line {Loop,A,RNode,C} = setup_active_timeout_sink(1, AutoClose), + {Loop,A,RNode,C} = setup_active_timeout_sink(1, AutoClose), inet:setopts(A, [{active, once}]), - ?line Mad = spawn_link(RNode,fun() -> mad_sender(C) end), - ?line {error,timeout} = - Loop(fun() -> - receive - {tcp, _Sock, _Data} -> - inet:setopts(A, [{active, once}]), - Res = gen_tcp:send(A,lists:duplicate(1000, $a)), - %erlang:display(Res), - Res; - Err -> - io:format("sock closed: ~p~n", [Err]), - Err - end - end), - unlink(Mad), + Mad = spawn_link(RNode,fun() -> mad_sender(C) end), + {error,timeout} = + Loop(fun() -> + receive + {tcp, _Sock, _Data} -> + inet:setopts(A, [{active, once}]), + Res = gen_tcp:send(A,lists:duplicate(1000, $a)), + %erlang:display(Res), + Res; + Err -> + io:format("sock closed: ~p~n", [Err]), + Err + end + end), + unlink(Mad), exit(Mad,kill), - ?line test_server:stop_node(RNode) + test_server:stop_node(RNode) end, BasicFun(false), flush(), @@ -2208,10 +2241,10 @@ get_max_diff() -> end. get_max_diff(Max) -> - T1 = millistamp(), + T1 = millis(), receive ok -> - Diff = millistamp() - T1, + Diff = millis() - T1, if Diff > Max -> get_max_diff(Diff); @@ -2219,7 +2252,7 @@ get_max_diff(Max) -> get_max_diff(Max) end; {error,timeout} -> - Diff = millistamp() - T1, + Diff = millis() - T1, if Diff > Max -> Diff; @@ -2227,29 +2260,29 @@ get_max_diff(Max) -> Max end after 10000 -> - exit(timeout) + exit(timeout) end. setup_closed_ao() -> Dir = filename:dirname(code:which(?MODULE)), {ok,R} = test_server:start_node(test_default_options_slave,slave, - [{args,"-pa " ++ Dir}]), + [{args,"-pa " ++ Dir}]), Host = list_to_atom(lists:nth(2,string:tokens(atom_to_list(node()),"@"))), {ok, L} = gen_tcp:listen(0, [{active,false},{packet,2}]), - Fun = fun(F) -> - receive - {From,X} when is_function(X) -> - From ! {self(),X()}, F(F); - die -> ok - end - end, + Fun = fun(F) -> + receive + {From,X} when is_function(X) -> + From ! {self(),X()}, F(F); + die -> ok + end + end, Pid = rpc:call(R,erlang,spawn,[fun() -> Fun(Fun) end]), {ok, Port} = inet:port(L), - Remote = fun(Fu) -> - Pid ! {self(), Fu}, - receive {Pid,X} -> X - end - end, + Remote = fun(Fu) -> + Pid ! {self(), Fu}, + receive {Pid,X} -> X + end + end, {ok, C} = Remote(fun() -> gen_tcp:connect(Host,Port, [{active,false},{packet,2}]) @@ -2257,113 +2290,109 @@ setup_closed_ao() -> {ok,A} = gen_tcp:accept(L), gen_tcp:send(A,"Hello"), {ok, "Hello"} = Remote(fun() -> gen_tcp:recv(C,0) end), - ok = Remote(fun() -> gen_tcp:close(C) end), - Loop2 = fun(_,_,_,0) -> + ok = Remote(fun() -> gen_tcp:close(C) end), + Loop2 = fun(_,_,_,0) -> {failure, timeout}; - (L2,{MA,MB},F2,N) -> - case F2() of - MA -> MA; - MB -> MB; - Other -> io:format("~p~n",[Other]), - receive after 1000 -> ok end, - L2(L2,{MA,MB},F2,N-1) - end + (L2,{MA,MB},F2,N) -> + case F2() of + MA -> MA; + MB -> MB; + Other -> io:format("~p~n",[Other]), + receive after 1000 -> ok end, + L2(L2,{MA,MB},F2,N-1) + end end, Loop = fun(Match2,F3) -> Loop2(Loop2,Match2,F3,10) end, test_server:stop_node(R), {Loop,A}. setup_timeout_sink(Timeout, AutoClose) -> - ?line Dir = filename:dirname(code:which(?MODULE)), - ?line {ok,R} = test_server:start_node(test_default_options_slave,slave, - [{args,"-pa " ++ Dir}]), - ?line Host = list_to_atom(lists:nth(2,string:tokens(atom_to_list(node()),"@"))), - ?line {ok, L} = gen_tcp:listen(0, [{active,false},{packet,2}, + Dir = filename:dirname(code:which(?MODULE)), + {ok,R} = test_server:start_node(test_default_options_slave,slave, + [{args,"-pa " ++ Dir}]), + Host = list_to_atom(lists:nth(2,string:tokens(atom_to_list(node()),"@"))), + {ok, L} = gen_tcp:listen(0, [{active,false},{packet,2}, {send_timeout,Timeout}, {send_timeout_close,AutoClose}]), - ?line Fun = fun(F) -> - receive - {From,X} when is_function(X) -> - From ! {self(),X()}, F(F); - die -> ok - end - end, - ?line Pid = rpc:call(R,erlang,spawn,[fun() -> Fun(Fun) end]), - ?line {ok, Port} = inet:port(L), - ?line Remote = fun(Fu) -> - Pid ! {self(), Fu}, - receive {Pid,X} -> X - end + Fun = fun(F) -> + receive + {From,X} when is_function(X) -> + From ! {self(),X()}, F(F); + die -> ok + end + end, + Pid = rpc:call(R,erlang,spawn,[fun() -> Fun(Fun) end]), + {ok, Port} = inet:port(L), + Remote = fun(Fu) -> + Pid ! {self(), Fu}, + receive {Pid,X} -> X + end end, - ?line {ok, C} = Remote(fun() -> + {ok, C} = Remote(fun() -> gen_tcp:connect(Host,Port, - [{active,false},{packet,2}]) + [{active,false},{packet,2}]) end), - ?line {ok,A} = gen_tcp:accept(L), - ?line gen_tcp:send(A,"Hello"), - ?line {ok, "Hello"} = Remote(fun() -> gen_tcp:recv(C,0) end), - ?line Loop2 = fun(_,_,0) -> - {failure, timeout}; - (L2,F2,N) -> + {ok,A} = gen_tcp:accept(L), + gen_tcp:send(A,"Hello"), + {ok, "Hello"} = Remote(fun() -> gen_tcp:recv(C,0) end), + Loop2 = fun(_,_,0) -> + {failure, timeout}; + (L2,F2,N) -> Ret = F2(), io:format("~p~n",[Ret]), case Ret of - ok -> receive after 1 -> ok end, + ok -> receive after 1 -> ok end, L2(L2,F2,N-1); Other -> Other - end + end end, - ?line Loop = fun(F3) -> Loop2(Loop2,F3,1000) end, + Loop = fun(F3) -> Loop2(Loop2,F3,1000) end, {Loop,A,R}. setup_active_timeout_sink(Timeout, AutoClose) -> - ?line Dir = filename:dirname(code:which(?MODULE)), - ?line {ok,R} = test_server:start_node(test_default_options_slave,slave, - [{args,"-pa " ++ Dir}]), - ?line Host = list_to_atom(lists:nth(2,string:tokens(atom_to_list(node()),"@"))), - ?line {ok, L} = gen_tcp:listen(0, [binary,{active,false},{packet,0},{nodelay, true},{keepalive, true}, + Dir = filename:dirname(code:which(?MODULE)), + {ok,R} = test_server:start_node(test_default_options_slave,slave, + [{args,"-pa " ++ Dir}]), + Host = list_to_atom(lists:nth(2,string:tokens(atom_to_list(node()),"@"))), + {ok, L} = gen_tcp:listen(0, [binary,{active,false},{packet,0},{nodelay, true},{keepalive, true}, {send_timeout,Timeout}, {send_timeout_close,AutoClose}]), - ?line Fun = fun(F) -> - receive - {From,X} when is_function(X) -> - From ! {self(),X()}, F(F); - die -> ok - end - end, - ?line Pid = rpc:call(R,erlang,spawn,[fun() -> Fun(Fun) end]), - ?line {ok, Port} = inet:port(L), - ?line Remote = fun(Fu) -> - Pid ! {self(), Fu}, - receive {Pid,X} -> X - end + Fun = fun(F) -> + receive + {From,X} when is_function(X) -> + From ! {self(),X()}, F(F); + die -> ok + end + end, + Pid = rpc:call(R,erlang,spawn,[fun() -> Fun(Fun) end]), + {ok, Port} = inet:port(L), + Remote = fun(Fu) -> + Pid ! {self(), Fu}, + receive {Pid,X} -> X + end end, - ?line {ok, C} = Remote(fun() -> + {ok, C} = Remote(fun() -> gen_tcp:connect(Host,Port, - [{active,false}]) + [{active,false}]) end), - ?line {ok,A} = gen_tcp:accept(L), - ?line gen_tcp:send(A,"Hello"), - ?line {ok, "H"++_} = Remote(fun() -> gen_tcp:recv(C,0) end), - ?line Loop2 = fun(_,_,0) -> - {failure, timeout}; - (L2,F2,N) -> + {ok,A} = gen_tcp:accept(L), + gen_tcp:send(A,"Hello"), + {ok, "H"++_} = Remote(fun() -> gen_tcp:recv(C,0) end), + Loop2 = fun(_,_,0) -> + {failure, timeout}; + (L2,F2,N) -> Ret = F2(), io:format("~p~n",[Ret]), case Ret of - ok -> receive after 1 -> ok end, + ok -> receive after 1 -> ok end, L2(L2,F2,N-1); Other -> Other - end + end end, - ?line Loop = fun(F3) -> Loop2(Loop2,F3,1000) end, + Loop = fun(F3) -> Loop2(Loop2,F3,1000) end, {Loop,A,R,C}. -millistamp() -> - {Mega, Secs, Micros} = erlang:now(), - (Micros div 1000) + Secs * 1000 + Mega * 1000000000. - has_superfluous_schedulers() -> case {erlang:system_info(schedulers), erlang:system_info(logical_processors)} of @@ -2378,22 +2407,22 @@ otp_7731(doc) -> "Leaking message from inet_drv {inet_reply,P,ok} " "when a socket sending resumes working after a send_timeout"; otp_7731(Config) when is_list(Config) -> - ?line ServerPid = spawn_link(?MODULE, otp_7731_server, [self()]), - ?line receive {ServerPid, ready, PortNum} -> ok end, + ServerPid = spawn_link(?MODULE, otp_7731_server, [self()]), + receive {ServerPid, ready, PortNum} -> ok end, - ?line {ok, Socket} = gen_tcp:connect("localhost", PortNum, - [binary, {active, false}, {packet, raw}, - {send_timeout, 1000}]), + {ok, Socket} = gen_tcp:connect("localhost", PortNum, + [binary, {active, false}, {packet, raw}, + {send_timeout, 1000}]), otp_7731_send(Socket), io:format("Sending complete...\n",[]), ServerPid ! {self(), recv}, - receive {ServerPid, ok} -> ok end, - + receive {ServerPid, ok} -> ok end, + io:format("Client waiting for leaking messages...\n",[]), %% Now make sure inet_drv does not leak any internal messages. receive Msg -> - ?line test_server:fail({unexpected, Msg}) + test_server:fail({unexpected, Msg}) after 1000 -> ok end, @@ -2403,15 +2432,15 @@ otp_7731(Config) when is_list(Config) -> otp_7731_send(Socket) -> Bin = <<1:10000>>, io:format("Client sending ~p bytes...\n",[size(Bin)]), - ?line case gen_tcp:send(Socket, Bin) of - ok -> otp_7731_send(Socket); - {error,timeout} -> ok - end. + case gen_tcp:send(Socket, Bin) of + ok -> otp_7731_send(Socket); + {error,timeout} -> ok + end. otp_7731_server(ClientPid) -> - ?line {ok, LSocket} = gen_tcp:listen(0, [binary, {packet, raw}, - {active, false}]), - ?line {ok, {_, PortNum}} = inet:sockname(LSocket), + {ok, LSocket} = gen_tcp:listen(0, [binary, {packet, raw}, + {active, false}]), + {ok, {_, PortNum}} = inet:sockname(LSocket), io:format("Listening on ~w with port number ~p\n", [LSocket, PortNum]), ClientPid ! {self(), ready, PortNum}, @@ -2433,7 +2462,7 @@ otp_7731_server(ClientPid) -> otp_7731_recv(Socket) -> - ?line case gen_tcp:recv(Socket, 0, 1000) of + case gen_tcp:recv(Socket, 0, 1000) of {ok, Bin} -> io:format("Server received ~p bytes\n",[size(Bin)]), otp_7731_recv(Socket); @@ -2452,21 +2481,21 @@ zombie_sockets(Config) when is_list(Config) -> register(zombie_collector,self()), Calls = 10, Server = spawn_link(?MODULE, zombie_server,[self(), Calls]), - ?line {Server, ready, PortNum} = receive Msg -> Msg end, + {Server, ready, PortNum} = receive Msg -> Msg end, io:format("Ports before = ~p\n",[lists:sort(erlang:ports())]), zombie_client_loop(Calls, PortNum), Ports = lists:sort(zombie_collector(Calls,[])), Server ! terminate, io:format("Collected ports = ~p\n",[Ports]), - ?line [] = zombies_alive(Ports, 10), + [] = zombies_alive(Ports, 10), timer:sleep(1000), ok. zombie_client_loop(0, _) -> ok; zombie_client_loop(N, PortNum) when is_integer(PortNum) -> - ?line {ok, Socket} = gen_tcp:connect("localhost", PortNum, - [binary, {active, false}, {packet, raw}]), - ?line gen_tcp:close(Socket), % to make server recv fail + {ok, Socket} = gen_tcp:connect("localhost", PortNum, + [binary, {active, false}, {packet, raw}]), + gen_tcp:close(Socket), % to make server recv fail zombie_client_loop(N-1, PortNum). @@ -2495,19 +2524,19 @@ zombies_alive(Ports, WaitSec) -> end. zombie_server(Pid, Calls) -> - ?line {ok, LSocket} = gen_tcp:listen(0, [binary, {packet, raw}, - {active, false}, {backlog, Calls}]), - ?line {ok, {_, PortNum}} = inet:sockname(LSocket), + {ok, LSocket} = gen_tcp:listen(0, [binary, {packet, raw}, + {active, false}, {backlog, Calls}]), + {ok, {_, PortNum}} = inet:sockname(LSocket), io:format("Listening on ~w with port number ~p\n", [LSocket, PortNum]), BigBin = list_to_binary(lists:duplicate(100*1024, 77)), Pid ! {self(), ready, PortNum}, zombie_accept_loop(LSocket, BigBin, Calls), - ?line terminate = receive Msg -> Msg end. + terminate = receive Msg -> Msg end. zombie_accept_loop(_, _, 0) -> ok; zombie_accept_loop(Socket, BigBin, Calls) -> - ?line case gen_tcp:accept(Socket) of + case gen_tcp:accept(Socket) of {ok, NewSocket} -> spawn_link(fun() -> zombie_serve_client(NewSocket, BigBin) end), zombie_accept_loop(Socket, BigBin, Calls-1); @@ -2517,29 +2546,27 @@ zombie_accept_loop(Socket, BigBin, Calls) -> zombie_serve_client(Socket, Bin) -> %%io:format("Got connection on ~p\n",[Socket]), - ?line gen_tcp:send(Socket, Bin), + gen_tcp:send(Socket, Bin), %%io:format("Sent data, waiting for reply on ~p\n",[Socket]), - ?line case gen_tcp:recv(Socket, 4) of + case gen_tcp:recv(Socket, 4) of {error,closed} -> ok; {error,econnaborted} -> ok % may be returned on Windows end, %%io:format("Closing ~p\n",[Socket]), - ?line gen_tcp:close(Socket), + gen_tcp:close(Socket), zombie_collector ! {closed, Socket}. - - otp_7816(suite) -> []; otp_7816(doc) -> "Hanging send on windows when sending iolist with more than 16 binaries."; otp_7816(Config) when is_list(Config) -> Client = self(), - ?line Server = spawn_link(fun()-> otp_7816_server(Client) end), - ?line receive {Server, ready, PortNum} -> ok end, + Server = spawn_link(fun()-> otp_7816_server(Client) end), + receive {Server, ready, PortNum} -> ok end, - ?line {ok, Socket} = gen_tcp:connect("localhost", PortNum, - [binary, {active, false}, {packet, 4}, - {send_timeout, 10}]), + {ok, Socket} = gen_tcp:connect("localhost", PortNum, + [binary, {active, false}, {packet, 4}, + {send_timeout, 10}]), %% We use the undocumented feature that sending can be resumed after %% a send_timeout without any data loss if the peer starts to receive data. %% Unless of course the 7816-bug is in affect, in which case the write event @@ -2549,9 +2576,9 @@ otp_7816(Config) when is_list(Config) -> io:format("Sending complete...\n",[]), - ?line ok = gen_tcp:close(Socket), + ok = gen_tcp:close(Socket), Server ! {self(), closed}, - ?line {Server, closed} = receive M -> M end. + {Server, closed} = receive M -> M end. otp_7816_send(Socket, BinNr, BinSize, Server) -> @@ -2559,7 +2586,7 @@ otp_7816_send(Socket, BinNr, BinSize, Server) -> SentBytes = otp_7816_send_data(Socket, Data, 0) * BinNr * BinSize, io:format("Client sent ~p bytes...\n",[SentBytes]), Server ! {self(),recv,SentBytes}, - ?line {Server, ok} = receive M -> M end. + {Server, ok} = receive M -> M end. @@ -2574,15 +2601,15 @@ otp_7816_send_data(Socket, Data, Loops) -> otp_7816_server(Client) -> - ?line {ok, LSocket} = gen_tcp:listen(0, [binary, {packet, 4}, + {ok, LSocket} = gen_tcp:listen(0, [binary, {packet, 4}, {active, false}]), - ?line {ok, {_, PortNum}} = inet:sockname(LSocket), + {ok, {_, PortNum}} = inet:sockname(LSocket), io:format("Listening on ~w with port number ~p\n", [LSocket, PortNum]), Client ! {self(), ready, PortNum}, - ?line {ok, CSocket} = gen_tcp:accept(LSocket), + {ok, CSocket} = gen_tcp:accept(LSocket), io:format("Server got connection...\n",[]), - ?line gen_tcp:close(LSocket), + gen_tcp:close(LSocket), otp_7816_server_loop(CSocket), @@ -2596,13 +2623,13 @@ otp_7816_server_loop(CSocket) -> {Client, recv, RecvBytes} -> io:format("Server start receiving...\n",[]), - ?line ok = otp_7816_recv(CSocket, RecvBytes), + ok = otp_7816_recv(CSocket, RecvBytes), Client ! {self(), ok}, otp_7816_server_loop(CSocket); {Client, closed} -> - ?line {error, closed} = gen_tcp:recv(CSocket, 0, 1000), + {error, closed} = gen_tcp:recv(CSocket, 0, 1000), Client ! {self(), closed} end. @@ -2611,7 +2638,7 @@ otp_7816_recv(_, 0) -> io:format("Server got all.\n",[]), ok; otp_7816_recv(CSocket, BytesLeft) -> - ?line case gen_tcp:recv(CSocket, 0, 1000) of + case gen_tcp:recv(CSocket, 0, 1000) of {ok, Bin} when byte_size(Bin) =< BytesLeft -> io:format("Server received ~p of ~p bytes.\n",[size(Bin), BytesLeft]), otp_7816_recv(CSocket, BytesLeft - byte_size(Bin)); @@ -2623,8 +2650,8 @@ otp_7816_recv(CSocket, BytesLeft) -> otp_8102(doc) -> ["Receive a packet with a faulty packet header"]; otp_8102(suite) -> []; otp_8102(Config) when is_list(Config) -> - ?line {ok, LSocket} = gen_tcp:listen(0, []), - ?line {ok, {_, PortNum}} = inet:sockname(LSocket), + {ok, LSocket} = gen_tcp:listen(0, []), + {ok, {_, PortNum}} = inet:sockname(LSocket), io:format("Listening on ~w with port number ~p\n", [LSocket, PortNum]), [otp_8102_do(LSocket, PortNum, otp_8102_packet(Type,Size)) @@ -2644,18 +2671,18 @@ otp_8102_packet({cdr,little}, Size) -> otp_8102_do(LSocket, PortNum, {Bin,PType}) -> io:format("Connect with packet option ~p ...\n",[PType]), - ?line {ok, RSocket} = gen_tcp:connect("localhost", PortNum, [binary, + {ok, RSocket} = gen_tcp:connect("localhost", PortNum, [binary, {packet,PType}, {active,true}]), - ?line {ok, SSocket} = gen_tcp:accept(LSocket), + {ok, SSocket} = gen_tcp:accept(LSocket), io:format("Got connection, sending ~p...\n",[Bin]), - ?line ok = gen_tcp:send(SSocket, Bin), + ok = gen_tcp:send(SSocket, Bin), io:format("Sending complete...\n",[]), - ?line {tcp_error,RSocket,emsgsize} = receive M -> M end, + {tcp_error,RSocket,emsgsize} = receive M -> M end, io:format("Got error msg, ok.\n",[]), gen_tcp:close(SSocket), @@ -2664,61 +2691,61 @@ otp_8102_do(LSocket, PortNum, {Bin,PType}) -> otp_9389(doc) -> ["Verify packet_size handles long HTTP header lines"]; otp_9389(suite) -> []; otp_9389(Config) when is_list(Config) -> - ?line {ok, LS} = gen_tcp:listen(0, [{active,false}]), - ?line {ok, {_, PortNum}} = inet:sockname(LS), + {ok, LS} = gen_tcp:listen(0, [{active,false}]), + {ok, {_, PortNum}} = inet:sockname(LS), io:format("Listening on ~w with port number ~p\n", [LS, PortNum]), OrigLinkHdr = "/" ++ string:chars($S, 8192), _Server = spawn_link( fun() -> - ?line {ok, S} = gen_tcp:accept(LS), - ?line ok = inet:setopts(S, [{packet_size, 16384}]), - ?line ok = otp_9389_loop(S, OrigLinkHdr), - ?line ok = gen_tcp:close(S) + {ok, S} = gen_tcp:accept(LS), + ok = inet:setopts(S, [{packet_size, 16384}]), + ok = otp_9389_loop(S, OrigLinkHdr), + ok = gen_tcp:close(S) end), - ?line {ok, S} = gen_tcp:connect("localhost", PortNum, + {ok, S} = gen_tcp:connect("localhost", PortNum, [binary, {active, false}]), Req = "GET / HTTP/1.1\r\n" ++ "Host: localhost\r\n" ++ "Link: " ++ OrigLinkHdr ++ "\r\n\r\n", - ?line ok = gen_tcp:send(S, Req), - ?line ok = inet:setopts(S, [{packet, http}]), - ?line {ok, {http_response, {1,1}, 200, "OK"}} = gen_tcp:recv(S, 0), - ?line ok = inet:setopts(S, [{packet, httph}, {packet_size, 16384}]), - ?line {ok, {http_header, _, 'Content-Length', _, "0"}} = gen_tcp:recv(S, 0), - ?line {ok, {http_header, _, "Link", _, LinkHdr}} = gen_tcp:recv(S, 0), - ?line true = (LinkHdr == OrigLinkHdr), + ok = gen_tcp:send(S, Req), + ok = inet:setopts(S, [{packet, http}]), + {ok, {http_response, {1,1}, 200, "OK"}} = gen_tcp:recv(S, 0), + ok = inet:setopts(S, [{packet, httph}, {packet_size, 16384}]), + {ok, {http_header, _, 'Content-Length', _, "0"}} = gen_tcp:recv(S, 0), + {ok, {http_header, _, "Link", _, LinkHdr}} = gen_tcp:recv(S, 0), + true = (LinkHdr == OrigLinkHdr), ok = gen_tcp:close(S), ok = gen_tcp:close(LS), ok. otp_9389_loop(S, OrigLinkHdr) -> - ?line ok = inet:setopts(S, [{active,once},{packet,http}]), + ok = inet:setopts(S, [{active,once},{packet,http}]), receive {http, S, {http_request, 'GET', _, _}} -> - ?line ok = otp_9389_loop(S, OrigLinkHdr, undefined) + ok = otp_9389_loop(S, OrigLinkHdr, undefined) after 3000 -> - ?line error({timeout,request_line}) + error({timeout,request_line}) end. otp_9389_loop(S, OrigLinkHdr, ok) -> - ?line Resp = "HTTP/1.1 200 OK\r\nContent-length: 0\r\n" ++ + Resp = "HTTP/1.1 200 OK\r\nContent-length: 0\r\n" ++ "Link: " ++ OrigLinkHdr ++ "\r\n\r\n", - ?line ok = gen_tcp:send(S, Resp); + ok = gen_tcp:send(S, Resp); otp_9389_loop(S, OrigLinkHdr, State) -> - ?line ok = inet:setopts(S, [{active,once}, {packet,httph}]), + ok = inet:setopts(S, [{active,once}, {packet,httph}]), receive {http, S, http_eoh} -> - ?line otp_9389_loop(S, OrigLinkHdr, ok); + otp_9389_loop(S, OrigLinkHdr, ok); {http, S, {http_header, _, "Link", _, LinkHdr}} -> - ?line LinkHdr = OrigLinkHdr, - ?line otp_9389_loop(S, OrigLinkHdr, State); + LinkHdr = OrigLinkHdr, + otp_9389_loop(S, OrigLinkHdr, State); {http, S, {http_header, _, _Hdr, _, _Val}} -> - ?line otp_9389_loop(S, OrigLinkHdr, State); + otp_9389_loop(S, OrigLinkHdr, State); {http, S, {http_error, Err}} -> - ?line error({error, Err}) + error({error, Err}) after 3000 -> - ?line error({timeout,header}) + error({timeout,header}) end. wrapping_oct(doc) -> @@ -2729,7 +2756,7 @@ wrapping_oct(Config) when is_list(Config) -> {ok,Sock} = gen_tcp:listen(0,[{active,false},{mode,binary}]), {ok,Port} = inet:port(Sock), spawn_link(?MODULE,oct_acceptor,[Sock]), - Res = oct_datapump(Port,16#1FFFFFFFF), + Res = oct_datapump(Port,16#10000FFFF), gen_tcp:close(Sock), ok = Res, ok. diff --git a/lib/kernel/test/inet_SUITE.erl b/lib/kernel/test/inet_SUITE.erl index 44a32fc1ec..c77de9316f 100644 --- a/lib/kernel/test/inet_SUITE.erl +++ b/lib/kernel/test/inet_SUITE.erl @@ -569,8 +569,11 @@ parse_address(Config) when is_list(Config) -> "::-1", "::g", "f:f11::10100:2", + "f:f11::01100:2", "::17000", + "::01700", "10000::", + "01000::", "::8:7:6:5:4:3:2:1", "8:7:6:5:4:3:2:1::", "8:7:6:5:4::3:2:1", diff --git a/lib/kernel/test/interactive_shell_SUITE.erl b/lib/kernel/test/interactive_shell_SUITE.erl index 3fb7c68886..89c574b025 100644 --- a/lib/kernel/test/interactive_shell_SUITE.erl +++ b/lib/kernel/test/interactive_shell_SUITE.erl @@ -718,8 +718,7 @@ toerl_loop(Port,Acc) -> end. millistamp() -> - {Mega, Secs, Micros} = erlang:now(), - (Micros div 1000) + Secs * 1000 + Mega * 1000000000. + erlang:monotonic_time(milli_seconds). get_data_within(Port, X, Acc) when X =< 0 -> ?dbg({get_data_within, X, Acc, ?LINE}), diff --git a/lib/kernel/test/rpc_SUITE.erl b/lib/kernel/test/rpc_SUITE.erl index 7adef49014..867b448b36 100644 --- a/lib/kernel/test/rpc_SUITE.erl +++ b/lib/kernel/test/rpc_SUITE.erl @@ -456,32 +456,33 @@ called_throws(Config) when is_list(Config) -> call_benchmark(Config) when is_list(Config) -> Timetrap = ?t:timetrap(?t:seconds(120)), - ?line PA = filename:dirname(code:which(?MODULE)), - ?line {ok, Node} = ?t:start_node(rpc_SUITE_call_benchmark, slave, - [{args, "-pa " ++ PA}]), + PA = filename:dirname(code:which(?MODULE)), + {ok, Node} = ?t:start_node(rpc_SUITE_call_benchmark, slave, + [{args, "-pa " ++ PA}]), Iter = case erlang:system_info(modified_timing_level) of undefined -> 10000; - _ -> 500 %Moified timing - spawn is slower + _ -> 500 %Modified timing - spawn is slower end, - ?line do_call_benchmark(Node, Iter), + Res = do_call_benchmark(Node, Iter), + ?t:stop_node(Node), ?t:timetrap_cancel(Timetrap), - ok. + Res. do_call_benchmark(Node, M) when is_integer(M), M > 0 -> - do_call_benchmark(Node, erlang:now(), 0, M). - -do_call_benchmark(Node, {A,B,C}, M, M) -> - ?line {D,E,F} = erlang:now(), - ?line T = float(D-A)*1000000.0 + float(E-B) + float(F-C)*0.000001, - ?line Q = 3.0 * float(M) / T, - ?line ?t:stop_node(Node), - {comment, - lists:flatten([float_to_list(Q)," RPC calls per second"])}; -do_call_benchmark(Node, Then, I, M) -> - ?line Node = rpc:call(Node, erlang, node, []), - ?line _ = rpc:call(Node, erlang, whereis, [rex]), - ?line 3 = rpc:call(Node, erlang, '+', [1,2]), - ?line do_call_benchmark(Node, Then, I+1, M). + {Micros,ok} = timer:tc(fun() -> + do_call_benchmark(Node, 0, M) + end), + Calls = 3*M, + S = io_lib:format("~p RPC calls/second", [Calls*1000000 div Micros]), + {comment,lists:flatten(S)}. + +do_call_benchmark(_Node, M, M) -> + ok; +do_call_benchmark(Node, I, M) -> + Node = rpc:call(Node, erlang, node, []), + _ = rpc:call(Node, erlang, whereis, [rex]), + 3 = rpc:call(Node, erlang, '+', [1,2]), + do_call_benchmark(Node, I+1, M). async_call(Config) when is_list(Config) -> Dog = ?t:timetrap(?t:seconds(120)), diff --git a/lib/megaco/doc/src/megaco.xml b/lib/megaco/doc/src/megaco.xml index dff1c3afc6..0a8dfe8a13 100644 --- a/lib/megaco/doc/src/megaco.xml +++ b/lib/megaco/doc/src/megaco.xml @@ -336,7 +336,7 @@ megaco_incr_timer() = #megaco_incr_timer{} <tag><c><![CDATA[request_keep_alive_timeout]]></c></tag> <item> <p>Specifies the timeout time for the request-keep-alive timer. </p> - <p>This timer is started when the <em>first</em> reply to an asynchroneous + <p>This timer is started when the <em>first</em> reply to an asynchronous request (issued using the <seealso marker="megaco#cast">megaco:cast/3</seealso> function) arrives. As long as this timer is running, replies will @@ -837,7 +837,7 @@ megaco_incr_timer() = #megaco_incr_timer{} <tag><c><![CDATA[request_keep_alive_timeout]]></c></tag> <item> <p>Specifies the timeout time for the request-keep-alive timer. </p> - <p>This timer is started when the <em>first</em> reply to an asynchroneous + <p>This timer is started when the <em>first</em> reply to an asynchronous request (issued using the <seealso marker="megaco#cast">megaco:cast/3</seealso> function) arrives. As long as this timer is running, replies will diff --git a/lib/megaco/src/app/megaco.app.src b/lib/megaco/src/app/megaco.app.src index 6ab85a1bbc..573b1857f6 100644 --- a/lib/megaco/src/app/megaco.app.src +++ b/lib/megaco/src/app/megaco.app.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2013. All Rights Reserved. +%% Copyright Ericsson AB 1999-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -113,8 +113,8 @@ {applications, [stdlib, kernel]}, {env, []}, {mod, {megaco_sup, []}}, - {runtime_dependencies, ["stdlib-2.0","runtime_tools-1.8.14","kernel-3.0", - "et-1.5","erts-6.0","debugger-4.0", + {runtime_dependencies, ["stdlib-2.5","runtime_tools-1.8.14","kernel-3.0", + "et-1.5","erts-7.0","debugger-4.0", "asn1-3.0"]} ]}. diff --git a/lib/megaco/src/app/megaco.appup.src b/lib/megaco/src/app/megaco.appup.src index 92504e8e87..1c55a92b55 100644 --- a/lib/megaco/src/app/megaco.appup.src +++ b/lib/megaco/src/app/megaco.appup.src @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2014. All Rights Reserved. +%% Copyright Ericsson AB 2001-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -183,11 +183,15 @@ %% | %% v %% 3.17.3 +%% | +%% v +%% 3.18 %% %% {"%VSN%", [ + {"3.17.3", [{restart_application,megaco}]}, {"3.17.2", []}, {"3.17.1", [{restart_application,megaco}]}, {"3.17.0.3", [{restart_application,megaco}]}, @@ -202,6 +206,7 @@ } ], [ + {"3.17.3", [{restart_application,megaco}]}, {"3.17.2", []}, {"3.17.1", [{restart_application,megaco}]}, {"3.17.0.3", [{restart_application,megaco}]}, diff --git a/lib/megaco/src/engine/megaco_trans_sender.erl b/lib/megaco/src/engine/megaco_trans_sender.erl index 710fef405a..e07f404289 100644 --- a/lib/megaco/src/engine/megaco_trans_sender.erl +++ b/lib/megaco/src/engine/megaco_trans_sender.erl @@ -672,8 +672,7 @@ to(To, Start) -> %% Time in milli seconds t() -> - {A,B,C} = erlang:now(), - A*1000000000+B*1000+(C div 1000). + erlang:monotonic_time(milli_seconds). warning_msg(F, A) -> ?megaco_warning("Transaction sender: " ++ F, A). diff --git a/lib/megaco/vsn.mk b/lib/megaco/vsn.mk index 8687d622e9..ede36e3fe6 100644 --- a/lib/megaco/vsn.mk +++ b/lib/megaco/vsn.mk @@ -2,7 +2,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2014. All Rights Reserved. +# Copyright Ericsson AB 1997-2015. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -18,6 +18,6 @@ # %CopyrightEnd% APPLICATION = megaco -MEGACO_VSN = 3.17.3 +MEGACO_VSN = 3.18 PRE_VSN = APP_VSN = "$(APPLICATION)-$(MEGACO_VSN)$(PRE_VSN)" diff --git a/lib/mnesia/doc/src/Makefile b/lib/mnesia/doc/src/Makefile index 6a72b98ebc..f41442f739 100644 --- a/lib/mnesia/doc/src/Makefile +++ b/lib/mnesia/doc/src/Makefile @@ -58,7 +58,6 @@ XML_CHAPTER_FILES = \ Mnesia_App_A.xml \ Mnesia_App_B.xml \ Mnesia_App_C.xml \ - Mnesia_App_D.xml \ notes.xml BOOK_FILES = book.xml diff --git a/lib/mnesia/doc/src/Mnesia_App_A.xml b/lib/mnesia/doc/src/Mnesia_App_A.xml deleted file mode 100644 index 62dbffa14a..0000000000 --- a/lib/mnesia/doc/src/Mnesia_App_A.xml +++ /dev/null @@ -1,87 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>1997</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - The contents of this file are subject to the Erlang Public License, - Version 1.1, (the "License"); you may not use this file except in - compliance with the License. You should have received a copy of the - Erlang Public License along with this software. If not, it can be - retrieved online at http://www.erlang.org/. - - Software distributed under the License is distributed on an "AS IS" - basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - the License for the specific language governing rights and limitations - under the License. - - </legalnotice> - - <title>Appendix A: Mnesia Error Messages</title> - <prepared>Claes Wikström, Hans Nilsson and Håkan Mattsson</prepared> - <responsible>Bjarne Däcker</responsible> - <docno></docno> - <approved>Bjarne Däcker</approved> - <checked>Bjarne Däcker</checked> - <date>96-11-20</date> - <rev>B</rev> - <file>Mnesia_App_A.xml</file> - </header> - <p>Whenever an operation returns an error in Mnesia, a description - of the error is available. For example, the functions - <c>mnesia:transaction(Fun)</c>, or <c>mnesia:create_table(N,L)</c> - may return the tuple <c>{aborted, Reason}</c>, where <c>Reason</c> - is a term describing the error. The following function is used to - retrieve more detailed information about the error: - </p> - <list type="bulleted"> - <item><c>mnesia:error_description(Error)</c></item> - </list> - - <section> - <title>Errors in Mnesia</title> - <p>The following is a list of valid errors in Mnesia.</p> - <list type="bulleted"> - <item><c>badarg</c>. Bad or invalid argument, possibly bad type. - </item> - <item><c>no_transaction</c>. Operation not allowed outside transactions. - </item> - <item><c>combine_error</c>. Table options were illegally combined. - </item> - <item><c>bad_index</c>. Index already exists, or was out of bounds. - </item> - <item><c>already_exists</c>. Schema option to be activated is already on. - </item> - <item><c>index_exists</c>. Some operations cannot be performed on tables with an index. - </item> - <item><c>no_exists</c>.; Tried to perform operation on non-existing (non-alive) item. - </item> - <item><c>system_limit</c>.; A system limit was exhausted. - </item> - <item><c>mnesia_down</c>. A transaction involves records on a - remote node which became unavailable before the transaction - was completed. Record(s) are no longer available elsewhere in - the network.</item> - <item><c>not_a_db_node</c>. A node was mentioned which does not exist in the schema.</item> - <item><c>bad_type</c>.; Bad type specified in argument.</item> - <item><c>node_not_running</c>. Node is not running.</item> - <item><c>truncated_binary_file</c>. Truncated binary in file.</item> - <item><c>active</c>. Some delete operations require that all active records are removed.</item> - <item><c>illegal</c>. Operation not supported on this record.</item> - </list> - <p>The following example illustrates a function which returns an error, and the method to retrieve more detailed error information. - </p> - <p>The function <c>mnesia:create_table(bar, [{attributes, 3.14}])</c> will return the tuple <c>{aborted,Reason}</c>, where <c>Reason</c> is the tuple - <c>{bad_type,bar,3.14000}</c>. - </p> - <p>The function <c>mnesia:error_description(Reason)</c>, returns the term - <c>{"Bad type on some provided arguments",bar,3.14000}</c> which is an error - description suitable - for display.</p> - </section> -</chapter> - diff --git a/lib/mnesia/doc/src/Mnesia_App_D.xmlsrc b/lib/mnesia/doc/src/Mnesia_App_A.xmlsrc index b7a4c270ad..21366766d8 100644 --- a/lib/mnesia/doc/src/Mnesia_App_D.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_App_A.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2002</year><year>2013</year> + <year>1997</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -13,31 +13,28 @@ compliance with the License. You should have received a copy of the Erlang Public License along with this software. If not, it can be retrieved online at http://www.erlang.org/. - + Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. - + </legalnotice> - <title>Appendix D: The Fragmented Table Hashing Call Back Interface</title> - <prepared>Håkan Mattsson</prepared> - <responsible></responsible> + <title>Appendix A: Backup Callback Interface</title> + <prepared>Claes Wikström, Hans Nilsson and Håkan Mattsson</prepared> + <responsible>Bjarne Däcker</responsible> <docno></docno> - <approved></approved> - <checked></checked> - <date></date> - <rev></rev> - <file>Mnesia_App_D.xml</file> + <approved>Bjarne Däcker</approved> + <checked>Bjarne Däcker</checked> + <date>97-05-27</date> + <rev>C</rev> + <file>Mnesia_App_A.xml</file> </header> <section> - <title>mnesia_frag_hash callback behavior</title> + <title>mnesia_backup Callback Behavior</title> <p></p> - <codeinclude file="../../src/mnesia_frag_hash.erl" tag="%header_doc_include" type="erl"></codeinclude> - <p></p> - <codeinclude file="../../src/mnesia_frag_hash.erl" tag="%impl_doc_include" type="erl"></codeinclude> + <codeinclude file="../../src/mnesia_backup.erl" tag="%0" type="erl"></codeinclude> </section> </chapter> - diff --git a/lib/mnesia/doc/src/Mnesia_App_B.xmlsrc b/lib/mnesia/doc/src/Mnesia_App_B.xmlsrc index f02e424ca4..9f65888190 100644 --- a/lib/mnesia/doc/src/Mnesia_App_B.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_App_B.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1998</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -21,21 +21,23 @@ </legalnotice> - <title>Appendix B: The Backup Call Back Interface</title> - <prepared>Claes Wikström, Hans Nilsson and Håkan Mattsson</prepared> - <responsible>Bjarne Däcker</responsible> + <title>Appendix B: Activity Access Callback Interface</title> + <prepared>Håkan Mattsson</prepared> + <responsible></responsible> <docno></docno> - <approved>Bjarne Däcker</approved> - <checked>Bjarne Däcker</checked> - <date>97-05-27</date> - <rev>C</rev> + <approved></approved> + <checked></checked> + <date></date> + <rev></rev> <file>Mnesia_App_B.xml</file> </header> <section> - <title>mnesia_backup callback behavior</title> + <title>mnesia_access Callback Behavior</title> <p></p> - <codeinclude file="../../src/mnesia_backup.erl" tag="%0" type="erl"></codeinclude> + <codeinclude file="../../src/mnesia_frag.erl" tag="%header_doc_include" type="erl"></codeinclude> + <p></p> + <codeinclude file="../../src/mnesia_frag.erl" tag="%impl_doc_include" type="erl"></codeinclude> </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_App_C.xmlsrc b/lib/mnesia/doc/src/Mnesia_App_C.xmlsrc index f7fefa36c4..eb7cc0431f 100644 --- a/lib/mnesia/doc/src/Mnesia_App_C.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_App_C.xmlsrc @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1998</year><year>2013</year> + <year>2002</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -21,7 +21,7 @@ </legalnotice> - <title>Appendix C: The Activity Access Call Back Interface</title> + <title>Appendix C: Fragmented Table Hashing Callback Interface</title> <prepared>Håkan Mattsson</prepared> <responsible></responsible> <docno></docno> @@ -33,11 +33,11 @@ </header> <section> - <title>mnesia_access callback behavior</title> + <title>mnesia_frag_hash Callback Behavior</title> <p></p> - <codeinclude file="../../src/mnesia_frag.erl" tag="%header_doc_include" type="erl"></codeinclude> + <codeinclude file="../../src/mnesia_frag_hash.erl" tag="%header_doc_include" type="erl"></codeinclude> <p></p> - <codeinclude file="../../src/mnesia_frag.erl" tag="%impl_doc_include" type="erl"></codeinclude> + <codeinclude file="../../src/mnesia_frag_hash.erl" tag="%impl_doc_include" type="erl"></codeinclude> </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_chap1.xml b/lib/mnesia/doc/src/Mnesia_chap1.xml index 540008cdc5..0e9b769557 100644 --- a/lib/mnesia/doc/src/Mnesia_chap1.xml +++ b/lib/mnesia/doc/src/Mnesia_chap1.xml @@ -31,235 +31,82 @@ <rev>C</rev> <file>Mnesia_chap1.xml</file> </header> - <p>This book describes the Mnesia DataBase Management - System (DBMS). <em>Mnesia</em> is a distributed Database Management - System, appropriate for telecommunications applications and other - Erlang applications which require continuous operation and soft - real-time properties. It is one section of the Open Telecom Platform - (OTP), which is a control system platform for building - telecommunications applications.</p> - - <section> - <title>About Mnesia</title> - <p>The management of data in telecommunications system has many - aspects whereof some, but not all, are addressed by traditional - commercial DBMSs (Data Base Management Systems). In particular the - very high level of fault tolerance which is required in many nonstop - systems, combined with requirements on the DBMS to run in the same - address space as the application, have led us to implement a brand new - DBMS. called Mnesia. Mnesia is implemented in, and very tightly - connected to, the programming language Erlang and it provides the - functionality that is necessary for the implementation of fault - tolerant telecommunications systems. Mnesia is a multiuser Distributed - DBMS specially made for industrial telecommunications applications - written in the symbolic programming language Erlang, which is also - the intended target language. Mnesia tries to address all of the data - management issues required for typical telecommunications systems and - it has a number of features that are not normally found in traditional - databases. <br></br> - - In telecommunications applications there are different needs - from the features provided by traditional DBMSs. The applications now - implemented in the Erlang language need a mixture of a broad range - of features, which generally are not satisfied by traditional DBMSs. - Mnesia is designed with requirements like the following in mind:</p> - <list type="ordered"> - <item>Fast real-time key/value lookup</item> - <item>Complicated non real-time queries mainly for - operation and maintenance</item> - <item>Distributed data due to distributed - applications</item> - <item>High fault tolerance</item> - <item>Dynamic re-configuration</item> - <item>Complex objects</item> - </list> - <p>What - sets Mnesia apart from most other DBMSs is that it is designed with - the typical data management problems of telecommunications applications - in mind. Hence Mnesia combines many concepts found in traditional - databases, such as transactions and queries with concepts found in data - management systems for telecommunications applications, such as very - fast real-time operations, configurable degree of fault tolerance (by - means of replication) and the ability to reconfigure the system without - stopping or suspending it. Mnesia is also interesting due to its tight - coupling to the programming language Erlang, thus almost turning Erlang - into a database programming language. This has many benefits, the - foremost is that - the impedance mismatch between data format used by the - DBMS and data format used by the programming language, which is used - to manipulate the data, completely disappears. <br></br> -</p> - </section> - - <section> - <title>The Mnesia DataBase Management System (DBMS)</title> - <p></p> + <p>The <c>Mnesia</c> application provides a heavy duty real-time + distributed database.</p> <section> - <title>Features</title> - <p>Mnesia contains the following features which combine to produce a fault-tolerant, - distributed database management system written in Erlang: - </p> + <title>Scope</title> + <p>This User's Guide describes how to + build <c>Mnesia</c> database applications, and how to integrate + and use the <c>Mnesia</c> database management system with + OTP. Programming constructs are described, and numerous + programming examples are included to illustrate the use of + <c>Mnesia</c>.</p> + <p>This User's Guide is organized as follows:</p> <list type="bulleted"> - <item>Database schema can be dynamically reconfigured at runtime. + <item><seealso marker="Mnesia_overview">Mnesia</seealso> + provides an introduction to + <c>Mnesia</c>. + </item> + <item><seealso marker="Mnesia_chap2">Getting Started</seealso> + introduces <c>Mnesia</c> with an example database. Examples + are included how to start an Erlang session, specify a + <c>Mnesia</c> database directory, initialize a database + schema, start <c>Mnesia</c>, and create tables. Initial + prototyping of record definitions is also discussed. + </item> + <item><seealso marker="Mnesia_chap3">Build a Mnesia + Database</seealso> more formally describes the steps + introduced in the previous section, namely the <c>Mnesia</c> + functions that define a database schema, start <c>Mnesia</c>, + and create the required tables. + </item> + <item><seealso marker="Mnesia_chap4">Transactions and Other Access Contexts</seealso> + describes the transactions properties that make <c>Mnesia</c> into + a fault tolerant, real-time distributed database management + system. This section also describes the concept of locking + to ensure consistency in tables, and "dirty + operations", or short cuts, which bypass the transaction system + to improve speed and reduce overheads. </item> - <item>Tables can be declared to have properties such as location, - replication, and persistence. + <item><seealso marker="Mnesia_chap5">Miscellaneous Mnesia + Features</seealso> describes features that enable the + construction of more complex database applications. These + features include indexing, checkpoints, distribution and fault + tolerance, disc-less nodes, replication manipulation, local + content tables, concurrency, and object-based programming in + <c>Mnesia</c>. </item> - <item>Tables can be moved or replicated to several nodes to improve - fault tolerance. The rest of the system can still access the tables - to read, write, and delete records. + <item><seealso marker="Mnesia_chap7">Mnesia System + Information</seealso> describes the files contained in the + <c>Mnesia</c> database directory, database configuration data, + core and table dumps, as well as the important subject of + backup, fall-back, and disaster recovery principles. </item> - <item>Table locations are transparent to the programmer. - Programs address table names and the system itself keeps track of - table locations. + <item><seealso marker="Mnesia_chap8">Combine Mnesia with + SNMP</seealso> is a short section that outlines <c>Mnesia</c> + integrated with SNMP. </item> - <item>Database transactions can be distributed, and a large number of - functions can be called within one transaction. + <item><seealso marker="Mnesia_App_A">Appendix A: Backup + Callback Interface</seealso> is a program listing of the + default implementation of this facility. </item> - <item>Several transactions can run concurrently, and their execution is - fully synchronized by the database management system. - Mnesia ensures that no two processes manipulate data simultaneously. + <item><seealso marker="Mnesia_App_B">Appendix B: Activity + Access Callback Interface</seealso> is a program outlining + one possible implementation of this facility. </item> - <item>Transactions can be assigned the property of being executed on - all nodes in the system, or on none. Transactions can also be bypassed - in favor of running so called "dirty operations", which reduce - overheads and run very fast. + <item><seealso marker="Mnesia_App_C">Appendix C: Fragmented + Table Hashing Callback Interface</seealso> is a program + outlining one possible implementation of this facility. </item> </list> - <p>Details of these features are described in the following sections.</p> - </section> - <p></p> - - <section> - <title>Add-on Applications</title> - <p>QLC and Mnesia Session can be used in conjunction with Mnesia to produce - specialized functions which enhance the operational ability of Mnesia. - Both Mnesia Session and QLC have their own documentation as part - of the OTP documentation set. Below are the main features of Mnesia Session - and QLC when used in conjunction with Mnesia:</p> - <list type="bulleted"> - <item><em>QLC</em> has the ability to optimize the query - compiler for the Mnesia Database Management System, essentially making - the DBMS more efficient.</item> - <item><em>QLC</em>, can be used as a database programming - language for Mnesia. It includes a notation called "list - comprehensions" and can be used to make complex database - queries over a set of tables.</item> - <item><em>Mnesia Session</em> is an interface for the Mnesia Database - Management System</item> - <item><em>Mnesia Session</em> enables access to the - Mnesia DBMS from foreign programming languages (i.e. other - languages than Erlang).</item> - </list> - <p></p> - - <section> - <title>When to Use Mnesia</title> - <p>Use Mnesia with the following types of applications: - </p> - <list type="bulleted"> - <item>Applications that need to replicate data. - </item> - <item>Applications that perform complicated searches on data. - </item> - <item>Applications that need to use atomic transactions to - update several records simultaneously. - </item> - <item>Applications that use soft real-time characteristics. - </item> - </list> - <p>On the other hand, Mnesia may not be appropriate with the - following types of applications: - </p> - <list type="bulleted"> - <item>Programs that process plain text or binary data files - </item> - <item>Applications that merely need a look-up dictionary - which can be stored to disc can utilize the standard - library module <c>dets</c>, which is a disc based version - of the module <c>ets</c>. - </item> - <item>Applications which need disc logging facilities can - utilize the module <c>disc_log</c> by preference. - </item> - <item>Not suitable for hard real time systems. - </item> - </list> - </section> - </section> - - <section> - <title>Scope and Purpose</title> - <p>This manual is included in the OTP document set. It describes - how to build Mnesia database applications, and how to integrate - and utilize the Mnesia database management system with - OTP. Programming constructs are described, and numerous - programming examples are included to illustrate the use of - Mnesia. - </p> </section> <section> <title>Prerequisites</title> - <p>Readers of this manual are assumed to be familiar with system - development principles and database management systems. Readers - are also assumed to be familiar with the Erlang programming - language.</p> - </section> - - <section> - <title>About This Book</title> - <p>This book contains the following chapters: - </p> - <list type="bulleted"> - <item>Chapter 2, "Getting Started with Mnesia", introduces - Mnesia with an example database. Examples are shown of how to - start an Erlang session, specify a Mnesia database directory, - initialize a database schema, start Mnesia, and create - tables. Initial prototyping of record definitions is also - discussed. - </item> - <item>Chapter 3, "Building a Mnesia Database", more formally - describes the steps introduced in Chapter 2, namely the Mnesia - functions which define a database schema, start Mnesia, and - create the required tables. - </item> - <item>Chapter 4, "Transactions and other access contexts", - describes the transactions properties which make Mnesia into a - fault tolerant, real-time distributed database management - system. This chapter also describes the concept of locking in - order to ensure consistency in tables, and so called "dirty - operations", or short cuts which bypass the transaction system - to improve speed and reduce overheads. - </item> - <item>Chapter 5, "Miscellaneous Mnesia Features", describes - features which enable the construction of more complex - database applications. These features includes indexing, - checkpoints, distribution and fault tolerance, disc-less - nodes, replication manipulation, local content tables, concurrency, - and object based programming in Mnesia. - </item> - <item>Chapter 6, "Mnesia System Information", describes the - files contained in the Mnesia database directory, database - configuration data, core and table dumps, as well as the - important subject of backup, fall-back, and disaster recovery - principles. - </item> - <item>Chapter 7, "Combining Mnesia with SNMP", is a short - chapter which outlines Mnesia integrated with SNMP. - </item> - <item>Appendix A, "Mnesia Errors Messages", lists Mnesia error - messages and their meanings. - </item> - <item>Appendix B, "The Backup Call Back Interface", is a - program listing of the default implementation of this facility. - </item> - <item>Appendix C, "The Activity Access Call Back Interface", - is a program outlining of one possible implementations of this facility. - </item> - </list> + <p>It is assumed that the reader is familiar with the Erlang + programming language, system development principles, and + database management systems.</p> </section> - </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc index f464135a89..14999228a8 100644 --- a/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap2.xmlsrc @@ -21,7 +21,7 @@ </legalnotice> - <title>Getting Started with Mnesia</title> + <title>Getting Started</title> <prepared>Claes Wikström, Hans Nilsson and Håkan Mattsson</prepared> <responsible></responsible> <docno></docno> @@ -31,27 +31,31 @@ <rev>C</rev> <file>Mnesia_chap2.xml</file> </header> - <p>This chapter introduces Mnesia. Following a brief discussion - about the first initial setup, a Mnesia database example is - demonstrated. This database example will be referenced in the - following chapters, where this example is modified in order to - illustrate various program constructs. In this chapter, the - following mandatory procedures are illustrated by examples: - </p> + <marker id="getting_started"></marker> + <p>This section introduces <c>Mnesia</c> with an example database. + This example is referenced in the + following sections, where the example is modified to + illustrate various program constructs. This section illustrates + the following mandatory procedures through examples:</p> <list type="bulleted"> - <item>Starting an Erlang session and specifying a directory for the - Mnesia database. + <item>Starting the Erlang session. </item> - <item>Initializing a database schema. + <item>Specifying the <c>Mnesia</c> directory where the database + is to be stored. + </item> + <item>Initializing a new database schema with an attribute that + specifies on which node, or nodes, that database is to operate. + </item> + <item>Starting <c>Mnesia</c>. + </item> + <item>Creating and populating the database tables. </item> - <item>Starting Mnesia and creating the required tables.</item> </list> <section> - <title>Starting Mnesia for the first time</title> - <p>Following is a simplified demonstration of a Mnesia system startup. This is the dialogue from the Erlang - shell: - </p> + <title>Starting Mnesia for the First Time</title> + <p>This section provides a simplified demonstration of a <c>Mnesia</c> + system startup. The dialogue from the Erlang shell is as follows:</p> <pre><![CDATA[ unix> erl -mnesia dir '"/tmp/funky"' Erlang (BEAM) emulator version 4.9 @@ -89,79 +93,62 @@ 0 transactions waits for other nodes: [] ok ]]></pre> - <p>In the example above the following actions were performed: - </p> + <p>In this example, the following actions are performed:</p> <list type="bulleted"> - <item>The Erlang system was started from the UNIX prompt - with a flag <c>-mnesia dir '"/tmp/funky"'</c>. This flag indicates - to Mnesia which directory will store the data. + <item><em>Step 1:</em> The Erlang system is started from the UNIX + prompt with a flag <c>-mnesia dir '"/tmp/funky"'</c>, which indicates + in which directory to store the data. </item> - <item>A new empty schema was initialized on the local node by evaluating - <c>mnesia:create_schema([node()]).</c> The schema contains - information about the database in general. This will be - thoroughly explained later on. + <item><em>Step 2:</em> A new empty schema is initialized on the local + node by evaluating + <seealso marker="mnesia#create_schema/1">mnesia:create_schema([node()])</seealso>. + The schema contains information about the database in general. + This is explained in detail later. </item> - <item>The DBMS was started by evaluating <c>mnesia:start()</c>. + <item><em>Step 3:</em> The DBMS is started by evaluating + <seealso marker="mnesia#start/0">mnesia:start()</seealso>. </item> - <item>A first table was created, called <c>funky</c> by evaluating - the expression <c>mnesia:create_table(funky, [])</c>. The table - was given default properties. + <item><em>Step 4:</em> A first table is created, called <c>funky</c>, + by evaluating the expression <c>mnesia:create_table(funky, [])</c>. + The table is given default properties. </item> - <item><c>mnesia:info()</c> was evaluated and subsequently displayed - information regarding the status of the database on the terminal. + <item><em>Step 5:</em> <seealso marker="mnesia#info/0">mnesia:info()</seealso> + is evaluated to + display information on the terminal about the status of the database. </item> </list> </section> <section> - <title>An Introductory Example</title> - <p>A Mnesia database is organized as a set of tables. + <title>Example</title> + <p>A <c>Mnesia</c> database is organized as a set of tables. Each table is populated with instances (Erlang records). - A table also has a number of properties, such as location and - persistence. - </p> - <p>In this example we shall: - </p> - <list type="bulleted"> - <item>Start an Erlang system, and specify the directory where - the database will be located. - </item> - <item>Initiate a new schema with an attribute that specifies - on which node, or nodes, the database will operate. - </item> - <item>Start Mnesia itself. - </item> - <item>Create and populate the database tables. - </item> - </list> + A table has also a number of properties, such as location and + persistence.</p> <section> - <title>The Example Database</title> + <title>Database</title> </section> - <p>In this database example, we will create the database and - relationships depicted in the following diagram. We will call this - database the <em>Company</em> database. - </p> + <p>This example shows how to create a database called <c>Company</c> + and the relationships shown in the following diagram:</p> <image file="company.gif"> <icaption>Company Entity-Relation Diagram</icaption> </image> - <p>The database model looks as follows: - </p> + <p>The database model is as follows:</p> <list type="bulleted"> - <item>There are three entities: employee, project, and - department. + <item>There are three entities: department, employee, and project. </item> <item> <p>There are three relationships between these entities:</p> <list type="ordered"> - <item>A department is managed by an employee, hence the - <em>manager</em> relationship. + <item>A department is managed by an employee, + hence the <c>manager</c> relationship. </item> <item>An employee works at a department, hence the - <em>at_dep</em> relationship. + <c>at_dep</c> relationship. </item> - <item>Each employee works on a number of projects, hence - the <em>in_proj</em> relationship. + <item>Each employee works on a number of projects, + hence the <c>in_proj</c> relationship. </item> </list> </item> @@ -169,35 +156,32 @@ <section> <title>Defining Structure and Content</title> - <p>We first enter our record definitions into a text file + <p>First the record definitions are entered into a text file named <c>company.hrl</c>. This file defines the following - structure for our sample database: - </p> + structure for the example database: </p> <codeinclude file="company.hrl" tag="%0" type="erl"></codeinclude> - <p>The structure defines six tables in our database. In Mnesia, - the function <c>mnesia:create_table(Name, ArgList)</c> is - used to create tables. <c>Name</c> is the table - name <em>Note:</em> The current version of Mnesia does - not require that the name of the table is the same as the record - name, See Chapter 4: - <seealso marker="Mnesia_chap4#recordnames_tablenames">Record Names Versus Table Names.</seealso></p> - <p>For example, the table - for employees will be created with the function - <c>mnesia:create_table(employee, [{attributes, record_info(fields, employee)}]).</c> The table + <p>The structure defines six tables in the database. In <c>Mnesia</c>, + the function + <seealso marker="mnesia#create_table/2">mnesia:create_table(Name, ArgList)</seealso> + creates tables. <c>Name</c> is the table name.</p> + <note><p>The current version of <c>Mnesia</c> does not require that + the name of the table is the same as the record name, see + <seealso marker="Mnesia_chap4#recordnames_tablenames">Record Names versus Table Names.</seealso>.</p></note> + <p>For example, the table for employees is created with the + function <c>mnesia:create_table(employee, + [{attributes, record_info(fields, employee)}])</c>. The table name <c>employee</c> matches the name for records specified - in <c>ArgList</c>. The expression <c>record_info(fields, RecordName)</c> is processed by the Erlang preprocessor and - evaluates to a list containing the names of the different - fields for a record. - </p> + in <c>ArgList</c>. The expression + <c>record_info(fields, RecordName)</c> is processed by the Erlang + preprocessor and evaluates to a list containing the names of the + different fields for a record.</p> </section> <section> - <title>The Program</title> - <p>The following shell interaction starts Mnesia and - initializes the schema for our <c>company</c> database: - </p> + <title>Program</title> + <p>The following shell interaction starts <c>Mnesia</c> and + initializes the schema for the <c>Company</c> database:</p> <pre> - % <input>erl -mnesia dir '"/ldisc/scratch/Mnesia.Company"'</input> Erlang (BEAM) emulator version 4.9 @@ -205,37 +189,39 @@ 1> mnesia:create_schema([node()]). ok 2> mnesia:start(). - ok - </pre> - <p>The following program module creates and populates previously defined tables: - </p> + ok</pre> + <p>The following program module creates and populates previously + defined tables:</p> <codeinclude file="company.erl" tag="%0" type="erl"></codeinclude> </section> <section> - <title>The Program Explained</title> - <p>The following commands and functions were used to initiate the - Company database: - </p> + <title>Program Explained</title> + <p>The following commands and functions are used to initiate the + <c>Company</c> database:</p> <list type="bulleted"> - <item><c>% erl -mnesia dir '"/ldisc/scratch/Mnesia.Company"'.</c> This is a UNIX - command line entry which starts the Erlang system. The flag - <c>-mnesia dir Dir</c> specifies the location of the + <item><c>% erl -mnesia dir '"/ldisc/scratch/Mnesia.Company"'</c>. + This is a UNIX + command-line entry that starts the Erlang system. The flag + <c>-mnesia dir Dir</c> specifies the location of the database directory. The system responds and waits for - further input with the prompt <em>1></em>. + further input with the prompt <c>1></c>. </item> - <item><c>mnesia:create_schema([node()]).</c> This function - has the format <c>mnesia:create_schema(DiscNodeList)</c> and - initiates a new schema. In this example, we have created a - non-distributed system using only one node. Schemas are fully - explained in Chapter 3:<seealso marker="Mnesia_chap3#def_schema">Defining a Schema</seealso>. + <item> + <seealso marker="mnesia#create_schema/1">mnesia:create_schema([node()])</seealso>. + This function + has the format <c>mnesia:create_schema(DiscNodeList)</c> and + initiates a new schema. In this example, a non-distributed system + using only one node is created. Schemas are fully explained in + <seealso marker="Mnesia_chap3#def_schema">Define a Schema</seealso>. </item> - <item><c>mnesia:start().</c> This function starts - Mnesia. This function is fully explained in Chapter 3: - <seealso marker="Mnesia_chap3#start_mnesia">Starting Mnesia</seealso>. + <item><seealso marker="mnesia#start/0">mnesia:start()</seealso>. + This function starts <c>Mnesia</c> and is fully explained in + <seealso marker="Mnesia_chap3#start_mnesia">Start Mnesia</seealso>. </item> </list> - <p>Continuing the dialogue with the Erlang shell will produce the following:</p> + <p>Continuing the dialogue with the Erlang shell produces the + following:</p> <pre><![CDATA[ 3> company:init(). {atomic,ok} @@ -271,63 +257,58 @@ 0 transactions waits for other nodes: [] ok ]]></pre> - <p>A set of tables is created: - </p> - <list type="bulleted"> - <item><c>mnesia:create_table(Name,ArgList)</c>. This - function is used to create the required database tables. The - options available with <c>ArgList</c> are explained in - Chapter 3: <seealso marker="Mnesia_chap3#create_tables">Creating New Tables</seealso>. </item> - </list> - <p>The <c>company:init/0</c> function creates our tables. Two tables are - of type <c>bag</c>. This is the <c>manager</c> relation as well - the <c>in_proj</c> relation. This shall be interpreted as: An + <p>A set of tables is created. The function + <seealso marker="mnesia#create_table/2">mnesia:create_table(Name, ArgList)</seealso> + creates the required database tables. The + options available with <c>ArgList</c> are explained in + <seealso marker="Mnesia_chap3#create_tables">Create New Tables</seealso>.</p> + <p>The function <c>company:init/0</c> creates the tables. Two tables + are of type <c>bag</c>. This is the <c>manager</c> relation as well + the <c>in_proj</c> relation. This is interpreted as: an employee can be manager over several departments, and an employee can participate in several projects. However, the <c>at_dep</c> - relation is <c>set</c> because an employee can only work in one department. - In this data model we have examples of relations that are one-to-one (<c>set</c>), - as well as one-to-many (<c>bag</c>). - </p> - <p><c>mnesia:info()</c> now indicates that a database - which has seven local tables, of which, six are our - user defined tables and one is the schema. - Six transactions have been committed, as six successful transactions were run when - creating the tables. - </p> - <p>To write a function which inserts an employee record into the database, there must be an - <c>at_dep</c> record and a set of <c>in_proj</c> records inserted. Examine the following - code used to complete this action: - </p> - <codeinclude file="company.erl" tag="%1" type="erl"></codeinclude> + relation is <c>set</c>, as an employee can only work in one department. + In this data model, there are examples of relations that are 1-to-1 + (<c>set</c>) and 1-to-many (<c>bag</c>).</p> + <p><seealso marker="mnesia#info/0">mnesia:info()</seealso> + now indicates that a database has seven + local tables, where six are the user-defined tables and one is + the schema. Six transactions have been committed, as six successful + transactions were run when creating the tables.</p> + <p>To write a function that inserts an employee record into the + database, there must be an <c>at_dep</c> record and a set of + <c>in_proj</c> records inserted. Examine the following + code used to complete this action:</p> + <codeinclude file="company.erl" tag="%1" type="erl"></codeinclude> <list type="bulleted"> <item> - <p><c>insert_emp(Emp, DeptId, ProjNames) -></c>. The - <c>insert_emp/3</c> arguments are:</p> + <p>The <c>insert_emp/3</c> arguments are as follows:</p> <list type="ordered"> <item><c>Emp</c> is an employee record. </item> - <item><c>DeptId</c> is the identity of the department where the employee is working. + <item><c>DeptId</c> is the identity of the department where + the employee works. </item> - <item><c>ProjNames</c> is a list of the names of the projects where the employee are working.</item> + <item><c>ProjNames</c> is a list of the names of the projects + where the employee works.</item> </list> </item> </list> - <p>The <c>insert_emp(Emp, DeptId, ProjNames) -></c> function - creates a <em>functional object</em>. Functional objects - are identified by the term <c>Fun</c>. The Fun is passed + <p>The function <c>insert_emp/3</c> creates a Functional Object (Fun). + <c>Fun</c> is passed as a single argument to the function - <c>mnesia:transaction(Fun)</c>. This means that Fun is - run as a transaction with the following properties: - </p> + <seealso marker="mnesia#transaction/2">mnesia:transaction(Fun)</seealso>. + This means that <c>Fun</c> is + run as a transaction with the following properties:</p> <list type="bulleted"> - <item>Fun either succeeds or fails completely. + <item>A <c>Fun</c> either succeeds or fails. </item> - <item>Code which manipulates the same data records can be + <item>Code that manipulates the same data records can be run concurrently without the different processes interfering with each other. </item> </list> - <p>The function can be used as:</p> + <p>The function can be used as follows:</p> <code type="none"> Emp = #employee{emp_no= 104732, name = klacke, @@ -335,20 +316,17 @@ sex = male, phone = 98108, room_no = {221, 015}}, - insert_emp(Me, 'B/SFR', [Erlang, mnesia, otp]). - </code> - <note> - <p>Functional Objects (Funs) are described in the - Erlang Reference Manual, "Fun Expressions". - </p> + insert_emp(Me, 'B/SFR', [Erlang, mnesia, otp]).</code> + <note><p>For information about Funs, see "Fun Expressions" in + section <c>Erlang Reference Manual</c> in System + Documentation..</p> </note> </section> <section> <title>Initial Database Content</title> - <p>After the insertion of the employee named <c>klacke</c> - we have the following records in the database: - </p> + <p>After the insertion of the employee named <c>klacke</c>, + the databse has the following records:</p> <marker id="table2_1"></marker> <table> <row> @@ -364,14 +342,14 @@ <cell align="left" valign="middle">klacke</cell> <cell align="left" valign="middle">7</cell> <cell align="left" valign="middle">male</cell> - <cell align="left" valign="middle">99586</cell> + <cell align="left" valign="middle">98108</cell> <cell align="left" valign="middle">{221, 015}</cell> </row> - <tcaption> -Employee</tcaption> + <tcaption>employee Database Record</tcaption> </table> - <p>An employee record has the following Erlang record/tuple - representation: <c>{employee, 104732, klacke, 7, male, 98108, {221, 015}}</c></p> + <p>This <c>employee</c> record has the Erlang record/tuple + representation + <c>{employee, 104732, klacke, 7, male, 98108, {221, 015}}</c>.</p> <marker id="table2_2"></marker> <table> <row> @@ -382,12 +360,10 @@ Employee</tcaption> <cell align="left" valign="middle">klacke</cell> <cell align="left" valign="middle">B/SFR</cell> </row> - <tcaption> -At_dep</tcaption> + <tcaption>at_dep Database Record</tcaption> </table> - <p>At_dep has the following Erlang tuple representation: - <c>{at_dep, klacke, 'B/SFR'}</c>. - </p> + <p>This <c>at_dep</c> record has the Erlang tuple representation + <c>{at_dep, klacke, 'B/SFR'}</c>.</p> <marker id="table3_3"></marker> <table> <row> @@ -406,39 +382,36 @@ At_dep</tcaption> <cell align="left" valign="middle">klacke</cell> <cell align="left" valign="middle">mnesia</cell> </row> - <tcaption> -In_proj</tcaption> + <tcaption>in_proj Database Record</tcaption> </table> - <p>In_proj has the following Erlang tuple representation: - <c>{in_proj, klacke, 'Erlang', klacke, 'otp', klacke, 'mnesia'}</c></p> - <p>There is no difference between rows in a table and Mnesia - records. Both concepts are the same and will be used - interchangeably throughout this book. - </p> - <p>A Mnesia table is populated by Mnesia records. For example, - the tuple <c>{boss, klacke, bjarne}</c> is a record. The - second element in this tuple is the key. In order to uniquely - identify a table row both the key and the table name is - needed. The term <em>object identifier</em>, - (oid) is sometimes used for the arity two tuple {Tab, Key}. The oid for - the <c>{boss, klacke, bjarne}</c> record is the arity two + <p>This <c>in_proj</c> record has the Erlang tuple representation + <c>{in_proj, klacke, 'Erlang', klacke, 'otp', klacke, + 'mnesia'}</c>.</p> + <p>There is no difference between rows in a table and <c>Mnesia</c> + records. Both concepts are the same and are used + interchangeably throughout this User's Guide.</p> + <p>A <c>Mnesia</c> table is populated by <c>Mnesia</c> records. For + example, the tuple <c>{boss, klacke, bjarne}</c> is a record. The + second element in this tuple is the key. To identify a table + uniquely, both the key and the table name is needed. + The term Object Identifier (OID) is + sometimes used for the arity two tuple {Tab, Key}. The OID for + the record <c>{boss, klacke, bjarne}</c> is the arity two tuple <c>{boss, klacke}</c>. The first element of the tuple is the type of the record and the second element is the key. An - oid can lead to zero, one, or more records depending on - whether the table type is <c>set</c> or <c>bag</c>. - </p> - <p>We were also able to insert the <c>{boss, klacke, bjarne}</c> record which contains an implicit reference to - another employee which does not yet exist in the - database. Mnesia does not enforce this. - </p> + OID can lead to zero, one, or more records depending on + whether the table type is <c>set</c> or <c>bag</c>.</p> + <p>The record <c>{boss, klacke, bjarne}</c> can also be inserted. + This record contains an implicit reference to + another employee that does not yet exist in the + database. <c>Mnesia</c> does not enforce this.</p> </section> <section> - <title>Adding Records and Relationships to the Database</title> - <p>After adding additional record to the Company database, we - may end up with the following records: - </p> - <p><em>Employees</em></p> + <title>Adding Records and Relationships to Database</title> + <p>After adding more records to the <c>Company</c> database, the + result can be the following records:</p> + <p><c>employees</c>:</p> <code type="none"> {employee, 104465, "Johnson Torbjorn", 1, male, 99184, {242,038}}. {employee, 107912, "Carlsson Tuula", 2, female,94556, {242,056}}. @@ -447,16 +420,13 @@ In_proj</tcaption> {employee, 104659, "Tornkvist Torbjorn", 2, male, 99514, {222,022}}. {employee, 104732, "Wikstrom Claes", 2, male, 99586, {221,015}}. {employee, 117716, "Fedoriw Anna", 1, female,99143, {221,031}}. - {employee, 115018, "Mattsson Hakan", 3, male, 99251, {203,348}}. - </code> - <p><em>Dept</em></p> + {employee, 115018, "Mattsson Hakan", 3, male, 99251, {203,348}}.</code> + <p><c>dept</c>:</p> <code type="none"> - {dept, 'B/SF', "Open Telecom Platform"}. {dept, 'B/SFP', "OTP - Product Development"}. - {dept, 'B/SFR', "Computer Science Laboratory"}. - </code> - <p><em>Projects</em></p> + {dept, 'B/SFR', "Computer Science Laboratory"}.</code> + <p><c>projects</c>:</p> <code type="none"> %% projects {project, erlang, 1}. @@ -465,23 +435,19 @@ In_proj</tcaption> {project, mnesia, 5}. {project, wolf, 6}. {project, documentation, 7}. - {project, www, 8}. - </code> - <p>The above three tables, titled <c>employees</c>, - <c>dept</c>, and <c>projects</c>, are the tables which are + {project, www, 8}.</code> + <p>These three tables, <c>employees</c>, <c>dept</c>, and + <c>projects</c>, are made up of real records. The following database content is - stored in the tables which is built on - relationships. These tables are titled <c>manager</c>, - <c>at_dep</c>, and <c>in_proj</c>. - </p> - <p><em>Manager</em></p> + stored in the tables and is built on + relationships. These tables are <c>manager</c>, + <c>at_dep</c>, and <c>in_proj</c>.</p> + <p><c>manager</c>:</p> <code type="none"> - {manager, 104465, 'B/SF'}. {manager, 104465, 'B/SFP'}. - {manager, 114872, 'B/SFR'}. - </code> - <p><em>At_dep</em></p> + {manager, 114872, 'B/SFR'}.</code> + <p><c>at_dep</c>:</p> <code type="none"> {at_dep, 104465, 'B/SF'}. {at_dep, 107912, 'B/SF'}. @@ -490,9 +456,8 @@ In_proj</tcaption> {at_dep, 104659, 'B/SFR'}. {at_dep, 104732, 'B/SFR'}. {at_dep, 117716, 'B/SFP'}. - {at_dep, 115018, 'B/SFP'}. - </code> - <p><em>In_proj</em></p> + {at_dep, 115018, 'B/SFP'}.</code> + <p><c>in_proj</c>:</p> <code type="none"> {in_proj, 104465, otp}. {in_proj, 107912, otp}. @@ -508,136 +473,118 @@ In_proj</tcaption> {in_proj, 117716, otp}. {in_proj, 117716, documentation}. {in_proj, 115018, otp}. - {in_proj, 115018, mnesia}. - </code> + {in_proj, 115018, mnesia}.</code> <p>The room number is an attribute of the employee - record. This is a structured attribute which consists of a + record. This is a structured attribute that consists of a tuple. The first element of the tuple identifies a corridor, - and the second element identifies the actual room in the - corridor. We could have chosen to represent this as a record + and the second element identifies the room in that + corridor. An alternative is to represent this as a record <c>-record(room, {corr, no}).</c> instead of an anonymous - tuple representation. - </p> - <p>The Company database is now initialized and contains - data. </p> + tuple representation.</p> + <p>The <c>Company</c> database is now initialized and contains + data.</p> </section> <section> <title>Writing Queries</title> - <p>Retrieving data from DBMS should usually be done with <c>mnesia:read/3</c> or - <c>mnesia:read/1</c> functions. The following function raises the salary:</p> + <p>Retrieving data from DBMS is usually to be done with the + functions + <seealso marker="mnesia#read/3">mnesia:read/3</seealso> or + <seealso marker="mnesia#read/2">mnesia:read/1</seealso>. + The following function raises the salary:</p> <codeinclude file="company.erl" tag="%5" type="erl"></codeinclude> - <p>Since we want to update the record using <c>mnesia:write/1</c> after we have - increased the salary we acquire a write lock (third argument to read) when we read the - record from the table. - </p> - <p>It is not always the case that we can directly read the values from the table, - we might need to search the table or several tables to get the data we want, this - is done by writing database queries. Queries are always more expensive operations - than direct lookups done with <c>mnesia:read</c> and should be avoided in performance - critical code.</p> - <p>There are two methods for writing database queries: - </p> + <p>Since it is desired to update the record using the function + <seealso marker="mnesia#write/1">mnesia:write/1</seealso> + after the salary has been increased, a write + lock (third argument to <c>read</c>) is acquired when the record from + the table is read.</p> + <p>To read the values from the table directly is not always possible. + It can be needed to search one or more tables to get the + wanted data, and this is done by writing database queries. Queries + are always more expensive operations than direct lookups done with + <c>mnesia:read</c>. Therefore, avoid queries in + performance-critical code.</p> + <p>Two methods are available for writing database queries:</p> <list type="bulleted"> - <item>Mnesia functions - </item> + <item><c>Mnesia</c> functions</item> <item>QLC</item> </list> <section> - <title>Mnesia functions </title> - <p></p> + <title>Using Mnesia Functions</title> <p>The following function extracts the names of the female employees - stored in the database: - </p> + stored in the database:</p> <pre> -mnesia:select(employee, [{#employee{sex = female, name = '$1', _ = '_'},[], ['$1']}]). - </pre> - <p>Select must always run within an activity such as a - transaction. To be able to call from the shell we might - construct a function as: - </p> +mnesia:select(employee, [{#employee{sex = female, name = '$1', _ = '_'},[], ['$1']}]).</pre> + <p><c>select</c> must always run within an activity, such as a + transaction. The following function can be constructed to call + from the shell:</p> <codeinclude file="company.erl" tag="%20" type="erl"></codeinclude> - <p>The select expression matches all entries in table employee with - the field sex set to female. - </p> - <p>This function can be called from the shell as follows: - </p> + <p>The <c>select</c> expression matches all entries in table + employee with the field <c>sex</c> set to <c>female</c>.</p> + <p>This function can be called from the shell as follows:</p> <pre> (klacke@gin)1> <input>company:all_females().</input> - {atomic, ["Carlsson Tuula", "Fedoriw Anna"]} - </pre> - <p>See also the <seealso marker="Mnesia_chap4#matching">Pattern Matching </seealso> - chapter for a description of select and its syntax. - </p> + {atomic, ["Carlsson Tuula", "Fedoriw Anna"]}</pre> + <p>For a description of <c>select</c> and its syntax, see + <seealso marker="Mnesia_chap4#matching">Pattern Matching</seealso>. + </p> </section> <section> <title>Using QLC </title> - <p>This section contains simple introductory examples - only. Refer to <em>QLC reference manual</em> for a - full description of the QLC query language. Using QLC - might be more expensive than using Mnesia functions directly but - offers a nice syntax. - </p> + <p>This section contains simple introductory examples only. For + a full description of the QLC query language, see the + <seealso marker="stdlib:qlc">qlc</seealso> manual page in + <c>STDLIB</c>.</p> + <p>Using QLC can be more expensive than using <c>Mnesia</c> + functions directly but offers a nice syntax.</p> <p>The following function extracts a list of female employees - from the database: - </p> + from the database:</p> <pre> Q = qlc:q([E#employee.name || E <![CDATA[<-]]> mnesia:table(employee), E#employee.sex == female]), - qlc:e(Q), - </pre> - <p>Accessing mnesia tables from a QLC list comprehension must + qlc:e(Q),</pre> + <p>Accessing <c>Mnesia</c> tables from a QLC list comprehension must always be done within a transaction. Consider the following - function: - </p> + function:</p> <codeinclude file="company.erl" tag="%2" type="erl"></codeinclude> - <p>This function can be called from the shell as follows: - </p> + <p>This function can be called from the shell as follows:</p> <pre> (klacke@gin)1> <input>company:females().</input> - {atomic, ["Carlsson Tuula", "Fedoriw Anna"]} - </pre> - <p>In traditional relational database terminology, the above - operation would be called a selection, followed by a projection. - </p> - <p>The list comprehension expression shown above contains a - number of syntactical elements. - </p> + {atomic, ["Carlsson Tuula", "Fedoriw Anna"]}</pre> + <p>In traditional relational database terminology, this + operation is called a selection, followed by a projection.</p> + <p>The previous list comprehension expression contains a + number of syntactical elements:</p> <list type="bulleted"> - <item>the first <c>[</c> bracket should be read as "build the - list" + <item>The first <c>[</c> bracket is read as "build the + list". </item> - <item>the <c>||</c> "such that" and the arrow <c><![CDATA[<-]]></c> should - be read as "taken from" + <item>The <c>||</c> "such that" and the arrow <c><![CDATA[<-]]></c> + is read as "taken from". </item> </list> - <p>Hence, the above list comprehension demonstrates the - formation of the list <c>E#employee.name</c> such that <c>E</c> is - taken from the table of employees and the <c>sex</c> attribute - of each records is equal with the atom <c>female</c>. - </p> - <p>The whole list comprehension must be given to the - <c>qlc:q/1</c> function. - </p> - <p>It is possible to combine list comprehensions with low - level Mnesia functions in the same transaction. If we want to - raise the salary of all female employees we execute: - </p> + <p>Hence, the previous list comprehension demonstrates the + formation of the list <c>E#employee.name</c> such that <c>E</c> is + taken from the table of employees, and attribute <c>sex</c> + of each record is equal to the atom <c>female</c>.</p> + <p>The whole list comprehension must be given to the function + <c>qlc:q/1</c>.</p> + <p>List comprehensions with low-level <c>Mnesia</c> functions + can be combined in the same transaction. To raise the + salary of all female employees, execute the following:</p> <codeinclude file="company.erl" tag="%4" type="erl"></codeinclude> <p>The function <c>raise_females/1</c> returns the tuple <c>{atomic, Number}</c>, where <c>Number</c> is the number of - female employees who received a salary increase. Should an error - occur, the value <c>{aborted, Reason}</c> is returned. In the - case of an error, Mnesia guarantees that the salary is not - raised for any employees at all. - </p> + female employees who received a salary increase. If an error + occurs, the value <c>{aborted, Reason}</c> is returned, and + <c>Mnesia</c> guarantees that the salary is not + raised for any employee.</p> + <p><em>Example:</em></p> <pre> - 33><input>company:raise_females(33).</input> - {atomic,2} - </pre> + {atomic,2}</pre> </section> </section> </section> diff --git a/lib/mnesia/doc/src/Mnesia_chap3.xml b/lib/mnesia/doc/src/Mnesia_chap3.xml deleted file mode 100644 index ae704b4199..0000000000 --- a/lib/mnesia/doc/src/Mnesia_chap3.xml +++ /dev/null @@ -1,556 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>1997</year><year>2013</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - The contents of this file are subject to the Erlang Public License, - Version 1.1, (the "License"); you may not use this file except in - compliance with the License. You should have received a copy of the - Erlang Public License along with this software. If not, it can be - retrieved online at http://www.erlang.org/. - - Software distributed under the License is distributed on an "AS IS" - basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - the License for the specific language governing rights and limitations - under the License. - - </legalnotice> - - <title>Building A Mnesia Database</title> - <prepared></prepared> - <responsible></responsible> - <docno></docno> - <approved></approved> - <checked></checked> - <date></date> - <rev></rev> - <file>Mnesia_chap3.xml</file> - </header> - <p>This chapter details the basic steps involved when designing - a Mnesia database and the programming constructs which make different - solutions available to the programmer. The chapter includes the following - sections: - </p> - <list type="bulleted"> - <item>defining a schema</item> - <item>the datamodel</item> - <item>starting Mnesia</item> - <item>creating new tables.</item> - </list> - - <section> - <marker id="def_schema"></marker> - <title>Defining a Schema</title> - <p>The configuration of a Mnesia system is described in the - schema. The schema is a special table which contains information - such as the table names and each table's - storage type, (i.e. whether a table should be stored in RAM, - on disc or possibly on both, as well as its location). - </p> - <p>Unlike data tables, information contained in schema tables can only be - accessed and modified by using the schema related functions - described in this section. - </p> - <p>Mnesia has various functions for defining the - database schema. It is possible to move tables, delete tables, - or reconfigure the layout of tables. - </p> - <p>An important aspect of these functions is that the system can access a - table while it is being reconfigured. For example, it is possible to move a - table and simultaneously perform write operations to the same - table. This feature is essential for applications that require - continuous service. - </p> - <p>The following section describes the functions available for schema management, - all of which return a tuple: - </p> - <list type="bulleted"> - <item><c>{atomic, ok}</c>; or, - </item> - <item><c>{aborted, Reason}</c> if unsuccessful.</item> - </list> - - <section> - <title>Schema Functions</title> - <list type="bulleted"> - <item><c>mnesia:create_schema(NodeList)</c>. This function is - used to initialize a new, empty schema. This is a mandatory - requirement before Mnesia can be started. Mnesia is a truly - distributed DBMS and the schema is a system table that is - replicated on all nodes in a Mnesia system. - The function will fail if a schema is already present on any of - the nodes in <c>NodeList</c>. This function requires Mnesia - to be stopped on the all - <c>db_nodes</c> contained in the parameter <c>NodeList</c>. - Applications call this function only once, - since it is usually a one-time activity to initialize a new - database. - </item> - <item><c>mnesia:delete_schema(DiscNodeList)</c>. This function - erases any old schemas on the nodes in - <c>DiscNodeList</c>. It also removes all old tables together - with all data. This function requires Mnesia to be stopped - on all <c>db_nodes</c>. - </item> - <item><c>mnesia:delete_table(Tab)</c>. This function - permanently deletes all replicas of table <c>Tab</c>. - </item> - <item><c>mnesia:clear_table(Tab)</c>. This function - permanently deletes all entries in table <c>Tab</c>. - </item> - <item><c>mnesia:move_table_copy(Tab, From, To)</c>. This - function moves the copy of table <c>Tab</c> from node - <c>From</c> to node <c>To</c>. The table storage type, - <c>{type}</c> is preserved, so if a RAM table is moved from - one node to another node, it remains a RAM table on the new - node. It is still possible for other transactions to perform - read and write operation to the table while it is being - moved. - </item> - <item><c>mnesia:add_table_copy(Tab, Node, Type)</c>. This - function creates a replica of the table <c>Tab</c> at node - <c>Node</c>. The <c>Type</c> argument must be either of the - atoms <c>ram_copies</c>, <c>disc_copies</c>, or - <c>disc_only_copies</c>. If we add a copy of the system - table <c>schema</c> to a node, this means that we want the - Mnesia schema to reside there as well. This action then - extends the set of nodes that comprise this particular - Mnesia system. - </item> - <item><c>mnesia:del_table_copy(Tab, Node)</c>. This function - deletes the replica of table <c>Tab</c> at node <c>Node</c>. - When the last replica of a table is removed, the table is - deleted. - </item> - <item> - <p><c>mnesia:transform_table(Tab, Fun, NewAttributeList, NewRecordName)</c>. This - function changes the format on all records in table - <c>Tab</c>. It applies the argument <c>Fun</c> to all - records in the table. <c>Fun</c> shall be a function which - takes a record of the old type, and returns the record of the new - type. The table key may not be changed.</p> - <code type="none"> --record(old, {key, val}). --record(new, {key, val, extra}). - -Transformer = - fun(X) when record(X, old) -> - #new{key = X#old.key, - val = X#old.val, - extra = 42} - end, -{atomic, ok} = mnesia:transform_table(foo, Transformer, - record_info(fields, new), - new), - </code> - <p>The <c>Fun</c> argument can also be the atom - <c>ignore</c>, it indicates that only the meta data about the table will - be updated. Usage of <c>ignore</c> is not recommended (since it creates - inconsistencies between the meta data and the actual data) but included - as a possibility for the user to do his own (off-line) transform.</p> - </item> - <item><c>change_table_copy_type(Tab, Node, ToType)</c>. This - function changes the storage type of a table. For example, a - RAM table is changed to a disc_table at the node specified - as <c>Node</c>.</item> - </list> - </section> - </section> - - <section> - <title>The Data Model</title> - <p>The data model employed by Mnesia is an extended - relational data model. Data is organized as a set of - tables and relations between different data records can - be modeled as additional tables describing the actual - relationships. - Each table contains instances of Erlang records - and records are represented as Erlang tuples. - </p> - <p>Object identifiers, also known as oid, are made up of a table name and a key. - For example, if we have an employee record represented by the tuple - <c>{employee, 104732, klacke, 7, male, 98108, {221, 015}}</c>. - This record has an object id, (Oid) which is the tuple - <c>{employee, 104732}</c>. - </p> - <p>Thus, each table is made up of records, where the first element - is a record name and the second element of the table is a key - which identifies the particular record in that table. The - combination of the table name and a key, is an arity two tuple - <c>{Tab, Key}</c> called the Oid. See Chapter 4:<seealso marker="Mnesia_chap4#recordnames_tablenames">Record Names Versus Table Names</seealso>, for more information - regarding the relationship between the record name and the table - name. - </p> - <p>What makes the Mnesia data model an extended relational model - is the ability to store arbitrary Erlang terms in the attribute - fields. One attribute value could for example be a whole tree of - oids leading to other terms in other tables. This - type of record is hard to model in traditional relational - DBMSs.</p> - </section> - - <section> - <marker id="start_mnesia"></marker> - <title>Starting Mnesia</title> - <p>Before we can start Mnesia, we must initialize an empty schema - on all the participating nodes. - </p> - <list type="bulleted"> - <item>The Erlang system must be started. - </item> - <item>Nodes with disc database schema must be defined and - implemented with the function <c>create_schema(NodeList).</c></item> - </list> - <p>When running a distributed system, with two or more - participating nodes, then the <c>mnesia:start( ).</c> function - must be executed on each participating node. Typically this would - be part of the boot script in an embedded environment. - In a test environment or an interactive environment, - <c>mnesia:start()</c> can also be used either from the - Erlang shell, or another program. - </p> - - <section> - <title>Initializing a Schema and Starting Mnesia</title> - <p>To use a known example, we illustrate how to run the - Company database described in Chapter 2 on two separate nodes, - which we call <c>a@gin</c> and <c>b@skeppet</c>. Each of these - nodes must have have a Mnesia directory as well as an - initialized schema before Mnesia can be started. There are two - ways to specify the Mnesia directory to be used: - </p> - <list type="bulleted"> - <item> - <p>Specify the Mnesia directory by providing an application - parameter either when starting the Erlang shell or in the - application script. Previously the following example was used - to create the directory for our Company database:</p> - <pre> -%<input>erl -mnesia dir '"/ldisc/scratch/Mnesia.Company"'</input> - </pre> - </item> - <item>If no command line flag is entered, then the Mnesia - directory will be the current working directory on the node - where the Erlang shell is started.</item> - </list> - <p>To start our Company database and get it running on the two - specified nodes, we enter the following commands: - </p> - <list type="ordered"> - <item> - <p>On the node called gin:</p> - <pre> - gin %<input>erl -sname a -mnesia dir '"/ldisc/scratch/Mnesia.company"'</input> - </pre> - </item> - <item> - <p>On the node called skeppet:</p> - <pre> -skeppet %<input>erl -sname b -mnesia dir '"/ldisc/scratch/Mnesia.company"'</input> - </pre> - </item> - <item> - <p>On one of the two nodes:</p> - <pre> -(a@gin1)><input>mnesia:create_schema([a@gin, b@skeppet]).</input> - </pre> - </item> - <item>The function <c>mnesia:start()</c> is called on both - nodes. - </item> - <item> - <p>To initialize the database, execute the following - code on one of the two nodes.</p> - <codeinclude file="company.erl" tag="%12" type="erl"></codeinclude> - </item> - </list> - <p>As illustrated above, the two directories reside on different nodes, because the - <c>/ldisc/scratch</c> (the "local" disc) exists on the two different - nodes. - </p> - <p>By executing these commands we have configured two Erlang - nodes to run the Company database, and therefore, initialize the - database. This is required only once when setting up, the next time the - system is started <c>mnesia:start()</c> is called - on both nodes, to initialize the system from disc. - </p> - <p>In a system of Mnesia nodes, every node is aware of the - current location of all tables. In this example, data is - replicated on both nodes and functions which manipulate the - data in our tables can be executed on either of the two nodes. - Code which manipulate Mnesia data behaves identically - regardless of where the data resides. - </p> - <p>The function <c>mnesia:stop()</c> stops Mnesia on the node - where the function is executed. Both the <c>start/0</c> and - the <c>stop/0</c> functions work on the "local" Mnesia system, - and there are no functions which start or stop a set of nodes. - </p> - </section> - - <section> - <title>The Start-Up Procedure</title> - <p>Mnesia is started by calling the following function: - </p> - <code type="none"> - mnesia:start(). - </code> - <p>This function initiates the DBMS locally. </p> - <p>The choice of configuration will alter the location and load - order of the tables. The alternatives are listed below: <br></br> -</p> - <list type="ordered"> - <item>Tables that are stored locally only, are initialized - from the local Mnesia directory. - </item> - <item>Replicated tables that reside locally - as well as somewhere else are either initiated from disc or - by copying the entire table from the other node depending on - which of the different replicas is the most recent. Mnesia - determines which of the tables is the most recent. - </item> - <item>Tables that reside on remote nodes are available to other nodes as soon - as they are loaded.</item> - </list> - <p>Table initialization is asynchronous, the function - call <c>mnesia:start()</c> returns the atom <c>ok</c> and - then starts to initialize the different tables. Depending on - the size of the database, this may take some time, and the - application programmer must wait for the tables that the - application needs before they can be used. This achieved by using - the function:</p> - <list type="bulleted"> - <item><c>mnesia:wait_for_tables(TabList, Timeout)</c></item> - </list> - <p>This function suspends the caller until all tables - specified in <c>TabList</c> are properly initiated. - </p> - <p>A problem can arise if a replicated table on one node is - initiated, but Mnesia deduces that another (remote) - replica is more recent than the replica existing on - the local node, the initialization procedure will not proceed. - In this situation, a call to to - <c>mnesia:wait_for_tables/2</c> suspends the caller until the - remote node has initiated the table from its local disc and - the node has copied the table over the network to the local node. - </p> - <p>This procedure can be time consuming however, the shortcut function - shown below will load all the tables from disc at a faster rate: - </p> - <list type="bulleted"> - <item><c>mnesia:force_load_table(Tab)</c>. This function forces - tables to be loaded from disc regardless of the network - situation.</item> - </list> - <p>Thus, we can assume that if an application - wishes to use tables <c>a</c> and <c>b</c>, then the - application must perform some action similar to the below code before it can utilize the tables. - </p> - <pre> - case mnesia:wait_for_tables([a, b], 20000) of - {timeout, RemainingTabs} -> - panic(RemainingTabs); - ok -> - synced - end. - </pre> - <warning> - <p>When tables are forcefully loaded from the local disc, - all operations that were performed on the replicated table - while the local node was down, and the remote replica was - alive, are lost. This can cause the database to become - inconsistent.</p> - </warning> - <p>If the start-up procedure fails, the - <c>mnesia:start()</c> function returns the cryptic tuple - <c>{error,{shutdown, {mnesia_sup,start,[normal,[]]}}}</c>. - Use command line arguments -boot start_sasl as argument to - the erl script in order to get more information - about the start failure. - </p> - </section> - </section> - - <section> - <marker id="create_tables"></marker> - <title>Creating New Tables</title> - <p>Mnesia provides one function to create new tables. This - function is: <c>mnesia:create_table(Name, ArgList).</c></p> - <p>When executing this function, it returns one of the following - responses: - </p> - <list type="bulleted"> - <item><c>{atomic, ok}</c> if the function executes - successfully - </item> - <item><c>{aborted, Reason}</c> if the function fails. - </item> - </list> - <p>The function arguments are: - </p> - <list type="bulleted"> - <item><c>Name</c> is the atomic name of the table. It is - usually the same name as the name of the records that - constitute the table. (See <c>record_name</c> for more - details.) - </item> - <item> - <p><c>ArgList</c> is a list of <c>{Key,Value}</c> tuples. - The following arguments are valid: - </p> - <list type="bulleted"> - <item> - <p><c>{type, Type}</c> where <c>Type</c> must be either of the - atoms <c>set</c>, <c>ordered_set</c> or <c>bag</c>. - The default value is - <c>set</c>. Note: currently 'ordered_set' - is not supported for 'disc_only_copies' tables. - A table of type <c>set</c> or <c>ordered_set</c> has either zero or - one record per key. Whereas a table of type <c>bag</c> can - have an arbitrary number of records per key. The key for - each record is always the first attribute of the record.</p> - <p>The following example illustrates the difference between - type <c>set</c> and <c>bag</c>: </p> - <pre> - f() -> F = fun() -> - mnesia:write({foo, 1, 2}), mnesia:write({foo, 1, 3}), - mnesia:read({foo, 1}) end, mnesia:transaction(F). </pre> - <p>This transaction will return the list <c>[{foo,1,3}]</c> if - the <c>foo</c> table is of type <c>set</c>. However, list - <c>[{foo,1,2}, {foo,1,3}]</c> will return if the table is - of type <c>bag</c>. Note the use of <c>bag</c> and - <c>set</c> table types. </p> - <p>Mnesia tables can never contain - duplicates of the same record in the same table. Duplicate - records have attributes with the same contents and key. - </p> - </item> - <item> - <p><c>{disc_copies, NodeList}</c>, where <c>NodeList</c> is a - list of the nodes where this table will reside on disc.</p> - <p>Write operations to a table replica of type - <c>disc_copies</c> will write data to the disc copy as well - as to the RAM copy of the table. </p> - <p>It is possible to have a - replicated table of type <c>disc_copies</c> on one node, and - the same table stored as a different type on another node. - The default value is <c>[]</c>. This arrangement is - desirable if we want the following operational - characteristics are required:</p> - <list type="ordered"> - <item>read operations must be very fast and performed in RAM - </item> - <item>all write operations must be written to persistent - storage.</item> - </list> - <p>A write operation on a <c>disc_copies</c> table - replica will be performed in two steps. First the write - operation is appended to a log file, then the actual - operation is performed in RAM. - </p> - </item> - <item> - <p><c>{ram_copies, NodeList}</c>, where <c>NodeList</c> is a - list of the nodes where this table is stored in RAM. The - default value for <c>NodeList</c> is <c>[node()]</c>. If the - default value is used to create a new table, it will be - located on the local node only. </p> - <p>Table replicas of type - <c>ram_copies</c> can be dumped to disc with the function - <c>mnesia:dump_tables(TabList)</c>. - </p> - </item> - <item><c>{disc_only_copies, NodeList}</c>. These table - replicas are stored on disc only and are therefore slower to - access. However, a disc only replica consumes less memory than - a table replica of the other two storage types. - </item> - <item><c>{index, AttributeNameList}</c>, where - <c>AttributeNameList</c> is a list of atoms specifying the - names of the attributes Mnesia shall build and maintain. An - index table will exist for every element in the list. The - first field of a Mnesia record is the key and thus need no - extra index. - <br></br> -The first field of a record is the second element of the - tuple, which is the representation of the record. - </item> - <item><c>{snmp, SnmpStruct}</c>. <c>SnmpStruct</c> is - described in the SNMP User Guide. Basically, if this attribute - is present in <c>ArgList</c> of <c>mnesia:create_table/2</c>, - the table is immediately accessible by means of the Simple - Network Management Protocol (SNMP). - <br></br> -It is easy to design applications which use SNMP to - manipulate and control the system. Mnesia provides a direct - mapping between the logical tables that make up an SNMP - control application and the physical data which make up a - Mnesia table. <c>[]</c> - is default. - </item> - <item><c>{local_content, true}</c> When an application needs a - table whose contents should be locally unique on each - node, - <c>local_content</c> tables may be used. The name of the - table is known to all Mnesia nodes, but its contents is - unique for each node. Access to this type of table must be - done locally. </item> - <item> - <p><c>{attributes, AtomList}</c> is a list of the attribute - names for the records that are supposed to populate the - table. The default value is the list <c>[key, val]</c>. The - table must at least have one extra attribute besides the - key. When accessing single attributes in a record, it is not - recommended to hard code the attribute names as atoms. Use - the construct <c>record_info(fields,record_name)</c> - instead. The expression - <c>record_info(fields,record_name)</c> is processed by the - Erlang macro pre-processor and returns a list of the - record's field names. With the record definition - <c>-record(foo, {x,y,z}).</c> the expression - <c>record_info(fields,foo)</c> is expanded to the list - <c>[x,y,z]</c>. Accordingly, it is possible to provide the - attribute names yourself, or to use the <c>record_info/2</c> - notation. </p> - <p>It is recommended that - the <c>record_info/2</c> notation be used as it is easier to - maintain the program and it will be more robust with regards - to future record changes. - </p> - </item> - <item> - <p><c>{record_name, Atom}</c> specifies the common name of - all records stored in the table. All records, stored in - the table, must have this name as their first element. - The <c>record_name</c> defaults to the name of the - table. For more information see Chapter 4:<seealso marker="Mnesia_chap4#recordnames_tablenames">Record Names Versus Table Names</seealso>.</p> - </item> - </list> - </item> - </list> - <p>As an example, assume we have the record definition:</p> - <pre> - -record(funky, {x, y}). - </pre> - <p>The below call would create a table which is replicated on two - nodes, has an additional index on the <c>y</c> attribute, and is - of type - <c>bag</c>.</p> - <pre> - mnesia:create_table(funky, [{disc_copies, [N1, N2]}, {index, - [y]}, {type, bag}, {attributes, record_info(fields, funky)}]). - </pre> - <p>Whereas a call to the below default code values: </p> - <pre> -mnesia:create_table(stuff, []) </pre> - <p>would return a table with a RAM copy on the - local node, no additional indexes and the attributes defaulted to - the list <c>[key,val]</c>.</p> - </section> -</chapter> - diff --git a/lib/mnesia/doc/src/Mnesia_chap3.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap3.xmlsrc new file mode 100644 index 0000000000..d965c98ab9 --- /dev/null +++ b/lib/mnesia/doc/src/Mnesia_chap3.xmlsrc @@ -0,0 +1,519 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>1997</year><year>2013</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>Build a Mnesia Database</title> + <prepared></prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date></date> + <rev></rev> + <file>Mnesia_chap3.xml</file> + </header> + <p>This section describes the basic steps when designing a + <c>Mnesia</c> database and the programming constructs that make different + solutions available to the programmer. The following topics are + included:</p> + <list type="bulleted"> + <item>Define a schema</item> + <item>Data model</item> + <item>Start <c>Mnesia</c></item> + <item>Create tables</item> + </list> + + <section> + <marker id="def_schema"></marker> + <title>Define a Schema</title> + <p>The configuration of a <c>Mnesia</c> system is described in a + schema. The schema is a special table that includes information + such as the table names and the storage type of each table + (that is, whether a table is to be stored in RAM, + on disc, or on both, as well as its location).</p> + <p>Unlike data tables, information in schema tables can only be + accessed and modified by using the schema-related functions + described in this section.</p> + <p><c>Mnesia</c> has various functions for defining the + database schema. Tables can be moved or deleted, and the + table layout can be reconfigured.</p> + <p>An important aspect of these functions is that the system can access + a table while it is being reconfigured. For example, it is possible + to move a + table and simultaneously perform write operations to the same + table. This feature is essential for applications that require + continuous service.</p> + <p>This section describes the functions available for schema management, + all which return either of the following tuples:</p> + <list type="bulleted"> + <item><c>{atomic, ok}</c> if successful</item> + <item><c>{aborted, Reason}</c> if unsuccessful</item> + </list> + + <section> + <title>Schema Functions</title> + <p>The schema functions are as follows:</p> + <list type="bulleted"> + <item><seealso marker="mnesia#create_schema/1">mnesia:create_schema(NodeList)</seealso> + initializes a new, empty schema. This is a mandatory requirement + before <c>Mnesia</c> can be started. <c>Mnesia</c> is a truly + distributed DBMS and the schema is a system table that is + replicated on all nodes in a <c>Mnesia</c> system. + This function fails if a schema is already present on any of + the nodes in <c>NodeList</c>. The function requires <c>Mnesia</c> + to be stopped on the all + <c>db_nodes</c> contained in parameter <c>NodeList</c>. + Applications call this function only once, as + it is usually a one-time activity to initialize a new database. + </item> + <item><seealso marker="mnesia#delete_schema/1">mnesia:delete_schema(DiscNodeList)</seealso> + erases any old schemas on the nodes in + <c>DiscNodeList</c>. It also removes all old tables together + with all data. This function requires <c>Mnesia</c> to be stopped + on all <c>db_nodes</c>. + </item> + <item><seealso marker="mnesia#delete_table/1">mnesia:delete_table(Tab)</seealso> + permanently deletes all replicas of table <c>Tab</c>. + </item> + <item><seealso marker="mnesia#clear_table/1">mnesia:clear_table(Tab)</seealso> + permanently deletes all entries in table <c>Tab</c>. + </item> + <item><seealso marker="mnesia#move_table_copy/3">mnesia:move_table_copy(Tab, From, To)</seealso> + moves the copy of table <c>Tab</c> from node + <c>From</c> to node <c>To</c>. The table storage type + <c>{type}</c> is preserved, so if a RAM table is moved from + one node to another, it remains a RAM table on the new + node. Other transactions can still perform read + and write operation to the table while it is being moved. + </item> + <item><seealso marker="mnesia#add_table_copy/3">mnesia:add_table_copy(Tab, Node, Type)</seealso> + creates a replica of table <c>Tab</c> at node + <c>Node</c>. Argument <c>Type</c> must be either of the + atoms <c>ram_copies</c>, <c>disc_copies</c>, or + <c>disc_only_copies</c>. If you add a copy of the system + table <c>schema</c> to a node, you want the <c>Mnesia</c> + schema to reside there as well. This action + extends the set of nodes that comprise this particular + <c>Mnesia</c> system. + </item> + <item><seealso marker="mnesia#del_table_copy/2">mnesia:del_table_copy(Tab, Node)</seealso> + deletes the replica of table <c>Tab</c> at node <c>Node</c>. + When the last replica of a table is removed, the table is + deleted. + </item> + <item> + <p><seealso marker="mnesia#transform_table/4">mnesia:transform_table(Tab, Fun, NewAttributeList, NewRecordName)</seealso> + changes the format on all records in table + <c>Tab</c>. It applies argument <c>Fun</c> to all + records in the table. <c>Fun</c> must be a function that + takes a record of the old type, and returns the record of the + new type. The table key must not be changed.</p> + <p><em>Example:</em></p> + <code type="none"> +-record(old, {key, val}). +-record(new, {key, val, extra}). + +Transformer = + fun(X) when record(X, old) -> + #new{key = X#old.key, + val = X#old.val, + extra = 42} + end, +{atomic, ok} = mnesia:transform_table(foo, Transformer, + record_info(fields, new), + new), +</code> + <p>Argument <c>Fun</c> can also be the atom + <c>ignore</c>, which indicates that only the metadata about + the table is updated. Use of <c>ignore</c> is not recommended + (as it creates inconsistencies between the metadata and the + actual data) but it is included as a possibility for the user + do to an own (offline) transform.</p> + </item> + <item><c>change_table_copy_type(Tab, Node, ToType)</c> + changes the storage type of a table. For example, a + RAM table is changed to a <c>disc_table</c> at the node specified + as <c>Node</c>.</item> + </list> + </section> + </section> + + <section> + <title>Data Model</title> + <p>The data model employed by <c>Mnesia</c> is an extended + relational data model. Data is organized as a set of + tables and relations between different data records can + be modeled as more tables describing the relationships. + Each table contains instances of Erlang records. + The records are represented as Erlang tuples.</p> + <p>Each Object Identifier (OID) is made up of a table name and a key. + For example, if an employee record is represented by the tuple + <c>{employee, 104732, klacke, 7, male, 98108, {221, 015}}</c>, + this record has an OID, which is the tuple + <c>{employee, 104732}</c>.</p> + <p>Thus, each table is made up of records, where the first element + is a record name and the second element of the table is a key, + which identifies the particular record in that table. The + combination of the table name and a key is an arity two tuple + <c>{Tab, Key}</c> called the OID. For more information about + the relationship beween the record name and the table name, see + <seealso marker="Mnesia_chap4#recordnames_tablenames">Record Names versus Table Names</seealso>. + </p> + <p>What makes the <c>Mnesia</c> data model an extended relational model + is the ability to store arbitrary Erlang terms in the attribute + fields. One attribute value can, for example, be a whole tree of + OIDs leading to other terms in other tables. This type + of record is difficult to model in traditional relational DBMSs.</p> + </section> + + <section> + <marker id="start_mnesia"></marker> + <title>Start Mnesia</title> + <p>Before starting <c>Mnesia</c>, the following must be done: + </p> + <list type="bulleted"> + <item>An empty schema must be initialized on all the + participating nodes.</item> + <item>The Erlang system must be started.</item> + <item>Nodes with disc database schema must be defined and + implemented with the function + <seealso marker="mnesia#create_schema/1">mnesia:create_schema(NodeList)</seealso>.</item> + </list> + <p>When running a distributed system with two or more + participating nodes, the function + <seealso marker="mnesia#start/0">mnesia:start()</seealso> + must be executed on each participating node. This would typically + be part of the boot script in an embedded environment. + In a test environment or an interactive environment, + <c>mnesia:start()</c> can also be used either from the + Erlang shell or another program. + </p> + + <section> + <title>Initialize a Schema and Start Mnesia</title> + <p>Let us use the example database <c>Company</c>, described in + <seealso marker="Mnesia_chap2#getting_started">Getting Started</seealso> to + illustrate how to run a database on two separate nodes, + called <c>a@gin</c> and <c>b@skeppet</c>. Each of these + nodes must have a <c>Mnesia</c> directory and an + initialized schema before <c>Mnesia</c> can be started. There are + two ways to specify the <c>Mnesia</c> directory to be used:</p> + <list type="bulleted"> + <item> + <p>Specify the <c>Mnesia</c> directory by providing an application + parameter either when starting the Erlang shell or in the + application script. Previously, the following example was used + to create the directory for the <c>Company</c> database:</p> + <pre> +%<input>erl -mnesia dir '"/ldisc/scratch/Mnesia.Company"'</input> + </pre> + </item> + <item>If no command-line flag is entered, the <c>Mnesia</c> + directory becomes the current working directory on the node + where the Erlang shell is started.</item> + </list> + <p>To start the <c>Company</c> database and get it running on the two + specified nodes, enter the following commands:</p> + <list type="ordered"> + <item> + <p>On the node <c>a@gin</c>:</p> + <pre> + gin %<input>erl -sname a -mnesia dir '"/ldisc/scratch/Mnesia.company"'</input></pre> + </item> + <item><p>On the node <c>b@skeppet</c>:</p> + <pre> +skeppet %<input>erl -sname b -mnesia dir '"/ldisc/scratch/Mnesia.company"'</input></pre> + </item> + <item> + <p>On one of the two nodes:</p> + <pre> +(a@gin)1><input>mnesia:create_schema([a@gin, b@skeppet]).</input></pre> + </item> + <item>The function + <seealso marker="mnesia#start/0">mnesia:start()</seealso> + is called on both nodes. + </item> + <item><p>To initialize the database, execute the following + code on one of the two nodes:</p> + <codeinclude file="company.erl" tag="%12" type="erl"></codeinclude> + </item> + </list> + <p>As illustrated, the two directories reside on different nodes, + because <c>/ldisc/scratch</c> (the "local" disc) exists on + the two different nodes.</p> + <p>By executing these commands, two Erlang nodes are configured to + run the <c>Company</c> database, and therefore, initialize the + database. This is required only once when setting up. The next time + the system is started, + <seealso marker="mnesia#start/0">mnesia:start()</seealso> + is called + on both nodes, to initialize the system from disc.</p> + <p>In a system of <c>Mnesia</c> nodes, every node is aware of the + current location of all tables. In this example, data is + replicated on both nodes and functions that manipulate the + data in the tables can be executed on either of the two nodes. + Code that manipulate <c>Mnesia</c> data behaves identically + regardless of where the data resides.</p> + <p>The function <seealso marker="mnesia#stop/0">mnesia:stop()</seealso> + stops <c>Mnesia</c> on the node + where the function is executed. The functions <c>mnesia:start/0</c> + and <c>mnesia:stop/0</c> work on the "local" <c>Mnesia</c> system. + No functions start or stop a set of nodes.</p> + </section> + + <section> + <title>Startup Procedure</title> + <p>Start <c>Mnesia</c> by calling the following function:</p> + <code type="none"> + mnesia:start().</code> + <p>This function initiates the DBMS locally.</p> + <p>The choice of configuration alters the location and load + order of the tables. The alternatives are as follows:</p> + <list type="ordered"> + <item>Tables that are only stored locally are initialized + from the local <c>Mnesia</c> directory. + </item> + <item>Replicated tables that reside locally + as well as somewhere else are either initiated from disc or + by copying the entire table from the other node, depending on + which of the different replicas are the most recent. <c>Mnesia</c> + determines which of the tables are the most recent. + </item> + <item>Tables that reside on remote nodes are available to other + nodes as soon as they are loaded.</item> + </list> + <p>Table initialization is asynchronous. The function + call <seealso marker="mnesia#start/0">mnesia:start()</seealso> + returns the atom <c>ok</c> and + then starts to initialize the different tables. Depending on + the size of the database, this can take some time, and the + application programmer must wait for the tables that the + application needs before they can be used. This is achieved by + using the function + <seealso marker="mnesia#wait_for_tables/2">mnesia:wait_for_tables(TabList, Timeout)</seealso>, + which suspends the caller until all tables + specified in <c>TabList</c> are properly initiated.</p> + <p>A problem can arise if a replicated table on one node is + initiated, but <c>Mnesia</c> deduces that another (remote) + replica is more recent than the replica existing on the + local node, and the initialization procedure does not proceed. + In this situation, a call to + <seealso marker="mnesia#wait_for_tables/2">mnesia:wait_for_tables/2</seealso>, + suspends the caller until the + remote node has initialized the table from its local disc and + the node has copied the table over the network to the local node.</p> + <p>However, this procedure can be time-consuming, the shortcut function + <seealso marker="mnesia#force_load_table/1">mnesia:force_load_table(Tab)</seealso> + loads all the tables from disc at a faster rate. The function forces + tables to be loaded from disc regardless of the network + situation.</p> + <p>Thus, it can be assumed that if an application wants to use + tables <c>a</c> and <c>b</c>, the application must perform + some action similar to following before it can use the tables:</p> + <pre> + case mnesia:wait_for_tables([a, b], 20000) of + {timeout, RemainingTabs} -> + panic(RemainingTabs); + ok -> + synced + end.</pre> + <warning> + <p>When tables are forcefully loaded from the local disc, + all operations that were performed on the replicated table + while the local node was down, and the remote replica was + alive, are lost. This can cause the database to become + inconsistent.</p> + </warning> + <p>If the startup procedure fails, the function + <seealso marker="mnesia#start/0">mnesia:start()</seealso> + returns the cryptic tuple + <c>{error,{shutdown, {mnesia_sup,start,[normal,[]]}}}</c>. + To get more information about the start failure, use + command-line arguments <c>-boot start_sasl</c> as argument to + the <c>erl</c> script.</p> + </section> + </section> + + <section> + <marker id="create_tables"></marker> + <title>Create Tables</title> + <p>The function + <seealso marker="mnesia#create_table/2">mnesia:create_table(Name, ArgList)</seealso> + creates tables. When executing this function, it returns one of + the following responses:</p> + <list type="bulleted"> + <item><c>{atomic, ok}</c> if the function executes successfully + </item> + <item><c>{aborted, Reason}</c> if the function fails + </item> + </list> + <p>The function arguments are as follows:</p> + <list type="bulleted"> + <item><c>Name</c> is the name of the table. It is + usually the same name as the name of the records that + constitute the table. For details, see <c>record_name</c>. + </item> + <item> + <p><c>ArgList</c> is a list of <c>{Key,Value}</c> tuples. + The following arguments are valid:</p> + <list type="bulleted"> + <item> + <p><c>{type, Type}</c>, where <c>Type</c> must be either of + the atoms <c>set</c>, <c>ordered_set</c>, or <c>bag</c>. + Default is <c>set</c>.</p> + <p>Notice that currently <c>ordered_set</c> is not + supported for <c>disc_only_copies</c> tables.</p> + <p>A table of type + <c>set</c> or <c>ordered_set</c> has either zero or + one record per key, whereas a table of type <c>bag</c> can + have an arbitrary number of records per key. The key for + each record is always the first attribute of the record.</p> + <p>The following example illustrates the difference between + type <c>set</c> and <c>bag</c>:</p> + <pre> + f() -> + F = fun() -> + mnesia:write({foo, 1, 2}), + mnesia:write({foo, 1, 3}), + mnesia:read({foo, 1}) + end, + mnesia:transaction(F).</pre> + <p>This transaction returns the list <c>[{foo,1,3}]</c> if + table <c>foo</c> is of type <c>set</c>. However, the list + <c>[{foo,1,2}, {foo,1,3}]</c> is returned if the table is + of type <c>bag</c>.</p> + <p><c>Mnesia</c> tables can never contain + duplicates of the same record in the same table. Duplicate + records have attributes with the same contents and key.</p> + </item> + <item> + <p><c>{disc_copies, NodeList}</c>, where <c>NodeList</c> is a + list of the nodes where this table is to reside on disc.</p> + <p>Write operations to a table replica of type + <c>disc_copies</c> write data to the disc copy and + to the RAM copy of the table.</p> + <p>It is possible to have a + replicated table of type <c>disc_copies</c> on one node, and + the same table stored as a different type on another node. + Default is <c>[]</c>. This arrangement is + desirable if the following operational + characteristics are required:</p> + <list type="ordered"> + <item>Read operations must be fast and performed in RAM.</item> + <item>All write operations must be written to persistent + storage.</item> + </list> + <p>A write operation on a <c>disc_copies</c> table + replica is performed in two steps. First the write + operation is appended to a log file, then the actual + operation is performed in RAM.</p> + </item> + <item> + <p><c>{ram_copies, NodeList}</c>, where <c>NodeList</c> is a + list of the nodes where this table is stored in RAM. + Default is <c>[node()]</c>. If the default value is used + to create a table, it is located on the local node only.</p> + <p>Table replicas of type + <c>ram_copies</c> can be dumped to disc with the function + <seealso marker="mnesia#dump_tables/1">mnesia:dump_tables(TabList)</seealso>.</p> + </item> + <item><c>{disc_only_copies, NodeList}</c>. These table + replicas are stored on disc only and are therefore slower to + access. However, a disc-only replica consumes less memory than + a table replica of the other two storage types. + </item> + <item><p><c>{index, AttributeNameList}</c>, where + <c>AttributeNameList</c> is a list of atoms specifying the + names of the attributes <c>Mnesia</c> is to build and maintain. + An index table exists for every element in the list. The first + field of a <c>Mnesia</c> record is the key and thus need no + extra index.</p> + <p>The first field of a record is the second element of the + tuple, which is the representation of the record.</p> + </item> + <item><p><c>{snmp, SnmpStruct}</c>. <c>SnmpStruct</c> is + described in the + <seealso marker="snmp:index">SNMP</seealso> User's Guide. + Basically, if this attribute is present in <c>ArgList</c> of + <seealso marker="mnesia#create_table/2">mnesia:create_table/2</seealso>, + the table is immediately accessible the SNMP.</p> + <p>It is easy to design applications that use SNMP to + manipulate and control the system. <c>Mnesia</c> provides a + direct mapping between the logical tables that make up an SNMP + control application and the physical data that makes up a + <c>Mnesia</c> table. The default value is <c>[]</c>.</p> + </item> + <item><c>{local_content, true}</c>. When an application needs a + table whose contents is to be locally unique on each node, + <c>local_content</c> tables can be used. The name of the + table is known to all <c>Mnesia</c> nodes, but its contents is + unique for each node. Access to this type of table must be + done locally.</item> + <item> + <p><c>{attributes, AtomList}</c> is a list of the attribute + names for the records that are supposed to populate the + table. Default is the list <c>[key, val]</c>. The + table must at least have one extra attribute besides the + key. When accessing single attributes in a record, it is not + recommended to hard code the attribute names as atoms. Use + the construct <c>record_info(fields, record_name)</c> + instead.</p> + <p>The expression + <c>record_info(fields, record_name)</c> is processed by the + Erlang preprocessor and returns a list of the + record field names. With the record definition + <c>-record(foo, {x,y,z}).</c>, the expression + <c>record_info(fields,foo)</c> is expanded to the list + <c>[x,y,z]</c>. It is therefore possible for you to provide + the attribute names or to use the <c>record_info/2</c> + notation.</p> + <p>It is recommended to use the <c>record_info/2</c> notation, + as it becomes easier to maintain the program and the program + becomes more robust with regards to future record changes.</p> + </item> + <item> + <p><c>{record_name, Atom}</c> specifies the common name of + all records stored in the table. All records stored in + the table must have this name as their first element. + <c>record_name</c> defaults to the name of the table. + For more information, see + <seealso marker="Mnesia_chap4#recordnames_tablenames">Record Names versus Table Names</seealso>.</p> + </item> + </list> + </item> + </list> + <p>As an example, consider the following record definition:</p> + <pre> + -record(funky, {x, y}).</pre> + <p>The following call would create a table that is replicated on two + nodes, has an extra index on attribute <c>y</c>, and is of type + <c>bag</c>.</p> + <pre> + mnesia:create_table(funky, [{disc_copies, [N1, N2]}, {index, + [y]}, {type, bag}, {attributes, record_info(fields, funky)}]).</pre> + <p>Whereas a call to the following default code values would return + a table with a RAM copy on the local node, no extra indexes, and the + attributes defaulted to the list <c>[key,val]</c>.</p> + <pre> +mnesia:create_table(stuff, [])</pre> + </section> +</chapter> diff --git a/lib/mnesia/doc/src/Mnesia_chap4.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap4.xmlsrc index a18f853662..1671faa9b6 100644 --- a/lib/mnesia/doc/src/Mnesia_chap4.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap4.xmlsrc @@ -31,163 +31,146 @@ <rev></rev> <file>Mnesia_chap4.xml</file> </header> - <p>This chapter describes the Mnesia transaction system and the - transaction properties which make Mnesia a fault tolerant, - distributed database management system. - </p> - <p>Also covered in this chapter are the locking functions, + <p>This section describes the <c>Mnesia</c> transaction system and + the transaction properties that make <c>Mnesia</c> a fault-tolerant, + distributed Database Management System (DBMS).</p> + <p>This section also describes the locking functions, including table locks and sticky locks, as well as alternative - functions which bypass the transaction system in favor of improved - speed and reduced overheads. These functions are called "dirty - operations". We also describe the usage of nested transactions. - This chapter contains the following sections: - </p> + functions that bypass the transaction system in favor of improved + speed and reduced overhead. These functions are called "dirty + operations". The use of nested transactions is also described. + The following topics are included:</p> <list type="bulleted"> - <item>transaction properties, which include atomicity, - consistency, isolation, and durability - </item> - <item>Locking - </item> - <item>Dirty operations - </item> - <item>Record names vs table names - </item> - <item>Activity concept and various access contexts - </item> - <item>Nested transactions - </item> - <item>Pattern matching - </item> - <item>Iteration - </item> + <item>Transaction properties, which include atomicity, + consistency, isolation, and durability</item> + <item>Locking</item> + <item>Dirty operations</item> + <item>Record names versus table names</item> + <item>Activity concept and various access contexts</item> + <item>Nested transactions</item> + <item>Pattern matching</item> + <item>Iteration</item> </list> <section> <marker id="trans_prop"></marker> <title>Transaction Properties</title> - <p>Transactions are an important tool when designing fault - tolerant, distributed systems. A Mnesia transaction is a mechanism + <p>Transactions are important when designing fault-tolerant, + distributed systems. A <c>Mnesia</c> transaction is a mechanism by which a series of database operations can be executed as one - functional block. The functional block which is run as a + functional block. The functional block that is run as a transaction is called a Functional Object (Fun), and this code can - read, write, or delete Mnesia records. The Fun is evaluated as a - transaction which either commits, or aborts. If a transaction - succeeds in executing Fun it will replicate the action on all nodes - involved, or abort if an error occurs. - </p> - <p>The following example shows a transaction which raises the - salary of certain employee numbers. - </p> + read, write, and delete <c>Mnesia</c> records. The Fun is evaluated + as a transaction that either commits or terminates. If a transaction + succeeds in executing the Fun, it replicates the action on all nodes + involved, or terminates if an error occurs.</p> + <p>The following example shows a transaction that raises the + salary of certain employee numbers:</p> <codeinclude file="company.erl" tag="%5" type="erl"></codeinclude> - <p>The transaction <c>raise(Eno, Raise) - ></c> contains a Fun - made up of four lines of code. This Fun is called by the statement - <c>mnesia:transaction(F)</c> and returns a value. - </p> - <p>The Mnesia transaction system facilitates the construction of + <p>The function <c>raise/2</c> contains a Fun + made up of four code lines. This Fun is called by the statement + <c>mnesia:transaction(F)</c> and returns a value.</p> + <p>The <c>Mnesia</c> transaction system facilitates the construction of reliable, distributed systems by providing the following important - properties: - </p> + properties:</p> <list type="bulleted"> - <item>The transaction handler ensures that a Fun which is placed - inside a transaction does not interfere with operations embedded + <item>The transaction handler ensures that a Fun, which is placed + inside a transaction, does not interfere with operations embedded in other transactions when it executes a series of operations on tables. </item> <item>The transaction handler ensures that either all operations in the transaction are performed successfully on all nodes atomically, or the transaction fails without permanent effect on - any of the nodes. + any node. </item> - <item>The Mnesia transactions have four important properties, - which we call <em>A</em>tomicity, - <em>C</em>onsistency,<em>I</em>solation, and - <em>D</em>urability, or ACID for short. These properties are - described in the following sub-sections.</item> + <item>The <c>Mnesia</c> transactions have four important properties, + called <em>A</em>tomicity, + <em>C</em>onsistency, <em>I</em>solation, and + <em>D</em>urability (ACID). These properties are + described in the following sections.</item> </list> <section> <title>Atomicity</title> - <p><em>Atomicity</em> means that database changes which are + <p>Atomicity means that database changes that are executed by a transaction take effect on all nodes involved, or - on none of the nodes. In other words, the transaction either - succeeds entirely, or it fails entirely. - </p> - <p>Atomicity is particularly important when we want to - atomically write more than one record in the same - transaction. The <c>raise/2</c> function, shown as an example - above, writes one record only. The <c>insert_emp/3</c> function, - shown in the program listing in Chapter 2, writes the record - <c>employee</c> as well as employee relations such as - <c>at_dep</c> and <c>in_proj</c> into the database. If we run - this latter code inside a transaction, then the transaction + on none of the nodes. That is, the transaction either + succeeds entirely, or it fails entirely.</p> + <p>Atomicity is important when it is needed to write + atomically more than one record in the same + transaction. The function <c>raise/2</c>, shown in the previous + example, writes one record only. The function <c>insert_emp/3</c>, + shown in the program listing in + <seealso marker="Mnesia_chap2#getting_started">Getting Started</seealso>, writes the record + <c>employee</c> as well as employee relations, such as + <c>at_dep</c> and <c>in_proj</c>, into the database. If this + latter code is run inside a transaction, the transaction handler ensures that the transaction either succeeds completely, - or not at all. - </p> - <p>Mnesia is a distributed DBMS where data can be replicated on - several nodes. In many such applications, it is important that a + or not at all.</p> + <p><c>Mnesia</c> is a distributed DBMS where data can be replicated + on several nodes. In many applications, it is important that a series of write operations are performed atomically inside a transaction. The atomicity property ensures that a transaction - take effect on all nodes, or none at all. </p> + takes effect on all nodes, or none.</p> </section> <section> <title>Consistency</title> - <p><em>Consistency</em>. This transaction property ensures that + <p>The consistency property ensures that a transaction always leaves the DBMS in a consistent state. For - example, Mnesia ensures that inconsistencies will not occur if - Erlang, Mnesia or the computer crashes while a write operation - is in progress. - </p> + example, <c>Mnesia</c> ensures that no inconsistencies occur if + Erlang, <c>Mnesia</c>, or the computer crashes while a write + operation is in progress.</p> </section> <section> <title>Isolation</title> - <p><em>Isolation</em>. This transaction property ensures that - transactions which execute on different nodes in a network, and - access and manipulate the same data records, will not interfere - with each other. - </p> - <p>The isolation property makes it possible to concurrently execute - the <c>raise/2</c> function. A classical problem in concurrency control - theory is the so called "lost update problem". - </p> - <p>The isolation property is extremely useful if the following - circumstances occurs where an employee (with an employee number - 123) and two processes, (P1 and P2), are concurrently trying to - raise the salary for the employee. The initial value of the - employees salary is, for example, 5. Process P1 then starts to execute, - it reads the employee record and adds 2 to the salary. At this - point in time, process P1 is for some reason preempted and - process P2 has the opportunity to run. P2 reads the record, adds 3 - to the salary, and finally writes a new employee record with - the salary set to 8. Now, process P1 start to run again and + <p>The isolation property ensures that + transactions that execute on different nodes in a network, and + access and manipulate the same data records, do not interfere + with each other. The isolation property makes it possible to + execute the function <c>raise/2</c> concurrently. A classical + problem in concurrency control theory is the "lost update + problem".</p> + <p>The isolation property is in particular useful if the following + circumstances occur where an employee (with employee number + 123) and two processes (P1 and P2) are concurrently trying to + raise the salary for the employee:</p> + <list type="bulleted"> + <item><em>Step 1:</em> The initial value of the employees salary + is, for example, 5. Process P1 starts to execute, reads the + employee record, and adds 2 to the salary.</item> + <item><em>Step 2:</em> Process P1 is for some reason pre-empted + and process P2 has the opportunity to run.</item> + <item><em>Step 3:</em> Process P2 reads the record, adds 3 to + the salary, and finally writes a new employee record with + the salary set to 8.</item> + <item><em>Step 4:</em> Process P1 starts to run again and writes its employee record with salary set to 7, thus effectively overwriting and undoing the work performed by - process P2. The update performed by P2 is lost. - </p> - <p>A transaction system makes it possible to concurrently - execute two or more processes which manipulate the same - record. The programmer does not need to check that the - updates are synchronous, this is overseen by the + process P2. The update performed by P2 is lost.</item> + </list> + <p>A transaction system makes it possible to execute two or more + processes concurrently that manipulate the same record. + The programmer does not need to check that the + updates are synchronous; this is overseen by the transaction handler. All programs accessing the database through - the transaction system may be written as if they had sole access - to the data. - </p> + the transaction system can be written as if they had sole access + to the data.</p> </section> <section> <title>Durability</title> - <p><em>Durability</em>. This transaction property ensures that + <p>The durability property ensures that changes made to the DBMS by a transaction are permanent. Once a - transaction has been committed, all changes made to the database - are durable - i.e. they are written safely to disc and will not - be corrupted or disappear. - </p> + transaction is committed, all changes made to the database are + durable, that is, they are written safely to disc and do not + become corrupted and do not disappear.</p> <note> - <p>The durability feature described does not entirely apply to - situations where Mnesia is configured as a "pure" primary memory - database. - </p> + <p>The described durability feature does not entirely apply to + situations where <c>Mnesia</c> is configured as a "pure" + primary memory database.</p> </note> </section> </section> @@ -195,397 +178,383 @@ <section> <title>Locking</title> <p>Different transaction managers employ different strategies to - satisfy the isolation property. Mnesia uses the standard technique - of two-phase locking. This means that locks are set on records - before they are read or written. Mnesia uses five different kinds - of locks. - </p> + satisfy the isolation property. <c>Mnesia</c> uses the standard + technique of two phase locking. That is, locks are set on records + before they are read or written. <c>Mnesia</c> uses the following + lock types:</p> <list type="bulleted"> <item><em>Read locks</em>. A read lock is set on one replica of a record before it can be read. </item> - <item><em>Write locks</em>. Whenever a transaction writes to an + <item><em>Write locks</em>. Whenever a transaction writes to a record, write locks are first set on all replicas of that - particular record. + particular record. </item> <item><em>Read table locks</em>. If a transaction traverses an - entire table in search for a record which satisfy some + entire table in search for a record that satisfies some particular property, it is most inefficient to set read locks on - the records, one by one. It is also very memory consuming, since - the read locks themselves may take up considerable space if the - table is very large. For this reason, Mnesia can set a read lock - on an entire table. + the records one by one. It is also memory consuming, as + the read locks themselves can take up considerable space if the + table is large. Therefore, <c>Mnesia</c> can set a read lock + on an entire table. </item> - <item><em>Write table locks</em>. If a transaction writes a - large number of records to one table, it is possible to set a - write lock on the entire table. + <item><em>Write table locks</em>. If a transaction writes many + records to one table, a write lock can be set on the entire table. </item> <item><em>Sticky locks</em>. These are write locks that stay in - place at a node after the transaction which initiated the lock - has terminated. </item> + place at a node after the transaction that initiated the lock + has terminated.</item> </list> - <p>Mnesia employs a strategy whereby functions such as - <c>mnesia:read/1</c> acquire the necessary locks dynamically as - the transactions execute. Mnesia automatically sets and releases - the locks and the programmer does not have to code these - operations. - </p> + <p><c>Mnesia</c> employs a strategy whereby functions, such as + <seealso marker="mnesia#read/1">mnesia:read/1</seealso> + acquire the necessary locks dynamically as + the transactions execute. <c>Mnesia</c> automatically sets and + releases the locks and the programmer does not need to code these + operations.</p> <p>Deadlocks can occur when concurrent processes set and release - locks on the same records. Mnesia employs a "wait-die" strategy to - resolve these situations. If Mnesia suspects that a deadlock can + locks on the same records. <c>Mnesia</c> employs a "wait-die" + strategy to resolve + these situations. If <c>Mnesia</c> suspects that a deadlock can occur when a transaction tries to set a lock, the transaction is - forced to release all its locks and sleep for a while. The - Fun in the transaction will be evaluated one more time. - </p> - <p>For this reason, it is important that the code inside the Fun given to - <c>mnesia:transaction/1</c> is pure. Some strange results can + forced to release all its locks and sleep for a while. The Fun + in the transaction is evaluated once more.</p> + <p>It is therefore important that the code inside the Fun given to + <seealso marker="mnesia#transaction/2"><c>mnesia:transaction/1</c></seealso> + is pure. Some strange results can occur if, for example, messages are sent by the transaction - Fun. The following example illustrates this situation: - </p> + Fun. The following example illustrates this situation:</p> <codeinclude file="company.erl" tag="%6" type="erl"></codeinclude> - <p>This transaction could write the text <c>"Trying to write ... "</c> a thousand times to the terminal. Mnesia does guarantee, - however, that each and every transaction will eventually run. As a - result, Mnesia is not only deadlock free, but also livelock - free. - </p> - <p>The Mnesia programmer cannot prioritize one particular - transaction to execute before other transactions which are waiting - to execute. As a result, the Mnesia DBMS transaction system is not - suitable for hard real time applications. However, Mnesia contains - other features that have real time properties. - </p> - <p>Mnesia dynamically sets and releases locks as - transactions execute, therefore, it is very dangerous to execute code with + <p>This transaction can write the text <c>"Trying to write ... "</c> + 1000 times to the terminal. However, <c>Mnesia</c> guarantees + that each transaction will eventually run. As a result, + <c>Mnesia</c> is not only deadlock free, but also livelock free.</p> + <p>The <c>Mnesia</c> programmer cannot prioritize one particular + transaction to execute before other transactions that are waiting + to execute. As a result, the <c>Mnesia</c> DBMS transaction system is + not suitable for hard real-time applications. However, <c>Mnesia</c> + contains other features that have real-time properties.</p> + <p><c>Mnesia</c> dynamically sets and releases locks as transactions + execute. It is therefore dangerous to execute code with transaction side-effects. In particular, a <c>receive</c> statement inside a transaction can lead to a situation where the transaction hangs and never returns, which in turn can cause locks - not to release. This situation could bring the whole system to a - standstill since other transactions which execute in other + not to release. This situation can bring the whole system to a + standstill, as other transactions that execute in other processes, or on other nodes, are forced to wait for the defective - transaction. - </p> - <p>If a transaction terminates abnormally, Mnesia will - automatically release the locks held by the transaction. - </p> - <p>We have shown examples of a number of functions that can be - used inside a transaction. The following list shows the - <em>simplest</em> Mnesia functions that work with transactions. It - is important to realize that these functions must be embedded in a + transaction.</p> + <p>If a transaction terminates abnormally, <c>Mnesia</c> + automatically releases the locks held by the transaction.</p> + <p>Up to now, examples of a number of functions that can be used + inside a transaction have been shown. The following list shows + the <em>simplest</em> <c>Mnesia</c> functions that work with + transactions. Notice that these functions must be embedded in a transaction. If no enclosing transaction (or other enclosing - Mnesia activity) exists, they will all fail. - </p> + <c>Mnesia</c> activity) exists, they all fail.</p> <list type="bulleted"> - <item><c>mnesia:transaction(Fun) -> {aborted, Reason} |{atomic, Value}</c>. This function executes one transaction with the - functional object <c>Fun</c> as the single parameter. + <item><seealso marker="mnesia#transaction/2">mnesia:transaction(Fun) -> {aborted, Reason} |{atomic, Value}</seealso> + executes one transaction with the + functional object <c>Fun</c> as the single parameter. </item> - <item><c>mnesia:read({Tab, Key}) -> transaction abort | RecordList</c>. This function reads all records with <c>Key</c> - as key from table <c>Tab</c>. This function has the same semantics + <item><seealso marker="mnesia#read/1">mnesia:read({Tab, Key}) -> transaction abort | RecordList</seealso> + reads all records with <c>Key</c> + as key from table <c>Tab</c>. This function has the same semantics regardless of the location of <c>Table</c>. If the table is of - type <c>bag</c>, the <c>read({Tab, Key})</c> can return an arbitrarily + type <c>bag</c>, <c>read({Tab, Key})</c> can return an arbitrarily long list. If the table is of type <c>set</c>, the list is - either of length one, or <c>[]</c>. + either of length one or <c>[]</c>. </item> - <item><c>mnesia:wread({Tab, Key}) -> transaction abort | RecordList</c>. This function behaves the same way as the - previously listed <c>read/1</c> function, except that it - acquires a write lock instead of a read lock. If we execute a - transaction which reads a record, modifies the record, and then + <item><seealso marker="mnesia#wread/1">mnesia:wread({Tab, Key}) -> transaction abort | RecordList</seealso> + behaves the same way as the + previously listed function <c>read/1</c>, except that it + acquires a write lock instead of a read lock. To execute a + transaction that reads a record, modifies the record, and then writes the record, it is slightly more efficient to set the - write lock immediately. In cases where we issue a - <c>mnesia:read/1</c>, followed by a <c>mnesia:write/1</c>, the - first read lock must be upgraded to a write lock when the write - operation is executed. + write lock immediately. When a <seealso marker="mnesia#read/1">mnesia:read/1</seealso> + is issued, followed by a + <seealso marker="mnesia#write/1">mnesia:write/1</seealso> + the first read lock must be upgraded to a write lock when the + write operation is executed. </item> - <item><c>mnesia:write(Record) -> transaction abort | ok</c>. This function writes a record into the database. The - <c>Record</c> argument is an instance of a record. The function - returns <c>ok</c>, or aborts the transaction if an error should - occur. + <item><seealso marker="mnesia#write/1">mnesia:write(Record) -> transaction abort | ok</seealso> + writes a record into the database. Argument + <c>Record</c> is an instance of a record. The function returns + <c>ok</c>, or terminates the transaction if an error occurs. </item> - <item><c>mnesia:delete({Tab, Key}) -> transaction abort | ok</c>. This - function deletes all records with the given key. + <item><seealso marker="mnesia#delete/1">mnesia:delete({Tab, Key}) -> transaction abort | ok</seealso> + deletes all records with the given key. </item> - <item><c>mnesia:delete_object(Record) -> transaction abort | ok</c>. This function deletes records with object id - <c>Record</c>. This function is used when we want to delete only - some records in a table of type <c>bag</c>. </item> + <item><seealso marker="mnesia#delete_object/1">mnesia:delete_object(Record) -> transaction abort | ok</seealso> + deletes records with the OID <c>Record</c>. Use this function to + delete only some records in a table of type <c>bag</c>.</item> </list> <section> <title>Sticky Locks</title> - <p>As previously stated, the locking strategy used by Mnesia is - to lock one record when we read a record, and lock all replicas - of a record when we write a record. However, there are - applications which use Mnesia mainly for its fault-tolerant - qualities, and these applications may be configured with one - node doing all the heavy work, and a standby node which is ready - to take over in case the main node fails. Such applications may + <p>As previously stated, the locking strategy used by <c>Mnesia</c> + is to lock one record when reading a record, and lock all replicas + of a record when writing a record. However, some + applications use <c>Mnesia</c> mainly for its fault-tolerant + qualities. These applications can be configured with one + node doing all the heavy work, and a standby node that is ready + to take over if the main node fails. Such applications can benefit from using sticky locks instead of the normal locking - scheme. - </p> - <p>A sticky lock is a lock which stays in place at a node after - the transaction which first acquired the lock has terminated. To - illustrate this, assume that we execute the following - transaction: - </p> + scheme.</p> + <p>A sticky lock is a lock that stays in place at a node, after + the transaction that first acquired the lock has terminated. To + illustrate this, assume that the following transaction is + executed:</p> <code type="none"> F = fun() -> mnesia:write(#foo{a = kalle}) end, - mnesia:transaction(F). - </code> + mnesia:transaction(F).</code> <p>The <c>foo</c> table is replicated on the two nodes <c>N1</c> - and <c>N2</c>. - <br></br> -Normal locking requires: - </p> + and <c>N2</c>.</p> + <p>Normal locking requires the following:</p> <list type="bulleted"> - <item>one network rpc (2 messages) to acquire the write lock + <item>One network RPC (two messages) to acquire the write lock </item> - <item>three network messages to execute the two-phase commit protocol. + <item>Three network messages to execute the two-phase commit + protocol </item> </list> - <p>If we use sticky locks, we must first change the code as follows: - </p> + <p>If sticky locks are used, the code must first be changed as + follows:</p> <code type="none"> - F = fun() -> mnesia:s_write(#foo{a = kalle}) end, - mnesia:transaction(F). - </code> - <p>This code uses the <c>s_write/1</c> function instead of the - <c>write/1</c> function. The <c>s_write/1</c> function sets a + mnesia:transaction(F).</code> + <p>This code uses the function + <seealso marker="mnesia#s_write/1">s_write/1</seealso> + instead of the function + <seealso marker="mnesia#write/1">write/1</seealso> + The function <c>s_write/1</c> sets a sticky lock instead of a normal lock. If the table is not replicated, sticky locks have no special effect. If the table is - replicated, and we set a sticky lock on node <c>N1</c>, this - lock will then stick to node <c>N1</c>. The next time we try to - set a sticky lock on the same record at node <c>N1</c>, Mnesia - will see that the lock is already set and will not do a network - operation in order to acquire the lock. - </p> - <p>It is much more efficient to set a local lock than it is to set - a networked lock, and for this reason sticky locks can benefit - application that use a replicated table and perform most of the - work on only one of the nodes. - </p> - <p>If a record is stuck at node <c>N1</c> and we try to set a + replicated, and a sticky lock is set on node <c>N1</c>, this + lock then sticks to node <c>N1</c>. The next time you try to + set a sticky lock on the same record at node <c>N1</c>, + <c>Mnesia</c> detects that the lock is already set and do no + network operation to acquire the lock.</p> + <p>It is more efficient to set a local lock than it is to set + a networked lock. Sticky locks can therefore benefit an + application that uses a replicated table and perform most of the + work on only one of the nodes.</p> + <p>If a record is stuck at node <c>N1</c> and you try to set a sticky lock for the record on node <c>N2</c>, the record must be - unstuck. This operation is expensive and will reduce performance. The unsticking is - done automatically if we issue <c>s_write/1</c> requests at - <c>N2</c>. - </p> + unstuck. This operation is expensive and reduces performance. + The unsticking is done automatically if you issue <c>s_write/1</c> + requests at <c>N2</c>.</p> </section> <section> <title>Table Locks</title> - <p>Mnesia supports read and write locks on whole tables as a + <p><c>Mnesia</c> supports read and write locks on whole tables as a complement to the normal locks on single records. As previously - stated, Mnesia sets and releases locks automatically, and the - programmer does not have to code these operations. However, - transactions which read and write a large number of records in a - specific table will execute more efficiently if we start the - transaction by setting a table lock on this table. This will - block other concurrent transactions from the table. The - following two function are used to set explicit table locks for - read and write operations: - </p> + stated, <c>Mnesia</c> sets and releases locks automatically, and + the programmer does not need to code these operations. However, + transactions that read and write many records in a + specific table execute more efficiently if the + transaction is started by setting a table lock on this table. This + blocks other concurrent transactions from the table. The + following two functions are used to set explicit table locks for + read and write operations:</p> <list type="bulleted"> - <item><c>mnesia:read_lock_table(Tab)</c> Sets a read lock on - the table <c>Tab</c></item> - <item><c>mnesia:write_lock_table(Tab)</c> Sets a write lock on - the table <c>Tab</c></item> + <item><seealso marker="mnesia#read_lock_table/1">mnesia:read_lock_table(Tab)</seealso> + sets a read lock on table <c>Tab</c>.</item> + <item><seealso marker="mnesia#write_lock_table/1">mnesia:write_lock_table(Tab)</seealso> + sets a write lock on table <c>Tab</c>.</item> </list> - <p>Alternate syntax for acquisition of table locks is as follows: - </p> + <p>Alternative syntax for acquisition of table locks is as + follows:</p> <code type="none"> mnesia:lock({table, Tab}, read) - mnesia:lock({table, Tab}, write) - </code> - <p>The matching operations in Mnesia may either lock the entire - table or just a single record (when the key is bound in the - pattern). - </p> + mnesia:lock({table, Tab}, write)</code> + <p>The matching operations in <c>Mnesia</c> can either lock the + entire table or only a single record (when the key is bound in + the pattern).</p> </section> <section> <title>Global Locks</title> <p>Write locks are normally acquired on all nodes where a replica of the table resides (and is active). Read locks are - acquired on one node (the local one if a local - replica exists). - </p> - <p>The function <c>mnesia:lock/2</c> is intended to support - table locks (as mentioned previously) - but also for situations when locks need to be - acquired regardless of how tables have been replicated: - </p> + acquired on one node (the local one if a local + replica exists).</p> + <p>The function + <seealso marker="mnesia#lock/2">mnesia:lock/2</seealso> + is intended to support table locks (as mentioned previously) + but also for situations when locks need to be + acquired regardless of how tables have been replicated:</p> <code type="none"> mnesia:lock({global, GlobalKey, Nodes}, LockKind) - LockKind ::= read | write | ... - </code> - <p>The lock is acquired on the LockItem on all Nodes in the - nodes list.</p> + LockKind ::= read | write | ...</code> + <p>The lock is acquired on <c>LockItem</c> on all nodes in the + node list.</p> </section> </section> <section> <title>Dirty Operations</title> <p>In many applications, the overhead of processing a transaction - may result in a loss of performance. Dirty operation are short - cuts which bypass much of the processing and increase the speed - of the transaction. - </p> - <p>Dirty operation are useful in many situations, for example in a datagram routing - application where Mnesia stores the routing table, and it is time + can result in a loss of performance. Dirty operation are short + cuts that bypass much of the processing and increase the speed + of the transaction.</p> + <p>Dirty operation are often useful, for example, in a + datagram routing application + where <c>Mnesia</c> stores the routing table, and it is time consuming to start a whole transaction every time a packet is - received. For this reason, Mnesia has functions which manipulate + received. <c>Mnesia</c> has therefore functions that manipulate tables without using transactions. This alternative - to processing is known as a dirty operation. However, it is important to - realize the trade-off in avoiding the overhead of transaction - processing: - </p> + to processing is known as a dirty operation. However, notice the + trade-off in avoiding the overhead of transaction processing:</p> <list type="bulleted"> - <item>The atomicity and the isolation properties of Mnesia are lost. + <item>The atomicity and the isolation properties of <c>Mnesia</c> + are lost. </item> <item>The isolation property is compromised, because other Erlang processes, which use transaction to manipulate the data, - do not get the benefit of isolation if we simultaneously use - dirty operations to read and write records from the same table. + do not get the benefit of isolation if dirty operations + simultaneously are used to read and write records from the same + table. </item> </list> <p>The major advantage of dirty operations is that they execute - much faster than equivalent operations that are processed as - functional objects within a transaction. - </p> + much faster than equivalent operations that are processed as + functional objects within a transaction.</p> <p>Dirty operations are written to disc if they are performed on a table of type - <c>disc_copies</c>, or type <c>disc_only_copies</c>. Mnesia also - ensures that all replicas of a table are updated if a - dirty write operation is performed on a table. - </p> - <p>A dirty operation will ensure a certain level of consistency. - For example, it is not possible for dirty operations to return - garbled records. Hence, each individual read or write operation - is performed in an atomic manner. - </p> - <p>All dirty functions execute a call to <c>exit({aborted, Reason})</c> on failure. Even if the following functions are - executed inside a transaction no locks will be acquired. The - following functions are available: - </p> + <c>disc_copies</c> or type <c>disc_only_copies</c>. <c>Mnesia</c> + also ensures that all replicas of a table are updated if a + dirty write operation is performed on a table.</p> + <p>A dirty operation ensures a certain level of consistency. + For example, dirty operations cannot return + garbled records. Hence, each individual read or write operation + is performed in an atomic manner.</p> + <p>All dirty functions execute a call to <c>exit({aborted, Reason})</c> + on failure. Even if the following functions are + executed inside a transaction no locks are acquired. The + following functions are available:</p> <list type="bulleted"> - <item><c>mnesia:dirty_read({Tab, Key})</c>. This function reads - record(s) from Mnesia. + <item><seealso marker="mnesia#dirty_read/1">mnesia:dirty_read({Tab, Key})</seealso> + reads one or more records from <c>Mnesia</c>. + </item> + <item><seealso marker="mnesia#dirty_write/1">mnesia:dirty_write(Record)</seealso> + writes the record <c>Record</c>. </item> - <item><c>mnesia:dirty_write(Record)</c>. This function writes - the record <c>Record</c></item> - <item><c>mnesia:dirty_delete({Tab, Key})</c>. This function deletes - record(s) with the key <c>Key</c>. + <item><seealso marker="mnesia#dirty_delete/1">mnesia:dirty_delete({Tab, Key})</seealso> + deletes one or more records with key <c>Key</c>. + </item> + <item><seealso marker="mnesia#dirty_delete_object/1">mnesia:dirty_delete_object(Record)</seealso> + is the dirty operation alternative to the function + <seealso marker="mnesia#delete_object/1">delete_object/1</seealso>. </item> - <item><c>mnesia:dirty_delete_object(Record)</c> This function is - the dirty operation alternative to the function - <c>delete_object/1</c></item> <item> - <p><c>mnesia:dirty_first(Tab)</c>. This function returns the - "first" key in the table <c>Tab</c>. </p> - <p>Records in <c>set</c> or <c>bag</c> tables are not sorted. - However, there is - a record order which is not known to the user. - This means that it is possible to traverse a table by means of - this function in conjunction with the <c>dirty_next/2</c> - function. - </p> - <p>If there are no records at all in the table, this function - will return the atom <c>'$end_of_table'</c>. It is not + <p><seealso marker="mnesia#dirty_first/1">mnesia:dirty_first(Tab)</seealso> + returns the "first" key in table <c>Tab</c>.</p> + <p>Records in <c>set</c> or <c>bag</c> tables are not sorted. + However, there is a record order that is unknown to the user. + This means that a table can be traversed by this function + with the function + <seealso marker="mnesia#dirty_next/2">mnesia:dirty_next/2</seealso>.</p> + <p>If there are no records in the table, this function + returns the atom <c>'$end_of_table'</c>. It is not recommended to use this atom as the key for any user - records. - </p> + records.</p> </item> - <item><c>mnesia:dirty_next(Tab, Key)</c>. This function returns - the "next" key in the table <c>Tab</c>. This function makes it + <item><p><seealso marker="mnesia#dirty_next/2">mnesia:dirty_next(Tab, Key)</seealso> + returns the "next" key in table <c>Tab</c>. This function makes it possible to traverse a table and perform some operation on all - records in the table. When the end of the table is reached the + records in the table. When the end of the table is reached, the special key <c>'$end_of_table'</c> is returned. Otherwise, the - function returns a key which can be used to read the actual - record. - <br></br> -The behavior is undefined if any process perform a write - operation on the table while we traverse the table with the - <c>dirty_next/2</c> function. This is because <c>write</c> - operations on a Mnesia table may lead to internal reorganizations - of the table itself. This is an implementation detail, but remember - the dirty functions are low level functions. + function returns a key that can be used to read the actual + record.</p> + <p>The behavior is undefined if any process performs a write + operation on the table while traversing the table with the + function + <seealso marker="mnesia#dirty_next/2">dirty_next/2</seealso> + This is because <c>write</c> + operations on a <c>Mnesia</c> table can lead to internal + reorganizations of the table itself. This is an implementation + detail, but remember that the dirty functions are low-level + functions.</p> </item> - <item><c>mnesia:dirty_last(Tab)</c> This function works exactly like - <c>mnesia:dirty_first/1</c> but returns the last object in - Erlang term order for the <c>ordered_set</c> table type. For + <item><seealso marker="mnesia#dirty_last/1">mnesia:dirty_last(Tab)</seealso> + works exactly like + <seealso marker="mnesia#dirty_first/1">mnesia:dirty_first/1</seealso> + but returns the last object in + Erlang term order for the table type <c>ordered_set</c>. For all other table types, <c>mnesia:dirty_first/1</c> and - <c>mnesia:dirty_last/1</c> are synonyms. + <c>mnesia:dirty_last/1</c> are synonyms. </item> - <item><c>mnesia:dirty_prev(Tab, Key)</c> This function works exactly like - <c>mnesia:dirty_next/2</c> but returns the previous object in - Erlang term order for the ordered_set table type. For - all other table types, <c>mnesia:dirty_next/2</c> and - <c>mnesia:dirty_prev/2</c> are synonyms. + <item><seealso marker="mnesia#dirty_prev/2">mnesia:dirty_prev(Tab, Key)</seealso> + works exactly like + <c>mnesia:dirty_next/2</c> but returns the previous object in + Erlang term order for the table type <c>ordered_set</c>. For + all other table types, <c>mnesia:dirty_next/2</c> and + <c>mnesia:dirty_prev/2</c> are synonyms. </item> <item> - <p><c>mnesia:dirty_slot(Tab, Slot)</c></p> - <p>Returns the list of records that are associated with Slot + <p><seealso marker="mnesia#dirty_slot/2">mnesia:dirty_slot(Tab, Slot)</seealso> + returns the list of records that are associated with <c>Slot</c> in a table. It can be used to traverse a table in a manner - similar to the <c>dirty_next/2</c> function. A table has a + similar to the function <c>dirty_next/2</c>. A table has a number of slots that range from zero to some unknown upper bound. The function <c>dirty_slot/2</c> returns the special atom <c>'$end_of_table'</c> when the end of the table is - reached. - <br></br> -The behavior of this function is undefined if the + reached.</p> + <p>The behavior of this function is undefined if the table is written on while being - traversed. <c>mnesia:read_lock_table(Tab)</c> may be used to - ensure that no transaction protected writes are performed - during the iteration. - </p> + traversed. The function + <seealso marker="mnesia#read_lock_table/1">mnesia:read_lock_table(Tab)</seealso> + can be used to ensure that no transaction-protected writes + are performed during the iteration.</p> </item> - <item> - <p><c>mnesia:dirty_update_counter({Tab,Key}, Val)</c>. </p> - <p>Counters are positive integers with a value greater than or - equal to zero. Updating a counter will add the <c>Val</c> and - the counter where <c>Val</c> is a positive or negative integer. - <br></br> - There exists no special counter records in - Mnesia. However, records on the form of <c>{TabName, Key, Integer}</c> can be used as counters, and can be - persistent. - </p> - <p>It is not possible to have transaction protected updates of - counter records. - </p> + <item><p><seealso marker="mnesia#dirty_update_counter/2">mnesia:dirty_update_counter({Tab,Key}, Val)</seealso>. + Counters are positive integers with a value greater than or + equal to zero. Updating a counter adds <c>Val</c> and the + counter where <c>Val</c> is a positive or negative integer.</p> + <p><c>Mnesia</c> has no special counter records. However, records + of the form <c>{TabName, Key, Integer}</c> can be used as + counters, and can be persistent.</p> + <p>Transaction-protected updates of counter records are not + possible.</p> <p>There are two significant differences when using this function instead of reading the record, performing the - arithmetic, and writing the record: - </p> + arithmetic, and writing the record:</p> <list type="ordered"> - <item>it is much more efficient + <item>It is much more efficient. </item> - <item>the <c>dirty_update_counter/2</c> function is - performed as an atomic operation although it is not protected by - a transaction. Accordingly, no table update is lost if two - processes simultaneously execute the - <c>dirty_update_counter/2</c> function. + <item>The funcion + <seealso marker="mnesia#dirty_update_counter/2">dirty_update_counter/2</seealso> + is performed as an atomic operation although it is not protected + by a transaction. Therfore no table update is lost if two + processes simultaneously execute the function + <c>dirty_update_counter/2</c>. </item> </list> </item> - <item><c>mnesia:dirty_match_object(Pat)</c>. This function is - the dirty equivalent of <c>mnesia:match_object/1</c>. + <item><seealso marker="mnesia#dirty_match_object/2">mnesia:dirty_match_object(Pat)</seealso> + is the dirty equivalent of + <seealso marker="mnesia#match_object/1">mnesia:match_object/1</seealso>. </item> - <item><c>mnesia:dirty_select(Tab, Pat)</c>. This function is - the dirty equivalent of <c>mnesia:select/2</c>. + <item><seealso marker="mnesia#dirty_select/2">mnesia:dirty_select(Tab, Pat)</seealso> + is the dirty equivalent of + <seealso marker="mnesia#select/2"> mnesia:select/2</seealso>. </item> - <item><c>mnesia:dirty_index_match_object(Pat, Pos)</c>. This - function is the dirty equivalent of - <c>mnesia:index_match_object/2</c>. + <item><seealso marker="mnesia#dirty_index_match_object/2">mnesia:dirty_index_match_object(Pat, Pos)</seealso> + is the dirty equivalent of + <seealso marker="mnesia#index_match_object/2">mnesia:index_match_object/2</seealso>. </item> - <item><c>mnesia:dirty_index_read(Tab, SecondaryKey, Pos)</c>. This - function is the dirty equivalent of <c>mnesia:index_read/3</c>. + <item><seealso marker="mnesia#dirty_index_read/3">mnesia:dirty_index_read(Tab, SecondaryKey, Pos)</seealso> + is the dirty equivalent of + <seealso marker="mnesia#index_read/3">mnesia:index_read/3</seealso>. </item> - <item><c>mnesia:dirty_all_keys(Tab)</c>. This function is the - dirty equivalent of <c>mnesia:all_keys/1</c>. + <item><seealso marker="mnesia#dirty_all_keys/1">mnesia:dirty_all_keys(Tab)</seealso> + is the dirty equivalent of <seealso marker="mnesia#all_keys/1"> +mnesia:all_keys/1</seealso>. </item> </list> </section> @@ -593,42 +562,38 @@ The behavior of this function is undefined if the <section> <marker id="recordnames_tablenames"></marker> <title>Record Names versus Table Names</title> - <p>In Mnesia, all records in a table must have the same name. All - the records must be instances of the same - record type. The record name does however not necessarily be - the same as the table name. Even though that it is the case in - the most of the examples in this document. If a table is created - without the <c>record_name</c> property the code below will - ensure all records in the tables have the same name as the table: - </p> + <p>In <c>Mnesia</c>, all records in a table must have the same name. + All the records must be instances of the same + record type. The record name, however, does not necessarily have + to be the same as the table name, although this is the case in + most of the examples in this User's Guide. If a table is created + without property <c>record_name</c>, the following code ensures + that all records in the tables have the same name as the table:</p> <code type="none"> - mnesia:create_table(subscriber, []) - </code> - <p>However, if the table is is created with an explicit record name - as argument, as shown below, it is possible to store subscriber - records in both of the tables regardless of the table names: - </p> + mnesia:create_table(subscriber, [])</code> + <p>However, if the table is created with an explicit record name + as argument, as shown in the following example, subscriber records + can be stored in both of the tables regardless of the table + names:</p> <code type="none"> TabDef = [{record_name, subscriber}], mnesia:create_table(my_subscriber, TabDef), - mnesia:create_table(your_subscriber, TabDef). - </code> - <p>In order to access such - tables it is not possible to use the simplified access functions - as described earlier in the document. For example, - writing a subscriber record into a table requires a - <c>mnesia:write/3</c>function instead of the simplified functions - <c>mnesia:write/1</c> and <c>mnesia:s_write/1</c>: - </p> + mnesia:create_table(your_subscriber, TabDef).</code> + <p>To access such tables, simplified access functions + (as described earlier) cannot be used. For example, + writing a subscriber record into a table requires the function + <seealso marker="mnesia#write/3">mnesia:write/3</seealso> + instead of the simplified functions + <seealso marker="mnesia#write/1">mnesia:write/1</seealso> + and + <seealso marker="mnesia#s_write/1">mnesia:s_write/1</seealso>:</p> <code type="none"> mnesia:write(subscriber, #subscriber{}, write) mnesia:write(my_subscriber, #subscriber{}, sticky_write) - mnesia:write(your_subscriber, #subscriber{}, write) - </code> - <p>The following simplified piece of code illustrates the + mnesia:write(your_subscriber, #subscriber{}, write)</code> + <p>The following simple code illustrates the relationship between the simplified access functions used in - most examples and their more flexible counterparts: - </p> + most of the examples and their more flexible counterparts:</p> <code type="none"> mnesia:dirty_write(Record) -> Tab = element(1, Record), @@ -676,7 +641,7 @@ The behavior of this function is undefined if the mnesia:s_delete_object(Record) -> Tab = element(1, Record), - mnesia:delete_object(Tab, Record. sticky_write). + mnesia:delete_object(Tab, Record, sticky_write). mnesia:read({Tab, Key}) -> mnesia:read(Tab, Key, read). @@ -690,217 +655,222 @@ The behavior of this function is undefined if the mnesia:index_match_object(Pattern, Attr) -> Tab = element(1, Pattern), - mnesia:index_match_object(Tab, Pattern, Attr, read). - </code> + mnesia:index_match_object(Tab, Pattern, Attr, read).</code> </section> <section> <title>Activity Concept and Various Access Contexts</title> - <p>As previously described, a functional object (Fun) performing - table access operations as listed below may be - passed on as arguments to the function - <c>mnesia:transaction/1,2,3</c>: - </p> + <p>As previously described, a Functional Object (Fun) performing + table access operations, as listed here, can be passed + on as arguments to the function + <seealso marker="mnesia#transaction/2">mnesia:transaction/1,2,3</seealso>: + </p> <list type="bulleted"> <item> - <p>mnesia:write/3 (write/1, s_write/1)</p> + <seealso marker="mnesia#write/3">mnesia:write/3 (write/1, s_write/1)</seealso> </item> <item> - <p>mnesia:delete/3 (delete/1, s_delete/1)</p> + <seealso marker="mnesia#delete/3">mnesia:delete/3</seealso> + (<seealso marker="mnesia#delete/1">mnesia:delete/1</seealso>, + <seealso marker="mnesia#s_delete/1">mnesia:s_delete/1</seealso>) </item> <item> - <p>mnesia:delete_object/3 (delete_object/1, s_delete_object/1)</p> + <seealso marker="mnesia#delete_object/3">mnesia:delete_object/3</seealso> + (<seealso marker="mnesia#delete_object/1">mnesia:delete_object/1</seealso>, + <seealso marker="mnesia#s_delete_object/1">mnesia:s_delete_object/1</seealso>) </item> <item> - <p>mnesia:read/3 (read/1, wread/1)</p> + <seealso marker="mnesia#read/3">mnesia:read/3</seealso> + (<seealso marker="mnesia#read/1">mnesia:read/1</seealso>, + <seealso marker="mnesia#wread/1">mnesia:wread/1</seealso>) </item> <item> - <p>mnesia:match_object/2 (match_object/1)</p> + <seealso marker="mnesia#match_object/3">mnesia:match_object/2</seealso> + (<seealso marker="mnesia#match_object/1">mnesia:match_object/1</seealso>) </item> <item> - <p>mnesia:select/3 (select/2)</p> + <seealso marker="mnesia#select/2">mnesia:select/3</seealso> + (<seealso marker="mnesia#select/2">mnesia:select/2</seealso>) </item> <item> - <p>mnesia:foldl/3 (foldl/4, foldr/3, foldr/4)</p> + <seealso marker="mnesia#foldl/3">mnesia:foldl/3</seealso> + (<c>mnesia:foldl/4</c>, + <seealso marker="mnesia#foldr/3">mnesia:foldr/3</seealso>, + <c>mnesia:foldr/4</c>) </item> <item> - <p>mnesia:all_keys/1</p> + <seealso marker="mnesia#all_keys/1">mnesia:all_keys/1</seealso> </item> <item> - <p>mnesia:index_match_object/4 (index_match_object/2)</p> + <seealso marker="mnesia#index_match_object/4">mnesia:index_match_object/4</seealso> + (<seealso marker="mnesia#index_match_object/2">mnesia:index_match_object/2</seealso>) </item> <item> - <p>mnesia:index_read/3</p> + <seealso marker="mnesia#index_read/3">mnesia:index_read/3</seealso> </item> <item> - <p>mnesia:lock/2 (read_lock_table/1, write_lock_table/1)</p> + <seealso marker="mnesia#lock/2">mnesia:lock/2</seealso> + (<seealso marker="mnesia#read_lock_table/1">mnesia:read_lock_table/1</seealso>, + <seealso marker="mnesia#write_lock_table/1">mnesia:write_lock_table/1</seealso>) </item> <item> - <p>mnesia:table_info/2</p> + <seealso marker="mnesia#table_info/2">mnesia:table_info/2</seealso> </item> </list> - <p>These functions will be performed in a - transaction context involving mechanisms like locking, logging, - replication, checkpoints, subscriptions, commit protocols - etc.However, the same function may also be - evaluated in other activity contexts. - <br></br> -The following activity access contexts are currently supported: - </p> + <p>These functions are performed in a + transaction context involving mechanisms, such as locking, logging, + replication, checkpoints, subscriptions, and commit protocols. + However, the same function can also be + evaluated in other activity contexts.</p> + <p>The following activity access contexts are currently supported:</p> <list type="bulleted"> - <item> - <p>transaction </p> - </item> - <item> - <p>sync_transaction</p> - </item> - <item> - <p>async_dirty</p> - </item> - <item> - <p>sync_dirty</p> - </item> - <item> - <p>ets</p> - </item> + <item><c>transaction</c></item> + <item><c>sync_transaction</c></item> + <item><c>async_dirty</c></item> + <item><c>sync_dirty</c></item> + <item><c>ets</c></item> </list> <p>By passing the same "fun" as argument to the function - <c>mnesia:sync_transaction(Fun [, Args])</c> it will be performed - in synced transaction context. Synced transactions waits until all + <seealso marker="mnesia#sync_transaction/3">mnesia:sync_transaction(Fun [, Args])</seealso> + it is performed + in synced transaction context. Synced transactions wait until all active replicas has committed the transaction (to disc) before - returning from the mnesia:sync_transaction call. Using - sync_transaction is useful for applications that are executing on - several nodes and want to be sure that the update is performed on - the remote nodes before a remote process is spawned or a message - is sent to a remote process, and also when combining transaction - writes with dirty_reads. This is also useful in situations where - an application performs frequent or voluminous updates which may - overload Mnesia on other nodes. - </p> + returning from the <c>mnesia:sync_transaction</c> call. Using + <c>sync_transaction</c> is useful in the following cases:</p> + <list type="bulleted"> + <item>When an application executes on several nodes and wants to + be sure that the update is performed on the remote nodes before + a remote process is spawned or a message is sent to a remote + process.</item> + <item>When a combining transaction writes with "dirty_reads", that + is, the functions <c>dirty_match_object</c>, <c>dirty_read</c>, + <c>dirty_index_read</c>, <c>dirty_select</c>, and so on.</item> + <item>When an application performs frequent or voluminous updates + that can overload <c>Mnesia</c> on other nodes.</item> + </list> <p>By passing the same "fun" as argument to the function - <c>mnesia:async_dirty(Fun [, Args])</c> it will be performed in - dirty context. The function calls will be mapped to the - corresponding dirty functions. This will still involve logging, - replication and subscriptions but there will be no locking, - local transaction storage or commit protocols involved. - Checkpoint retainers will be updated but will be updated - "dirty". Thus, they will be updated asynchronously. The - functions will wait for the operation to be performed on one - node but not the others. If the table resides locally no waiting - will occur. - </p> + <seealso marker="mnesia#async_dirty/2">mnesia:async_dirty(Fun [, Args])</seealso>, + it is performed in dirty context. The function calls are mapped to + the corresponding dirty functions. This still involves logging, + replication, and subscriptions but no locking, + local transaction storage, or commit protocols are involved. + Checkpoint retainers are updated but updated + "dirty". Thus, they are updated asynchronously. The + functions wait for the operation to be performed on one + node but not the others. If the table resides locally, no waiting + occurs.</p> <p>By passing the same "fun" as an argument to the function - <c>mnesia:sync_dirty(Fun [, Args])</c> it will be performed in - almost the same context as <c>mnesia:async_dirty/1,2</c>. The - difference is that the operations are performed - synchronously. The caller will wait for the updates to be - performed on all active replicas. Using sync_dirty is useful for - applications that are executing on several nodes and want to be - sure that the update is performed on the remote nodes before a remote - process is spawned or a message is sent to a remote process. This - is also useful in situations where an application performs frequent or - voluminous updates which may overload Mnesia on other - nodes. - </p> - <p>You can check if your code is executed within a transaction with - <c>mnesia:is_transaction/0</c>, it returns <c>true</c> when called - inside a transaction context and false otherwise.</p> - - <p>Mnesia tables with storage type RAM_copies and disc_copies - are implemented internally as "ets-tables" and - it is possible for applications to access the these tables - directly. This is only recommended if all options have been weighed - and the possible outcomes are understood. By passing the earlier - mentioned "fun" to the function - <c>mnesia:ets(Fun [, Args])</c> it will be performed but in a very raw - context. The operations will be performed directly on the - local ets tables assuming that the local storage type are - RAM_copies and that the table is not replicated on other - nodes. Subscriptions will not be triggered nor - checkpoints updated, but this operation is blindingly fast. Disc resident - tables should not be updated with the ets-function since the - disc will not be updated. - </p> - <p>The Fun may also be passed as an argument to the function - <c>mnesia:activity/2,3,4</c> which enables usage of customized + <seealso marker="mnesia#sync_dirty/2">mnesia:sync_dirty(Fun [, Args])</seealso>, + it is performed in almost the same context as the function + <seealso marker="mnesia#async_dirty/2">mnesia:async_dirty/1,2</seealso>. + The difference is that the operations are performed + synchronously. The caller waits for the updates to be + performed on all active replicas. Using <c>mnesia:sync_dirty/1,2</c> + is useful in the following cases:</p> + <list type="bulleted"> + <item>When an application executes on several nodes and wants to + be sure that the update is performed on the remote nodes before + a remote process is spawned or a message is sent to a remote + process.</item> + <item>When an application performs frequent or voluminous updates + that can overload <c>Mnesia</c> on the nodes.</item> + </list> + <p>To check if your code is executed within a transaction, use + the function + <seealso marker="mnesia#is_transaction/0">mnesia:is_transaction/0</seealso>. + It returns <c>true</c> when called + inside a transaction context, otherwise <c>false</c>.</p> + <p><c>Mnesia</c> tables with storage type <c>RAM_copies</c> and + <c>disc_copies</c> are implemented internally as + <c>ets</c> tables. Applications can access the these tables + directly. This is only + recommended if all options have been weighed and the possible + outcomes are understood. By passing the earlier mentioned "fun" + to the function + <seealso marker="mnesia#ets/2">mnesia:ets(Fun [, Args])</seealso>, + it is performed but in a raw + context. The operations are performed directly on the + local <c>ets</c> tables, assuming that the local storage type is + <c>RAM_copies</c> and that the table is not replicated on other + nodes.</p> + <p>Subscriptions are not triggered and no checkpoints are updated, + but this operation is blindingly fast. Disc resident + tables are not to be updated with the <c>ets</c> function, as the + disc is not updated.</p> + <p>The Fun can also be passed as an argument to the function + <seealso marker="mnesia#activity-4">mnesia:activity/2,3,4</seealso>, + which enables use of customized activity access callback modules. It can either be obtained - directly by stating the module name as argument or implicitly - by usage of the <c>access_module</c> configuration parameter. A - customized callback module may be used for several purposes, - such as providing triggers, integrity constraints, run time - statistics, or virtual tables. - <br></br> - The callback module does - not have to access real Mnesia tables, it is free to do whatever - it likes as long as the callback interface is fulfilled. - <br></br> - In Appendix C "The Activity Access Call Back Interface" the source - code for one alternate implementation is provided - (mnesia_frag.erl). The context sensitive function - <c>mnesia:table_info/2</c> may be used to provide virtual - information about a table. One usage of this is to perform + directly by stating the module name as argument, or implicitly + by use of configuration parameter <c>access_module</c>. A + customized callback module can be used for several purposes, + such as providing triggers, integrity constraints, runtime + statistics, or virtual tables.</p> + <p>The callback module does not have + to access real <c>Mnesia</c> tables, it is free to do whatever + it wants as long as the callback interface is fulfilled.</p> + <p><seealso marker="Mnesia_App_B">Appendix B, + Activity Access Callback Interface</seealso> provides the + source code, <c>mnesia_frag.erl</c>, for one alternative + implementation. The context-sensitive function + <seealso marker="mnesia#table_info/2">mnesia:table_info/2</seealso> + can be used to provide virtual + information about a table. One use of this is to perform <c>QLC</c> queries within an activity context with a customized callback module. By providing table information about - table indices and other <c>QLC</c> requirements, - <c>QLC</c> may be used as a generic query language to - access virtual tables. - </p> - <p>QLC queries may be performed in all these activity - contexts (transaction, sync_transaction, async_dirty, sync_dirty - and ets). The ets activity will only work if the table has no - indices. - </p> + table indexes and other <c>QLC</c> requirements, <c>QLC</c> can + be used as a generic query language to access virtual tables.</p> + <p>QLC queries can be performed in all these activity + contexts (<c>transaction</c>, <c>sync_transaction</c>, + <c>async_dirty</c>, <c>sync_dirty</c>, and <c>ets</c>). The + <c>ets</c> activity only works if the table has no indexes.</p> <note> - <p>The mnesia:dirty_* function always executes with - async_dirty semantics regardless of which activity access contexts - are invoked. They may even invoke contexts without any - enclosing activity access context.</p> + <p>The function <c>mnesia:dirty_*</c> always executes with + <c>async_dirty</c> semantics regardless of which activity + access contexts that are started. It can even start contexts + without any enclosing activity access context.</p> </note> </section> <section> - <title>Nested transactions</title> - <p>Transactions may be nested in an arbitrary fashion. A child transaction - must run in the same process as its parent. When a child transaction - aborts, the caller of the child transaction will get the - return value <c>{aborted, Reason}</c> and any work performed - by the child will be erased. If a child transaction commits, the - records written by the child will be propagated to the parent. - </p> + <title>Nested Transactions</title> + <p>Transactions can be nested in an arbitrary fashion. A child + transaction must run in the same process as its parent. When a + child transaction terminates, the caller of the child transaction + gets return value <c>{aborted, Reason}</c> and any work performed + by the child is erased. If a child transaction commits, the + records written by the child are propagated to the parent.</p> <p>No locks are released when child transactions terminate. Locks - created by a sequence of nested transactions are kept until - the topmost transaction terminates. Furthermore, any updates - performed by a nested transaction are only propagated + created by a sequence of nested transactions are kept until + the topmost transaction terminates. Furthermore, any update + performed by a nested transaction is only propagated in such a manner so that the parent of the nested transaction - sees the updates. No final commitment will be done until - the top level transaction is terminated. + sees the updates. No final commitment is done until + the top-level transaction terminates. So, although a nested transaction returns <c>{atomic, Val}</c>, - if the enclosing parent transaction is aborted, the entire - nested operation is aborted. - </p> + if the enclosing parent transaction terminates, the entire + nested operation terminates.</p> <p>The ability to have nested transaction with identical semantics - as top level transaction makes it easier to write - library functions that manipulate mnesia tables. - </p> - <p>Say for example that we have a function that adds a - new subscriber to a telephony system:</p> + as top-level transaction makes it easier to write + library functions that manipulate <c>Mnesia</c> tables.</p> + <p>Consider a function that adds a subscriber to a telephony + system:</p> <pre> add_subscriber(S) -> mnesia:transaction(fun() -> - case mnesia:read( .......... - </pre> + case mnesia:read( ..........</pre> <p>This function needs to be called as a transaction. - Now assume that we wish to write a function that - both calls the <c>add_subscriber/1</c> function and + Assume that you wish to write a function that + both calls the function <c>add_subscriber/1</c> and is in itself protected by the context of a transaction. - By simply calling the <c>add_subscriber/1</c> from within - another transaction, a nested transaction is created. - </p> - <p>It is also possible to mix different activity access contexts while nesting, - but the dirty ones (async_dirty,sync_dirty and ets) will inherit the transaction - semantics if they are called inside a transaction and thus it will grab locks and - use two or three phase commit. - </p> + By calling <c>add_subscriber/1</c> from within + another transaction, a nested transaction is created.</p> + <p>Also, different activity access contexts can be mixed while + nesting. However, the dirty ones (<c>async_dirty</c>, + <c>sync_dirty</c>, and <c>ets</c>) inherit the transaction + semantics if they are called inside a transaction and thus + grab locks and use two or three phase commit.</p> + <p><em>Example:</em></p> <pre> add_subscriber(S) -> mnesia:transaction(fun() -> @@ -915,17 +885,18 @@ The following activity access contexts are currently supported: mnesia:read({some_tab, some_data}), mnesia:transaction(fun() -> %% In a transaction context. - case mnesia:read( ..) ..end), end). - </pre> + case mnesia:read( ..) ..end), end).</pre> </section> <section> <title>Pattern Matching</title> <marker id="matching"></marker> - <p>When it is not possible to use <c>mnesia:read/3</c> Mnesia + <p>When the function + <seealso marker="mnesia#read/3">mnesia:read/3</seealso> + cannot be used, <c>Mnesia</c> provides the programmer with several functions for matching - records against a pattern. The most useful functions of these are: - </p> + records against a pattern. The most useful ones + are the following:</p> <code type="none"> mnesia:select(Tab, MatchSpecification, LockKind) -> transaction abort | [ObjectList] @@ -934,170 +905,178 @@ The following activity access contexts are currently supported: mnesia:select(Cont) -> transaction abort | {[Object],Continuation} | '$end_of_table' mnesia:match_object(Tab, Pattern, LockKind) -> - transaction abort | RecordList - </code> - <p>These functions matches a <c>Pattern</c> against all records in - table <c>Tab</c>. In a <c>mnesia:select</c> call <c>Pattern</c> is - a part of <c>MatchSpecification</c> described below. It is not - necessarily performed as an exhaustive search of the entire - table. By utilizing indices and bound values in the key of the - pattern, the actual work done by the function may be condensed - into a few hash lookups. Using <c>ordered_set</c> tables may reduce the - search space if the keys are partially bound. - </p> + transaction abort | RecordList</code> + <p>These functions match a <c>Pattern</c> against all records in + table <c>Tab</c>. In a + <seealso marker="mnesia#select/2">mnesia:select</seealso> + call, <c>Pattern</c> is + a part of <c>MatchSpecification</c> described in the following. It + is not necessarily performed as an exhaustive search of the entire + table. By using indexes and bound values in the key of the + pattern, the actual work done by the function can be condensed + into a few hash lookups. Using <c>ordered_set</c> tables can reduce + the search space if the keys are partially bound.</p> <p>The pattern provided to the functions must be a valid record, and the first element of the provided tuple must be the <c>record_name</c> of the table. The special element <c>'_'</c> matches any data structure in Erlang (also known as an Erlang - term). The special elements <c><![CDATA['$<number>']]></c> behaves as Erlang - variables i.e. matches anything and binds the first occurrence and - matches the coming occurrences of that variable against the bound value. - </p> - <p>Use the function <c>mnesia:table_info(Tab, wild_pattern)</c> - to obtain a basic pattern which matches all records in a table + term). The special elements <c><![CDATA['$<number>']]></c> + behave as Erlang variables, that is, they match anything, + bind the first occurrence, and match the + coming occurrences of that variable against the bound value.</p> + <p>Use function + <seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, wild_pattern)</seealso> + to obtain a basic pattern, which matches all records in a table, or use the default value in record creation. - Do not make the pattern hard coded since it will make your code more - vulnerable to future changes of the record definition. - </p> + Do not make the pattern hard-coded, as this makes the code more + vulnerable to future changes of the record definition.</p> + <p><em>Example:</em></p> <code type="none"> Wildpattern = mnesia:table_info(employee, wild_pattern), %% Or use - Wildpattern = #employee{_ = '_'}, - </code> - <p>For the employee table the wild pattern will look like:</p> + Wildpattern = #employee{_ = '_'},</code> + <p>For the employee table, the wild pattern looks as follows:</p> <code type="none"> - {employee, '_', '_', '_', '_', '_',' _'}. - </code> - <p>In order to constrain the match you must replace some + {employee, '_', '_', '_', '_', '_',' _'}.</code> + <p>To constrain the match, it is needed to replace some of the <c>'_'</c> elements. The code for matching out - all female employees, looks like: - </p> + all female employees looks as follows:</p> <code type="none"> Pat = #employee{sex = female, _ = '_'}, F = fun() -> mnesia:match_object(Pat) end, - Females = mnesia:transaction(F). - </code> - <p>It is also possible to use the match function if we want to - check the equality of different attributes. Assume that we want - to find all employees which happens to have a employee number - which is equal to their room number: - </p> + Females = mnesia:transaction(F).</code> + <p>The match function can also be used to check the equality of + different attributes. For example, to find all employees with + an employee number equal to their room number:</p> <code type="none"> Pat = #employee{emp_no = '$1', room_no = '$1', _ = '_'}, F = fun() -> mnesia:match_object(Pat) end, - Odd = mnesia:transaction(F). - </code> - <p>The function <c>mnesia:match_object/3</c> lacks some important - features that <c>mnesia:select/3</c> have. For example + Odd = mnesia:transaction(F).</code> + <p>The function + <seealso marker="mnesia#match_object/3">mnesia:match_object/3</seealso> + lacks some important features that + <seealso marker="mnesia#select/2">mnesia:select/3</seealso> + have. For example, <c>mnesia:match_object/3</c> can only return the matching records, - and it can not express constraints other then equality. - If we want to find the names of the male employees on the second floor - we could write: - </p> + and it cannot express constraints other than equality. To find + the names of the male employees on the second floor:</p> <codeinclude file="company.erl" tag="%21" type="erl"></codeinclude> - <p>Select can be used to add additional constraints and create - output which can not be done with <c>mnesia:match_object/3</c>. </p> - <p>The second argument to select is a <c>MatchSpecification</c>. - A <c>MatchSpecification</c> is list of <c>MatchFunctions</c>, where + <p>The function <c>select</c> can be used to add more constraints + and create output that cannot be done with + <c>mnesia:match_object/3</c>.</p> + <p>The second argument to <c>select</c> is a <c>MatchSpecification</c>. + A <c>MatchSpecification</c> is a list of <c>MatchFunction</c>s, where each <c>MatchFunction</c> consists of a tuple containing - <c>{MatchHead, MatchCondition, MatchBody}</c>. <c>MatchHead</c> - is the same pattern used in <c>mnesia:match_object/3</c> - described above. <c>MatchCondition</c> is a list of additional - constraints applied to each record, and <c>MatchBody</c> is used - to construct the return values. - </p> - <p>A detailed explanation of match specifications can be found in - the <em>Erts users guide: Match specifications in Erlang </em>, - and the ets/dets documentations may provide some additional - information. - </p> - <p>The functions <c>select/4</c> and <c>select/1</c> are used to - get a limited number of results, where the <c>Continuation</c> - are used to get the next chunk of results. Mnesia uses the - <c>NObjects</c> as an recommendation only, thus more or less - results then specified with <c>NObjects</c> may be returned in - the result list, even the empty list may be returned despite there - are more results to collect. - </p> + <c>{MatchHead, MatchCondition, MatchBody}</c>:</p> + <list type="bulleted"> + <item><c>MatchHead</c> is the same pattern as used in + <c>mnesia:match_object/3</c> described earlier.</item> + <item><c>MatchCondition</c> is a list of extra constraints + applied to each record.</item> + <item><c>MatchBody</c> constructs the return values.</item> + </list> + <p>For details about the match specifications, see + "Match Specifications in Erlang" in + <seealso marker="erts:index">ERTS</seealso> User's Guide. + For more information, see the + <seealso marker="stdlib:ets">ets</seealso> and + <seealso marker="stdlib:dets">dets</seealso> + manual pages in <c>STDLIB</c>.</p> + <p>The functions + <seealso marker="mnesia#select/4">select/4</seealso> and + <seealso marker="mnesia#select/2">select/1</seealso> + are used to + get a limited number of results, where <c>Continuation</c> + gets the next chunk of results. <c>Mnesia</c> uses + <c>NObjects</c> as a recommendation only. Thus, more or less + results than specified with <c>NObjects</c> can be returned in + the result list, even the empty list can be returned even + if there are more results to collect.</p> <warning> <p>There is a severe performance penalty in using - <c>mnesia:select/[1|2|3|4]</c> after any modifying operations - are done on that table in the same transaction, i.e. avoid using - <c>mnesia:write/1</c> or <c>mnesia:delete/1</c> before a - <c>mnesia:select</c> in the same transaction.</p> + <c>mnesia:select/[1|2|3|4]</c> after any modifying operation + is done on that table in the same transaction. That is, avoid + using + <seealso marker="mnesia#write/1">mnesia:write/1</seealso> or + <seealso marker="mnesia#delete/1">mnesia:delete/1</seealso> + before <c>mnesia:select</c> in the same transaction.</p> </warning> <p>If the key attribute is bound in a pattern, the match operation - is very efficient. However, if the key attribute in a pattern is - given as <c>'_'</c>, or <c>'$1'</c>, the whole <c>employee</c> + is efficient. However, if the key attribute in a pattern is + given as <c>'_'</c> or <c>'$1'</c>, the whole <c>employee</c> table must be searched for records that match. Hence if the table is - large, this can become a time consuming operation, but it can be - remedied with indices (refer to Chapter 5: <seealso marker="Mnesia_chap5#indexing">Indexing</seealso>) if - <c>mnesia:match_object</c> is used. - </p> - <p>QLC queries can also be used to search Mnesia tables. By - using <c>mnesia:table/[1|2]</c> as the generator inside a QLC - query you let the query operate on a mnesia table. Mnesia - specific options to <c>mnesia:table/2</c> are {lock, Lock}, - {n_objects,Integer} and {traverse, SelMethod}. The <c>lock</c> - option specifies whether mnesia should acquire a read or write - lock on the table, and <c>n_objects</c> specifies how many - results should be returned in each chunk to QLC. The last option is - <c>traverse</c> and it specifies which function mnesia should - use to traverse the table. Default <c>select</c> is used, but by using - <c>{traverse, {select, MatchSpecification}}</c> as an option to - <c>mnesia:table/2</c> the user can specify it's own view of the - table. - </p> - <p>If no options are specified a read lock will acquired and 100 - results will be returned in each chunk, and select will be used - to traverse the table, i.e.: - </p> + large, this can become a time-consuming operation, but it can be + remedied with indexes (see + <seealso marker="Mnesia_chap5#indexing">Indexing</seealso>) + if the function + <seealso marker="mnesia#match_object/1">mnesia:match_object</seealso> + is used.</p> + <p>QLC queries can also be used to search <c>Mnesia</c> tables. By + using the function + <seealso marker="mnesia#table/1">mnesia:table/[1|2]</seealso> + as the generator inside a QLC + query, you let the query operate on a <c>Mnesia</c> table. + <c>Mnesia</c>-specific options to <c>mnesia:table/2</c> are + <c>{lock, Lock}</c>, <c>{n_objects,Integer}</c>, and + <c>{traverse, SelMethod}</c>:</p> + <list type="bulleted"> + <item><c>lock</c> specifies whether <c>Mnesia</c> is to acquire a + read or write lock on the table.</item> + <item><c>n_objects</c> specifies how many results are to be + returned in each chunk to QLC.</item> + <item><c>traverse</c> specifies which function <c>Mnesia</c> is + to use to traverse the table. Default <c>select</c> is used, but + by using <c>{traverse, {select, MatchSpecification}}</c> as an + option to + <seealso marker="mnesia#table/1">mnesia:table/2</seealso> + the user can specify its own view of the table.</item> + </list> + <p>If no options are specified, a read lock is acquired, 100 + results are returned in each chunk, and <c>select</c> is used + to traverse the table, that is:</p> <code type="none"> mnesia:table(Tab) -> - mnesia:table(Tab, [{n_objects,100},{lock, read}, {traverse, select}]). - </code> - <p>The function <c>mnesia:all_keys(Tab)</c> returns all keys in a - table.</p> + mnesia:table(Tab, [{n_objects,100},{lock, read}, {traverse, select}]).</code> + <p>The function + <seealso marker="mnesia#all_keys/1">mnesia:all_keys(Tab)</seealso> + returns all keys in a table.</p> </section> <section> <title>Iteration</title> <marker id="iteration"></marker> - <p>Mnesia provides a couple of functions which iterates over all - the records in a table. - </p> + <p><c>Mnesia</c> provides the following functions that iterate over all + the records in a table:</p> <code type="none"> mnesia:foldl(Fun, Acc0, Tab) -> NewAcc | transaction abort mnesia:foldr(Fun, Acc0, Tab) -> NewAcc | transaction abort mnesia:foldl(Fun, Acc0, Tab, LockType) -> NewAcc | transaction abort - mnesia:foldr(Fun, Acc0, Tab, LockType) -> NewAcc | transaction abort - </code> - <p>These functions iterate over the mnesia table <c>Tab</c> and - apply the function <c>Fun</c> to each record. The <c>Fun</c> - takes two arguments, the first argument is a record from the - table and the second argument is the accumulator. The - <c>Fun</c> return a new accumulator. </p> - <p>The first time the <c>Fun</c> is applied <c>Acc0</c> will - be the second argument. The next time the <c>Fun</c> is called - the return value from the previous call, will be used as the - second argument. The term the last call to the Fun returns - will be the return value of the <c>fold[lr]</c> function. - </p> - <p>The difference between <c>foldl</c> and <c>foldr</c> is the - order the table is accessed for <c>ordered_set</c> tables, - for every other table type the functions are equivalent. - </p> - <p><c>LockType</c> specifies what type of lock that shall be + mnesia:foldr(Fun, Acc0, Tab, LockType) -> NewAcc | transaction abort</code> + <p>These functions iterate over the <c>Mnesia</c> table <c>Tab</c> + and apply the function <c>Fun</c> to each record. <c>Fun</c> + takes two arguments, the first is a record from the + table, and the second is the accumulator. + <c>Fun</c> returns a new accumulator.</p> + <p>The first time <c>Fun</c> is applied, <c>Acc0</c> is + the second argument. The next time <c>Fun</c> is called, + the return value from the previous call is used as the + second argument. The term the last call to <c>Fun</c> returns + is the return value of the function + <seealso marker="mnesia#foldl/3">mnesia:foldl/3</seealso> or + <seealso marker="mnesia#foldr/3">mnesia:foldr/3</seealso>.</p> + <p>The difference between these functions is the + order the table is accessed for <c>ordered_set</c> tables. + For other table types the functions are equivalent.</p> + <p><c>LockType</c> specifies what type of lock that is to be acquired for the iteration, default is <c>read</c>. If - records are written or deleted during the iteration a write - lock should be acquired. </p> - <p>These functions might be used to find records in a table - when it is impossible to write constraints for - <c>mnesia:match_object/3</c>, or when you want to perform - some action on certain records. - </p> - <p>For example finding all the employees who has a salary - below 10 could look like:</p> + records are written or deleted during the iteration, a write + lock is to be acquired.</p> + <p>These functions can be used to find records in a table + when it is impossible to write constraints for the function + <seealso marker="mnesia#match_object/3">mnesia:match_object/3</seealso>, + or when you want to perform some action on certain records.</p> + <p>For example, finding all the employees who have a salary + less than 10 can look as follows:</p> <code type="none"><![CDATA[ find_low_salaries() -> Constraint = @@ -1109,7 +1088,7 @@ The following activity access contexts are currently supported: Find = fun() -> mnesia:foldl(Constraint, [], employee) end, mnesia:transaction(Find). ]]></code> - <p>Raising the salary to 10 for everyone with a salary below 10 + <p>To raise the salary to 10 for everyone with a salary less than 10 and return the sum of all raises:</p> <code type="none"><![CDATA[ increase_low_salaries() -> @@ -1124,48 +1103,54 @@ The following activity access contexts are currently supported: IncLow = fun() -> mnesia:foldl(Increase, 0, employee, write) end, mnesia:transaction(IncLow). ]]></code> - <p>A lot of nice things can be done with the iterator functions - but some caution should be taken about performance and memory - utilization for large tables. </p> - <p>Call these iteration functions on nodes that contain a replica of the - table. Each call to the function <c>Fun</c> access the table and if the table - resides on another node it will generate a lot of unnecessary - network traffic. </p> - <p>Mnesia also provides some functions that make it possible for - the user to iterate over the table. The order of the - iteration is unspecified if the table is not of the <c>ordered_set</c> - type. </p> + <p>Many nice things can be done with the iterator functions but take + some caution about performance and memory use for large tables.</p> + <p>Call these iteration functions on nodes that contain a replica of + the table. Each call to the function <c>Fun</c> access the table + and if the table resides on another node it generates much + unnecessary network traffic.</p> + <p><c>Mnesia</c> also provides some functions that make it possible + for the user to iterate over the table. The order of the iteration + is unspecified if the table is not of type <c>ordered_set</c>:</p> <code type="none"> mnesia:first(Tab) -> Key | transaction abort mnesia:last(Tab) -> Key | transaction abort mnesia:next(Tab,Key) -> Key | transaction abort mnesia:prev(Tab,Key) -> Key | transaction abort - mnesia:snmp_get_next_index(Tab,Index) -> {ok, NextIndex} | endOfTable - </code> - <p>The order of first/last and next/prev are only valid for - <c>ordered_set</c> tables, for all other tables, they are synonyms. - When the end of the table is reached the special key + mnesia:snmp_get_next_index(Tab,Index) -> {ok, NextIndex} | endOfTable</code> + <p>The order of <c>first</c>/<c>last</c> and <c>next</c>/<c>prev</c> + is only valid for + <c>ordered_set</c> tables, they are synonyms for other tables. + When the end of the table is reached, the special key <c>'$end_of_table'</c> is returned.</p> <p>If records are written and deleted during the traversal, use - <c>mnesia:fold[lr]/4</c> with a <c>write</c> lock. Or - <c>mnesia:write_lock_table/1</c> when using first and next.</p> + the function + <seealso marker="mnesia#foldl">mnesia:foldl/3</seealso> or + <seealso marker="mnesia#foldr">mnesia:foldr/3</seealso> + with a <c>write</c> lock. Or the function + <seealso marker="mnesia#write_lock_table/1">mnesia:write_lock_table/1</seealso> + when using <c>first</c> and <c>next</c>.</p> <p>Writing or deleting in transaction context creates a local copy - of each modified record, so modifying each record in a large - table uses a lot of memory. Mnesia will compensate for every + of each modified record. Thus, modifying each record in a large + table uses much memory. <c>Mnesia</c> compensates for every written or deleted record during the iteration in a transaction - context, which may reduce the performance. If possible avoid writing + context, which can reduce the performance. If possible, avoid writing or deleting records in the same transaction before iterating over the table.</p> - <p>In dirty context, i.e. <c>sync_dirty</c> or <c>async_dirty</c>, + <p>In dirty context, that is, <c>sync_dirty</c> or <c>async_dirty</c>, the modified records are not stored in a local copy; instead, - each record is updated separately. This generates a lot of + each record is updated separately. This generates much network traffic if the table has a replica on another node and has all the other drawbacks that dirty operations - have. Especially for the <c>mnesia:first/1</c> and - <c>mnesia:next/2</c> commands, the same drawbacks as described - above for <c>dirty_first</c> and <c>dirty_next</c> applies, i.e. - no writes to the table should be done during iteration.</p> - <p></p> + have. Especially for commands + <seealso marker="mnesia#first/1">mnesia:first/1</seealso> and + <seealso marker="mnesia#next/2">mnesia:next/2</seealso>, + the same drawbacks as described previously for + <seealso marker="mnesia#dirty_first/1">mnesia:dirty_first/1</seealso> + and + <seealso marker="mnesia#dirty_next/2">mnesia:dirty_next/2</seealso> + applies, that + is, no writing to the table is to be done during iteration.</p> </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc index 127c23e0f7..813731e0b8 100644 --- a/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap5.xmlsrc @@ -31,222 +31,205 @@ <rev></rev> <file>Mnesia_chap5.xml</file> </header> - <p>The earlier chapters of this User Guide described how to get - started with Mnesia, and how to build a Mnesia database. In this - chapter, we will describe the more advanced features available - when building a distributed, fault tolerant Mnesia database. This - chapter contains the following sections: - </p> + + <p>The previous sections describe how to get started + with <c>Mnesia</c> and how to build a <c>Mnesia</c> database. This + section describes the more advanced features available + when building a distributed, fault-tolerant <c>Mnesia</c> database. + The following topics are included:</p> <list type="bulleted"> - <item>Indexing - </item> - <item>Distribution and Fault Tolerance - </item> - <item>Table fragmentation. - </item> - <item>Local content tables. - </item> - <item>Disc-less nodes. - </item> - <item>More about schema management - </item> - <item>Debugging a Mnesia application - </item> - <item>Concurrent Processes in Mnesia - </item> - <item>Prototyping - </item> - <item>Object Based Programming with Mnesia. - </item> + <item>Indexing</item> + <item>Distribution and fault tolerance</item> + <item>Table fragmentation</item> + <item>Local content tables</item> + <item>Disc-less nodes</item> + <item>More about schema management</item> + <item><c>Mnesia</c> event handling</item> + <item>Debugging <c>Mnesia</c> applications</item> + <item>Concurrent processes in <c>Mnesia</c></item> + <item>Prototyping</item> + <item>Object-based programming with <c>Mnesia</c></item> </list> <section> <marker id="indexing"></marker> <title>Indexing</title> - <p>Data retrieval and matching can be performed very efficiently - if we know the key for the record. Conversely, if the key is not - known, all records in a table must be searched. The larger the - table the more time consuming it will become. To remedy this - problem Mnesia's indexing capabilities are used to improve data - retrieval and matching of records. - </p> - <p>The following two functions manipulate indexes on existing tables: - </p> + <p>Data retrieval and matching can be performed efficiently + if the key for the record is known. Conversely, if the key is + unknown, all records in a table must be searched. The larger the + table, the more time consuming it becomes. To remedy this + problem, <c>Mnesia</c> indexing capabilities are used to improve + data retrieval and matching of records.</p> + <p>The following two functions manipulate indexes on existing + tables:</p> <list type="bulleted"> - <item><c>mnesia:add_table_index(Tab, AttributeName) -> {aborted, R} |{atomic, ok}</c></item> - <item><c>mnesia:del_table_index(Tab, AttributeName) -> {aborted, R} |{atomic, ok}</c></item> - </list> - <p>These functions create or delete a table index on field - defined by <c>AttributeName</c>. To illustrate this, add an - index to the table definition <c>(employee, {emp_no, name, salary, sex, phone, room_no}</c>, which is the example table - from the Company database. The function - which adds an index on the element <c>salary</c> can be expressed in - the following way: - </p> - <list type="ordered"> - <item><c>mnesia:add_table_index(employee, salary)</c></item> + <item><seealso marker="mnesia#add_table_index/2">mnesia:add_table_index(Tab, AttributeName) + -> {aborted, R} |{atomic, ok}</seealso></item> + <item><seealso marker="mnesia#del_table_index/2">mnesia:del_table_index(Tab, AttributeName) + -> {aborted, R} |{atomic, ok}</seealso></item> </list> - <p>The indexing capabilities of Mnesia are utilized with the - following three functions, which retrieve and match records on the - basis of index entries in the database. - </p> + <p>These functions create or delete a table index on a field + defined by <c>AttributeName</c>. To illustrate this, add an + index to the table definition <c>(employee, {emp_no, name, + salary, sex, phone, room_no})</c>, which is the example table + from the <c>Company</c> database. The function that + adds an index on element <c>salary</c> can be expressed + as <c>mnesia:add_table_index(employee, salary)</c>.</p> + <p>The indexing capabilities of <c>Mnesia</c> are used with the + following three functions, which retrieve and match records + based on index entries in the database:</p> <list type="bulleted"> - <item><c>mnesia:index_read(Tab, SecondaryKey, AttributeName) -> transaction abort | RecordList</c>. - Avoids an exhaustive search of the entire table, by looking up - the <c>SecondaryKey</c> in the index to find the primary keys. + <item> + <seealso marker="mnesia#index_read/3">mnesia:index_read(Tab, SecondaryKey, AttributeName) + -> transaction abort | RecordList</seealso> + avoids an exhaustive search of the entire table, by looking up + <c>SecondaryKey</c> in the index to find the primary keys. </item> - <item><c>mnesia:index_match_object(Pattern, AttributeName) -> transaction abort | RecordList</c> - Avoids an exhaustive search of the entire table, by looking up + <item> + <seealso marker="mnesia#index_match_object/2">mnesia:index_match_object(Pattern, AttributeName) + -> transaction abort | RecordList</seealso> + avoids an exhaustive search of the entire table, by looking up the secondary key in the index to find the primary keys. - The secondary key is found in the <c>AttributeName</c> field of - the <c>Pattern</c>. The secondary key must be bound. + The secondary key is found in field <c>AttributeName</c> of + <c>Pattern</c>. The secondary key must be bound. </item> - <item><c>mnesia:match_object(Pattern) -> transaction abort | RecordList</c> - Uses indices to avoid exhaustive search of the entire table. - Unlike the other functions above, this function may utilize + <item> + <seealso marker="mnesia#match_object/1">mnesia:match_object(Pattern) + -> transaction abort | RecordList</seealso> + uses indexes to avoid exhaustive search of the entire table. + Unlike the previous functions, this function can use any index as long as the secondary key is bound.</item> </list> <p>These functions are further described and exemplified in - Chapter 4: <seealso marker="Mnesia_chap4#matching">Pattern matching</seealso>. - </p> + <seealso marker="Mnesia_chap4#matching">Pattern Matching</seealso>. + </p> </section> <section> <title>Distribution and Fault Tolerance</title> - <p>Mnesia is a distributed, fault tolerant DBMS. It is possible - to replicate tables on different Erlang nodes in a variety of - ways. The Mnesia programmer does not have to state + <p><c>Mnesia</c> is a distributed, fault-tolerant DBMS. Tables + can be replicated on different Erlang nodes in various + ways. The <c>Mnesia</c> programmer does not need to state where the different tables reside, only the names of the - different tables are specified in the program code. This is - known as "location transparency" and it is an important - concept. In particular: - </p> + different tables need to be specified in the program code. This + is known as "location transparency" and is an important + concept. In particular:</p> <list type="bulleted"> - <item>A program will work regardless of the - location of the data. It makes no difference whether the data - resides on the local node, or on a remote node. <em>Note:</em> The program - will run slower if the data is located on a remote node. + <item><p>A program works regardless of the data + location. It makes no difference whether the data + resides on the local node or on a remote node.</p> + <p>Notice that the program runs slower if the data + is located on a remote node.</p> </item> <item>The database can be reconfigured, and tables can be - moved between nodes. These operations do not effect the user + moved between nodes. These operations do not affect the user programs. </item> </list> - <p>We have previously seen that each table has a number of - system attributes, such as <c>index</c> and - <c>type</c>. - </p> + <p>It has previously been shown that each table has a number of + system attributes, such as <c>index</c> and <c>type</c>.</p> <p>Table attributes are specified when the table is created. For - example, the following function will create a new table with two - RAM replicas: - </p> + example, the following function creates a table with two + RAM replicas:</p> <pre> mnesia:create_table(foo, [{ram_copies, [N1, N2]}, - {attributes, record_info(fields, foo)}]). - </pre> + {attributes, record_info(fields, foo)}]).</pre> <p>Tables can also have the following properties, - where each attribute has a list of Erlang nodes as its value. - </p> + where each attribute has a list of Erlang nodes as its value:</p> <list type="bulleted"> <item> - <p><c>ram_copies</c>. The value of the node list is a list of - Erlang nodes, and a RAM replica of the table will reside on - each node in the list. This is a RAM replica, and it is - important to realize that no disc operations are performed when - a program executes write operations to these replicas. However, - should permanent RAM replicas be a requirement, then the + <p><c>ram_copies</c>. The value of the node list is a list + of Erlang nodes, and a RAM replica of the table resides on + each node in the list.</p> + <p>Notice that no disc operations are performed when + a program executes write operations to these replicas. + However, if permanent RAM replicas are required, the following alternatives are available:</p> <list type="ordered"> - <item>The <c>mnesia:dump_tables/1</c> function can be used - to dump RAM table replicas to disc. + <item>The function + <seealso marker="mnesia#dump_tables/1">mnesia:dump_tables/1</seealso> + can be used to dump RAM table replicas to disc. </item> - <item>The table replicas can be backed up; either from - RAM, or from disc if dumped there with the above - function. + <item>The table replicas can be backed up, either from + RAM, or from disc if dumped there with this function. </item> </list> </item> <item><c>disc_copies</c>. The value of the attribute is a list - of Erlang nodes, and a replica of the table will reside both + of Erlang nodes, and a replica of the table resides both in RAM and on disc on each node in the list. Write operations - addressed to the table will address both the RAM and the disc + addressed to the table address both the RAM and the disc copy of the table. </item> <item><c>disc_only_copies</c>. The value of the attribute is a - list of Erlang nodes, and a replica of the table will reside + list of Erlang nodes, and a replica of the table resides only as a disc copy on each node in the list. The major disadvantage of this type of table replica is the access speed. The major advantage is that the table does not occupy space in memory. </item> </list> - <p>It is also possible to set and change table properties on - existing tables. Refer to Chapter 3: <seealso marker="Mnesia_chap3#def_schema">Defining the Schema</seealso> for full - details. - </p> + <p>In addition, table properties can be set and changed. + For details, see + <seealso marker="Mnesia_chap3#def_schema">Define a Schema</seealso>. + </p> <p>There are basically two reasons for using more than one table - replica: fault tolerance, or speed. It is worthwhile to note + replica: fault tolerance and speed. Notice that table replication provides a solution to both of these - system requirements. - </p> - <p>If we have two active table replicas, all information is - still available if one of the replicas fail. This can be a very + system requirements.</p> + <p>If there are two active table replicas, all information is + still available if one replica fails. This can be an important property in many applications. Furthermore, if a table - replica exists at two specific nodes, applications which execute + replica exists at two specific nodes, applications that execute at either of these nodes can read data from the table without - accessing the network. Network operations are considerably - slower and consume more resources than local operations. - </p> + accessing the network. Network operations are considerably + slower and consume more resources than local operations.</p> <p>It can be advantageous to create table replicas for a - distributed application which reads data often, but writes data - seldom, in order to achieve fast read operations on the local + distributed application that reads data often, but writes data + seldom, to achieve fast read operations on the local node. The major disadvantage with replication is the increased time to write data. If a table has two replicas, every write operation must access both table replicas. Since one of these write operations must be a network operation, it is considerably more expensive to perform a write operation to a replicated - table than to a non-replicated table. - </p> + table than to a non-replicated table.</p> </section> <section> <title>Table Fragmentation</title> <section> - <title>The Concept</title> - <p>A concept of table fragmentation has been introduced in - order to cope with very large tables. The idea is to split a - table into several more manageable fragments. Each fragment - is implemented as a first class Mnesia table and may be - replicated, have indices etc. as any other table. But the - tables may neither have <c>local_content</c> nor have the - <c>snmp</c> connection activated. - </p> - <p>In order to be able to access a record in a fragmented - table, Mnesia must determine to which fragment the - actual record belongs. This is done by the - <c>mnesia_frag</c> module, which implements the - <c>mnesia_access</c> callback behaviour. Please, read the - documentation about <c>mnesia:activity/4</c> to see how - <c>mnesia_frag</c> can be used as a <c>mnesia_access</c> - callback module. - </p> - <p>At each record access <c>mnesia_frag</c> first computes - a hash value from the record key. Secondly the name of the - table fragment is determined from the hash value. And - finally the actual table access is performed by the same + <title>Concept</title> + <p>A concept of table fragmentation has been introduced + to cope with large tables. The idea is to split a + table into several manageable fragments. Each fragment is + implemented as a first class <c>Mnesia</c> table and can be + replicated, have indexes, and so on, as any other table. But + the tables cannot have <c>local_content</c> or have the + <c>snmp</c> connection activated.</p> + <p>To be able to access a record in a fragmented + table, <c>Mnesia</c> must determine to which fragment the + actual record belongs. This is done by module + <c>mnesia_frag</c>, which implements the <c>mnesia_access</c> + callback behavior. It is recommended to read the + documentation about the function + <seealso marker="mnesia#activity/4">mnesia:activity/4</seealso> + to see how <c>mnesia_frag</c> + can be used as a <c>mnesia_access</c> callback module.</p> + <p>At each record access, <c>mnesia_frag</c> first computes + a hash value from the record key. Second, the name of the + table fragment is determined from the hash value. + Finally the actual table access is performed by the same functions as for non-fragmented tables. When the key is not known beforehand, all fragments are searched for - matching records. Note: In <c>ordered_set</c> tables - the records will be ordered per fragment, and the - the order is undefined in results returned by select and - match_object. - </p> - <p>The following piece of code illustrates - how an existing Mnesia table is converted to be a - fragmented table and how more fragments are added later on. - </p> + matching records.</p> + <p>Notice that in <c>ordered_set</c> tables, the records + are ordered per fragment, and the the order is undefined in + results returned by <c>select</c> and <c>match_object</c>.</p> + <p>The following code illustrates how a <c>Mnesia</c> table is + converted to be a fragmented table and how more fragments + are added later:</p> <code type="none"><![CDATA[ Eshell V4.7.3.3 (abort with ^G) (a@sam)1> mnesia:start(). @@ -299,102 +282,96 @@ ok <section> <title>Fragmentation Properties</title> - <p>There is a table property called - <c>frag_properties</c> and may be read with - <c>mnesia:table_info(Tab, frag_properties)</c>. The - fragmentation properties is a list of tagged tuples with - the arity 2. By default the list is empty, but when it is - non-empty it triggers Mnesia to regard the table as - fragmented. The fragmentation properties are: - </p> + <p>The table property <c>frag_properties</c> can be read with + the function + <seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, frag_properties)</seealso>. + The fragmentation properties are a list of tagged tuples with + arity 2. By default the list is empty, but when it is + non-empty it triggers <c>Mnesia</c> to regard the table as + fragmented. The fragmentation properties are as follows:</p> <taglist> <tag><c>{n_fragments, Int}</c></tag> <item> <p><c>n_fragments</c> regulates how many fragments - that the table currently has. This property may explicitly + that the table currently has. This property can explicitly be set at table creation and later be changed with <c>{add_frag, NodesOrDist}</c> or - <c>del_frag</c>. <c>n_fragment</c>s defaults to <c>1</c>. - </p> + <c>del_frag</c>. <c>n_fragments</c> defaults to <c>1</c>.</p> </item> <tag><c>{node_pool, List}</c></tag> <item> - <p>The node pool contains a list of nodes and may + <p>The node pool contains a list of nodes and can explicitly be set at table creation and later be changed - with <c>{add_node, Node}</c> or <c>{del_node, Node}</c>. At table creation Mnesia tries to distribute + with <c>{add_node, Node}</c> or <c>{del_node, Node}</c>. + At table creation <c>Mnesia</c> tries to distribute the replicas of each fragment evenly over all the nodes in - the node pool. Hopefully all nodes will end up with the + the node pool. Hopefully all nodes end up with the same number of replicas. <c>node_pool</c> defaults to the - return value from <c>mnesia:system_info(db_nodes)</c>. - </p> + return value from the function + <seealso marker="mnesia#system_info/1">mnesia:system_info(db_nodes)</seealso>.</p> </item> <tag><c>{n_ram_copies, Int}</c></tag> <item> <p>Regulates how many <c>ram_copies</c> replicas - that each fragment should have. This property may - explicitly be set at table creation. The default is + that each fragment is to have. This property can + explicitly be set at table creation. Defaults is <c>0</c>, but if <c>n_disc_copies</c> and <c>n_disc_only_copies</c> also are <c>0</c>, - <c>n_ram_copies</c> will default be set to <c>1</c>. - </p> + <c>n_ram_copies</c> defaults to <c>1</c>.</p> </item> <tag><c>{n_disc_copies, Int}</c></tag> <item> - <p>Regulates how many <c>disc_copies</c> replicas - that each fragment should have. This property may - explicitly be set at table creation. The default is <c>0</c>. - </p> + <p>Regulates how many <c>disc_copies</c> replicas that + each fragment is to have. This property can explicitly + be set at table creation. Default is <c>0</c>.</p> </item> <tag><c>{n_disc_only_copies, Int}</c></tag> <item> <p>Regulates how many <c>disc_only_copies</c> replicas - that each fragment should have. This property may - explicitly be set at table creation. The default is <c>0</c>. - </p> + that each fragment is to have. This property can + explicitly be set at table creation. Defaults is + <c>0</c>.</p> </item> <tag><c>{foreign_key, ForeignKey}</c></tag> <item> - <p><c>ForeignKey</c> may either be the atom + <p><c>ForeignKey</c> can either be the atom <c>undefined</c> or the tuple <c>{ForeignTab, Attr}</c>, - where <c>Attr</c> denotes an attribute which should be + where <c>Attr</c> denotes an attribute that is to be interpreted as a key in another fragmented table named - <c>ForeignTab</c>. Mnesia will ensure that the number of + <c>ForeignTab</c>. <c>Mnesia</c> ensures that the number of fragments in this table and in the foreign table are - always the same. When fragments are added or deleted - Mnesia will automatically propagate the operation to all - fragmented tables that has a foreign key referring to this + always the same.</p> + <p>When fragments are added or deleted, <c>Mnesia</c> + automatically propagates the operation to all + fragmented tables that have a foreign key referring to this table. Instead of using the record key to determine which - fragment to access, the value of the <c>Attr</c> field is - used. This feature makes it possible to automatically - co-locate records in different tables to the same - node. <c>foreign_key</c> defaults to - <c>undefined</c>. However if the foreign key is set to - something else it will cause the default values of the + fragment to access, the value of field <c>Attr</c> is + used. This feature makes it possible to colocate records + automatically in different tables to the same node. + <c>foreign_key</c> defaults to + <c>undefined</c>. However, if the foreign key is set to + something else, it causes the default values of the other fragmentation properties to be the same values as - the actual fragmentation properties of the foreign table. - </p> + the actual fragmentation properties of the foreign table.</p> </item> <tag><c>{hash_module, Atom}</c></tag> <item> - <p>Enables definition of an alternate hashing scheme. - The module must implement the <c>mnesia_frag_hash</c> - callback behaviour (see the reference manual). This - property may explicitly be set at table creation. - The default is <c>mnesia_frag_hash</c>.</p> - <p>Older tables that was created before the concept of - user defined hash modules was introduced, uses - the <c>mnesia_frag_old_hash</c> module in order to - be backwards compatible. The <c>mnesia_frag_old_hash</c> - is still using the poor deprecated <c>erlang:hash/1</c> - function. - </p> + <p>Enables definition of an alternative hashing scheme. + The module must implement the + <seealso marker="mnesia_frag_hash">mnesia_frag_hash</seealso> + callback behavior. This property can explicitly be set at + table creation. Default is <c>mnesia_frag_hash</c>.</p> + <p>Older tables, that were created before the concept of + user-defined hash modules was introduced, use module + <c>mnesia_frag_old_hash</c> to be backwards compatible. + <c>mnesia_frag_old_hash</c> still uses the poor + deprecated function <c>erlang:hash/1</c>.</p> </item> <tag><c>{hash_state, Term}</c></tag> <item> - <p>Enables a table specific parameterization - of a generic hash module. This property may explicitly - be set at table creation. - The default is <c>undefined</c>.</p> + <p>Enables a table-specific parameterization of a + generic hash module. This property can explicitly be set + at table creation. Default is <c>undefined</c>.</p> <code type="none"><![CDATA[ Eshell V4.7.3.3 (abort with ^G) (a@sam)1> mnesia:start(). @@ -463,177 +440,159 @@ ok <title>Management of Fragmented Tables</title> <p>The function <c>mnesia:change_table_frag(Tab, Change)</c> is intended to be used for reconfiguration of fragmented - tables. The <c>Change</c> argument should have one of the - following values: - </p> + tables. Argument <c>Change</c> is to have one of the + following values:</p> <taglist> <tag><c>{activate, FragProps}</c></tag> <item> <p>Activates the fragmentation properties of an - existing table. <c>FragProps</c> should either contain - <c>{node_pool, Nodes}</c> or be empty. - </p> + existing table. <c>FragProps</c> is either to contain + <c>{node_pool, Nodes}</c> or be empty.</p> </item> <tag><c>deactivate</c></tag> <item> <p>Deactivates the fragmentation properties of a - table. The number of fragments must be <c>1</c>. No other - tables may refer to this table in its foreign key. - </p> + table. The number of fragments must be <c>1</c>. No other + table can refer to this table in its foreign key.</p> </item> <tag><c>{add_frag, NodesOrDist}</c></tag> <item> - <p>Adds one new fragment to a fragmented table. All - records in one of the old fragments will be rehashed and - about half of them will be moved to the new (last) - fragment. All other fragmented tables, which refers to this - table in their foreign key, will automatically get a new - fragment, and their records will also be dynamically - rehashed in the same manner as for the main table. - </p> - <p>The <c>NodesOrDist</c> argument may either be a list - of nodes or the result from <c>mnesia:table_info(Tab, frag_dist)</c>. The <c>NodesOrDist</c> argument is + <p>Adds a fragment to a fragmented table. All + records in one of the old fragments are rehashed and + about half of them are moved to the new (last) + fragment. All other fragmented tables, which refer to this + table in their foreign key, automatically get a new + fragment. Also, their records are dynamically + rehashed in the same manner as for the main table.</p> + <p>Argument <c>NodesOrDist</c> can either be a list of + nodes or the result from the function + <seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, frag_dist)</seealso>. + Argument <c>NodesOrDist</c> is assumed to be a sorted list with the best nodes to host new replicas first in the list. The new fragment - will get the same number of replicas as the first - fragment (see <c>n_ram_copies</c>, <c>n_disc_copies</c> + gets the same number of replicas as the first + fragment (see <c>n_ram_copies</c>, <c>n_disc_copies</c>, and <c>n_disc_only_copies</c>). The <c>NodesOrDist</c> list must at least contain one element for each - replica that needs to be allocated. - </p> + replica that needs to be allocated.</p> </item> <tag><c>del_frag</c></tag> <item> - <p>Deletes one fragment from a fragmented table. All - records in the last fragment will be moved to one of the other - fragments. All other fragmented tables which refers to - this table in their foreign key, will automatically lose - their last fragment and their records will also be + <p>Deletes a fragment from a fragmented table. All + records in the last fragment are moved to one of the other + fragments. All other fragmented tables, which refer to + this table in their foreign key, automatically lose + their last fragment. Also, their records are dynamically rehashed in the same manner as for the main - table. - </p> + table.</p> </item> <tag><c>{add_node, Node}</c></tag> <item> - <p>Adds a new node to the <c>node_pool</c>. The new - node pool will affect the list returned from - <c>mnesia:table_info(Tab, frag_dist)</c>. - </p> + <p>Adds a node to <c>node_pool</c>. The new + node pool affects the list returned from the function + <seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, frag_dist)</seealso>. + </p> </item> <tag><c>{del_node, Node}</c></tag> <item> - <p>Deletes a new node from the <c>node_pool</c>. The - new node pool will affect the list returned from - <c>mnesia:table_info(Tab, frag_dist)</c>.</p> + <p>Deletes a node from <c>node_pool</c>. The new + node pool affects the list returned from the function + <seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, frag_dist)</seealso>. + </p> </item> </taglist> </section> <section> <title>Extensions of Existing Functions</title> - <p>The function <c>mnesia:create_table/2</c> is used to - create a brand new fragmented table, by setting the table - property <c>frag_properties</c> to some proper values. - </p> - <p>The function <c>mnesia:delete_table/1</c> is used to - delete a fragmented table including all its - fragments. There must however not exist any other - fragmented tables which refers to this table in their foreign key. - </p> - <p>The function <c>mnesia:table_info/2</c> now understands - the <c>frag_properties</c> item. - </p> - <p>If the function <c>mnesia:table_info/2</c> is invoked in - the activity context of the <c>mnesia_frag</c> module, - information of several new items may be obtained: - </p> + <p>The function + <seealso marker="mnesia#create_table/2">mnesia:create_table/2</seealso> + creates a brand new fragmented table, by setting table + property <c>frag_properties</c> to some proper values.</p> + <p>The function + <seealso marker="mnesia#delete_table/1">mnesia:delete_table/1</seealso> + deletes a fragmented table including all its + fragments. There must however not exist any other fragmented + tables that refer to this table in their foreign key.</p> + <p>The function + <seealso marker="mnesia#table_info/2">mnesia:table_info/2</seealso> + now understands item <c>frag_properties</c>.</p> + <p>If the function <c>mnesia:table_info/2</c> is started in + the activity context of module <c>mnesia_frag</c>, + information of several new items can be obtained:</p> <taglist> <tag><c>base_table</c></tag> - <item> - <p>the name of the fragmented table - </p> - </item> + <item>The name of the fragmented table</item> <tag><c>n_fragments</c></tag> - <item> - <p>the actual number of fragments - </p> - </item> + <item>The actual number of fragments</item> <tag><c>node_pool</c></tag> - <item> - <p>the pool of nodes - </p> - </item> + <item>The pool of nodes</item> <tag><c>n_ram_copies</c></tag> <item></item> <tag><c>n_disc_copies</c></tag> <item></item> <tag><c>n_disc_only_copies</c></tag> <item> - <p>the number of replicas with storage type - <c>ram_copies</c>, <c>disc_copies</c> and <c>disc_only_copies</c> + <p>The number of replicas with storage type <c>ram_copies</c>, + <c>disc_copies</c>, and <c>disc_only_copies</c>, respectively. The actual values are dynamically derived from the first fragment. The first fragment serves as a - pro-type and when the actual values needs to be computed - (e.g. when adding new fragments) they are simply - determined by counting the number of each replicas for - each storage type. This means, when the functions - <c>mnesia:add_table_copy/3</c>, - <c>mnesia:del_table_copy/2</c> and<c>mnesia:change_table_copy_type/2</c> are applied on the - first fragment, it will affect the settings on + protype. When the actual values need to be computed + (for example, when adding new fragments) they are + determined by counting the number of each replica for + each storage type. This means that when the functions + <seealso marker="mnesia#add_table_copy/3">mnesia:add_table_copy/3</seealso>, + + <seealso marker="mnesia#del_table_copy/2">mnesia:del_table_copy/2</seealso>, + and + <seealso marker="mnesia#change_table_copy_type/3">mnesia:change_table_copy_type/2</seealso> are applied on the + first fragment, it affects the settings on <c>n_ram_copies</c>, <c>n_disc_copies</c>, and - <c>n_disc_only_copies</c>. - </p> + <c>n_disc_only_copies</c>.</p> </item> <tag><c>foreign_key</c></tag> <item> - <p>the foreign key. - </p> + <p>The foreign key</p> </item> <tag><c>foreigners</c></tag> <item> - <p>all other tables that refers to this table in - their foreign key. - </p> + <p>All other tables that refer to this table in + their foreign key</p> </item> <tag><c>frag_names</c></tag> <item> - <p>the names of all fragments. - </p> + <p>The names of all fragments</p> </item> <tag><c>frag_dist</c></tag> <item> - <p>a sorted list of <c>{Node, Count}</c> tuples - which is sorted in increasing <c>Count</c> order. The + <p>A sorted list of <c>{Node, Count}</c> tuples + that are sorted in increasing <c>Count</c> order. <c>Count</c> is the total number of replicas that this fragmented table hosts on each <c>Node</c>. The list - always contains at least all nodes in the - <c>node_pool</c>. The nodes which not belongs to the - <c>node_pool</c> will be put last in the list even if - their <c>Count</c> is lower. - </p> + always contains at least all nodes in + <c>node_pool</c>. Nodes that do not belong to + <c>node_pool</c> are put last in the list even if + their <c>Count</c> is lower.</p> </item> <tag><c>frag_size</c></tag> <item> - <p>a list of <c>{Name, Size}</c> tuples where - <c>Name</c> is a fragment <c>Name</c> and <c>Size</c> is - how many records it contains. - </p> + <p>A list of <c>{Name, Size}</c> tuples, where + <c>Name</c> is a fragment <c>Name</c>, and <c>Size</c> is + how many records it contains</p> </item> <tag><c>frag_memory</c></tag> <item> - <p>a list of <c>{Name, Memory}</c> tuples where - <c>Name</c> is a fragment <c>Name</c> and <c>Memory</c> is - how much memory it occupies. - </p> + <p>A list of <c>{Name, Memory}</c> tuples, where + <c>Name</c> is a fragment <c>Name</c>, and <c>Memory</c> is + how much memory it occupies</p> </item> <tag><c>size</c></tag> <item> - <p>total size of all fragments - </p> + <p>Total size of all fragments</p> </item> <tag><c>memory</c></tag> <item> - <p>the total memory of all fragments</p> + <p>Total memory of all fragments</p> </item> </taglist> </section> @@ -642,42 +601,45 @@ ok <title>Load Balancing</title> <p>There are several algorithms for distributing records in a fragmented table evenly over a - pool of nodes. No one is best, it simply depends of the - application needs. Here follows some examples of - situations which may need some attention: - </p> - <p><c>permanent change of nodes</c> when a new permanent - <c>db_node</c> is introduced or dropped, it may be time to - change the pool of nodes and re-distribute the replicas - evenly over the new pool of nodes. It may also be time to - add or delete a fragment before the replicas are re-distributed. - </p> - <p><c>size/memory threshold</c> when the total size or + pool of nodes. No one is best, it depends on the + application needs. The following examples of + situations need some attention:</p> + <list type="bulleted"> + <item><c>permanent change of nodes</c>. When a new permanent + <c>db_node</c> is introduced or dropped, it can be time to + change the pool of nodes and redistribute the replicas + evenly over the new pool of nodes. It can also be time to + add or delete a fragment before the replicas are redistributed. + </item> + <item><c>size/memory threshold</c>. When the total size or total memory of a fragmented table (or a single - fragment) exceeds some application specific threshold, it - may be time to dynamically add a new fragment in order - obtain a better distribution of records. - </p> - <p><c>temporary node down</c> when a node temporarily goes - down it may be time to compensate some fragments with new - replicas in order to keep the desired level of - redundancy. When the node comes up again it may be time to - remove the superfluous replica. - </p> - <p><c>overload threshold</c> when the load on some node is - exceeds some application specific threshold, it may be time to - either add or move some fragment replicas to nodes with lesser - load. Extra care should be taken if the table has a foreign - key relation to some other table. In order to avoid severe - performance penalties, the same re-distribution must be - performed for all of the related tables. - </p> - <p>Use <c>mnesia:change_table_frag/2</c> to add new fragments + fragment) exceeds some application-specific threshold, it + can be time to add a new fragment dynamically to + obtain a better distribution of records. + </item> + <item><c>temporary node down</c>. When a node temporarily goes + down, it can be time to compensate some fragments with new + replicas to keep the desired level of + redundancy. When the node comes up again, it can be time to + remove the superfluous replica. + </item> + <item><c>overload threshold</c>. When the load on some node + exceeds some application-specific threshold, it can be time to + either add or move some fragment replicas to nodes with lower + load. Take extra care if the table has a foreign + key relation to some other table. To avoid severe + performance penalties, the same redistribution must be + performed for all the related tables. + </item> + </list> + <p>Use the function + <c>mnesia:change_table_frag/2</c> to add new fragments and apply the usual schema manipulation functions (such as - <c>mnesia:add_table_copy/3</c>, <c>mnesia:del_table_copy/2</c> - and <c>mnesia:change_table_copy_type/2</c>) on each fragment - to perform the actual re-distribution. - </p> + <seealso marker="mnesia#add_table_copy/3">mnesia:add_table_copy/3</seealso>, + <seealso marker="mnesia#del_table_copy/2">mnesia:del_table_copy/2</seealso>, + and + <seealso marker="mnesia#change_table_copy_type/3">mnesia:change_table_copy_type/2</seealso>) + on each fragment to perform the actual redistribution.</p> </section> </section> @@ -685,356 +647,369 @@ ok <title>Local Content Tables</title> <p>Replicated tables have the same content on all nodes where they are replicated. However, it is sometimes advantageous to - have tables but different content on different nodes. - </p> - <p>If we specify the attribute <c>{local_content, true}</c> when - we create the table, the table will reside on the nodes where - we specify that the table shall exist, but the write operations on the - table will only be performed on the local copy. - </p> - <p>Furthermore, when the table is initialized at start-up, the - table will only be initialized locally, and the table - content will not be copied from another node. - </p> + have tables, but different content on different nodes.</p> + <p>If attribute <c>{local_content, true}</c> is specified when + you create the table, the table resides on the nodes where you + specify the table to exist, but the write operations on the + table are only performed on the local copy.</p> + <p>Furthermore, when the table is initialized at startup, the + table is only initialized locally, and the table + content is not copied from another node.</p> </section> <section> - <title>Disc-less Nodes</title> - <p>It is possible to run Mnesia on nodes that do not have a - disc. It is of course not possible to have replicas - of neither <c>disc_copies</c>, nor <c>disc_only_copies</c> - on such nodes. This especially troublesome for the - <c>schema</c> table since Mnesia need the schema in order - to initialize itself. - </p> - <p>The schema table may, as other tables, reside on one or - more nodes. The storage type of the schema table may either - be <c>disc_copies</c> or <c>ram_copies</c> - (not <c>disc_only_copies</c>). At - start-up Mnesia uses its schema to determine with which - nodes it should try to establish contact. If any - of the other nodes are already started, the starting node + <title>Disc-Less Nodes</title> + <p><c>Mnesia</c> can be run on nodes that do not have a disc. + Replicas of <c>disc_copies</c> or <c>disc_only_copies</c> are + not possible on such nodes. This is especially troublesome for + the <c>schema</c> table, as <c>Mnesia</c> needs the schema + to initialize itself.</p> + <p>The schema table can, as other tables, reside on one or + more nodes. The storage type of the schema table can either + be <c>disc_copies</c> or <c>ram_copies</c> + (but not <c>disc_only_copies</c>). At + startup, <c>Mnesia</c> uses its schema to determine with which + nodes it is to try to establish contact. If any + other node is started already, the starting node merges its table definitions with the table definitions brought from the other nodes. This also applies to the - definition of the schema table itself. The application - parameter <c>extra_db_nodes</c> contains a list of nodes which - Mnesia also should establish contact with besides the ones - found in the schema. The default value is the empty list - <c>[]</c>. - </p> + definition of the schema table itself. Application + parameter <c>extra_db_nodes</c> contains a list of nodes that + <c>Mnesia</c> also is to establish contact with besides those + found in the schema. Default is <c>[]</c> (empty list).</p> <p>Hence, when a disc-less node needs to find the schema - definitions from a remote node on the network, we need to supply - this information through the application parameter <c>-mnesia extra_db_nodes NodeList</c>. Without this - configuration parameter set, Mnesia will start as a single node - system. It is also possible to use <c>mnesia:change_config/2</c> - to assign a value to 'extra_db_nodes' and force a connection - after mnesia have been started, i.e. - mnesia:change_config(extra_db_nodes, NodeList). - </p> - <p>The application parameter schema_location controls where - Mnesia will search for its schema. The parameter may be one of - the following atoms: - </p> + definitions from a remote node on the network, this + information must be supplied through application parameter + <c>-mnesia extra_db_nodes NodeList</c>. Without this + configuration parameter set, <c>Mnesia</c> starts as a single + node system. Also, the function + <seealso marker="mnesia#change_config/2">mnesia:change_config/2</seealso> + can be used to assign a value to <c>extra_db_nodes</c> and force + a connection after <c>Mnesia</c> has been started, that is, + <c>mnesia:change_config(extra_db_nodes, NodeList)</c>.</p> + <p>Application parameter <c>schema_location</c> controls where + <c>Mnesia</c> searches for its schema. The parameter can be one + of the following atoms:</p> <taglist> <tag><c>disc</c></tag> <item> <p>Mandatory disc. The schema is assumed to be located - on the Mnesia directory. And if the schema cannot be found, - Mnesia refuses to start. - </p> + in the <c>Mnesia</c> directory. If the schema cannot be found, + <c>Mnesia</c> refuses to start.</p> </item> <tag><c>ram</c></tag> <item> - <p>Mandatory ram. The schema resides in ram - only. At start-up a tiny new schema is generated. This - default schema contains just the definition of the schema - table and only resides on the local node. Since no other - nodes are found in the default schema, the configuration - parameter <c>extra_db_nodes</c> must be used in order to let the - node share its table definitions with other nodes. (The - <c>extra_db_nodes</c> parameter may also be used on disc-full nodes.) - </p> + <p>Mandatory RAM. The schema resides in RAM + only. At startup, a tiny new schema is generated. This + default schema contains only the definition of the schema + table and resides on the local node only. Since no other + nodes are found in the default schema, configuration + parameter <c>extra_db_nodes</c> must be used to let the + node share its table definitions with other nodes. (Parameter + <c>extra_db_nodes</c> can also be used on disc-full nodes.)</p> </item> <tag><c>opt_disc</c></tag> <item> - <p>Optional disc. The schema may reside on either disc - or ram. If the schema is found on disc, Mnesia starts as a - disc-full node (the storage type of the schema table is - disc_copies). If no schema is found on disc, Mnesia starts - as a disc-less node (the storage type of the schema table is - ram_copies). The default value for the application parameter - is - <c>opt_disc</c>. </p> + <p>Optional disc. The schema can reside on either disc or + RAM. If the schema is found on disc, <c>Mnesia</c> starts as + a disc-full node (the storage type of the schema table is + disc_copies). If no schema is found on disc, <c>Mnesia</c> + starts as a disc-less node (the storage type of the schema + table is <c>ram_copies</c>). The default for the + application parameter is <c>opt_disc</c>.</p> </item> </taglist> - <p>When the <c>schema_location</c> is set to opt_disc the - function <c>mnesia:change_table_copy_type/3</c> may be used to - change the storage type of the schema. - This is illustrated below: - </p> + <p>When <c>schema_location</c> is set to <c>opt_disc</c>, the + function + <seealso marker="mnesia#change_table_copy_type/3">mnesia:change_table_copy_type/3</seealso> + can be used to change the storage type of the schema. + This is illustrated as follows:</p> <pre> 1> mnesia:start(). ok 2> mnesia:change_table_copy_type(schema, node(), disc_copies). - {atomic, ok} - </pre> - <p>Assuming that the call to <c>mnesia:start</c> did not - find any schema to read on the disc, then Mnesia has started - as a disc-less node, and then changed it to a node that - utilizes the disc to locally store the schema. - </p> + {atomic, ok}</pre> + <p>Assuming that the call to + <seealso marker="mnesia#start/0">mnesia:start/0</seealso> does not + find any schema to read on the disc, <c>Mnesia</c> starts + as a disc-less node, and then change it to a node that + use the disc to store the schema locally.</p> </section> <section> - <title>More Schema Management</title> - <p>It is possible to add and remove nodes from a Mnesia system. - This can be done by adding a copy of the schema to those nodes. - </p> - <p>The functions <c>mnesia:add_table_copy/3</c> and - <c>mnesia:del_table_copy/2</c> may be used to add and delete - replicas of the schema table. Adding a node to the list - of nodes where the schema is replicated will affect two - things. First it allows other tables to be replicated to - this node. Secondly it will cause Mnesia to try to contact - the node at start-up of disc-full nodes. - </p> - <p>The function call <c>mnesia:del_table_copy(schema, mynode@host)</c> deletes the node 'mynode@host' from the - Mnesia system. The call fails if mnesia is running on - 'mynode@host'. The other mnesia nodes will never try to connect - to that node again. Note, if there is a disc - resident schema on the node 'mynode@host', the entire mnesia - directory should be deleted. This can be done with - <c>mnesia:delete_schema/1</c>. If - mnesia is started again on the the node 'mynode@host' and the - directory has not been cleared, mnesia's behaviour is undefined. - </p> - <p>If the storage type of the schema is ram_copies, i.e, we - have disc-less node, Mnesia - will not use the disc on that particular node. The disc - usage is enabled by changing the storage type of the table - <c>schema</c> to disc_copies. - </p> - <p>New schemas are - created explicitly with <c>mnesia:create_schema/1</c> or implicitly - by starting Mnesia without a disc resident schema. Whenever - a table (including the schema table) is created it is - assigned its own unique cookie. The schema table is not created with - <c>mnesia:create_table/2</c> as normal tables. - </p> - <p>At start-up Mnesia connects different nodes to each other, - then they exchange table definitions with each other and the - table definitions are merged. During the merge procedure Mnesia + <title>More about Schema Management</title> + <p>Nodes can be added to and removed from a <c>Mnesia</c> system. + This can be done by adding a copy of the schema to those nodes.</p> + <p>The functions + <seealso marker="mnesia#add_table_copy/3">mnesia:add_table_copy/3</seealso> + and + <seealso marker="mnesia#del_table_copy/2">mnesia:del_table_copy/2</seealso> + can be used to add and delete + replicas of the schema table. Adding a node to the list of + nodes where the schema is replicated affects the following:</p> + <list type="bulleted"> + <item>It allows other tables to be replicated to this node. + </item> + <item>It causes <c>Mnesia</c> to try to contact the node at + startup of disc-full nodes. + </item> + </list> + <p>The function call <c>mnesia:del_table_copy(schema, + mynode@host)</c> deletes node <c>mynode@host</c> from the + <c>Mnesia</c> system. The call fails if <c>Mnesia</c> is running + on <c>mynode@host</c>. The other <c>Mnesia</c> nodes never try to + connect to that node again. Notice that if there is a disc resident + schema on node <c>mynode@host</c>, the entire <c>Mnesia</c> + directory is to be deleted. This is done with the function + <seealso marker="mnesia#delete_schema/1">mnesia:delete_schema/1</seealso>. + If <c>Mnesia</c> is started again + on node <c>mynode@host</c> and the directory has not been + cleared, the behavior of <c>Mnesia</c> is undefined.</p> + <p>If the storage type of the schema is <c>ram_copies</c>, + that is, a disc-less node, <c>Mnesia</c> + does not use the disc on that particular node. The disc + use is enabled by changing the storage type of table + <c>schema</c> to <c>disc_copies</c>.</p> + <p>New schemas are created explicitly with the function + <seealso marker="mnesia#create_schema/1">mnesia:create_schema/1</seealso> + or implicitly by starting + <c>Mnesia</c> without a disc resident schema. Whenever + a table (including the schema table) is created, it is + assigned its own unique cookie. The schema table is not created + with the function + <seealso marker="mnesia#create_table/2">mnesia:create_table/2</seealso> + as normal tables.</p> + <p>At startup, <c>Mnesia</c> connects different nodes to each other, + then they exchange table definitions with each other, and the table + definitions are merged. During the merge procedure, <c>Mnesia</c> performs a sanity test to ensure that the table definitions are - compatible with each other. If a table exists on several nodes - the cookie must be the same, otherwise Mnesia will shutdown one - of the nodes. This unfortunate situation will occur if a table + compatible with each other. If a table exists on several nodes, + the cookie must be the same, otherwise <c>Mnesia</c> shut down one + of the nodes. This unfortunate situation occurs if a table has been created on two nodes independently of each other while - they were disconnected. To solve the problem, one of the tables - must be deleted (as the cookies differ we regard it to be two - different tables even if they happen to have the same name). - </p> - <p>Merging different versions of the schema table, does not + they were disconnected. To solve this, one of the tables + must be deleted (as the cookies differ, it is regarded to be two + different tables even if they have the same name).</p> + <p>Merging different versions of the schema table does not always require the cookies to be the same. If the storage - type of the schema table is disc_copies, the cookie is - immutable, and all other db_nodes must have the same - cookie. When the schema is stored as type ram_copies, + type of the schema table is <c>disc_copies</c>, the cookie is + immutable, and all other <c>db_nodes</c> must have the same + cookie. When the schema is stored as type <c>ram_copies</c>, its cookie can be replaced with a cookie from another node - (ram_copies or disc_copies). The cookie replacement (during - merge of the schema table definition) is performed each time - a RAM node connects to another node. - </p> - <p><c>mnesia:system_info(schema_location)</c> and - <c>mnesia:system_info(extra_db_nodes)</c> may be used to determine - the actual values of schema_location and extra_db_nodes - respectively. <c>mnesia:system_info(use_dir)</c> may be used to - determine whether Mnesia is actually using the Mnesia - directory. <c>use_dir</c> may be determined even before - Mnesia is started. The function <c>mnesia:info/0</c> may now be - used to printout some system information even before Mnesia - is started. When Mnesia is started the function prints out - more information. - </p> - <p>Transactions which update the definition of a table, - requires that Mnesia is started on all nodes where the - storage type of the schema is disc_copies. All replicas of + (<c>ram_copies</c> or <c>disc_copies</c>). The cookie replacement + (during merge of the schema table definition) is performed each + time a RAM node connects to another node.</p> + <p>Further, the following applies:</p> + <list type ="bulleted"> + <item><seealso marker="mnesia#system_info/1">mnesia:system_info(schema_location)</seealso> + and + <seealso marker="mnesia#system_info/1">mnesia:system_info(extra_db_nodes)</seealso> + can be used to determine the actual values of <c>schema_location</c> + and <c>extra_db_nodes</c>, respectively. + </item> + <item><seealso marker="mnesia#system_info/1">mnesia:system_info(use_dir)</seealso> + can be used to determine whether <c>Mnesia</c> is actually + using the <c>Mnesia</c> directory. + </item> + <item><c>use_dir</c> can be determined even before + <c>Mnesia</c> is started. + </item> + </list> + <p>The function <seealso marker="mnesia#info/0">mnesia:info/0</seealso> + can now be used to print + some system information even before <c>Mnesia</c> is started. + When <c>Mnesia</c> is started, the function prints more + information.</p> + <p>Transactions that update the definition of a table + requires that <c>Mnesia</c> is started on all nodes where the + storage type of the schema is <c>disc_copies</c>. All replicas of the table on these nodes must also be loaded. There are a - few exceptions to these availability rules. Tables may be - created and new replicas may be added without starting all - of the disc-full nodes. New replicas may be added before all - other replicas of the table have been loaded, it will suffice - when one other replica is active. - </p> + few exceptions to these availability rules:</p> + <list type="bulleted"> + <item>Tables can be created and new replicas can be added + without starting all the disc-full nodes. + </item> + <item>New replicas can be added before all other replicas of + the table have been loaded, provided that at least one other + replica is active. + </item> + </list> </section> <section> <marker id="event_handling"></marker> <title>Mnesia Event Handling</title> - <p>System events and table events are the two categories of events - that Mnesia will generate in various situations. - </p> - <p>It is possible for user process to subscribe on the - events generated by Mnesia. - We have the following two functions:</p> + <p>System events and table events are the two event categories + that <c>Mnesia</c> generates in various situations.</p> + <p>A user process can subscribe on the events generated by + <c>Mnesia</c>. The following two functions are provided:</p> <taglist> - <tag><c>mnesia:subscribe(Event-Category)</c></tag> - <item> - <p>Ensures that a copy of all events of type - <c>Event-Category</c> are sent to the calling process. - </p> - </item> - <tag><c>mnesia:unsubscribe(Event-Category)</c></tag> + <tag><seealso marker="mnesia#subscribe/1">mnesia:subscribe(Event-Category)</seealso> + </tag> + <item>Ensures that a copy of all events of type + <c>Event-Category</c> are sent to the calling process</item> + <tag><seealso marker="mnesia#unsubscribe/1">mnesia:unsubscribe(Event-Category)</seealso> + </tag> <item>Removes the subscription on events of type - <c>Event-Category</c></item> + <c>Event-Category</c> + </item> </taglist> - <p><c>Event-Category</c> may either be the atom <c>system</c>, the atom <c>activity</c>, or - one of the tuples <c>{table, Tab, simple}</c>, <c>{table, Tab, detailed}</c>. The old event-category <c>{table, Tab}</c> is the same - event-category as <c>{table, Tab, simple}</c>. - The subscribe functions activate a subscription + <p><c>Event-Category</c> can be either of the following:</p> + <list type="bulleted"> + <item>The atom <c>system</c> + </item> + <item>The atom <c>activity</c> + </item> + <item>The tuple <c>{table, Tab, simple}</c> + </item> + <item>The tuple <c>{table, Tab, detailed}</c> + </item> + </list> + <p>The old event category <c>{table, Tab}</c> is the same + event category as <c>{table, Tab, simple}</c>.</p> + <p>The subscribe functions activate a subscription of events. The events are delivered as messages to the process - evaluating the <c>mnesia:subscribe/1</c> function. The syntax of - system events is <c>{mnesia_system_event, Event}</c>, - <c>{mnesia_activity_event, Event}</c> for activity events, and - <c>{mnesia_table_event, Event}</c> for table events. What the various - event types mean is described below.</p> - <p>All system events are subscribed by Mnesia's - gen_event handler. The default gen_event handler is - <c>mnesia_event</c>. But it may be changed by using the application - parameter <c>event_module</c>. The value of this parameter must be - the name of a module implementing a complete handler - as specified by the <c>gen_event</c> module in - STDLIB. <c>mnesia:system_info(subscribers)</c> and - <c>mnesia:table_info(Tab, subscribers)</c> may be used to determine - which processes are subscribed to various - events. - </p> + evaluating the function + <seealso marker="mnesia#subscribe/1">mnesia:subscribe/1</seealso> + The syntax is as follows:</p> + <list type="bulleted"> + <item><c>{mnesia_system_event, Event}</c> for system events + </item> + <item><c>{mnesia_activity_event, Event}</c> for activity events + </item> + <item><c>{mnesia_table_event, Event}</c> for table events + </item> + </list> + <p>The event types are described in the next sections.</p> + <p>All system events are subscribed by the <c>Mnesia</c> + <c>gen_event</c> handler. The default <c>gen_event</c> handler + is <c>mnesia_event</c>, but it can be changed by using + application parameter <c>event_module</c>. The value of this + parameter must be the name of a module implementing a complete + handler, as specified by the + <seealso marker="stdlib:gen_event">gen_event</seealso> module + in <c>STDLIB</c>.</p> + <p><seealso marker="mnesia#system_info/1">mnesia:system_info(subscribers)</seealso> + and + <seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, subscribers)</seealso> + can be used to determine which processes are subscribed to + various events.</p> <section> <title>System Events</title> - <p>The system events are detailed below:</p> + <p>The system events are as follows:</p> <taglist> <tag><c>{mnesia_up, Node}</c></tag> - <item> - <p>Mnesia has been started on a node. - Node is the name of the node. By default this event is ignored. - </p> + <item>Mnesia is started on a node. <c>Node</c> is the node + name. By default this event is ignored. </item> <tag><c>{mnesia_down, Node}</c></tag> - <item> - <p>Mnesia has been stopped on a node. - Node is the name of the node. By default this event is - ignored. - </p> + <item>Mnesia is stopped on a node. <c>Node</c> is the node + name. By default this event is ignored. </item> <tag><c>{mnesia_checkpoint_activated, Checkpoint}</c></tag> - <item> - <p>a checkpoint with the name - <c>Checkpoint</c> has been activated and that the current node is - involved in the checkpoint. Checkpoints may be activated - explicitly with <c>mnesia:activate_checkpoint/1</c> or implicitly - at backup, adding table replicas, internal transfer of data - between nodes etc. By default this event is ignored. - </p> + <item>A checkpoint with the name <c>Checkpoint</c> is + activated and the current node is involved in the + checkpoint. Checkpoints can be activated explicitly with + the function + <seealso marker="mnesia#activate_checkpoint/1">mnesia:activate_checkpoint/1</seealso> + or implicitly at + backup, when adding table replicas, at internal transfer of + data between nodes, and so on. By default this event is + ignored. </item> <tag><c>{mnesia_checkpoint_deactivated, Checkpoint}</c></tag> - <item> - <p>A checkpoint with the name - <c>Checkpoint</c> has been deactivated and that the current node was - involved in the checkpoint. Checkpoints may explicitly be - deactivated with <c>mnesia:deactivate/1</c> or implicitly when the - last replica of a table (involved in the checkpoint) - becomes unavailable, e.g. at node down. By default this - event is ignored. - </p> + <item>A checkpoint with the name <c>Checkpoint</c> is + deactivated and the current node is involved in the + checkpoint. Checkpoints can be deactivated explicitly with + the function + <seealso marker="mnesia#deactivate_checkpoint/1">mnesia:deactivate/1</seealso> + or implicitly when the last + replica of a table (involved in the checkpoint) becomes + unavailable, for example, at node-down. By default this + event is ignored. </item> <tag><c>{mnesia_overload, Details}</c></tag> - <item> - <p>Mnesia on the current node is - overloaded and the subscriber should take action. - </p> + <item><p><c>Mnesia</c> on the current node is + overloaded and the subscriber is to take action.</p> <p>A typical overload situation occurs when the - applications are performing more updates on disc - resident tables than Mnesia is able to handle. Ignoring - this kind of overload may lead into a situation where + applications perform more updates on disc resident + tables than <c>Mnesia</c> can handle. Ignoring + this kind of overload can lead to a situation where the disc space is exhausted (regardless of the size of - the tables stored on disc). - <br></br> - Each update is appended to - the transaction log and occasionally(depending of how it + the tables stored on disc).</p> + <p>Each update is appended to the transaction log and + occasionally (depending on how it is configured) dumped to the tables files. The table file storage is more compact than the transaction log storage, especially if the same record is updated - over and over again. If the thresholds for dumping the - transaction log have been reached before the previous - dump was finished an overload event is triggered. - </p> + repeatedly. If the thresholds for dumping the + transaction log are reached before the previous + dump is finished, an overload event is triggered.</p> <p>Another typical overload situation is when the transaction manager cannot commit transactions at the - same pace as the applications are performing updates of - disc resident tables. When this happens the message - queue of the transaction manager will continue to grow + same pace as the applications perform updates of + disc resident tables. When this occurs, the message + queue of the transaction manager continues to grow until the memory is exhausted or the load - decreases. - </p> - <p>The same problem may occur for dirty updates. The overload - is detected locally on the current node, but its cause may - be on another node. Application processes may cause heavy - loads if any table are residing on other nodes (replicated or not). By default this event - is reported to the error_logger. - </p> + decreases.</p> + <p>The same problem can occur for dirty updates. The overload + is detected locally on the current node, but its cause can + be on another node. Application processes can cause high + load if any table resides on another node (replicated + or not). By default this event + is reported to <c>error_logger.</c></p> </item> <tag><c>{inconsistent_database, Context, Node}</c></tag> - <item> - <p>Mnesia regards the database as - potential inconsistent and gives its applications a chance - to recover from the inconsistency, e.g. by installing a - consistent backup as fallback and then restart the system - or pick a <c>MasterNode</c> from <c>mnesia:system_info(db_nodes)</c>) - and invoke <c>mnesia:set_master_node([MasterNode])</c>. By default an - error is reported to the error logger. - </p> + <item><c>Mnesia</c> regards the database as potential + inconsistent and gives its applications a chance to + recover from the inconsistency. For example, by installing a + consistent backup as fallback and then restart the system. + An alternative is to pick a <c>MasterNode</c> from + <seealso marker="mnesia#system_info/1">mnesia:system_info(db_nodes)</seealso> + and invoke + <seealso marker="mnesia#set_master_nodes/1">mnesia:set_master_node([MasterNode])</seealso>. + By default an error is reported to <c>error_logger</c>. </item> <tag><c>{mnesia_fatal, Format, Args, BinaryCore}</c></tag> <item> - <p>Mnesia has encountered a fatal error - and will (in a short period of time) be terminated. The reason for - the fatal error is explained in Format and Args which may - be given as input to <c>io:format/2</c> or sent to the - error_logger. By default it will be sent to the - error_logger. <c>BinaryCore</c> is a binary containing a summary of - Mnesia's internal state at the time the when the fatal error was - encountered. By default the binary is written to a - unique file name on current directory. On RAM nodes the - core is ignored. - </p> + <p><c>Mnesia</c> detected a fatal error and + terminates soon. The fault reason is explained in + <c>Format</c> and <c>Args</c>, which can be given as input + to <c>io:format/2</c> or sent to <c>error_logger</c>. By + default it is sent to <c>error_logger</c>.</p> + <p><c>BinaryCore</c> is a binary containing a summary of the + <c>Mnesia</c> internal state at the time when the fatal + error was detected. By default the binary is written to a + unique filename on the current directory. On RAM nodes, the + core is ignored.</p> </item> <tag><c>{mnesia_info, Format, Args}</c></tag> - <item> - <p>Mnesia has detected something that - may be of interest when debugging the system. This is explained - in <c>Format</c> and <c>Args</c> which may appear - as input to <c>io:format/2</c> or sent to the error_logger. By - default this event is printed with <c>io:format/2</c>. - </p> + <item><c>Mnesia</c> detected something that can be of + interest when debugging the system. This is explained in + <c>Format</c> and <c>Args</c>, which can appear as input + to <c>io:format/2</c> or sent to <c>error_logger</c>. By + default this event is printed with <c>io:format/2</c>. </item> <tag><c>{mnesia_error, Format, Args}</c></tag> - <item> - <p>Mnesia has encountered an error. The - reason for the error is explained i <c>Format</c> and <c>Args</c> - which may be given as input to <c>io:format/2</c> or sent to the - error_logger. By default this event is reported to the error_logger. - </p> + <item><c>Mnesia</c> has detected an error. The fault reason is + explained in <c>Format</c> and <c>Args</c>, which can be + given as input to <c>io:format/2</c> or sent to + <c>error_logger</c>. By default this event is reported to + <c>error_logger</c>. </item> <tag><c>{mnesia_user, Event}</c></tag> - <item> - <p>An application has invoked the - function <c>mnesia:report_event(Event)</c>. <c>Event</c> may be any Erlang - data structure. When tracing a system of Mnesia applications - it is useful to be able to interleave Mnesia's own events with - application related events that give information about the - application context. Whenever the application starts with - a new and demanding Mnesia activity or enters a - new and interesting phase in its execution it may be a good idea - to use <c>mnesia:report_event/1</c>. </p> + <item>An application started the function + <seealso marker="mnesia#report_event/1">mnesia:report_event(Event)</seealso>. + <c>Event</c> can be + any Erlang data structure. When tracing a system of + <c>Mnesia</c> applications, it is useful to be able to + interleave own events of <c>Mnesia</c> with application-related + events that give information about the application context. + Whenever the application starts with a new and demanding + <c>Mnesia</c> activity, or enters a new and interesting + phase in its execution, it can be a good idea to use + <c>mnesia:report_event/1</c>. </item> </taglist> </section> @@ -1045,80 +1020,86 @@ ok <taglist> <tag><c>{complete, ActivityID}</c></tag> <item> - <p>This event occurs when a transaction that caused a modification to the database - has completed. It is useful for determining when a set of table events - (see below) caused by a given activity have all been sent. Once the this event - has been received, it is guaranteed that no further table events with the same - ActivityID will be received. Note that this event may still be received even - if no table events with a corresponding ActivityID were received, depending on + <p>This event occurs when a transaction that caused a modification + to the database is completed. It is useful for determining when + a set of table events (see the next section), caused by a given + activity, have been sent. Once this event is received, it is + guaranteed that no further table events with the same + <c>ActivityID</c> will be received. Notice that this event can + still be received even if no table events with a corresponding + <c>ActivityID</c> were received, depending on the tables to which the receiving process is subscribed.</p> - <p>Dirty operations always only contain one update and thus no activity event is sent.</p> + <p>Dirty operations always contain only one update and thus no + activity event is sent.</p> </item> </taglist> </section> <section> <title>Table Events</title> - <p>The final category of events are table events, which are - events related to table updates. There are two types of table - events simple and detailed. - </p> - <p>The simple table events are tuples looking like this: - <c>{Oper, Record, ActivityId}</c>. Where <c>Oper</c> is the - operation performed. <c>Record</c> is the record involved in the - operation and <c>ActivityId</c> is the identity of the - transaction performing the operation. Note that the name of the - record is the table name even when the <c>record_name</c> has - another setting. The various table related events that may - occur are: - </p> + <p>Table events are events related to table updates. There are + two types of table events, simple and detailed.</p> + <p>The <em>simple table events</em> are tuples like + <c>{Oper, Record, ActivityId}</c>, where:</p> + <list type="bulleted"> + <item><c>Oper</c> is the operation performed. + </item> + <item><c>Record</c> is the record involved in the operation. + </item> + <item><c>ActivityId</c> is the identity of the transaction + performing the operation. + </item> + </list> + <p>Notice that the record name is the table name even when + <c>record_name</c> has another setting.</p> + <p>The table-related events that can occur are as follows:</p> <taglist> <tag><c>{write, NewRecord, ActivityId}</c></tag> - <item> - <p>a new record has been written. - NewRecord contains the new value of the record. - </p> + <item>A new record has been written. <c>NewRecord</c> contains + the new record value. </item> <tag><c>{delete_object, OldRecord, ActivityId}</c></tag> - <item> - <p>a record has possibly been deleted - with <c>mnesia:delete_object/1</c>. <c>OldRecord</c> - contains the value of the old record as stated as argument - by the application. Note that, other records with the same - key may be remaining in the table if it is a bag. - </p> + <item>A record has possibly been deleted with + <seealso marker="mnesia#delete_object/1">mnesia:delete_object/1</seealso>. + <c>OldRecord</c> + contains the value of the old record, as stated as argument + by the application. Notice that other records with the same + key can remain in the table if it is of type <c>bag</c>. </item> <tag><c>{delete, {Tab, Key}, ActivityId}</c></tag> - <item> - <p>one or more records possibly has - been deleted. All records with the key Key in the table - <c>Tab</c> have been deleted. </p> + <item>One or more records have possibly been deleted. + All records with the key <c>Key</c> in the table + <c>Tab</c> have been deleted. </item> </taglist> - <p>The detailed table events are tuples looking like - this: <c>{Oper, Table, Data, [OldRecs], ActivityId}</c>. - Where <c>Oper</c> is the operation - performed. <c>Table</c> is the table involved in the operation, - <c>Data</c> is the record/oid written/deleted. - <c>OldRecs</c> is the contents before the operation. - and <c>ActivityId</c> is the identity of the transaction - performing the operation. - The various table related events that may occur are: - </p> + <p>The <em>detailed table events</em> are tuples like + <c>{Oper, Table, Data, [OldRecs], ActivityId}</c>, where:</p> + <list type="bulleted"> + <item><c>Oper</c> is the operation performed. + </item> + <item><c>Table</c> is the table involved in the operation. + </item> + <item><c>Data</c> is the record/OID written/deleted. + </item> + <item><c>OldRecs</c> is the contents before the operation. + </item> + <item><c>ActivityId</c> is the identity of the transaction + performing the operation. + </item> + </list> + <p>The table-related events that can occur are as follows:</p> <taglist> <tag><c>{write, Table, NewRecord, [OldRecords], ActivityId}</c></tag> - <item> - <p>a new record has been written. - NewRecord contains the new value of the record and OldRecords - contains the records before the operation is performed. - Note that the new content is dependent on the type of the table.</p> + <item>A new record has been written. <c>NewRecord</c> contains + the new record value and <c>OldRecords</c> contains the + records before the operation is performed. Notice that the + new content depends on the table type. </item> <tag><c>{delete, Table, What, [OldRecords], ActivityId}</c></tag> - <item> - <p>records has possibly been deleted - <c>What</c> is either {Table, Key} or a record {RecordName, Key, ...} - that was deleted. - Note that the new content is dependent on the type of the table.</p> + <item>Records have possibly been deleted. <c>What</c> is + either <c>{Table, Key}</c> or a record + <c>{RecordName, Key, ...}</c> that was deleted. Notice + that the new content depends on the table type. </item> </taglist> </section> @@ -1126,69 +1107,55 @@ ok <section> <title>Debugging Mnesia Applications</title> - <p>Debugging a Mnesia application can be difficult due to a number of reasons, primarily related + <p>Debugging a <c>Mnesia</c> application can be difficult + for various reasons, primarily related to difficulties in understanding how the transaction - and table load mechanisms work. An other source of - confusion may be the semantics of nested transactions. - </p> - <p>We may set the debug level of Mnesia by calling: - </p> - <list type="bulleted"> - <item><c>mnesia:set_debug_level(Level)</c></item> - </list> - <p>Where the parameter <c>Level</c> is: - </p> + and table load mechanisms work. Another source of + confusion can be the semantics of nested transactions.</p> + <p>The debug level of <c>Mnesia</c> is set by calling the function + <seealso marker="mnesia#set_debug_level/1">mnesia:set_debug_level(Level)</seealso>, + where <c>Level</c>is one of the following:</p> <taglist> <tag><c>none</c></tag> - <item> - <p>no trace outputs at all. This is the default. - </p> + <item>No trace outputs. This is the default. </item> <tag><c>verbose</c></tag> - <item> - <p>activates tracing of important debug events. These - debug events will generate <c>{mnesia_info, Format, Args}</c> - system events. Processes may subscribe to these events with - <c>mnesia:subscribe/1</c>. The events are always sent to Mnesia's - event handler. - </p> + <item>Activates tracing of important debug events. These + events generate <c>{mnesia_info, Format, Args}</c> + system events. Processes can subscribe to these events with + the function + <seealso marker="mnesia#subscribe/1">mnesia:subscribe/1</seealso>. + The events are always sent to the <c>Mnesia</c> event handler. </item> <tag><c>debug</c></tag> - <item> - <p>activates all events at the verbose level plus - traces of all debug events. These debug events will generate - <c>{mnesia_info, Format, Args}</c> system events. Processes may - subscribe to these events with <c>mnesia:subscribe/1</c>. The - events are always sent to Mnesia's event handler. On this - debug level Mnesia's event handler starts subscribing - updates in the schema table. - </p> + <item>Activates all events at the verbose level plus + traces of all debug events. These debug events generate + <c>{mnesia_info, Format, Args}</c> system events. Processes + can subscribe to these events with <c>mnesia:subscribe/1</c>. + The events are always sent to the <c>Mnesia</c> event handler. + On this debug level, the <c> Mnesia</c> event handler starts + subscribing to updates in the schema table. </item> <tag><c>trace</c></tag> - <item> - <p>activates all events at the debug level. On this - debug level Mnesia's event handler starts subscribing - updates on all Mnesia tables. This level is only intended - for debugging small toy systems, since many large - events may be generated.</p> + <item>Activates all events at the debug level. On this + level, the <c>Mnesia</c> event handler starts subscribing to + updates on all <c>Mnesia</c> tables. This level is intended + only for debugging small toy systems, as many large + events can be generated. </item> <tag><c>false</c></tag> - <item> - <p>is an alias for none.</p> + <item>An alias for none. </item> <tag><c>true</c></tag> - <item> - <p>is an alias for debug.</p> + <item>An alias for debug. </item> </taglist> - <p>The debug level of Mnesia itself, is also an application - parameter, thereby making it possible to start an Erlang system - in order to turn on Mnesia debug in the initial - start-up phase by using the following code: - </p> + <p>The debug level of <c>Mnesia</c> itself is also an application + parameter, making it possible to start an Erlang system + to turn on <c>Mnesia</c> debug in the initial + startup phase by using the following code:</p> <pre> - % erl -mnesia debug verbose - </pre> + % erl -mnesia debug verbose</pre> </section> <section> @@ -1196,85 +1163,81 @@ ok <p>Programming concurrent Erlang systems is the subject of a separate book. However, it is worthwhile to draw attention to the following features, which permit concurrent processes to - exist in a Mnesia system. - </p> - <p>A group of functions or processes can be called within a - transaction. A transaction may include statements that read, - write or delete data from the DBMS. A large number of such + exist in a <c>Mnesia</c> system:</p> + <list type="bulleted"> + <item><p>A group of functions or processes can be called within a + transaction. A transaction can include statements that read, + write, or delete data from the DBMS. Many such transactions can run concurrently, and the programmer does not - have to explicitly synchronize the processes which manipulate - the data. All programs accessing the database through the - transaction system may be written as if they had sole access to - the data. This is a very desirable property since all + need to explicitly synchronize the processes that manipulate + the data.</p> + <p>All programs accessing the database through the + transaction system can be written as if they had sole access to + the data. This is a desirable property, as all synchronization is taken care of by the transaction handler. If a program reads or writes data, the system ensures that no other - program tries to manipulate the same data at the same time. - </p> - <p>It is possible to move tables, delete tables or reconfigure - the layout of a table in various ways. An important aspect of - the actual implementation of these functions is that it is - possible for user programs to continue to use a table while it - is being reconfigured. For example, it is possible to - simultaneously move a table and perform write operations to the - table . This is important for many applications that - require continuously available services. Refer to Chapter 4: - <seealso marker="Mnesia_chap4#trans_prop">Transactions and other access contexts</seealso> for more information. - </p> + program tries to manipulate the same data at the same time.</p> + </item> + <item>Tables can be moved or deleted, and the layout of a table + can be reconfigured in various ways. An important aspect of + the implementation of these functions is that user programs + can continue to use a table while it + is being reconfigured. For example, it is possible to move a + table and perform write operations to the table at the same + time. This is important for many applications that require + continuously available services. For more information, see + <seealso marker="Mnesia_chap4#trans_prop">Transactions and Other Access Contexts</seealso>. + </item> + </list> </section> <section> <title>Prototyping</title> - <p>If and when we decide that we would like to start and manipulate - Mnesia, it is often easier to write the definitions and + <p>If and when you would like to start and manipulate + <c>Mnesia</c>, it is often easier to write the definitions and data into an ordinary text file. Initially, no tables and no data exist, or which - tables are required. At the initial stages of prototyping it - is prudent write all data into one file, process - that file and have the data in the file inserted into the database. - It is possible to initialize Mnesia with data read from a text file. - We have the following two functions to work with text files. - </p> + tables are required. At the initial stages of prototyping, it + is prudent to write all data into one file, process that + file, and have the data in the file inserted into the database. + <c>Mnesia</c> can be initialized with data read from a text file. + The following two functions can be used to work with text + files.</p> <list type="bulleted"> <item> - <p><c>mnesia:load_textfile(Filename)</c> Which loads a - series of local table definitions and data found in the file - into Mnesia. This function also starts Mnesia and possibly - creates a new schema. The function only operates on the - local node. - </p> + <seealso marker="mnesia#load_textfile/1">mnesia:load_textfile(Filename)</seealso> + loads a series of local table definitions and data found in the + file into <c>Mnesia</c>. This function also starts <c>Mnesia</c> + and possibly creates a new schema. The function operates + on the local node only. </item> <item> - <p><c>mnesia:dump_to_textfile(Filename)</c> Dumps - all local tables of a mnesia system into a text file which can - then be edited (by means of a normal text editor) and then - later reloaded.</p> + <seealso marker="mnesia#dump_to_textfile/1">mnesia:dump_to_textfile(Filename)</seealso> + dumps all local + tables of a <c>Mnesia</c> system into a text file, which + can be edited (with a normal text editor) and later reloaded. </item> </list> - <p>These functions are of course much slower than the ordinary - store and load functions of Mnesia. However, this is mainly intended for minor experiments - and initial prototyping. The major advantages of these functions is that they are very easy - to use. - </p> - <p>The format of the text file is: - </p> + <p>These functions are much slower than the ordinary store and + load functions of <c>Mnesia</c>. However, this is mainly intended + for minor experiments and initial prototyping. The major + advantage of these functions is that they are easy to use.</p> + <p>The format of the text file is as follows:</p> <pre> {tables, [{Typename, [Options]}, {Typename2 ......}]}. - {Typename, Attribute1, Atrribute2 ....}. - {Typename, Attribute1, Atrribute2 ....}. - </pre> + {Typename, Attribute1, Attribute2 ....}. + {Typename, Attribute1, Attribute2 ....}.</pre> <p><c>Options</c> is a list of <c>{Key,Value}</c> tuples conforming - to the options we could give to <c>mnesia:create_table/2</c>. - </p> - <p>For example, if we want to start playing with a small - database for healthy foods, we enter then following data into - the file <c>FRUITS</c>. - </p> + to the options that you can give to + <seealso marker="mnesia#create_table/2">mnesia:create_table/2</seealso>. + </p> + <p>For example, to start playing with a small database for healthy + foods, enter the following data into file <c>FRUITS</c>:</p> <codeinclude file="FRUITS" tag="%0" type="erl"></codeinclude> - <p>The following session with the Erlang shell then shows how - to load the fruits database. - </p> + <p>The following session with the Erlang shell shows how + to load the <c>FRUITS</c> database:</p> <pre><![CDATA[ % erl Erlang (BEAM) emulator version 4.9 @@ -1311,54 +1274,49 @@ ok ok 3> ]]></pre> - <p>Where we can see that the DBMS was initiated from a - regular text file. - </p> + <p>It can be seen that the DBMS was initiated from a + regular text file.</p> </section> <section> - <title>Object Based Programming with Mnesia</title> - <p>The Company database introduced in Chapter 2 has three tables - which store records (employee, dept, project), and three tables - which store relationships (manager, at_dep, in_proj). This is a - normalized data model, which has some advantages over a - non-normalized data model. - </p> - <p>It is more efficient to do a + <title>Object-Based Programming with Mnesia</title> + <p>The <c>Company</c> database, introduced in + <seealso marker="Mnesia_chap2#getting_started">Getting Started</seealso>, + has three tables that store records (<c>employee</c>, + <c>dept</c>, <c>project</c>), and three tables that store + relationships (<c>manager</c>, <c>at_dep</c>, <c>in_proj</c>). + This is a normalized data model, which has some advantages over + a non-normalized data model.</p> + <p>It is more efficient to do a generalized search in a normalized database. Some operations are also easier to perform on a normalized data model. For example, - we can easily remove one project, as the following example - illustrates: - </p> + one project can easily be removed, as the following example + illustrates:</p> <codeinclude file="company.erl" tag="%13" type="erl"></codeinclude> <p>In reality, data models are seldom fully normalized. A realistic alternative to a normalized database model would be - a data model which is not even in first normal form. Mnesia - is very suitable for applications such as telecommunications, - because it is easy to organize data in a very flexible manner. A - Mnesia database is always organized as a set of tables. Each - table is filled with rows/objects/records. What sets Mnesia - apart is that individual fields in a record can contain any type - of compound data structures. An individual field in a record can - contain lists, tuples, functions, and even record code. - </p> + a data model that is not even in first normal form. <c>Mnesia</c> + is suitable for applications such as telecommunications, + because it is easy to organize data in a flexible manner. A + <c>Mnesia</c> database is always organized as a set of tables. + Each table is filled with rows, objects, and records. + What sets <c>Mnesia</c> apart is that individual fields in + a record can contain any type of + compound data structures. An individual field in a record can + contain lists, tuples, functions, and even record code.</p> <p>Many telecommunications applications have unique requirements - on lookup times for certain types of records. If our Company - database had been a part of a telecommunications system, then it - could be that the lookup time of an employee <em>together</em> - with a list of the projects the employee is working on, should - be minimized. If this was the case, we might choose a - drastically different data model which has no direct - relationships. We would only have the records themselves, and - different records could contain either direct references to - other records, or they could contain other records which are not - part of the Mnesia schema. - </p> - <p>We could create the following record definitions: - </p> + on lookup times for certain types of records. If the <c>Company</c> + database had been a part of a telecommunications system, it + could be to minimize the lookup time of an employee + <em>together</em> with a list of the projects the employee is + working on. If this is the case, a drastically different data model + without direct relationships can be chosen. You would then have + only the records themselves, and different records could contain + either direct references to other records, or contain other + records that are not part of the <c>Mnesia</c> schema.</p> + <p>The following record definitions can be created:</p> <codeinclude file="company_o.hrl" tag="%0" type="erl"></codeinclude> - <p>An record which describes an employee might look like this: - </p> + <p>A record that describes an employee can look as follows:</p> <pre> Me = #employee{emp_no= 104732, name = klacke, @@ -1368,50 +1326,43 @@ ok room_no = {221, 015}, dept = 'B/SFR', projects = [erlang, mnesia, otp], - manager = 114872}, - </pre> - <p>This model only has three different tables, and the employee - records contain references to other records. We have the following - references in the record. - </p> + manager = 114872},</pre> + <p>This model has only three different tables, and the employee + records contain references to other records. The record has the + following references:</p> <list type="bulleted"> - <item><c>'B/SFR'</c> refers to a <c>dept</c> record. + <item><c>'B/SFR'</c> refers to a <c>dept</c> record. </item> - <item><c>[erlang, mnesia, otp]</c>. This is a list of three - direct references to three different <c>projects</c> records. + <item><c>[erlang, mnesia, otp]</c> is a list of three + direct references to three different <c>projects</c> records. </item> - <item><c>114872</c>. This refers to another employee record. + <item><c>114872</c> refers to another employee record. </item> </list> - <p>We could also use the Mnesia record identifiers (<c>{Tab, Key}</c>) - as references. In this case, the <c>dept</c> attribute would be - set to the value <c>{dept, 'B/SFR'}</c> instead of - <c>'B/SFR'</c>. - </p> + <p>The <c>Mnesia</c> record identifiers (<c>{Tab, Key}</c>) can + also be used as references. In this case, attribute <c>dept</c> + would be set to value <c>{dept, 'B/SFR'}</c> instead of + <c>'B/SFR'</c>.</p> <p>With this data model, some operations execute considerably - faster than they do with the normalized data model in our - Company database. On the other hand, some other operations + faster than they do with the normalized data model in the + <c>Company</c> database. However, some other operations become much more complicated. In particular, it becomes more difficult to ensure that records do not contain dangling - pointers to other non-existent, or deleted, records. - </p> + pointers to other non-existent, or deleted, records.</p> <p>The following code exemplifies a search with a non-normalized - data model. To find all employees at department - <c>Dep</c> with a salary higher than <c>Salary</c>, use the following code: - </p> + data model. To find all employees at department <c>Dep</c> with + a salary higher than <c>Salary</c>, use the following code:</p> <codeinclude file="company_o.erl" tag="%9" type="erl"></codeinclude> - <p>This code is not only easier to write and to understand, but it - also executes much faster. - </p> - <p>It is easy to show examples of code which executes faster if - we use a non-normalized data model, instead of a normalized - model. The main reason for this is that fewer tables are - required. For this reason, we can more easily combine data from - different tables in join operations. In the above example, the - <c>get_emps/2</c> function was transformed from a join operation - into a simple query which consists of a selection and a projection - on one single table. - </p> + <p>This code is easier to write and to understand, and it + also executes much faster.</p> + <p>It is easy to show examples of code that executes faster if + a non-normalized data model is used, instead of a normalized + model. The main reason is that fewer tables are required. + Therefore, data from different tables can more easily be + combined in join operations. In the previous example, the + function <c>get_emps/2</c> is transformed from a join operation + into a simple query, which consists of a selection and a + projection on one single table.</p> </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_chap7.xmlsrc b/lib/mnesia/doc/src/Mnesia_chap7.xmlsrc index 4458cd3919..573ca79106 100644 --- a/lib/mnesia/doc/src/Mnesia_chap7.xmlsrc +++ b/lib/mnesia/doc/src/Mnesia_chap7.xmlsrc @@ -32,47 +32,61 @@ <file>Mnesia_chap7.xml</file> </header> + <p>The following topics are included:</p> + <list type="bulleted"> + <item>Database configuration data</item> + <item>Core dumps</item> + <item>Dumping tables</item> + <item>Checkpoints</item> + <item>Startup files, log file, and data files</item> + <item>Loading tables at startup</item> + <item>Recovery from communication failure</item> + <item>Recovery of transactions</item> + <item>Backup, restore, fallback, and disaster recovery</item> + </list> + <section> <title>Database Configuration Data</title> <p>The following two functions can be used to retrieve system - information. They are described in detail in the reference manual. - </p> + information. For details, see the Reference Manual.</p> <list type="bulleted"> - <item><c>mnesia:table_info(Tab, Key) -></c><c>Info | exit({aborted, Reason})</c>. - Returns information about one table. Such as the - current size of the table, on which nodes it resides etc. + <item><seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, Key) + -> Info | exit({aborted,Reason})</seealso> + returns information about one table, for example, + the current size of the table and on which nodes it resides. </item> - <item><c>mnesia:system_info(Key) -> </c><c>Info | exit({aborted, Reason})</c>. - Returns information about the Mnesia system. For example, transaction - statistics, db_nodes, configuration parameters etc. + <item><seealso marker="mnesia#system_info/1">mnesia:system_info(Key) + -> Info | exit({aborted, Reason})</seealso> + returns information about the <c>Mnesia</c> system, + for example, transaction statistics, <c>db_nodes</c>, and + configuration parameters. </item> </list> </section> <section> <title>Core Dumps</title> - <p>If Mnesia malfunctions, system information is dumped to a file - named <c>MnesiaCore.Node.When</c>. The type of system + <p>If <c>Mnesia</c> malfunctions, system information is dumped to + file <c>MnesiaCore.Node.When</c>. The type of system information contained in this file can also be generated with - the function <c>mnesia_lib:coredump()</c>. If a Mnesia system - behaves strangely, it is recommended that a Mnesia core dump - file be included in the bug report.</p> + the function <c>mnesia_lib:coredump()</c>. If a <c>Mnesia</c> + system behaves strangely, it is recommended that a <c>Mnesia</c> + core dump file is included in the bug report.</p> </section> <section> <title>Dumping Tables</title> <p>Tables of type <c>ram_copies</c> are by definition stored in - memory only. It is possible, however, to dump these tables to - disc, either at regular intervals, or before the system is - shutdown. The function <c>mnesia:dump_tables(TabList)</c> dumps - all replicas of a set of RAM tables to disc. The tables can be - accessed while being dumped to disc. To dump the tables to - disc all replicas must have the storage type <c>ram_copies</c>. - </p> - <p>The table content is placed in a .DCD file on the - disc. When the Mnesia system is started, the RAM table will - initially be loaded with data from its .DCD file. - </p> + memory only. However, these tables can be dumped to + disc, either at regular intervals or before the system is + shut down. The function + <seealso marker="mnesia#dump_tables/1">mnesia:dump_tables(TabList)</seealso> + dumps all replicas of a set of RAM tables to disc. The tables can be + accessed while being dumped to disc. To dump the tables to disc, + all replicas must have the storage type <c>ram_copies</c>.</p> + <p>The table content is placed in a <c>.DCD</c> file on the + disc. When the <c>Mnesia</c> system is started, the RAM table + is initially loaded with data from its <c>.DCD</c> file.</p> </section> <section> @@ -80,137 +94,128 @@ <title>Checkpoints</title> <p>A checkpoint is a transaction consistent state that spans over one or more tables. When a checkpoint is activated, the system - will remember the current content of the set of tables. The + remembers the current content of the set of tables. The checkpoint retains a transaction consistent state of the tables, allowing the tables to be read and updated while the checkpoint - is active. A checkpoint is typically used to + is active. A checkpoint is typically used to back up tables to external media, but they are also used - internally in Mnesia for other purposes. Each checkpoint is - independent and a table may be involved in several checkpoints - simultaneously. - </p> - <p>Each table retains its old contents in a checkpoint retainer - and for performance critical applications, it may be important + internally in <c>Mnesia</c> for other purposes. Each checkpoint + is independent and a table can be involved in several checkpoints + simultaneously.</p> + <p>Each table retains its old contents in a checkpoint retainer. + For performance critical applications, it can be important to realize the processing overhead associated with checkpoints. - In a worst case scenario, the checkpoint retainer will consume - even more memory than the table itself. Each update will also be + In a worst case scenario, the checkpoint retainer consumes + more memory than the table itself. Also, each update becomes slightly slower on those nodes where checkpoint - retainers are attached to the tables. - </p> - <p>For each table it is possible to choose if there should be one + retainers are attached to the tables.</p> + <p>For each table, it is possible to choose if there is to be one checkpoint retainer attached to all replicas of the table, or if it is enough to have only one checkpoint retainer attached to a single replica. With a single checkpoint retainer per table, the - checkpoint will consume less memory, but it will be vulnerable - to node crashes. With several redundant checkpoint retainers the - checkpoint will survive as long as there is at least one active - checkpoint retainer attached to each table. - </p> - <p>Checkpoints may be explicitly deactivated with the function - <c>mnesia:deactivate_checkpoint(Name)</c>, where <c>Name</c> is + checkpoint consumes less memory, but it is vulnerable + to node crashes. With several redundant checkpoint retainers, the + checkpoint survives as long as there is at least one active + checkpoint retainer attached to each table.</p> + <p>Checkpoints can be explicitly deactivated with the function + <seealso marker="mnesia#deactivate_checkpoint/1">mnesia:deactivate_checkpoint(Name)</seealso>, + where <c>Name</c> is the name of an active checkpoint. This function returns - <c>ok</c> if successful, or <c>{error, Reason}</c> in the case - of an error. All tables in a checkpoint must be attached to at + <c>ok</c> if successful or <c>{error, Reason}</c> if there is + an error. All tables in a checkpoint must be attached to at least one checkpoint retainer. The checkpoint is automatically - de-activated by Mnesia, when any table lacks a checkpoint - retainer. This may happen when a node goes down or when a - replica is deleted. Use the <c>min</c> and - <c>max</c> arguments described below, to control the degree of - checkpoint retainer redundancy. - </p> - <p>Checkpoints are activated with the function <marker id="mnesia:chkpt(Args)"></marker> -<c>mnesia:activate_checkpoint(Args)</c>, - where <c>Args</c> is a list of the following tuples: - </p> + deactivated by <c>Mnesia</c>, when any table lacks a checkpoint + retainer. This can occur when a node goes down or when a + replica is deleted. Use arguments <c>min</c> and + <c>max</c> (described in the following list) to control the + degree of checkpoint retainer redundancy.</p> + <marker id="mnesia:chkpt(Args)"></marker> + <p>Checkpoints are activated with the function + <seealso marker="mnesia#activate_checkpoint/1">mnesia:activate_checkpoint(Args)</seealso>, + where <c>Args</c> is a list of the following tuples:</p> <list type="bulleted"> - <item><c>{name,Name}</c>. <c>Name</c> specifies a temporary name - of the checkpoint. The name may be re-used when the checkpoint - has been de-activated. If no name is specified, a name is + <item><c>{name,Name}</c>, where <c>Name</c> specifies a temporary + name of the checkpoint. The name can be reused when the checkpoint + has been deactivated. If no name is specified, a name is generated automatically. </item> - <item><c>{max,MaxTabs}</c>. <c>MaxTabs</c> is a list of tables - which will be included in the checkpoint. The default is - <c>[]</c> (an empty list). For these tables, the redundancy - will be maximized. The old contents of the table will be + <item><c>{max,MaxTabs}</c>, where <c>MaxTabs</c> is a list of + tables that are to be included in the checkpoint. Default is + <c>[]</c> (empty list). For these tables, the redundancy + is maximized. The old content of the table is retained in the checkpoint retainer when the main table is - updated by the applications. The checkpoint becomes more fault + updated by the applications. The checkpoint is more fault tolerant if the tables have several replicas. When new - replicas are added by means of the schema manipulation - function <c>mnesia:add_table_copy/3</c>, it will also - attach a local checkpoint retainer. + replicas are added by the schema manipulation function + <seealso marker="mnesia#add_table_copy/3">mnesia:add_table_copy/3</seealso> + it also attaches a local checkpoint retainer. </item> - <item><c>{min,MinTabs}</c>. <c>MinTabs</c> is a list of tables - that should be included in the checkpoint. The default is - <c>[]</c>. For these tables, the redundancy will be minimized, - and there will be a single checkpoint retainer per table, + <item><c>{min,MinTabs}</c>, where <c>MinTabs</c> is a list of + tables that are to be included in the checkpoint. Default + is <c>[]</c>. For these tables, the redundancy is minimized, + and there is to be single checkpoint retainer per table, preferably at the local node. </item> - <item><c>{allow_remote,Bool}</c>. <c>false</c> means that all - checkpoint retainers must be local. If a table does not reside - locally, the checkpoint cannot be activated. <c>true</c> - allows checkpoint retainers to be allocated on any node. The - defaults is <c>true</c>. + <item><c>{allow_remote,Bool}</c>, where <c>false</c> means that + all checkpoint retainers must be local. If a table does not + reside locally, the checkpoint cannot be activated. <c>true</c> + allows checkpoint retainers to be allocated on any node. + Default is <c>true</c>. </item> <item><c>{ram_overrides_dump,Bool}</c>. This argument only applies to tables of type <c>ram_copies</c>. <c>Bool</c> - specifies if the table state in RAM should override the table + specifies if the table state in RAM is to override the table state on disc. <c>true</c> means that the latest committed records in RAM are included in the checkpoint retainer. These are the records that the application accesses. <c>false</c> - means that the records on the disc .DAT file are - included in the checkpoint retainer. These are the records - that will be loaded on start-up. Default is <c>false</c>.</item> + means that the records on the disc <c>.DAT</c> file are + included in the checkpoint retainer. These records are + loaded on startup. Default is <c>false</c>.</item> </list> - <p>The <c>mnesia:activate_checkpoint(Args)</c> returns one of the - following values: - </p> + <p>The function + <seealso marker="mnesia#activate_checkpoint/1">mnesia:activate_checkpoint(Args)</seealso> + returns one of the following values:</p> <list type="bulleted"> <item><c>{ok, Name, Nodes}</c></item> - <item><c>{error, Reason}</c>.</item> + <item><c>{error, Reason}</c></item> </list> - <p><c>Name</c> is the name of the checkpoint, and <c>Nodes</c> are - the nodes where the checkpoint is known. - </p> + <p><c>Name</c> is the checkpoint name. <c>Nodes</c> are + the nodes where the checkpoint is known.</p> <p>A list of active checkpoints can be obtained with the following - functions: - </p> + functions:</p> <list type="bulleted"> - <item><c>mnesia:system_info(checkpoints)</c>. This function + <item><seealso marker="mnesia#system_info/1">mnesia:system_info(checkpoints)</seealso> returns all active checkpoints on the current node.</item> - <item><c>mnesia:table_info(Tab,checkpoints)</c>. This function + <item><seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, checkpoints)</seealso> returns active checkpoints on a specific table.</item> </list> </section> <section> - <title>Files</title> - <p>This section describes the internal files which are created and maintained by the Mnesia system, - in particular, the workings of the Mnesia log is described. - </p> + <title>Startup Files, Log File, and Data Files</title> + <p>This section describes the internal files that are created + and maintained by the <c>Mnesia</c> system. In particular, + the workings of the <c>Mnesia</c> log are described.</p> <section> - <title>Start-Up Files</title> - </section> - <p>In Chapter 3 we detailed the following pre-requisites for - starting Mnesia (refer Chapter 3: <seealso marker="Mnesia_chap3#start_mnesia">Starting Mnesia</seealso>: - </p> + <title>Startup Files</title> + <p><seealso marker="Mnesia_chap3#start_mnesia">Start Mnesia</seealso> + states the following prerequisites + for starting <c>Mnesia</c>:</p> <list type="bulleted"> - <item>We must start an Erlang session and specify a Mnesia - directory for our database. + <item>An Erlang session must be started and a <c>Mnesia</c> + directory must be specified for the database. </item> - <item>We must initiate a database schema, using the function - <c>mnesia:create_schema/1</c>. + <item>A database schema must be initiated, using the function + <seealso marker="mnesia#create_schema/1">mnesia:create_schema/1</seealso>. </item> </list> - <p>The following example shows how these tasks are performed: - </p> - <list type="ordered"> - <item> - <pre> -% <input>erl -sname klacke -mnesia dir '"/ldisc/scratch/klacke"'</input> </pre> - </item> - <item> - <pre> + <p>The following example shows how these tasks are performed:</p> + <p><em>Step 1:</em> Start an Erlang session and specify a + <c>Mnesia</c> directory for the database:</p> + <pre> +% <input>erl -sname klacke -mnesia dir '"/ldisc/scratch/klacke"'</input></pre> + <pre> Erlang (BEAM) emulator version 4.9 Eshell V4.9 (abort with ^G) @@ -218,679 +223,700 @@ Eshell V4.9 (abort with ^G) ok (klacke@gin)2> <input>^Z</input> -Suspended </pre> - <p>We can inspect the Mnesia directory to see what files have been created. Enter the following command: - </p> - <pre> +Suspended</pre> + <p><em>Step 2:</em> You can inspect the <c>Mnesia</c> directory + to see what files have been created:</p> + <pre> % <input>ls -l /ldisc/scratch/klacke</input> --rw-rw-r-- 1 klacke staff 247 Aug 12 15:06 FALLBACK.BUP </pre> - <p>The response shows that the file FALLBACK.BUP has been created. This is called a backup file, and it contains an initial schema. If we had specified more than one node in the <c>mnesia:create_schema/1</c> function, identical backup files would have been created on all nodes. - </p> - </item> - <item> - <p>Continue by starting Mnesia:</p> - <pre> +-rw-rw-r-- 1 klacke staff 247 Aug 12 15:06 FALLBACK.BUP</pre> + <p>The response shows that the file <c>FALLBACK.BUP</c> has + been created. This is called a backup file, and it contains + an initial schema. If more than one node in the function + <seealso marker="mnesia#create_schema/1">mnesia:create_schema/1</seealso> + had been specified, identical + backup files would have been created on all nodes.</p> + <p><em>Step 3:</em> Start <c>Mnesia</c>:</p> + <pre> (klacke@gin)3><input>mnesia:start( ).</input> -ok </pre> - <p>We can now see the following listing in the Mnesia directory: - </p> - <pre> +ok</pre> + <p><em>Step 4:</em> You can see the following listing in + the <c>Mnesia</c> directory:</p> + <pre> -rw-rw-r-- 1 klacke staff 86 May 26 19:03 LATEST.LOG --rw-rw-r-- 1 klacke staff 34507 May 26 19:03 schema.DAT </pre> - <p>The schema in the backup file FALLBACK.BUP has been used to generate the file <c>schema.DAT.</c> Since we have no other disc resident tables than the schema, no other data files were created. The file FALLBACK.BUP was removed after the successful "restoration". We also see a number of files that are for internal use by Mnesia. - </p> - </item> - <item> - <p>Enter the following command to create a table:</p> - <pre> +-rw-rw-r-- 1 klacke staff 34507 May 26 19:03 schema.DAT</pre> + <p>The schema in the backup file <c>FALLBACK.BUP</c> has been + used to generate the file <c>schema.DAT</c>. Since there are + no other disc resident tables than the schema, no other data + files were created. The file <c>FALLBACK.BUP</c> was removed + after the successful "restoration". You also see some files + that are for internal use by <c>Mnesia</c>.</p> + <p><em>Step 5:</em> Create a table:</p> + <pre> (klacke@gin)4> <input>mnesia:create_table(foo,[{disc_copies, [node()]}]).</input> -{atomic,ok} </pre> - <p>We can now see the following listing in the Mnesia directory: - </p> - <pre> +{atomic,ok}</pre> + <p><em>Step 6:</em> You can see the following listing in + the <c>Mnesia</c> directory:</p> + <pre> % <input>ls -l /ldisc/scratch/klacke</input> -rw-rw-r-- 1 klacke staff 86 May 26 19:07 LATEST.LOG -rw-rw-r-- 1 klacke staff 94 May 26 19:07 foo.DCD --rw-rw-r-- 1 klacke staff 6679 May 26 19:07 schema.DAT </pre> - <p>Where a file <c>foo.DCD</c> has been created. This file will eventually store - all data that is written into the <c>foo</c> table.</p> - </item> - </list> +-rw-rw-r-- 1 klacke staff 6679 May 26 19:07 schema.DAT</pre> + <p>The file <c>foo.DCD</c> has been created. This file will + eventually store all data that is written into the + <c>foo</c> table.</p> + </section> <section> - <title>The Log File</title> - <p>When starting Mnesia, a .LOG file called <c>LATEST.LOG</c> - was created and placed in the database directory. This file is - used by Mnesia to log disc based transactions. This includes all - transactions that write at least one record in a table which is - of storage type <c>disc_copies</c>, or - <c>disc_only_copies</c>. It also includes all operations which - manipulate the schema itself, such as creating new tables. The - format of the log can vary with different implementations of - Mnesia. The Mnesia log is currently implemented with the - standard library module <c>disc_log</c>. - </p> - <p>The log file will grow continuously and must be dumped at - regular intervals. "Dumping the log file" means that Mnesia will - perform all the operations listed in the log and place the - records in the corresponding .DAT, .DCD and .DCL data files. For - example, if the operation "write record <c>{foo, 4, elvis, 6}</c>" - is listed in the log, Mnesia inserts the operation into the - file <c>foo.DCL</c>, later when Mnesia thinks the .DCL has become to large - the data is moved to the .DCD file. - The dumping operation can be time consuming - if the log is very large. However, it is important to realize - that the Mnesia system continues to operate during log dumps. - </p> - <p>By default Mnesia either dumps the log whenever 100 records have - been written in the log or when 3 minutes have passed. + <title>Log File</title> + <p>When starting <c>Mnesia</c>, a <c>.LOG</c> file called + <c>LATEST.LOG</c> is created + and placed in the database directory. This file is used by + <c>Mnesia</c> to log disc-based transactions. This includes all + transactions that write at least one record in a table that is + of storage type <c>disc_copies</c> or <c>disc_only_copies</c>. + The file also includes all operations that + manipulate the schema itself, such as creating new tables. + The log format can vary with different implementations of + <c>Mnesia</c>. The <c>Mnesia</c> log is currently implemented + in the standard library module + <seealso marker="kernel:disk_log">disk_log</seealso> in + <c>Kernel</c>.</p> + <p>The log file grows continuously and must be dumped at + regular intervals. "Dumping the log file" means that <c>Mnesia</c> + performs all the operations listed in the log and place the + records in the corresponding <c>.DAT</c>, <c>.DCD</c>, and + <c>.DCL</c> data files. For example, if the operation "write + record <c>{foo, 4, elvis, 6}</c>" is listed in the log, + <c>Mnesia</c> inserts the operation into the file + <c>foo.DCL</c>. Later, when <c>Mnesia</c> thinks that the + <c>.DCL</c> file is too large, the data is moved to the + <c>.DCD</c> file. The dumping operation can be time consuming + if the log is large. Notice that the <c>Mnesia</c> system + continues to operate during log dumps.</p> + <p>By default <c>Mnesia</c> either dumps the log whenever + 100 records have + been written in the log or when three minutes have passed. This is controlled by the two application parameters <c>-mnesia dump_log_write_threshold WriteOperations</c> and - <c>-mnesia dump_log_time_threshold MilliSecs</c>. - </p> + <c>-mnesia dump_log_time_threshold MilliSecs</c>.</p> <p>Before the log is dumped, the file <c>LATEST.LOG</c> is renamed to <c>PREVIOUS.LOG</c>, and a new <c>LATEST.LOG</c> file is created. Once the log has been successfully dumped, the file - <c>PREVIOUS.LOG</c> is deleted. - </p> - <p>The log is also dumped at start-up and whenever a schema - operation is performed. - </p> + <c>PREVIOUS.LOG</c> is deleted.</p> + <p>The log is also dumped at startup and whenever a schema + operation is performed.</p> </section> <section> - <title>The Data Files</title> - <p>The directory listing also contains one .DAT file. This contain - the schema itself, contained in the <c>schema.DAT</c> - file. The DAT files are indexed files, and it is efficient to - insert and search for records in these files with a specific - key. The .DAT files are used for the schema and for <c>disc_only_copies</c> - tables. The Mnesia data files are currently implemented with the - standard library module <c>dets</c>, and all operations which - can be performed on <c>dets</c> files can also be performed on - the Mnesia data files. For example, <c>dets</c> contains a - function <c>dets:traverse/2</c> which can be used to view the - contents of a Mnesia DAT file. However, this can only be done - when Mnesia is not running. So, to view a our schema file, we - can: </p> + <title>Data Files</title> + <p>The directory listing also contains one <c>.DAT</c> file, + which contains the schema itself, contained in the + <c>schema.DAT</c> file. The <c>DAT</c> files are indexed + files, and it is efficient to insert and search for records + in these files with a specific key. The <c>.DAT</c> files + are used for the schema and for <c>disc_only_copies</c> + tables. The <c>Mnesia</c> data files are currently implemented + in the standard library module + <seealso marker="stdlib:dets">dets</seealso> in + <c>STDLIB</c>.</p> + <p>All operations that can be performed on <c>dets</c> files + can also be performed on the <c>Mnesia</c> data files. For + example, <c>dets</c> contains the function + <c>dets:traverse/2</c>, which can be used to view the + contents of a <c>Mnesia</c> <c>DAT</c> file. However, this + can only be done when <c>Mnesia</c> is not running. So, to + view the schema file, do as follows;</p> <pre> {ok, N} = dets:open_file(schema, [{file, "./schema.DAT"},{repair,false}, {keypos, 2}]), F = fun(X) -> io:format("~p~n", [X]), continue end, dets:traverse(N, F), -dets:close(N). </pre> - <note> - <p>Refer to the Reference Manual, <c>std_lib</c> for information about <c>dets</c>.</p> - </note> +dets:close(N).</pre> <warning> - <p>The DAT files must always be opened with the <c>{repair, false}</c> - option. This ensures that these files are not - automatically repaired. Without this option, the database may - become inconsistent, because Mnesia may - believe that the files were properly closed. Refer to the reference - manual for information about the configuration parameter - <c>auto_repair</c>.</p> + <p>The <c>DAT</c> files must always be opened with option + <c>{repair, false}</c>. This ensures that these files are not + automatically repaired. Without this option, the database can + become inconsistent, because <c>Mnesia</c> can believe that + the files were properly closed. For information about + configuration parameter <c>auto_repair</c>, see the + Reference Manual.</p> </warning> <warning> - <p>It is recommended that Data files are not tampered with while Mnesia is - running. While not prohibited, the behavior of Mnesia is unpredictable. </p> + <p>It is recommended that the data files are not tampered + with while <c>Mnesia</c> is running. While not prohibited, + the behavior of <c>Mnesia</c> is unpredictable.</p> </warning> - <p>The <c>disc_copies</c> tables are stored on disk with .DCL and .DCD files, - which are standard disk_log files. - </p> + <p>The <c>disc_copies</c> tables are stored on disk with + <c>.DCL</c> and <c>.DCD</c> files, which are standard + <c>disk_log</c> files.</p> </section> </section> <section> - <title>Loading of Tables at Start-up</title> - <p>At start-up Mnesia loads tables in order to make them accessible - for its applications. Sometimes Mnesia decides to load all tables - that reside locally, and sometimes the tables may not be - accessible until Mnesia brings a copy of the table - from another node. - </p> - <p>To understand the behavior of Mnesia at start-up it is - essential to understand how Mnesia reacts when it loses contact - with Mnesia on another node. At this stage, Mnesia cannot distinguish - between a communication failure and a "normal" node down. <br></br> - - When this happens, Mnesia will assume that the other node is no longer running. - Whereas, in reality, the communication between the nodes has merely failed. - </p> - <p>To overcome this situation, simply try to restart the ongoing transactions that are - accessing tables on the failing node, and write a <c>mnesia_down</c> entry to a log file. - </p> - <p>At start-up, it must be noted that all tables residing on nodes - without a <c>mnesia_down</c> entry, may have fresher replicas. - Their replicas may have been updated after the termination - of Mnesia on the current node. In order to catch up with the latest + <title>Loading Tables at Startup</title> + <p>At startup, <c>Mnesia</c> loads tables to make them accessible + for its applications. Sometimes <c>Mnesia</c> decides to load + all tables that reside locally, and sometimes the tables are + not accessible until <c>Mnesia</c> brings a copy of the table + from another node.</p> + <p>To understand the behavior of <c>Mnesia</c> at startup, it is + essential to understand how <c>Mnesia</c> reacts when it loses + contact with <c>Mnesia</c> on another node. At this stage, + <c>Mnesia</c> cannot distinguish between a communication + failure and a "normal" node-down. When this occurs, + <c>Mnesia</c> assumes that the other node is no longer running, + whereas, in reality, the communication between the nodes has + failed.</p> + <p>To overcome this situation, try to restart the ongoing + transactions that are accessing tables on the failing node, + and write a <c>mnesia_down</c> entry to a log file.</p> + <p>At startup, notice that all tables residing on nodes + without a <c>mnesia_down</c> entry can have fresher replicas. + Their replicas can have been updated after the termination of + <c>Mnesia</c> on the current node. To catch up with the latest updates, transfer a copy of the table from one of these other - "fresh" nodes. If you are unlucky, other nodes may be down - and you must wait for the table to be - loaded on one of these nodes before receiving a fresh copy of - the table. - </p> + "fresh" nodes. If you are unlucky, other nodes can be down + and you must wait for the table to be loaded on one of these + nodes before receiving a fresh copy of the table.</p> <p>Before an application makes its first access to a table, - <c>mnesia:wait_for_tables(TabList, Timeout)</c> ought to be executed + <seealso marker="mnesia#wait_for_tables/2">mnesia:wait_for_tables(TabList, Timeout)</seealso> + is to be executed to ensure that the table is accessible from the local node. If - the function times out the application may choose to force a + the function times out, the application can choose to force a load of the local replica with - <c>mnesia:force_load_table(Tab)</c> and deliberately lose all - updates that may have been performed on the other nodes while - the local node was down. If - Mnesia already has loaded the table on another node or intends - to do so, we will copy the table from that node in order to - avoid unnecessary inconsistency. - </p> + <seealso marker="mnesia#force_load_table/1">mnesia:force_load_table(Tab)</seealso> + and deliberately lose all + updates that can have been performed on the other nodes while + the local node was down. If <c>Mnesia</c> + has loaded the table on another node already, or intends + to do so, copy the table from that node to + avoid unnecessary inconsistency.</p> <warning> - <p>Keep in mind that it is only - one table that is loaded by <c>mnesia:force_load_table(Tab)</c> - and since committed transactions may have caused updates in - several tables, the tables may now become inconsistent due to - the forced load.</p> + <p>Only one table is loaded by + <seealso marker="mnesia#force_load_table/1">mnesia:force_load_table(Tab)</seealso>. + Since committed + transactions can have caused updates in several tables, the + tables can become inconsistent because of the forced load.</p> </warning> - <p>The allowed <c>AccessMode</c> of a table may be defined to - either be <c>read_only</c> or <c>read_write</c>. And it may be - toggled with the function <c>mnesia:change_table_access_mode(Tab, AccessMode)</c> in runtime. <c>read_only</c> tables and - <c>local_content</c> tables will always be loaded locally, since - there are no need for copying the table from other nodes. Other - tables will primary be loaded remotely from active replicas on - other nodes if the table already has been loaded there, or if - the running Mnesia already has decided to load the table there. - </p> - <p>At start up, Mnesia will assume that its local replica is the - most recent version and load the table from disc if either - situation is detected: - </p> + <p>The allowed <c>AccessMode</c> of a table can be defined to be + <c>read_only</c> or <c>read_write</c>. It can be toggled with + the function + <seealso marker="mnesia#change_table_access_mode/2"> + mnesia:change_table_access_mode(Tab, AccessMode)</seealso> + in runtime. <c>read_only</c> tables and + <c>local_content</c> tables are always loaded locally, as + there is no need for copying the table from other nodes. Other + tables are primarily loaded remotely from active replicas on + other nodes if the table has been loaded there already, or if + the running <c>Mnesia</c> has decided to load the table there + already.</p> + <p>At startup, <c>Mnesia</c> assumes that its local replica is the + most recent version and loads the table from disc if either of + the following situations is detected:</p> <list type="bulleted"> - <item><c>mnesia_down</c> is returned from all other nodes that holds a disc - resident replica of the table; or,</item> - <item>if all replicas are <c>ram_copies</c></item> + <item><c>mnesia_down</c> is returned from all other nodes that + hold a disc resident replica of the table.</item> + <item>All replicas are <c>ram_copies</c>.</item> </list> - <p>This is normally a wise decision, but it may turn out to - be disastrous if the nodes have been disconnected due to a - communication failure, since Mnesia's normal table load - mechanism does not cope with communication failures. - </p> - <p>When Mnesia is loading many tables the default load - order. However, it is possible to - affect the load order by explicitly changing the - <c>load_order</c> property for the tables, with the function - <c>mnesia:change_table_load_order(Tab, LoadOrder)</c>. The - <c>LoadOrder</c> is by default <c>0</c> for all tables, but it - can be set to any integer. The table with the highest - <c>load_order</c> will be loaded first. Changing load order is + <p>This is normally a wise decision, but it can be disastrous + if the nodes have been disconnected because of a communication + failure, as the <c>Mnesia</c> normal table load + mechanism does not cope with communication failures.</p> + <p>When <c>Mnesia</c> loads many tables, the default load order + is used. However, the load order + can be affected, by explicitly changing property + <c>load_order</c> for the tables, with the function + <seealso marker="mnesia#change_table_load_order/2"> + mnesia:change_table_load_order(Tab, LoadOrder)</seealso>. + <c>LoadOrder</c> is by default <c>0</c> for all tables, but + it can be set to any integer. The table with the highest + <c>load_order</c> is loaded first. Changing the load order is especially useful for applications that need to ensure early - availability of fundamental tables. Large peripheral - tables should have a low load order value, perhaps set - below 0. - </p> + availability of fundamental tables. Large peripheral tables + are to have a low load order value, perhaps less than <c>0</c></p> </section> <section> <title>Recovery from Communication Failure</title> - <p>There are several occasions when Mnesia may detect that the - network has been partitioned due to a communication failure. - </p> - <p>One is when Mnesia already is up and running and the Erlang - nodes gain contact again. Then Mnesia will try to contact Mnesia - on the other node to see if it also thinks that the network has - been partitioned for a while. If Mnesia on both nodes has logged - <c>mnesia_down</c> entries from each other, Mnesia generates a - system event, called <c>{inconsistent_database, running_partitioned_network, Node}</c> which is sent to Mnesia's - event handler and other possible subscribers. The default event - handler reports an error to the error logger. - </p> - <p>Another occasion when Mnesia may detect that the network has - been partitioned due to a communication failure, is at start-up. - If Mnesia detects that both the local node and another node received - <c>mnesia_down</c> from each other it generates a - <c>{inconsistent_database, starting_partitioned_network, Node}</c> system event and acts as described above. - </p> + <p>There are several occasions when <c>Mnesia</c> can detect + that the network has been partitioned because of a + communication failure, for example:</p> + <list type="bulleted"> + <item><c>Mnesia</c> is operational already and the Erlang nodes + gain contact again. Then <c>Mnesia</c> tries to contact + <c>Mnesia</c> on the other node to see if it also thinks that + the network has been partitioned for a while. If <c>Mnesia</c> + on both nodes has logged <c>mnesia_down</c> entries from each + other, <c>Mnesia</c> generates a system event, called + <c>{inconsistent_database, running_partitioned_network, Node}</c>, + which is sent to the <c>Mnesia</c> event handler and other + possible subscribers. The default event + handler reports an error to the error logger. + </item> + <item>If <c>Mnesia</c> detects at startup that both the local + node and another node received <c>mnesia_down</c> from each + other, <c>Mnesia</c> generates an + <c>{inconsistent_database, starting_partitioned_network, Node}</c> + system event and acts as described in the previous item. + </item> + </list> <p>If the application detects that there has been a communication - failure which may have caused an inconsistent database, it may - use the function <c>mnesia:set_master_nodes(Tab, Nodes)</c> to - pinpoint from which nodes each table may be loaded.</p> - <p>At start-up Mnesia's normal table load algorithm will be - bypassed and the table will be loaded from one of the master + failure that can have caused an inconsistent database, it can + use the function + <seealso marker="mnesia#set_master_nodes/2">mnesia:set_master_nodes(Tab, Nodes)</seealso> + to pinpoint from which nodes each table can be loaded.</p> + <p>At startup, the <c>Mnesia</c> normal table load algorithm is + bypassed and the table is loaded from one of the master nodes defined for the table, regardless of potential - <c>mnesia_down</c> entries in the log. The <c>Nodes</c> may only - contain nodes where the table has a replica and if it is empty, - the master node recovery mechanism for the particular table will - be reset and the normal load mechanism will be used when next - restarting. - </p> - <p>The function <c>mnesia:set_master_nodes(Nodes)</c> sets master - nodes for all tables. For each table it will determine its - replica nodes and invoke <c>mnesia:set_master_nodes(Tab, TabNodes)</c> with those replica nodes that are included in the - <c>Nodes</c> list (i.e. <c>TabNodes</c> is the intersection of + <c>mnesia_down</c> entries in the log. <c>Nodes</c> can only + contain nodes where the table has a replica. If <c>Nodes</c> + is empty, the master node recovery mechanism for the particular + table is reset and the normal load mechanism is used at the + next restart.</p> + <p>The function + <seealso marker="mnesia#set_master_nodes/1">mnesia:set_master_nodes(Nodes)</seealso> + sets master + nodes for all tables. For each table it determines its replica + nodes and starts + <seealso marker="mnesia#set_master_nodes/2">mnesia:set_master_nodes(Tab, TabNodes)</seealso> + with those replica nodes that are included in the <c>Nodes</c> + list (that is, <c>TabNodes</c> is the intersection of <c>Nodes</c> and the replica nodes of the table). If the - intersection is empty the master node recovery mechanism for the - particular table will be reset and the normal load mechanism - will be used at next restart. - </p> - <p>The functions <c>mnesia:system_info(master_node_tables)</c> and - <c>mnesia:table_info(Tab, master_nodes)</c> may be used to - obtain information about the potential master nodes. - </p> - <p>Determining which data to keep after communication failure is outside - the scope of Mnesia. One approach would be to determine which "island" - contains a majority of the nodes. Using the <c>{majority,true}</c> option - for critical tables can be a way of ensuring that nodes that are not part - of a "majority island" are not able to update those tables. Note that this - constitutes a reduction in service on the minority nodes. This would be - a tradeoff in favour of higher consistency guarantees.</p> - <p>The function <c>mnesia:force_load_table(Tab)</c> may be used to - force load the table regardless of which table load mechanism - is activated. - </p> + intersection is empty, the master node recovery mechanism for + the particular table is reset and the normal load mechanism + is used at the next restart.</p> + <p>The functions + <seealso marker="mnesia#system_info/1">mnesia:system_info(master_node_tables)</seealso> + and + <seealso marker="mnesia#table_info/2">mnesia:table_info(Tab, master_nodes)</seealso> + can be used to + obtain information about the potential master nodes.</p> + <p>Determining what data to keep after a communication failure + is outside the scope of <c>Mnesia</c>. One approach is to + determine which "island" contains most of the nodes. Using + option <c>{majority,true}</c> for critical tables can be a way + to ensure that nodes that are not part of a "majority island" + cannot update those tables. Notice that this constitutes a + reduction in service on the minority nodes. This would be a + tradeoff in favor of higher consistency guarantees.</p> + <p>The function + <seealso marker="mnesia#force_load_table/1">mnesia:force_load_table(Tab)</seealso> + can be used to force load the table regardless of which table + load mechanism that is activated.</p> </section> <section> <title>Recovery of Transactions</title> - <p>A Mnesia table may reside on one or more nodes. When a table is - updated, Mnesia will ensure that the updates will be replicated - to all nodes where the table resides. If a replica happens to be - inaccessible for some reason (e.g. due to a temporary node down), - Mnesia will then perform the replication later. - </p> - <p>On the node where the application is started, there will be a + <p>A <c>Mnesia</c> table can reside on one or more nodes. When a + table is updated, <c>Mnesia</c> ensures that the updates are + replicated to all nodes where the table resides. If a replica is + inaccessible (for example, because of a temporary node-down), + <c>Mnesia</c> performs the replication later.</p> + <p>On the node where the application is started, there is a transaction coordinator process. If the transaction is - distributed, there will also be a transaction participant process on - all the other nodes where commit work needs to be performed. - </p> - <p>Internally Mnesia uses several commit protocols. The selected - protocol depends on which table that has been updated in - the transaction. If all the involved tables are symmetrically - replicated, (i.e. they all have the same <c>ram_nodes</c>, - <c>disc_nodes</c> and <c>disc_only_nodes</c> currently + distributed, there is also a transaction participant process on + all the other nodes where commit-work needs to be performed.</p> + <p>Internally <c>Mnesia</c> uses several commit protocols. The + selected protocol depends on which table that has been updated + in the transaction. If all the involved tables are symmetrically + replicated (that is, they all have the same <c>ram_nodes</c>, + <c>disc_nodes</c>, and <c>disc_only_nodes</c> currently accessible from the coordinator node), a lightweight transaction - commit protocol is used. - </p> + commit protocol is used.</p> <p>The number of messages that the - transaction coordinator and its participants needs to exchange - is few, since Mnesia's table load mechanism takes care of the - transaction recovery if the commit protocol gets + transaction coordinator and its participants need to exchange + is few, as the <c>Mnesia</c> table load mechanism takes care of + the transaction recovery if the commit protocol gets interrupted. Since all involved tables are replicated - symmetrically the transaction will automatically be recovered by - loading the involved tables from the same node at start-up of a - failing node. We do not really care if the transaction was - aborted or committed as long as we can ensure the ACID - properties. The lightweight commit protocol is non-blocking, - i.e. the surviving participants and their coordinator will - finish the transaction, regardless of some node crashes in the - middle of the commit protocol or not. - </p> - <p>If a node goes down in the middle of a dirty operation the - table load mechanism will ensure that the update will be - performed on all replicas or none. Both asynchronous dirty + symmetrically, the transaction is automatically recovered by + loading the involved tables from the same node at startup of a + failing node. It does not matter if the transaction was + committed or terminated as long as the ACID properties can be + ensured. The lightweight commit protocol is non-blocking, + that is, the surviving participants and their coordinator + finish the transaction, even if any node crashes in the + middle of the commit protocol.</p> + <p>If a node goes down in the middle of a dirty operation, the + table load mechanism ensures that the update is + performed on all replicas, or none. Both asynchronous dirty updates and synchronous dirty updates use the same recovery - principle as lightweight transactions. - </p> + principle as lightweight transactions.</p> <p>If a transaction involves updates of asymmetrically replicated tables or updates of the schema table, a heavyweight commit - protocol will be used. The heavyweight commit protocol is able - to finish the transaction regardless of how the tables are - replicated. The typical usage of a heavyweight transaction is - when we want to move a replica from one node to another. Then we - must ensure that the replica either is entirely moved or left as - it was. We must never end up in a situation with replicas on both - nodes or no node at all. Even if a node crashes in the middle of - the commit protocol, the transaction must be guaranteed to be + protocol is used. This protocol can + finish the transaction regardless of how the tables are + replicated. The typical use of a heavyweight transaction is + when a replica is to be moved from one node to another. Then + ensure that the replica either is entirely moved or left as + it was. Do never end up in a situation with replicas on both + nodes, or on no node at all. Even if a node crashes in the middle + of the commit protocol, the transaction must be guaranteed to be atomic. The heavyweight commit protocol involves more messages between the transaction coordinator and its participants than - a lightweight protocol and it will perform recovery work at - start-up in order to finish the abort or commit work. - </p> + a lightweight protocol, and it performs recovery work at + startup to finish the terminating or commit work.</p> <p>The heavyweight commit protocol is also non-blocking, which allows the surviving participants and their coordinator to finish the transaction regardless (even if a node crashes in the - middle of the commit protocol). When a node fails at start-up, - Mnesia will determine the outcome of the transaction and - recover it. Lightweight protocols, heavyweight protocols and dirty updates, are - dependent on other nodes to be up and running in order to make the - correct heavyweight transaction recovery decision. - </p> - <p>If Mnesia has not started on some of the nodes that are involved in the - transaction AND neither the local node or any of the already - running nodes know the outcome of the transaction, Mnesia will - by default wait for one. In the worst case scenario all other - involved nodes must start before Mnesia can make the correct decision - about the transaction and finish its start-up. - </p> - <p>This means that Mnesia (on one node)may hang if a double fault occurs, i.e. when two nodes crash simultaneously - and one attempts to start when the other refuses to - start e.g. due to a hardware error. - </p> - <p>It is possible to specify the maximum time that Mnesia - will wait for other nodes to respond with a transaction - recovery decision. The configuration parameter - <c>max_wait_for_decision</c> defaults to infinity (which may - cause the indefinite hanging as mentioned above) but if it is - set to a definite time period (eg.three minutes), Mnesia will then enforce a - transaction recovery decision if needed, in order to allow - Mnesia to continue with its start-up procedure. </p> - <p>The downside of an enforced transaction recovery decision, is that the decision may be - incorrect, due to insufficient information regarding the other nodes' - recovery decisions. This may result in an - inconsistent database where Mnesia has committed the transaction - on some nodes but aborted it on others. </p> - <p>In fortunate cases the inconsistency will only appear in tables belonging to a specific - application, but if a schema transaction has been inconsistently - recovered due to the enforced transaction recovery decision, the - effects of the inconsistency can be fatal. + middle of the commit protocol). When a node fails at startup, + <c>Mnesia</c> determines the outcome of the transaction and + recovers it. Lightweight protocols, heavyweight protocols, and + dirty updates, are dependent on other nodes to be operational + to make the correct heavyweight transaction recovery decision.</p> + <p>If <c>Mnesia</c> has not started on some of the nodes that + are involved in the transaction <em>and</em> neither the + local node nor any of the already running nodes know the + outcome of the transaction, <c>Mnesia</c> waits for one, + by default. In the worst case scenario, all other involved + nodes must start before <c>Mnesia</c> can make the correct + decision about the transaction and finish its startup.</p> + <p>Thus, <c>Mnesia</c> (on one node) can hang if a double fault + occurs, that is, when two nodes crash simultaneously + and one attempts to start when the other refuses to + start, for example, because of a hardware error.</p> + <p>The maximum time that <c>Mnesia</c> waits for other nodes to + respond with a transaction recovery decision can be specified. + The configuration parameter <c>max_wait_for_decision</c> + defaults to <c>infinity</c>, which can cause the indefinite + hanging as mentioned earlier. However, if the parameter is + set to a definite time period (for example, three minutes), + <c>Mnesia</c> then enforces a transaction recovery decision, + if needed, to allow <c>Mnesia</c> to continue with its startup + procedure.</p> + <p>The downside of an enforced transaction recovery decision is + that the decision can be incorrect, because of insufficient + information about the recovery decisions from the other nodes. + This can result in an inconsistent database where <c>Mnesia</c> + has committed the transaction on some nodes but terminated it + on others.</p> + <p>In fortunate cases, the inconsistency is only visible in + tables belonging to a specific application. However, if a + schema transaction is inconsistently recovered because of + the enforced transaction recovery decision, the + effects of the inconsistency can be fatal. However, if the higher priority is availability rather than - consistency, then it may be worth the risk. </p> - <p>If Mnesia - encounters a inconsistent transaction decision a - <c>{inconsistent_database, bad_decision, Node}</c> system event - will be generated in order to give the application a chance to - install a fallback or other appropriate measures to resolve the inconsistency. The default - behavior of the Mnesia event handler is the same as if the - database became inconsistent as a result of partitioned network (see - above). - </p> + consistency, it can be worth the risk.</p> + <p>If <c>Mnesia</c> detects an inconsistent transaction decision, + an <c>{inconsistent_database, bad_decision, Node}</c> system event + is generated to give the application a chance to install a + fallback or other appropriate measures to resolve the + inconsistency. The default behavior of the <c>Mnesia</c> + event handler is the same as if the database became + inconsistent as a result of partitioned network (as + described earlier).</p> </section> <section> - <title>Backup, Fallback, and Disaster Recovery</title> - <p>The following functions are used to backup data, to install a - backup as fallback, and for disaster recovery. - </p> + <title>Backup, Restore, Fallback, and Disaster Recovery</title> + <p>The following functions are used to back up data, to install + a backup as fallback, and for disaster recovery:</p> <list type="bulleted"> - <item><c>mnesia:backup_checkpoint(Name, Opaque, [Mod])</c>. This - function performs a backup of the tables included in the - checkpoint. + <item> + <seealso marker="mnesia#backup_checkpoint/2">mnesia:backup_checkpoint(Name, Opaque, [Mod])</seealso> + performs a backup of the tables included in the checkpoint. </item> - <item><c>mnesia:backup(Opaque, [Mod])</c>. This function - activates a new checkpoint which covers all Mnesia tables and + <item> + <seealso marker="mnesia#backup/1">mnesia:backup(Opaque, [Mod])</seealso> + activates a new + checkpoint that covers all <c>Mnesia</c> tables and performs a backup. It is performed with maximum degree of - redundancy (also refer to the function <seealso marker="#checkpoints">mnesia:activate_checkpoint(Args)</seealso>, - <c>{max, MaxTabs} and {min, MinTabs}).</c></item> - <item><c>mnesia:traverse_backup(Source,[SourceMod,]</c><c>Target,[TargetMod,]Fun,Ac)</c>. This function can be used - to read an existing backup, create a new backup from an - existing one, or to copy a backup from one type media to - another. + redundancy (see also the function + <seealso marker="#checkpoints">mnesia:activate_checkpoint(Args)</seealso>, + <c>{max, MaxTabs} and {min, MinTabs})</c>. </item> - <item><c>mnesia:uninstall_fallback()</c>. This function removes - previously installed fallback files. + <item> + <seealso marker="mnesia#traverse_backup/4">mnesia:traverse_backup(Source, [SourceMod,] Target, [TargetMod,] Fun, Acc)</seealso> + can be used to read an existing backup, create a backup from an + existing one, or to copy a backup from one type media to another. + </item> + <item> + <seealso marker="mnesia#uninstall_fallback/0">mnesia:uninstall_fallback()</seealso> + removes previously installed fallback files. </item> - <item><c>mnesia:restore(Opaque, Args)</c>. This function + <item> + <seealso marker="mnesia#restore/2">mnesia:restore(Opaque, Args)</seealso> restores a set of tables from a previous backup. </item> - <item><c>mnesia:install_fallback(Opaque, [Mod])</c>. This - function can be configured to restart the Mnesia and reload data - tables, and possibly schema tables, from an existing + <item> + <seealso marker="mnesia#install_fallback/1">mnesia:install_fallback(Opaque, [Mod])</seealso> + can be configured to restart <c>Mnesia</c> and the reload data + tables, and possibly the schema tables, from an existing backup. This function is typically used for disaster recovery - purposes, when data or schema tables are corrupted.</item> + purposes, when data or schema tables are corrupted. + </item> </list> - <p>These functions are explained in the following - sub-sections. Also refer to the the section <seealso marker="#checkpoints">Checkpoints</seealso> in this chapter, which - describes the two functions used to activate and de-activate - checkpoints. - </p> + <p>These functions are explained in the following sections. + See also <seealso marker="#checkpoints">Checkpoints</seealso>, + which describes the two functions used + to activate and deactivate checkpoints.</p> <section> <title>Backup</title> - <p>Backup operation are performed with the following functions: - </p> + <p>Backup operation are performed with the following functions:</p> <list type="bulleted"> - <item><c>mnesia:backup_checkpoint(Name, Opaque, [Mod])</c></item> - <item><c>mnesia:backup(Opaque, [Mod])</c></item> - <item><c>mnesia:traverse_backup(Source, [SourceMod,],</c><c>Target,[TargetMod,]Fun,Acc)</c>.</item> + <item> + <seealso marker="mnesia#backup_checkpoint/2">mnesia:backup_checkpoint(Name, Opaque, [Mod])</seealso> + </item> + <item> + <seealso marker="mnesia#backup/1">mnesia:backup(Opaque, [Mod])</seealso> + </item> + <item> + <seealso marker="mnesia#traverse_backup/4">mnesia:traverse_backup(Source, [SourceMod,] Target, [TargetMod,] Fun, Acc)</seealso> + </item> </list> <p>By default, the actual access to the backup media is - performed via the <c>mnesia_backup</c> module for both read + performed through module <c>mnesia_backup</c> for both read and write. Currently <c>mnesia_backup</c> is implemented with - the standard library module <c>disc_log</c>, but it is possible to write - your own module with the same interface as - <c>mnesia_backup</c> and configure Mnesia so the alternate - module performs the actual accesses to the backup media. This - means that the user may put the backup on medias that Mnesia + the standard library module <c>disc_log</c>. However, you + can write your own module with the same interface as + <c>mnesia_backup</c> and configure <c>Mnesia</c> so that + the alternative module performs the actual accesses to + the backup media. The user can + therefore put the backup on a media that <c>Mnesia</c> does not know about, possibly on hosts where Erlang is not - running. Use the configuration parameter <c><![CDATA[-mnesia backup_module <module>]]></c> for this purpose. </p> - <p>The source - for a backup is an activated checkpoint. The backup function - most commonly used is <c>mnesia:backup_checkpoint(Name, Opaque,[Mod])</c>. This function returns either <c>ok</c>, or - <c>{error,Reason}</c>. It has the following arguments: - </p> + running. Use configuration parameter + <c><![CDATA[-mnesia backup_module <module>]]></c> + for this purpose.</p> + <p>The source for a backup is an activated checkpoint. + The backup function + <seealso marker="mnesia#backup_checkpoint/2">mnesia:backup_checkpoint(Name, Opaque,[Mod])</seealso> + is most commonly used and returns <c>ok</c> or + <c>{error,Reason}</c>. It has the following arguments:</p> <list type="bulleted"> - <item><c>Name</c> is the name of an activated - checkpoint. Refer to the section <seealso marker="#checkpoints">Checkpoints</seealso> in this chapter, the - function <c>mnesia:activate_checkpoint(ArgList)</c> for - details on how to include table names in checkpoints. + <item><c>Name</c> is the name of an activated checkpoint. + For details on how to include table names in checkpoints, + see the function <c>mnesia:activate_checkpoint(ArgList)</c> + in <seealso marker="#checkpoints">Checkpoints</seealso>. </item> - <item><c>Opaque</c>. Mnesia does not interpret this argument, - but it is forwarded to the backup module. The Mnesia default - backup module, <c>mnesia_backup</c> interprets this argument - as a local file name. + <item><c>Opaque</c>. <c>Mnesia</c> does not interpret this + argument, but it is forwarded to the backup module. The + <c>Mnesia</c> default backup module <c>mnesia_backup</c> + interprets this argument as a local filename. </item> - <item><c>Mod</c>. The name of an alternate backup module. + <item><c>Mod</c> is the name of an alternative backup module. </item> </list> - <p>The function <c>mnesia:backup(Opaque[, Mod])</c> activates a - new checkpoint which covers all Mnesia tables with maximum - degree of redundancy and performs a backup. Maximum + <p>The function + <seealso marker="mnesia#backup/1">mnesia:backup(Opaque [,Mod])</seealso> + activates a + new checkpoint that covers all <c>Mnesia</c> tables with + maximum degree of redundancy and performs a backup. Maximum redundancy means that each table replica has a checkpoint - retainer. Tables with the <c>local_contents</c> property are - backed up as they - look on the current node. - </p> - <p>It is possible to iterate over a backup, either for the - purpose of transforming it into a new backup, or just reading - it. The function <c>mnesia:traverse_backup(Source, [SourceMod,]</c><c>Target, [TargeMod,] Fun, Acc)</c> which normally returns <c>{ok, LastAcc}</c>, is used for both of these purposes. - </p> + retainer. Tables with property <c>local_contents</c> are + backed up as they look on the current node.</p> + <p>You can iterate over a backup, either to transform it + into a new backup, or only read it. The function + <seealso marker="mnesia#traverse_backup/4">mnesia:traverse_backup(Source, [SourceMod,] Target, [TargetMod,] Fun, Acc)</seealso>, + which normally returns <c>{ok, LastAcc}</c>, + is used for both of these purposes.</p> <p>Before the traversal starts, the source backup media is opened with <c>SourceMod:open_read(Source)</c>, and the target backup media is opened with - <c>TargetMod:open_write(Target)</c>. The arguments are: - </p> + <c>TargetMod:open_write(Target)</c>. The arguments are as + follows:</p> <list type="bulleted"> <item><c>SourceMod</c> and <c>TargetMod</c> are module names. </item> <item><c>Source</c> and <c>Target</c> are opaque data used exclusively by the modules <c>SourceMod</c> and - <c>TargetMod</c> for the purpose of initializing the backup - medias. + <c>TargetMod</c> for initializing the backup medias. </item> <item><c>Acc</c> is an initial accumulator value. </item> <item><c>Fun(BackupItems, Acc)</c> is applied to each item in - the backup. The Fun must return a tuple <c>{ValGoodBackupItems, NewAcc}</c>, where <c>ValidBackupItems</c> is a list of valid - backup items, and <c>NewAcc</c> is a new accumulator value. + the backup. The Fun must return a tuple + <c>{ValGoodBackupItems, NewAcc}</c>, + where <c>ValidBackupItems</c> is a list of valid + backup items. <c>NewAcc</c> is a new accumulator value. The <c>ValidBackupItems</c> are written to the target backup with the function <c>TargetMod:write/2</c>. </item> - <item><c>LastAcc</c> is the last accumulator value. I.e. + <item><c>LastAcc</c> is the last accumulator value, that is, the last <c>NewAcc</c> value that was returned by <c>Fun</c>. </item> </list> - <p>It is also possible to perform a read-only traversal of the - source backup without updating a target backup. If - <c>TargetMod==read_only</c>, then no target backup is accessed - at all. - </p> + <p>Also, a read-only traversal of the source backup can be + performed without updating a target backup. If + <c>TargetMod==read_only</c>, no target backup is accessed.</p> <p>By setting <c>SourceMod</c> and <c>TargetMod</c> to different - modules it is possible to copy a backup from one kind of backup - media to another. - </p> - <p>Valid <c>BackupItems</c> are the following tuples: - </p> + modules, a backup can be copied from one backup + media to another.</p> + <p>Valid <c>BackupItems</c> are the following tuples:</p> <list type="bulleted"> <item><c>{schema, Tab}</c> specifies a table to be deleted. </item> <item><c>{schema, Tab, CreateList}</c> specifies a table to be - created. See <c>mnesia_create_table/2</c> for more - information about <c>CreateList</c>. + created. For more information about <c>CreateList</c>, see + <seealso marker="mnesia#create_table/2">mnesia:create_table/2</seealso>. </item> <item><c>{Tab, Key}</c> specifies the full identity of a record - to be deleted. + to be deleted. </item> <item><c>{Record}</c> specifies a record to be inserted. It - can be a tuple with <c>Tab</c> as first field. Note that the + can be a tuple with <c>Tab</c> as first field. Notice that the record name is set to the table name regardless of what - <c>record_name</c> is set to. + <c>record_name</c> is set to. </item> </list> <p>The backup data is divided into two sections. The first - section contains information related to the schema. All schema - related items are tuples where the first field equals the atom - schema. The second section is the record section. It is not - possible to mix schema records with other records and all schema - records must be located first in the backup. - </p> - <p>The schema itself is a table and will possibly be included in - the backup. All nodes where the schema table resides are - regarded as a <c>db_node</c>. - </p> - <p>The following example illustrates how - <c>mnesia:traverse_backup</c> can be used to rename a db_node in - a backup file: - </p> + section contains information related to the schema. All + schema-related items are tuples where the first field equals + the atom schema. The second section is the record section. + Schema records cannot be mixed with other records and all + schema records must be located first in the backup.</p> + <p>The schema itself is a table and is possibly included in + the backup. Each node where the schema table resides is + regarded as a <c>db_node</c>.</p> + <p>The following example shows how + <seealso marker="mnesia#traverse_backup/4">mnesia:traverse_backup</seealso> + can be used to rename a <c>db_node</c> in a backup file:</p> <codeinclude file="bup.erl" tag="%0" type="erl"></codeinclude> </section> <section> <title>Restore</title> - <p>Tables can be restored on-line from a backup without - restarting Mnesia. A restore is performed with the function - <c>mnesia:restore(Opaque,Args)</c>, where <c>Args</c> can - contain the following tuples: - </p> + <p>Tables can be restored online from a backup without + restarting <c>Mnesia</c>. A restore is performed with the + function + <seealso marker="mnesia#restore/2">mnesia:restore(Opaque, Args)</seealso>, + where <c>Args</c> can contain the following tuples:</p> <list type="bulleted"> <item><c>{module,Mod}</c>. The backup module <c>Mod</c> is used to access the backup media. If omitted, the default - backup module will be used.</item> - <item><c>{skip_tables, TableList}</c> Where <c>TableList</c> - is a list of tables which should not be read from the backup.</item> - <item><c>{clear_tables, TableList}</c> Where <c>TableList</c> - is a list of tables which should be cleared, before the - records from the backup are inserted, i.e. all records in + backup module is used. + </item> + <item><c>{skip_tables, TableList}</c>, where <c>TableList</c> + is a list of tables, which is not to be read from the backup. + </item> + <item><c>{clear_tables, TableList}</c>, where <c>TableList</c> + is a list of tables, which is to be cleared before the + records from the backup are inserted. That is, all records in the tables are deleted before the tables are restored. Schema information about the tables is not cleared or read - from backup.</item> - <item><c>{keep_tables, TableList}</c> Where <c>TableList</c> - is a list of tables which should be not be cleared, before - the records from the backup are inserted, i.e. the records - in the backup will be added to the records in the table. + from the backup. + </item> + <item><c>{keep_tables, TableList}</c>, where <c>TableList</c> + is a list of tables, which is not to be cleared before + the records from the backup are inserted. That is, the records + in the backup are added to the records in the table. Schema information about the tables is not cleared or read - from backup.</item> - <item><c>{recreate_tables, TableList}</c> Where <c>TableList</c> - is a list of tables which should be re-created, before the - records from the backup are inserted. The tables are first - deleted and then created with the schema information from the - backup. All the nodes in the backup needs to be up and running.</item> - <item><c>{default_op, Operation}</c> Where <c>Operation</c> is - one of the following operations <c>skip_tables</c>, - <c>clear_tables</c>, <c>keep_tables</c> or - <c>recreate_tables</c>. The default operation specifies - which operation should be used on tables from the backup - which are not specified in any of the lists above. - If omitted, the operation <c>clear_tables</c> will be used. </item> + from the backup. + </item> + <item><c>{recreate_tables, TableList}</c>, where <c>TableList</c> + is a list of tables, which is to be recreated before the + records from the backup are inserted. The tables are first + deleted and then created with the schema information from the + backup. All the nodes in the backup need to be operational. + </item> + <item><c>{default_op, Operation}</c>, where <c>Operation</c> is + one of the operations <c>skip_tables</c>, + <c>clear_tables</c>, <c>keep_tables</c>, or + <c>recreate_tables</c>. The default operation specifies + which operation is to be used on tables from the backup + that are not specified in any of the previous lists. + If omitted, the operation <c>clear_tables</c> is used. + </item> </list> <p>The argument <c>Opaque</c> is forwarded to the backup module. It returns <c>{atomic, TabList}</c> if successful, or the - tuple <c>{aborted, Reason}</c> in the case of an error. - <c>TabList</c> is a list of the restored tables. Tables which - are restored are write locked for the duration of the restore - operation. However, regardless of any lock conflict caused by + tuple <c>{aborted, Reason}</c> if there is an error. + <c>TabList</c> is a list of the restored tables. Tables that + are restored are write-locked during the restore + operation. However, regardless of any lock conflict caused by this, applications can continue to do their work during the - restore operation. - </p> + restore operation.</p> <p>The restoration is performed as a single transaction. If the - database is very large, it may not be possible to restore it - online. In such a case the old database must be restored by - installing a fallback, and then restart. - </p> + database is large, it cannot always be restored + online. The old database must then be restored by + installing a fallback, followed by a restart.</p> </section> <section> - <title>Fallbacks</title> - <p>The function <c>mnesia:install_fallback(Opaque, [Mod])</c> is - used to install a backup as fallback. It uses the backup module + <title>Fallback</title> + <p>The function + <seealso marker="mnesia#install_fallback/2">mnesia:install_fallback(Opaque, [Mod])</seealso> + installs a backup as fallback. It uses the backup module <c>Mod</c>, or the default backup module, to access the backup - media. This function returns <c>ok</c> if successful, or - <c>{error, Reason}</c> in the case of an error. - </p> - <p>Installing a fallback is a distributed operation that is + media. The function returns <c>ok</c> if successful, or + <c>{error, Reason}</c> if there is an error.</p> + <p>Installing a fallback is a distributed operation, which is <em>only</em> performed on all <c>db_nodes</c>. The fallback - is used to restore the database the next time the system is - started. If a Mnesia node with a fallback installed detects that - Mnesia on another node has died for some reason, it will - unconditionally terminate itself. - </p> + restores the database the next time the system is started. + If a <c>Mnesia</c> node with a fallback installed detects that + <c>Mnesia</c> on another node has died, it + unconditionally terminates itself.</p> <p>A fallback is typically used when a system upgrade is performed. A system typically involves the installation of new - software versions, and Mnesia tables are often transformed into - new layouts. If the system crashes during an upgrade, it is - highly probable re-installation of the old - applications will be required and restoration of the database - to its previous state. This can be done if a backup is performed and - installed as a fallback before the system upgrade begins. - </p> - <p>If the system upgrade fails, Mnesia must be restarted on all - <c>db_nodes</c> in order to restore the old database. The - fallback will be automatically de-installed after a successful - start-up. The function <c>mnesia:uninstall_fallback()</c> may - also be used to de-install the fallback after a + software versions, and <c>Mnesia</c> tables are often transformed + into new layouts. If the system crashes during an upgrade, it is + highly probable that reinstallation of the old applications is + required, and restoration of the database to its previous state. + This can be done if a backup is performed and + installed as a fallback before the system upgrade begins.</p> + <p>If the system upgrade fails, <c>Mnesia</c> must be restarted + on all <c>db_nodes</c> to restore the old database. The + fallback is automatically deinstalled after a successful + startup. The function + <seealso marker="mnesia#uninstall_fallback/0">mnesia:uninstall_fallback()</seealso> + can also be used to deinstall the fallback after a successful system upgrade. Again, this is a distributed - operation that is either performed on all <c>db_nodes</c>, or - none. Both the installation and de-installation of fallbacks - require Erlang to be up and running on all <c>db_nodes</c>, but - it does not matter if Mnesia is running or not. - </p> + operation that is either performed on all <c>db_nodes</c> or + none. Both the installation and deinstallation of fallbacks + require Erlang to be operational on all <c>db_nodes</c>, but + it does not matter if <c>Mnesia</c> is running or not.</p> </section> <section> <title>Disaster Recovery</title> - <p>The system may become inconsistent as a result of a power - failure. The UNIX <c>fsck</c> feature can possibly repair the - file system, but there is no guarantee that the file contents - will be consistent. - </p> - <p>If Mnesia detects that a file has not been properly closed, - possibly as a result of a power failure, it will attempt to - repair the bad file in a similar manner. Data may be lost, but - Mnesia can be restarted even if the data is inconsistent. The - configuration parameter <c><![CDATA[-mnesia auto_repair <bool>]]></c> can be - used to control the behavior of Mnesia at start-up. If - <c><![CDATA[<bool>]]></c> has the value <c>true</c>, Mnesia will attempt to - repair the file; if <c><![CDATA[<bool>]]></c> has the value <c>false</c>, - Mnesia will not restart if it detects a suspect file. This - configuration parameter affects the repair behavior of log - files, DAT files, and the default backup media. - </p> - <p>The configuration parameter <c><![CDATA[-mnesia dump_log_update_in_place <bool>]]></c> controls the safety level of - the <c>mnesia:dump_log()</c> function. By default, Mnesia will - dump the transaction log directly into the DAT files. If a power - failure happens during the dump, this may cause the randomly - accessed DAT files to become corrupt. If the parameter is set to - <c>false</c>, Mnesia will copy the DAT files and target the dump + <p>The system can become inconsistent as a result of a power + failure. The UNIX feature <c>fsck</c> can possibly repair the + file system, but there is no guarantee that the file content + is consistent.</p> + <p>If <c>Mnesia</c> detects that a file has not been properly + closed, possibly as a result of a power failure, it tries to + repair the bad file in a similar manner. Data can be lost, but + <c>Mnesia</c> can be restarted even if the data is inconsistent. + Configuration parameter + <c><![CDATA[-mnesia auto_repair <bool>]]></c> can be used + to control the behavior of <c>Mnesia</c> at startup. If + <c><![CDATA[<bool>]]></c> has the value <c>true</c>, + <c>Mnesia</c> tries to repair the file. If + <c><![CDATA[<bool>]]></c> has the value <c>false</c>, + <c>Mnesia</c> does not restart if it detects a suspect file. + This configuration parameter affects the repair behavior of log + files, <c>DAT</c> files, and the default backup media.</p> + <p>Configuration parameter + <c><![CDATA[-mnesia dump_log_update_in_place <bool>]]></c> + controls the safety level of the function + <seealso marker="mnesia#dump_log/0">mnesia:dump_log()</seealso> + By default, <c>Mnesia</c> dumps the + transaction log directly into the <c>DAT</c> files. If a power + failure occurs during the dump, this can cause the randomly + accessed <c>DAT</c> files to become corrupt. If the parameter + is set to <c>false</c>, <c>Mnesia</c> copies the <c>DAT</c> + files and target the dump to the new temporary files. If the dump is successful, the - temporary files will be renamed to their normal DAT + temporary files are renamed to their normal <c>DAT</c> suffixes. The possibility for unrecoverable inconsistencies in - the data files will be much smaller with this strategy. On the - other hand, the actual dumping of the transaction log will be + the data files becomes much smaller with this strategy. + However, the actual dumping of the transaction log becomes considerably slower. The system designer must decide whether - speed or safety is the higher priority. - </p> - <p>Replicas of type <c>disc_only_copies</c> will only be + speed or safety is the higher priority.</p> + <p>Replicas of type <c>disc_only_copies</c> are only affected by this parameter during the initial dump of the log - file at start-up. When designing applications which have - <em>very</em> high requirements, it may be appropriate not to + file at startup. When designing applications with + <em>very</em> high requirements, it can be appropriate not to use <c>disc_only_copies</c> tables at all. The reason for this is the random access nature of normal operating system files. If - a node goes down for reason for a reason such as a power - failure, these files may be corrupted because they are not - properly closed. The DAT files for <c>disc_only_copies</c> are - updated on a per transaction basis. - </p> - <p>If a disaster occurs and the Mnesia database has been - corrupted, it can be reconstructed from a backup. This should be - regarded as a last resort, since the backup contains old data. The - data is hopefully consistent, but data will definitely be lost - when an old backup is used to restore the database. - </p> + a node goes down for a reason such as a power + failure, these files can be corrupted because they are not + properly closed. The <c>DAT</c> files for <c>disc_only_copies</c> + are updated on a per transaction basis.</p> + <p>If a disaster occurs and the <c>Mnesia</c> database is + corrupted, it can be reconstructed from a backup. Regard + this as a last resort, as the backup contains old data. The + data is hopefully consistent, but data is definitely lost + when an old backup is used to restore the database.</p> </section> </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_chap8.xml b/lib/mnesia/doc/src/Mnesia_chap8.xml index d35dd0c539..55309177b1 100644 --- a/lib/mnesia/doc/src/Mnesia_chap8.xml +++ b/lib/mnesia/doc/src/Mnesia_chap8.xml @@ -21,7 +21,7 @@ </legalnotice> - <title>Combining Mnesia with SNMP</title> + <title>Combine Mnesia with SNMP</title> <prepared>Claes Wikström, Hans Nilsson and Håkan Mattsson</prepared> <responsible></responsible> <docno></docno> @@ -33,32 +33,29 @@ </header> <section> - <title>Combining Mnesia and SNMP </title> + <title>Combine Mnesia and SNMP</title> <p>Many telecommunications applications must be controlled and reconfigured remotely. It is sometimes an advantage to perform this remote control with an open protocol such as the Simple Network Management Protocol (SNMP). The alternatives to this would - be: - </p> + be the following:</p> <list type="bulleted"> - <item>Not being able to control the application remotely at all. + <item>Not being able to control the application remotely </item> - <item>Using a proprietary control protocol. + <item>Using a proprietary control protocol </item> - <item>Using a bridge which maps control messages in a + <item>Using a bridge that maps control messages in a proprietary protocol to a standardized management protocol and - vice versa. + conversely </item> </list> - <p>All of these approaches have different advantages and - disadvantages. Mnesia applications can easily be opened to the - SNMP protocol. It is possible to establish a direct one-to-one - mapping between Mnesia tables and SNMP tables. This - means that a Mnesia table can be configured to be <em>both</em> - a Mnesia table and an SNMP table. A number of functions to - control this behavior are described in the Mnesia reference - manual. - </p> + <p>All these approaches have different advantages and + disadvantages. <c>Mnesia</c> applications can easily be opened to + the SNMP protocol. A direct 1-to-1 mapping can be established + between <c>Mnesia</c> tables and SNMP tables. This means + that a <c>Mnesia</c> table can be configured to be <em>both</em> + a <c>Mnesia</c> table and an SNMP table. A number of functions to + control this behavior are described in the Reference Manual.</p> </section> </chapter> diff --git a/lib/mnesia/doc/src/Mnesia_overview.xml b/lib/mnesia/doc/src/Mnesia_overview.xml new file mode 100644 index 0000000000..4758612053 --- /dev/null +++ b/lib/mnesia/doc/src/Mnesia_overview.xml @@ -0,0 +1,192 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>1997</year><year>2013</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>Mnesia</title> + <prepared>Claes Wikström, Hans Nilsson and Håkan Mattsson</prepared> + <responsible>Bjarne Däcker</responsible> + <docno></docno> + <approved>Bjarne Däcker</approved> + <checked>Bjarne Däcker</checked> + <date></date> + <rev>C</rev> + <file>Mnesia_overview.xml</file> + </header> + + <p>The management of data in telecommunications system has many + aspects, thereof some, but not all, are addressed by traditional + commercial Database Management Systems (DBMSs). In particular the + high level of fault tolerance that is required in many nonstop + systems, combined with requirements on the DBMS to run in the same + address space as the application, have led us to implement a new + DBMS, called <c>Mnesia</c>.</p> + <p><c>Mnesia</c> is implemented in, and tightly connected to Erlang. + It provides the functionality that is necessary for the + implementation of fault tolerant telecommunications systems.</p> + <p><c>Mnesia</c> is a multiuser distributed DBMS specially made for + industrial telecommunications applications written in Erlang, + which is also the intended target language. + <c>Mnesia</c> tries to address all the data + management issues required for typical telecommunications systems. + It has a number of features that are not normally found in traditional + databases.</p> + <p>In telecommunications applications, there are different needs + from the features provided by traditional DBMSs. The applications now + implemented in Erlang need a mixture of a broad range + of features, which generally are not satisfied by traditional DBMSs. + <c>Mnesia</c> is designed with requirements like the following in + mind:</p> + <list type="ordered"> + <item>Fast real-time key/value lookup + </item> + <item>Complicated non-real-time queries mainly for + operation and maintenance + </item> + <item>Distributed data because of distributed applications + </item> + <item>High fault tolerance + </item> + <item>Dynamic reconfiguration + </item> + <item>Complex objects + </item> + </list> + <p><c>Mnesia</c> is designed with the typical data management problems + of telecommunications applications in mind. This sets <c>Mnesia</c> + apart from most other DBMS. Hence <c>Mnesia</c> + combines many concepts found in traditional databases such as + transactions and queries with concepts found in data management + systems for telecommunications applications, for example:</p> + <list type="bulleted"> + <item>Fast real-time operations + </item> + <item>Configurable degree of fault tolerance (by replication) + </item> + <item>The ability to reconfigure the system without stopping or + suspending it. + </item> + </list> + <p><c>Mnesia</c> is also interesting because of its tight coupling to + Erlang, thus almost turning Erlang into a database programming + language. This has many benefits, the foremost is that + the impedance mismatch between the data format used by the DBMS + and the data format used by the programming language, which is used + to manipulate the data, completely disappears.</p> + + <section> + <title>Mnesia Database Management System (DBMS)</title> + <section> + <title>Features</title> + <p><c>Mnesia</c> contains the following features that combine to + produce a fault-tolerant, distributed DBMS written in Erlang: + </p> + <list type="bulleted"> + <item>Database schema can be dynamically reconfigured at runtime. + </item> + <item>Tables can be declared to have properties such as location, + replication, and persistence. + </item> + <item>Tables can be moved or replicated to several nodes to improve + fault tolerance. The rest of the system can still access the tables + to read, write, and delete records. + </item> + <item>Table locations are transparent to the programmer. + Programs address table names and the system itself keeps track of + table locations. + </item> + <item>Database transactions can be distributed, and many + functions can be called within one transaction. + </item> + <item>Several transactions can run concurrently, and their execution + is fully synchronized by the DBMS. <c>Mnesia</c> ensures that no + two processes manipulate data simultaneously. + </item> + <item>Transactions can be assigned the property of being executed on + all nodes in the system, or on none. Transactions can also be + bypassed in favor of running "dirty operations", which reduce + overheads and run fast. + </item> + </list> + <p>Details of these features are described in the following sections.</p> + </section> + + <section> + <title>Add-On Application</title> + <p>Query List Comprehension (QLC) can be used with <c>Mnesia</c> + to produce specialized functions that enhance the operational + ability of <c>Mnesia</c>. QLC has its own documentation as part + of the OTP documentation set. The main features of QLC + when used with <c>Mnesia</c> are as follows:</p> + <list type="bulleted"> + <item>QLC can optimize the query compiler for the <c>Mnesia</c> + DBMS, essentially making the DBMS more efficient. + </item> + <item>QLC can be used as a database programming + language for <c>Mnesia</c>. It includes a notation called "list + comprehensions" and can be used to make complex database + queries over a set of tables. + </item> + </list> + <p>For information about QLC, see the + <seealso marker="stdlib:qlc">qlc</seealso> manual page + in <c>STDLIB</c>.</p> + </section> + + <section> + <title>When to Use Mnesia</title> + <p>Use <c>Mnesia</c> with the following types of applications:</p> + <list type="bulleted"> + <item>Applications that need to replicate data. + </item> + <item>Applications that perform complicated searches on data. + </item> + <item>Applications that need to use atomic transactions to + update several records simultaneously. + </item> + <item>Applications that use soft real-time characteristics. + </item> + </list> + <p><c>Mnesia</c> is not as appropriate with the + following types of applications:</p> + <list type="bulleted"> + <item>Programs that process plain text or binary data files. + </item> + <item>Applications that merely need a look-up dictionary that + can be stored to disc. Those applications use the standard + library module <c>dets</c>, which is a disc-based version + of the module <c>ets</c>. For information about <c>dets</c>, + see the <seealso marker="stdlib:dets">dets</seealso> + manual page in <c>STDLIB</c>. + </item> + <item>Applications that need disc logging facilities. + Those applications can + use the module <c>disk_log</c> by preference. For + information about <c>disk_log</c>, see the + <seealso marker="kernel:disk_log">disk_log</seealso> + manual page in <c>Kernel</c>. + </item> + <item>Hard real-time systems. + </item> + </list> + </section> + </section> +</chapter> diff --git a/lib/mnesia/doc/src/mnesia.xml b/lib/mnesia/doc/src/mnesia.xml index 856a7594a7..1f5c62a5e6 100644 --- a/lib/mnesia/doc/src/mnesia.xml +++ b/lib/mnesia/doc/src/mnesia.xml @@ -32,291 +32,254 @@ <file></file> </header> <module>mnesia</module> - <modulesummary>A Distributed Telecommunications DBMS </modulesummary> + <modulesummary>A distributed telecommunications DBMS</modulesummary> <description> - <p><c>Mnesia</c> is a distributed DataBase Management System (DBMS), - appropriate for telecommunications applications and other Erlang - applications which require continuous operation and exhibit soft - real-time properties. - </p> - <p>Listed below are some of the most important and attractive capabilities, Mnesia provides: - </p> + + <p>The following are some of the most important and attractive + capabilities provided by <c>Mnesia</c>:</p> <list type="bulleted"> - <item> - <p>A relational/object hybrid data model which is - suitable for telecommunications applications. - </p> + <item>A relational/object hybrid data model that is suitable + for telecommunications applications. </item> - <item> - <p>A specifically designed DBMS query language, QLC (as an add-on library). - </p> + <item>A DBMS query language, Query List Comprehension (QLC) as + an add-on library. </item> - <item> - <p>Persistence. Tables may be coherently kept on disc as - well as in main memory. - </p> + <item>Persistence. Tables can be coherently kept on disc and + in the main memory. </item> - <item> - <p>Replication. Tables may be replicated at several nodes. - </p> + <item>Replication. Tables can be replicated at several nodes. </item> - <item> - <p>Atomic transactions. A series of table manipulation - operations can be grouped into a single atomic - transaction. - </p> + <item>Atomic transactions. A series of table manipulation + operations can be grouped into a single atomic transaction. </item> - <item> - <p>Location transparency. Programs can be written without - knowledge of the actual location of data. - </p> + <item>Location transparency. Programs can be written without + knowledge of the actual data location. </item> - <item> - <p>Extremely fast real time data searches. - </p> + <item>Extremely fast real-time data searches. </item> - <item> - <p>Schema manipulation routines. It is possible to - reconfigure the DBMS at runtime without stopping the - system. - </p> + <item>Schema manipulation routines. The DBMS can be + reconfigured at runtime without stopping the system. </item> </list> - <p>This Reference Manual describes the Mnesia API. This includes - functions used to define and manipulate Mnesia tables. - </p> - <p>All functions documented in these pages can be used in any - combination with queries using the list comprehension notation. The - query notation is described in the QLC's man page. - </p> - <p>Data in Mnesia is organized as a set of tables. Each table - has a name which must be an atom. Each table is made up of - Erlang records. The user is responsible for the record - definitions. Each table also has a set of properties. Below - are some of the properties that are associated with each - table: - </p> + <p>This Reference Manual describes the <c>Mnesia</c> API. This + includes functions that define and manipulate <c>Mnesia</c> + tables.</p> + <p>All functions in this Reference Manual can be used in any + combination with queries using the list comprehension notation. + For information about the query notation, see the + <seealso marker="stdlib:qlc">qlc</seealso> + manual page in <c>STDLIB</c>.</p> + <p>Data in <c>Mnesia</c> is organized as a set of tables. Each table + has a name that must be an atom. Each table is made up of + Erlang records. The user is responsible for the record + definitions. Each table also has a set of properties. The + following are some of the properties that are associated with each + table:</p> <list type="bulleted"> <item> - <p><c>type</c>. Each table can either have 'set', - 'ordered_set' or 'bag' semantics. Note: currently 'ordered_set' - is not supported for 'disc_only_copies'. If a table is of type - 'set' it means that each key leads to either one or zero - records. <br></br> -If a new item is inserted with the same key as - an existing record, the old record is overwritten. On the - other hand, if a table is of type 'bag', each key can map to - several records. However, all records in type bag tables are - unique, only the keys may be duplicated. - </p> + <p><c>type</c>. Each table can have <c>set</c>, + <c>ordered_set</c>, or <c>bag</c> semantics. Notice that + currently <c>ordered_set</c> is not supported for + <c>disc_only_copies</c>.</p> + <p>If a table is of type <c>set</c>, each key leads to + either one or zero records.</p> + <p>If a new item is inserted with the same key as an + existing record, the old record is overwritten. However, + if a table is of type <c>bag</c>, each key can map to + several records. All records in type <c>bag</c> tables are + unique, only the keys can be duplicated.</p> </item> <item> <p><c>record_name</c>. All records stored in a table must - have the same name. You may say that the records must be - instances of the same record type. - </p> + have the same name. The records must be instances of the + same record type.</p> </item> <item> - <p><c>ram_copies</c> A table can be replicated on a number - of Erlang nodes. The <c>ram_copies</c> property specifies a - list of Erlang nodes where RAM copies are kept. These - copies can be dumped to disc at regular intervals. However, + <p><c>ram_copies</c>. A table can be replicated on a number + of Erlang nodes. Property <c>ram_copies</c> specifies a + list of Erlang nodes where RAM copies are kept. These + copies can be dumped to disc at regular intervals. However, updates to these copies are not written to disc on a - transaction basis. - </p> + transaction basis.</p> </item> <item> - <p><c>disc_copies</c> The <c>disc_copies</c> property + <p><c>disc_copies</c>. This property specifies a list of Erlang nodes where the table is kept in - RAM as well as on disc. All updates of the table are - performed on the actual table and are also logged to disc. + RAM and on disc. All updates of the table are + performed in the actual table and are also logged to disc. If a table is of type <c>disc_copies</c> at a certain node, - it means that the entire table is resident in RAM memory as - well as on disc. Each transaction performed on the table is - appended to a LOG file as well as written into the RAM - table. - </p> + the entire table is resident in RAM memory and on disc. + Each transaction performed on the table is appended to a + <c>LOG</c> file and written into the RAM table.</p> </item> <item> - <p><c>disc_only_copies</c> Some, or all, table replicas + <p><c>disc_only_copies</c>. Some, or all, table replicas can be kept on disc only. These replicas are considerably - slower than the RAM based replicas. - </p> + slower than the RAM-based replicas.</p> </item> <item> - <p><c>index</c> This is a list of attribute names, or + <p><c>index</c>. This is a list of attribute names, or integers, which specify the tuple positions on which - Mnesia shall build and maintain an extra index table. - </p> + <c>Mnesia</c> is to build and maintain an extra index + table.</p> </item> <item> - <p><c>local_content</c> When an application requires + <p><c>local_content</c>. When an application requires tables whose contents are local to each node, - <c>local_content</c> tables may be used. The name of the - table is known to all Mnesia nodes, but its contents are + <c>local_content</c> tables can be used. The table name + is known to all <c>Mnesia</c> nodes, but its content is unique on each node. This means that access to such a table - must be done locally. Set the <c>local_content</c> field to - <c>true</c> if you want to enable the <c>local_content</c> - behavior. The default is <c>false</c>. - </p> + must be done locally. Set field <c>local_content</c> to + <c>true</c> to enable the <c>local_content</c> + behavior. Default is <c>false</c>.</p> </item> <item> - <p><c>majority</c> This attribute can be either <c>true</c> or - <c>false</c> (default is <c>false</c>). When <c>true</c>, a majority - of the table replicas must be available for an update to succeed. - Majority checking can be enabled on tables with mission-critical data, - where it is vital to avoid inconsistencies due to network splits. - </p> + <p><c>majority</c>. This attribute is <c>true</c> or + <c>false</c>; default is <c>false</c>. When <c>true</c>, + a majority of the table replicas must be available for an + update to succeed. Majority checking can be enabled on + tables with mission-critical data, where it is vital to + avoid inconsistencies because of network splits.</p> </item> <item> - <p><c>snmp</c> Each (set based) Mnesia table can be - automatically turned into an SNMP ordered table as well. - This property specifies the types of the SNMP keys. - </p> + <p><c>snmp</c>. Each (set-based) <c>Mnesia</c> table can be + automatically turned into a Simple Network Management + Protocol (SNMP) ordered table as well. + This property specifies the types of the SNMP keys.</p> </item> <item> <p><c>attributes</c>. The names of the attributes for the - records that are inserted in the table. - </p> + records that are inserted in the table.</p> </item> </list> - <p>See <c>mnesia:create_table/2</c> about the complete set of - table properties and their details. - </p> - <p>This document uses a table of persons to illustrate various - examples. The following record definition is assumed: - </p> + <p>For information about the complete set of table properties + and their details, see <c>mnesia:create_table/2</c>.</p> + <p>This Reference Manual uses a table of persons to illustrate + various examples. The following record definition is assumed:</p> <code type="none"> -record(person, {name, age = 0, address = unknown, salary = 0, - children = []}), - </code> - <p>The first attribute of the record is the primary key, or key - for short. - </p> - <p>The function descriptions are sorted in alphabetic order. <em>Hint:</em> - start to read about <c>mnesia:create_table/2</c>, - <c>mnesia:lock/2</c> and <c>mnesia:activity/4</c> before you continue on - and learn about the rest. - </p> - <p>Writing or deleting in transaction context creates a local copy - of each modified record during the transaction. During iteration, - i.e. <c>mnesia:fold[lr]/4</c> <c>mnesia:next/2</c> <c>mnesia:prev/2</c> - <c>mnesia:snmp_get_next_index/2</c>, mnesia will compensate for - every written or deleted record, which may reduce the - performance. If possible avoid writing or deleting records in - the same transaction before iterating over the table. - </p> + children = []}),</code> + <p>The first record attribute is the primary key, or key + for short.</p> + <p>The function descriptions are sorted in alphabetical order. + It is recommended to start to read about + <c>mnesia:create_table/2</c>, <c>mnesia:lock/2</c>, and + <c>mnesia:activity/4</c> before you continue and learn + about the rest.</p> + <p>Writing or deleting in transaction-context creates a local + copy of each modified record during the transaction. During + iteration, that is, <c>mnesia:fold[lr]/4</c>, + <c>mnesia:next/2</c>, <c>mnesia:prev/2</c>, and + <c>mnesia:snmp_get_next_index/2</c>, <c>Mnesia</c> + compensates for every written or deleted record, which can + reduce the performance.</p> + <p>If possible, avoid writing or deleting records in the same + transaction before iterating over the table.</p> </description> + <funcs> <func> - <name>abort(Reason) -> transaction abort </name> - <fsummary>Abort the current transaction.</fsummary> + <name>abort(Reason) -> transaction abort</name> + <fsummary>Terminates the current transaction.</fsummary> <desc> - <p>Makes the transaction silently + <p>Makes the transaction silently return the tuple <c>{aborted, Reason}</c>. - The abortion of a Mnesia transaction means that - an exception will be thrown to an enclosing <c>catch</c>. + Termination of a <c>Mnesia</c> transaction means that + an exception is thrown to an enclosing <c>catch</c>. Thus, the expression <c>catch mnesia:abort(x)</c> does - not abort the transaction. </p> + not terminate the transaction.</p> </desc> </func> <func> <name>activate_checkpoint(Args) -> {ok,Name,Nodes} | {error,Reason}</name> - <fsummary>Activate a checkpoint.</fsummary> + <fsummary>Activates a checkpoint.</fsummary> <desc> - <p>A checkpoint is a consistent view of the system. + <marker id="activate_checkpoint"></marker> + <p>A checkpoint is a consistent view of the system. A checkpoint can be activated on a set of tables. - This checkpoint can then be traversed and will - present a view of the system as it existed at the time when - the checkpoint was activated, even if the tables are being or have been - manipulated. - </p> - <p><c>Args</c> is a list of the following tuples: - </p> + This checkpoint can then be traversed and + presents a view of the system as it existed at the time when + the checkpoint was activated, even if the tables are + being or have been manipulated.</p> + <p><c>Args</c> is a list of the following tuples:</p> <list type="bulleted"> <item> - <p><c>{name,Name}</c>. <c>Name</c> of checkpoint. Each - checkpoint must have a name which is unique to the + <p><c>{name,Name}</c>. <c>Name</c> is the checkpoint name. + Each checkpoint must have a name that is unique to the associated nodes. The name can be reused only once the checkpoint has been deactivated. By default, a name - which is probably unique is generated. - </p> + that is probably unique is generated.</p> </item> <item> <p><c>{max,MaxTabs}</c>. <c>MaxTabs</c> is a list of - tables that should be included in the checkpoint. The - default is []. For these tables, the redundancy will be - maximized and checkpoint information will be retained together + tables that are to be included in the checkpoint. + Default is <c>[]</c>. For these tables, the redundancy is + maximized and checkpoint information is retained together with all replicas. The checkpoint becomes more fault tolerant if the tables have several replicas. When a new - replica is added by means of the schema manipulation - function <c>mnesia:add_table_copy/3</c>, a retainer will - also be attached automatically. - </p> + replica is added by the schema manipulation + function <c>mnesia:add_table_copy/3</c>, a retainer is + also attached automatically.</p> </item> <item> <p><c>{min,MinTabs}</c>. <c>MinTabs</c> is a list of - tables that should be included in the checkpoint. The - default is []. For these tables, the redundancy will be - minimized and the checkpoint information will only be retained - with one replica, preferably on the local node. - </p> + tables that are to be included in the checkpoint. + Default is []. For these tables, the redundancy is + minimized and the checkpoint information is only retained + with one replica, preferably on the local node.</p> </item> <item> <p><c>{allow_remote,Bool}</c>. <c>false</c> means that all retainers must be local. The checkpoint cannot be activated if a table does not reside locally. <c>true</c> allows retainers to be allocated on any - node. Default is set to <c>true</c>. - </p> + node. Default is <c>true</c>.</p> </item> <item> <p><c>{ram_overrides_dump,Bool}</c>. Only applicable - for <c>ram_copies</c>. <c>Bool</c> allows you to choose - to backup the table state as it is in RAM, or as it is on - disc. <c>true</c> means that the latest committed - records in RAM should be included in the checkpoint. - These are the records that the application accesses. - <c>false</c> means that the records dumped to DAT files - should be included in the checkpoint. These are the - records that will be loaded at startup. Default is - <c>false</c>. - </p> + for <c>ram_copies</c>. <c>Bool</c> allows you to choose + to back up the table state as it is in RAM, or as it is + on disc. <c>true</c> means that the latest committed + records in RAM are to be included in the checkpoint. + These are the records that the application accesses. + <c>false</c> means that the records dumped to <c>DAT</c> + files are to be included in the checkpoint. These + records are loaded at startup. Default is <c>false</c>.</p> </item> </list> <p>Returns <c>{ok,Name,Nodes}</c> or <c>{error,Reason}</c>. - <c>Name</c> is the (possibly generated) name of the - checkpoint. <c>Nodes</c> are the nodes that + <c>Name</c> is the (possibly generated) checkpoint name. + <c>Nodes</c> are the nodes that are involved in the checkpoint. Only nodes that keep a - checkpoint retainer know about the checkpoint. - </p> + checkpoint retainer know about the checkpoint.</p> </desc> </func> <func> <name>activity(AccessContext, Fun [, Args]) -> ResultOfFun | exit(Reason)</name> - <fsummary>Execute <c>Fun</c>in <c>AccessContext</c>.</fsummary> + <fsummary>Executes <c>Fun</c> in <c>AccessContext</c>.</fsummary> <desc> - <p>Invokes <c>mnesia:activity(AccessContext, Fun, Args, AccessMod)</c> where <c>AccessMod</c> is the default + <marker id="activity_2_3"></marker> + <p>Calls <c>mnesia:activity(AccessContext, Fun, Args, + AccessMod)</c>, where <c>AccessMod</c> is the default access callback module obtained by <c>mnesia:system_info(access_module)</c>. <c>Args</c> - defaults to the empty list <c>[]</c>.</p> + defaults to <c>[]</c> (empty list).</p> </desc> </func> <func> <name>activity(AccessContext, Fun, Args, AccessMod) -> ResultOfFun | exit(Reason)</name> - <fsummary>Execute <c>Fun</c>in <c>AccessContext</c>.</fsummary> - <desc> - <p>This function executes the functional object <c>Fun</c> - with the arguments <c>Args</c>. - </p> - <p>The code which executes inside the activity can - consist of a series of table manipulation functions, which is - performed in a <c>AccessContext</c>. Currently, the following - access contexts are supported: - </p> + <fsummary>Executes <c>Fun</c> in <c>AccessContext</c>.</fsummary> + <desc> + <marker id="activity_4"></marker> + <p>Executes the functional object <c>Fun</c> + with argument <c>Args</c>.</p> + <p>The code that executes inside the activity can + consist of a series of table manipulation functions, which are + performed in an <c>AccessContext</c>. Currently, the following + access contexts are supported:</p> <taglist> <tag><c>transaction</c></tag> <item> @@ -324,10 +287,10 @@ If a new item is inserted with the same key as </item> <tag><c>{transaction, Retries}</c></tag> <item> - <p>Invokes <c>mnesia:transaction(Fun, Args, Retries)</c>. Note that the result from the <c>Fun</c> is - returned if the transaction was successful (atomic), - otherwise the function exits with an abort reason. - </p> + <p>Calls <c>mnesia:transaction(Fun, Args, Retries)</c>. + Notice that the result from <c>Fun</c> is + returned if the transaction is successful (atomic), + otherwise the function exits with an abort reason.</p> </item> <tag><c>sync_transaction</c></tag> <item> @@ -335,540 +298,499 @@ If a new item is inserted with the same key as </item> <tag><c>{sync_transaction, Retries}</c></tag> <item> - <p>Invokes <c>mnesia:sync_transaction(Fun, Args, Retries)</c>. Note that the result from the <c>Fun</c> is - returned if the transaction was successful (atomic), - otherwise the function exits with an abort reason. - </p> + <p>Calls <c>mnesia:sync_transaction(Fun, Args, Retries)</c>. + Notice that the result from <c>Fun</c> is + returned if the transaction is successful (atomic), + otherwise the function exits with an abort reason.</p> </item> <tag><c>async_dirty</c></tag> <item> - <p>Invokes <c>mnesia:async_dirty(Fun, Args)</c>. - </p> + <p>Calls <c>mnesia:async_dirty(Fun, Args)</c>.</p> </item> <tag><c>sync_dirty</c></tag> <item> - <p>Invokes <c>mnesia:sync_dirty(Fun, Args)</c>. - </p> + <p>Calls <c>mnesia:sync_dirty(Fun, Args)</c>.</p> </item> <tag><c>ets</c></tag> <item> - <p>Invokes <c>mnesia:ets(Fun, Args)</c>. - </p> + <p>Calls <c>mnesia:ets(Fun, Args)</c>.</p> </item> </taglist> <p>This function (<c>mnesia:activity/4</c>) differs in an - important aspect from the <c>mnesia:transaction</c>, + important way from the functions <c>mnesia:transaction</c>, <c>mnesia:sync_transaction</c>, - <c>mnesia:async_dirty</c>, <c>mnesia:sync_dirty</c> and - <c>mnesia:ets</c> functions. The <c>AccessMod</c> argument - is the name of a callback module which implements the - <c>mnesia_access</c> behavior. - </p> - <p>Mnesia will forward calls to the following functions: - </p> + <c>mnesia:async_dirty</c>, <c>mnesia:sync_dirty</c>, and + <c>mnesia:ets</c>. Argument <c>AccessMod</c> + is the name of a callback module, which implements the + <c>mnesia_access</c> behavior.</p> + <p><c>Mnesia</c> forwards calls to the following functions:</p> <list type="bulleted"> - <item> - <p>mnesia:lock/2 (read_lock_table/1, write_lock_table/1)</p> + <item>mnesia:lock/2 (read_lock_table/1, write_lock_table/1) </item> - <item> - <p>mnesia:write/3 (write/1, s_write/1)</p> + <item>mnesia:write/3 (write/1, s_write/1) </item> - <item> - <p>mnesia:delete/3 (delete/1, s_delete/1)</p> + <item>mnesia:delete/3 (delete/1, s_delete/1) </item> - <item> - <p>mnesia:delete_object/3 (delete_object/1, s_delete_object/1)</p> + <item>mnesia:delete_object/3 (delete_object/1, s_delete_object/1) </item> - <item> - <p>mnesia:read/3 (read/1, wread/1)</p> + <item>mnesia:read/3 (read/1, wread/1) </item> - <item> - <p>mnesia:match_object/3 (match_object/1)</p> + <item>mnesia:match_object/3 (match_object/1) </item> - <item> - <p>mnesia:all_keys/1</p> + <item>mnesia:all_keys/1 </item> - <item> - <p>mnesia:first/1</p> + <item>mnesia:first/1 </item> - <item> - <p>mnesia:last/1</p> + <item>mnesia:last/1 </item> - <item> - <p>mnesia:prev/2</p> + <item>mnesia:prev/2 </item> - <item> - <p>mnesia:next/2</p> + <item>mnesia:next/2 </item> - <item> - <p>mnesia:index_match_object/4 (index_match_object/2)</p> + <item>mnesia:index_match_object/4 (index_match_object/2) </item> - <item> - <p>mnesia:index_read/3</p> + <item>mnesia:index_read/3 </item> - <item> - <p>mnesia:table_info/2</p> + <item>mnesia:table_info/2 </item> </list> - <p>to the corresponding: - </p> + <p>to the corresponding:</p> <list type="bulleted"> - <item> - <p>AccessMod:lock(ActivityId, Opaque, LockItem, LockKind)</p> + <item>AccessMod:lock(ActivityId, Opaque, LockItem, LockKind) </item> - <item> - <p>AccessMod:write(ActivityId, Opaque, Tab, Rec, LockKind)</p> + <item>AccessMod:write(ActivityId, Opaque, Tab, Rec, LockKind) </item> - <item> - <p>AccessMod:delete(ActivityId, Opaque, Tab, Key, LockKind)</p> + <item>AccessMod:delete(ActivityId, Opaque, Tab, Key, LockKind) </item> - <item> - <p>AccessMod:delete_object(ActivityId, Opaque, Tab, RecXS, LockKind)</p> + <item>AccessMod:delete_object(ActivityId, Opaque, Tab, RecXS, + LockKind) </item> - <item> - <p>AccessMod:read(ActivityId, Opaque, Tab, Key, LockKind)</p> + <item>AccessMod:read(ActivityId, Opaque, Tab, Key, LockKind) </item> - <item> - <p>AccessMod:match_object(ActivityId, Opaque, Tab, Pattern, LockKind)</p> + <item>AccessMod:match_object(ActivityId, Opaque, Tab, Pattern, + LockKind) </item> - <item> - <p>AccessMod:all_keys(ActivityId, Opaque, Tab, LockKind)</p> + <item>AccessMod:all_keys(ActivityId, Opaque, Tab, LockKind) </item> - <item> - <p>AccessMod:first(ActivityId, Opaque, Tab)</p> + <item>AccessMod:first(ActivityId, Opaque, Tab) </item> - <item> - <p>AccessMod:last(ActivityId, Opaque, Tab)</p> + <item>AccessMod:last(ActivityId, Opaque, Tab) </item> - <item> - <p>AccessMod:prev(ActivityId, Opaque, Tab, Key)</p> + <item>AccessMod:prev(ActivityId, Opaque, Tab, Key) </item> - <item> - <p>AccessMod:next(ActivityId, Opaque, Tab, Key)</p> + <item>AccessMod:next(ActivityId, Opaque, Tab, Key) </item> - <item> - <p>AccessMod:index_match_object(ActivityId, Opaque, Tab, Pattern, Attr, LockKind)</p> + <item>AccessMod:index_match_object(ActivityId, Opaque, Tab, + Pattern, Attr, LockKind) </item> - <item> - <p>AccessMod:index_read(ActivityId, Opaque, Tab, SecondaryKey, Attr, LockKind)</p> + <item>AccessMod:index_read(ActivityId, Opaque, Tab, + SecondaryKey, Attr, LockKind) </item> - <item> - <p>AccessMod:table_info(ActivityId, Opaque, Tab, InfoItem)</p> + <item>AccessMod:table_info(ActivityId, Opaque, Tab, InfoItem) </item> </list> - <p>where <c>ActivityId</c> is a record which represents the - identity of the enclosing Mnesia activity. The first field - (obtained with <c>element(1, ActivityId)</c> contains an - atom which may be interpreted as the type of the activity: - <c>'ets'</c>, <c>'async_dirty'</c>, <c>'sync_dirty'</c> or - <c>'tid'</c>. <c>'tid'</c> means that the activity is a + <p><c>ActivityId</c> is a record that represents the identity + of the enclosing <c>Mnesia</c> activity. The first field + (obtained with <c>element(1, ActivityId)</c>) contains an + atom, which can be interpreted as the activity type: + <c>ets</c>, <c>async_dirty</c>, <c>sync_dirty</c>, or + <c>tid</c>. <c>tid</c> means that the activity is a transaction. The structure of the rest of the identity - record is internal to Mnesia. - </p> - <p><c>Opaque</c> is an opaque data structure which is internal - to Mnesia.</p> + record is internal to <c>Mnesia</c>.</p> + <p><c>Opaque</c> is an opaque data structure that is internal + to <c>Mnesia</c>.</p> </desc> </func> <func> <name>add_table_copy(Tab, Node, Type) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Copy a table to a remote node.</fsummary> + <fsummary>Copies a table to a remote node.</fsummary> <desc> - <p>This function makes another copy of a table at the - node <c>Node</c>. The <c>Type</c> argument must be - either of the atoms <c>ram_copies</c>, <c>disc_copies</c>, - or + <marker id="add_table_copy"></marker> + <p>Makes another copy of a table at the node <c>Node</c>. + Argument <c>Type</c> must be either of the atoms + <c>ram_copies</c>, <c>disc_copies</c>, or <c>disc_only_copies</c>. For example, the following call - ensures that a disc replica of the <c>person</c> table also - exists at node <c>Node</c>.</p> + ensures that a disc replica of the <c>person</c> table also + exists at node <c>Node</c>:</p> <code type="none"> -mnesia:add_table_copy(person, Node, disc_copies) - </code> +mnesia:add_table_copy(person, Node, disc_copies)</code> <p>This function can also be used to add a replica of the table named <c>schema</c>.</p> </desc> </func> <func> <name>add_table_index(Tab, AttrName) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Create an index for a table. </fsummary> - <desc> - <p>Table indices can and should be used whenever the user - wants to frequently use some other field than the key field - to look up records. If this other field has an index - associated with it, these lookups can occur in constant time - and space. For example, if our application wishes to use - the age field of persons to efficiently find all person with - a specific age, it might be a good idea to have an index on - the age field. This can be accomplished with the following + <fsummary>Creates an index for a table.</fsummary> + <desc> + <marker id="add_table_index"></marker> + <p>Table indexes can be used whenever the user + wants to use frequently some other field than the key field + to look up records. If this other field has an associated + index, these lookups can occur in constant time + and space. For example, if your application wishes to use + field <c>age</c> to find efficiently all persons with + a specific age, it can be a good idea to have an index on + field <c>age</c>. This can be done with the following call:</p> <code type="none"> -mnesia:add_table_index(person, age) - </code> - <p>Indices do not come free, they occupy space which is - proportional to the size of the table. They also cause insertions - into the table to execute slightly slower. </p> +mnesia:add_table_index(person, age)</code> + <p>Indexes do not come for free. They occupy space that is + proportional to the table size, and they cause insertions + into the table to execute slightly slower.</p> </desc> </func> <func> <name>all_keys(Tab) -> KeyList | transaction abort</name> - <fsummary>Return all keys in a table.</fsummary> + <fsummary>Returns all keys in a table.</fsummary> <desc> - <p>This function returns a list of all keys in the table - named <c>Tab</c>. The semantics of this function is context - sensitive. See <c>mnesia:activity/4</c> for more information. In - transaction context it acquires a read lock on the entire + <marker id="all_keys"></marker> + <p>Returns a list of all keys in the table named <c>Tab</c>. + The semantics of this function is context-sensitive. + For more information, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a read lock on the entire table.</p> </desc> </func> <func> - <name>async_dirty(Fun, [, Args]) -> ResultOfFun | exit(Reason)</name> - <fsummary>Call the Fun in a context which is not protected by a transaction.</fsummary> + <name>async_dirty(Fun, [, Args]) -> ResultOfFun | exit(Reason)</name> + <fsummary>Calls the <c>Fun</c> in a context that is not protected by a transaction.</fsummary> <desc> - <p>Call the <c>Fun</c> in a context which is not protected - by a transaction. The Mnesia function calls performed in the - <c>Fun</c> are mapped to the corresponding dirty - functions. This still involves logging, replication and + <marker id="async_dirty"></marker> + <p>Calls the <c>Fun</c> in a context that is not protected by + a transaction. The <c>Mnesia</c> function calls performed in + the <c>Fun</c> are mapped to the corresponding dirty + functions. This still involves logging, replication, and subscriptions, but there is no locking, local transaction storage, or commit protocols involved. Checkpoint retainers - and indices are updated, but they will be updated dirty. As - for normal mnesia:dirty_* operations, the operations are - performed semi-asynchronously. See - <c>mnesia:activity/4</c> and the Mnesia User's Guide for - more details. - </p> - <p>It is possible to manipulate the Mnesia tables without + and indexes are updated, but they are updated dirty. As + for normal <c>mnesia:dirty_*</c> operations, the operations + are performed semi-asynchronously. For details, see + <c>mnesia:activity/4</c> and the User's Guide.</p> + <p>The <c>Mnesia</c> tables can be manipulated without using transactions. This has some serious disadvantages, but - is considerably faster since the transaction manager is not + is considerably faster, as the transaction manager is not involved and no locks are set. A dirty operation does, - however, guarantee a certain level of consistency and it is - not possible for the dirty operations to return garbled - records. All dirty operations provide location transparency - to the programmer and a program does not have to be aware of - the whereabouts of a certain table in order to function. - </p> - <p><em>Note:</em>It is more than 10 times more efficient to read records dirty - than within a transaction. - </p> - <p>Depending on the application, it may be a good idea to use + however, guarantee a certain level of consistency, and + the dirty operations cannot return garbled records. + All dirty operations provide location transparency + to the programmer, and a program does not have to be aware + of the whereabouts of a certain table to function.</p> + <p>Notice that it is more than ten times more efficient to + read records dirty than within a transaction.</p> + <p>Depending on the application, it can be a good idea to use the dirty functions for certain operations. Almost all - Mnesia functions which can be called within transactions - have a dirty equivalent which is much more - efficient. However, it must be noted that it is possible for - the database to be left in an inconsistent state if dirty - operations are used to update it. Dirty operations should - only be used for performance reasons when it is absolutely - necessary. </p> - <p><em>Note:</em> Calling (nesting) a <c>mnesia:[a]sync_dirty</c> - inside a transaction context will inherit the transaction semantics. - </p> + <c>Mnesia</c> functions that can be called within + transactions have a dirty equivalent, which is much more + efficient.</p> + <p>However, notice that there is a risk that the database can + be left in an inconsistent state if dirty operations are + used to update it. Dirty operations are only to be used + for performance reasons when it is absolutely necessary.</p> + <p>Notice that calling (nesting) <c>mnesia:[a]sync_dirty</c> + inside a transaction-context inherits the transaction + semantics.</p> </desc> </func> <func> <name>backup(Opaque [, BackupMod]) -> ok | {error,Reason}</name> - <fsummary>Back up all tables in the database.</fsummary> + <fsummary>Backs up all tables in the database.</fsummary> <desc> - <p>Activates a new checkpoint covering all Mnesia tables, - including the schema, with maximum degree of redundancy and - performs a backup using <c>backup_checkpoint/2/3</c>. The + <marker id="backup"></marker> + <p>Activates a new checkpoint covering all <c>Mnesia</c> tables, + including the schema, with maximum degree of redundancy, and + performs a backup using <c>backup_checkpoint/2/3</c>. The default value of the backup callback module <c>BackupMod</c> is obtained by <c>mnesia:system_info(backup_module)</c>.</p> </desc> </func> <func> - <name>backup_checkpoint(Name, Opaque [, BackupMod]) -> ok | {error,Reason}</name> - <fsummary>Back up all tables in a checkpoint.</fsummary> + <name>backup_checkpoint(Name, Opaque [, BackupMod]) -> ok | {error,Reason}</name> + <fsummary>Backs up all tables in a checkpoint.</fsummary> <desc> - <p>The tables are backed up to external media using the backup + <marker id="backup_checkpoint"></marker> + <p>The tables are backed up to external media using backup module <c>BackupMod</c>. Tables with the local contents property are backed up as they exist on the current - node. <c>BackupMod</c> is the default backup callback + node. <c>BackupMod</c> is the default backup callback module obtained by - <c>mnesia:system_info(backup_module)</c>. See the User's - Guide about the exact callback interface (the - <c>mnesia_backup behavior</c>).</p> + <c>mnesia:system_info(backup_module)</c>. For information + about the exact callback interface (the + <c>mnesia_backup behavior</c>), see the User's Guide.</p> </desc> </func> <func> <name>change_config(Config, Value) -> {error, Reason} | {ok, ReturnValue}</name> - <fsummary>Change a configuration parameter.</fsummary> + <fsummary>Changes a configuration parameter.</fsummary> <desc> - <p>The <c>Config</c> should be an atom of the following - configuration parameters: </p> + <marker id="change_config"></marker> + <p><c>Config</c> is to be an atom of the following + configuration parameters:</p> <taglist> <tag><c>extra_db_nodes</c></tag> <item> - <p><c>Value</c> is a list of nodes which Mnesia should try to connect to. - The <c>ReturnValue</c> will be those nodes in - <c>Value</c> that Mnesia are connected to. - <br></br> -Note: This function shall only be used to connect to newly started ram nodes - (N.D.R.S.N.) with an empty schema. If for example it is used after the network - have been partitioned it may lead to inconsistent tables. - <br></br> -Note: Mnesia may be connected to other nodes than those - returned in <c>ReturnValue</c>.</p> + <p><c>Value</c> is a list of nodes that <c>Mnesia</c> + is to try to connect to. <c>ReturnValue</c> is those + nodes in <c>Value</c> that <c>Mnesia</c> is connected + to.</p> + <p>Notice that this function must only be used to connect + to newly started RAM nodes (N.D.R.S.N.) with an empty + schema. If, for example, this function is used after + the network has been partitioned, it can lead to + inconsistent tables.</p> + <p>Notice that <c>Mnesia</c> can be connected to other + nodes than those returned in <c>ReturnValue</c>.</p> </item> <tag><c>dc_dump_limit</c></tag> <item> - <p><c>Value</c> is a number. See description in - <c>Configuration Parameters</c> below. - The <c>ReturnValue</c> is the new value. Note this configuration parameter - is not persistent, it will be lost when mnesia stopped.</p> + <p><c>Value</c> is a number. See the description in + <seealso marker="#configuration_parameters">Section + Configuration Parameters</seealso>. <c>ReturnValue</c> + is the new value. Notice that this configuration + parameter is not persistent. It is lost when + <c>Mnesia</c> has stopped.</p> </item> </taglist> </desc> </func> <func> <name>change_table_access_mode(Tab, AccessMode) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Change the access mode for the table.</fsummary> - <desc> - <p>The <c>AcccessMode</c> is by default the atom - <c>read_write</c> but it may also be set to the atom - <c>read_only</c>. If the <c>AccessMode</c> is set to - <c>read_only</c>, it means that it is not possible to perform - updates to the table. At startup Mnesia always loads + <fsummary>Changes the access mode for the table.</fsummary> + <desc> + <marker id="change_table_access_mode"></marker> + <p><c>AcccessMode</c> is by default the atom + <c>read_write</c> but it can also be set to the atom + <c>read_only</c>. If <c>AccessMode</c> is set to + <c>read_only</c>, updates to the table cannot be + performed. At startup, <c>Mnesia</c> always loads <c>read_only</c> tables locally regardless of when and if - Mnesia was terminated on other nodes.</p> + <c>Mnesia</c> is terminated on other nodes.</p> </desc> </func> <func> <name>change_table_copy_type(Tab, Node, To) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Change the storage type of a table.</fsummary> + <fsummary>Changes the storage type of a table.</fsummary> <desc> + <marker id="change_table_copy_type"></marker> <p>For example:</p> <code type="none"> -mnesia:change_table_copy_type(person, node(), disc_copies) - </code> - <p>Transforms our <c>person</c> table from a RAM table into - a disc based table at <c>Node</c>. - </p> - <p>This function can also be used to change the storage type of - the table named <c>schema</c>. The schema table can only - have <c>ram_copies</c> or <c>disc_copies</c> as the storage type. If the - storage type of the schema is <c>ram_copies</c>, no other table - can be disc resident on that node.</p> +mnesia:change_table_copy_type(person, node(), disc_copies)</code> + <p>Transforms the <c>person</c> table from a RAM table into + a disc-based table at <c>Node</c>.</p> + <p>This function can also be used to change the storage type + of the table named <c>schema</c>. The schema table can only + have <c>ram_copies</c> or <c>disc_copies</c> as the storage + type. If the storage type of the schema is <c>ram_copies</c>, + no other table can be disc-resident on that node.</p> </desc> </func> <func> <name>change_table_load_order(Tab, LoadOrder) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Change the load order priority for the table.</fsummary> + <fsummary>Changes the load order priority for the table.</fsummary> <desc> + <marker id="change_table_load_order"></marker> <p>The <c>LoadOrder</c> priority is by default <c>0</c> (zero) - but may be set to any integer. The tables with the highest - <c>LoadOrder</c> priority will be loaded first at startup.</p> + but can be set to any integer. The tables with the highest + <c>LoadOrder</c> priority are loaded first at startup.</p> </desc> </func> <func> <name>change_table_majority(Tab, Majority) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Change the majority check setting for the table.</fsummary> + <fsummary>Changes the majority check setting for the table.</fsummary> <desc> - <p><c>Majority</c> must be a boolean; the default is <c>false</c>. - When <c>true</c>, a majority of the table's replicas must be available - for an update to succeed. When used on fragmented tables, <c>Tab</c> - must be the name base table. Directly changing the majority setting on - individual fragments is not allowed.</p> + <p><c>Majority</c> must be a boolean. Default is <c>false</c>. + When <c>true</c>, a majority of the table replicas must be + available for an update to succeed. When used on fragmented + tables, <c>Tab</c> must be the base table name. Directly + changing the majority setting on individual fragments is + not allowed.</p> </desc> </func> <func> <name>clear_table(Tab) -> {aborted, R} | {atomic, ok}</name> <fsummary>Deletes all entries in a table.</fsummary> <desc> + <marker id="clear_table"></marker> <p>Deletes all entries in the table <c>Tab</c>.</p> </desc> </func> <func> <name>create_schema(DiscNodes) -> ok | {error,Reason}</name> - <fsummary>Create a brand new schema on the specified nodes.</fsummary> + <fsummary>Creates a new schema on the specified nodes.</fsummary> <desc> + <marker id="create_schema"></marker> <p>Creates a new database on disc. Various files are - created in the local Mnesia directory of each node. Note - that the directory must be unique for each node. Two nodes - may never share the same directory. If possible, use a local - disc device in order to improve performance.</p> + created in the local <c>Mnesia</c> directory of each node. + Notice that the directory must be unique for each node. + Two nodes must never share the same directory. If possible, + use a local disc device to improve performance.</p> <p><c>mnesia:create_schema/1</c> fails if any of the Erlang nodes given as <c>DiscNodes</c> are not alive, if - Mnesia is running on anyone of the nodes, or if anyone of - the nodes already has a schema. Use + <c>Mnesia</c> is running on any of the nodes, or if any + of the nodes already have a schema. Use <c>mnesia:delete_schema/1</c> to get rid of old faulty - schemas. - </p> - <p><em>Note:</em> Only nodes with disc should be - included in <c>DiscNodes</c>. Disc-less nodes, that is nodes - where all tables including the schema only resides in RAM, - may not be included.</p> + schemas.</p> + <p>Notice that only nodes with disc are to be included in + <c>DiscNodes</c>. Disc-less nodes, that is, nodes where + all tables including the schema only resides in RAM, + must not be included.</p> </desc> </func> <func> <name>create_table(Name, TabDef) -> {atomic, ok} | {aborted, Reason}</name> - <fsummary>Create a Mnesia table called <c>Name</c>with properties as described by the argument <c>TabDef</c>.</fsummary> + <fsummary>Creates a <c>Mnesia</c> table called <c>Name</c>with properties as described by argument <c>TabDef</c>.</fsummary> <desc> - <p>This function creates a Mnesia table called <c>Name</c> - according to the - argument <c>TabDef</c>. This list must be a list of - <c>{Item, Value}</c> tuples, where the following values are - allowed:</p> + <marker id="create_table"></marker> + <p>Creates a <c>Mnesia</c> table called + <c>Name</c> according to argument <c>TabDef</c>. This + list must be a list of <c>{Item, Value}</c> tuples, + where the following values are allowed:</p> <list type="bulleted"> <item> - <p><c>{access_mode, Atom}</c>. The access mode is by - default the atom <c>read_write</c> but it may also be - set to the atom <c>read_only</c>. If the - <c>AccessMode</c> is set to <c>read_only</c>, it means - that it is not possible to perform updates to the table. - </p> - <p>At startup Mnesia always loads <c>read_only</c> tables - locally regardless of when and if Mnesia was terminated - on other nodes. This argument returns the access mode of - the table. The access mode may either be read_only or - read_write. - </p> - </item> - <item> - <p><c>{attributes, AtomList}</c> a list of the + <p><c>{access_mode, Atom}</c>. The access mode is by + default the atom <c>read_write</c> but it can also be + set to the atom <c>read_only</c>. If <c>AccessMode</c> + is set to <c>read_only</c>, updates to the table + cannot be performed.</p> + <p>At startup, <c>Mnesia</c> always loads <c>read_only</c> + table locally regardless of when and if <c>Mnesia</c> is + terminated on other nodes. This argument returns the + access mode of the table. The access mode can be + <c>read_only</c> or <c>read_write</c>.</p> + </item> + <item> + <p><c>{attributes, AtomList}</c> is a list of the attribute names for the records that are supposed to - populate the table. The default value is <c>[key, val]</c>. The table must have at least one extra - attribute in addition to the key. - </p> - <p>When accessing single attributes in a record, it is not - necessary, or even recommended, to hard code any - attribute names as atoms. Use the construct - <c>record_info(fields, RecordName)</c> instead. It can be - used for records of type <c>RecordName</c></p> + populate the table. Default is <c>[key, val]</c>. + The table must at least have one extra attribute in + addition to the key.</p> + <p>When accessing single attributes in a record, it is + not necessary, or even recommended, to hard code any + attribute names as atoms. Use construct + <c>record_info(fields, RecordName)</c> instead. It can + be used for records of type <c>RecordName</c>.</p> </item> <item> <p><c>{disc_copies, Nodelist}</c>, where <c>Nodelist</c> is a list of the nodes where this table - is supposed to have disc copies. If a table replica is + is supposed to have disc copies. If a table replica is of type <c>disc_copies</c>, all write operations on this - particular replica of the table are written to disc as - well as to the RAM copy of the table. - </p> - <p>It is possible - to have a replicated table of type <c>disc_copies</c> - on one node, and another type on another node. The - default value is <c>[]</c></p> + particular replica of the table are written to disc and + to the RAM copy of the table.</p> + <p>It is possible to have a replicated table of type + <c>disc_copies</c> on one node and another type on + another node. Default is <c>[]</c>.</p> </item> <item> <p><c>{disc_only_copies, Nodelist}</c>, where <c>Nodelist</c> is a list of the nodes where this table - is supposed to have <c>disc_only_copies</c>. A disc only - table replica is kept on disc only and unlike the other - replica types, the contents of the replica will not + is supposed to have <c>disc_only_copies</c>. A disc only + table replica is kept on disc only and unlike the other + replica types, the contents of the replica do not reside in RAM. These replicas are considerably slower - than replicas held in RAM. - </p> + than replicas held in RAM.</p> </item> <item> <p><c>{index, Intlist}</c>, where <c>Intlist</c> is a list of attribute names (atoms) or - record fields for which Mnesia shall build and maintain - an extra index table. The <c>qlc</c> query compiler may - or may not utilize any additional indices while - processing queries on a table. - </p> + record fields for which <c>Mnesia</c> is to build and + maintain an extra index table. The <c>qlc</c> query + compiler <em>may</em> be able to optimize queries + if there are indexes available.</p> </item> <item> <p><c>{load_order, Integer}</c>. The load order - priority is by default <c>0</c> (zero) but may be set to - any integer. The tables with the highest load order - priority will be loaded first at startup. - </p> - </item> - <item> - <p><c>{majority, Flag}</c>, where <c>Flag</c> must be a boolean. - If <c>true</c>, any (non-dirty) update to the table will abort unless - a majority of the table's replicas are available for the commit. - When used on a fragmented table, all fragments will be given - the same majority setting. - </p> - </item> + priority is by default <c>0</c> (zero) but can be set + to any integer. The tables with the highest load order + priority are loaded first at startup.</p> + </item> + <item> + <p><c>{majority, Flag}</c>, where <c>Flag</c> must be a + boolean. If <c>true</c>, any (non-dirty) update to the + table is aborted, unless a majority of the table + replicas are available for the commit. When used on a + fragmented table, all fragments are given the same + the same majority setting.</p> + </item> <item> <p><c>{ram_copies, Nodelist}</c>, where <c>Nodelist</c> is a list of the nodes where this table - is supposed to have RAM copies. A table replica of type - <c>ram_copies</c> is obviously not written to disc on a - per transaction basis. It is possible to dump - <c>ram_copies</c> replicas to disc with the function - <c>mnesia:dump_tables(Tabs)</c>. The default value for - this attribute is <c>[node()]</c>. - </p> + is supposed to have RAM copies. A table replica of type + <c>ram_copies</c> is not written to disc on a per + transaction basis. <c>ram_copies</c> replicas can be + dumped to disc with the function + <c>mnesia:dump_tables(Tabs)</c>. Default value for + this attribute is <c>[node()]</c>.</p> </item> <item> <p><c>{record_name, Name}</c>, where <c>Name</c> must - be an atom. All records, stored in the table, must have + be an atom. All records stored in the table must have this name as the first element. It defaults to the same - name as the name of the table. - </p> - </item> - <item> - <p><c>{snmp, SnmpStruct}</c>. See - <c>mnesia:snmp_open_table/2</c> for a description of - <c>SnmpStruct</c>. If this attribute is present in the - <c>ArgList</c> to <c>mnesia:create_table/2</c>, the - table is immediately accessible by means of the Simple - Network Management Protocol (SNMP). This means that - applications which use SNMP to manipulate and control - the system can be designed easily, since Mnesia provides - a direct mapping between the logical tables that make up - an SNMP control application and the physical data which - makes up a Mnesia table. - </p> - </item> - <item> - <p><c>{storage_properties, [{Backend, Properties}]</c>. - Forwards additional properties to the backend storage. - <c>Backend</c> can currently be <c>ets</c> or <c>dets</c> and - <c>Properties</c> is a list of options sent to the backend storage - during table creation. <c>Properties</c> may not contain properties - already used by mnesia such as <c>type</c> or <c>named_table</c>. - </p> - <p>For example:</p> - <code type="none"> + name as the table name.</p> + </item> + <item> + <p><c>{snmp, SnmpStruct}</c>. For a description of + <c>SnmpStruct</c>, see <c>mnesia:snmp_open_table/2</c>. + If this attribute is present in <c>ArgList</c> to + <c>mnesia:create_table/2</c>, the table is immediately + accessible by SNMP. Therefore applications that use + SNMP to manipulate and control the system can be + designed easily, since <c>Mnesia</c> provides a + direct mapping between the logical tables that make up + an SNMP control application and the physical data that + makes up a <c>Mnesia</c> table.</p> + </item> + <item> + <p><c>{storage_properties, [{Backend, Properties}]</c> + forwards more properties to the back end storage. + <c>Backend</c> can currently be <c>ets</c> or <c>dets</c>. + <c>Properties</c> is a list of options sent to the + back end storage during table creation. <c>Properties</c> + cannot contain properties already used by <c>Mnesia</c>, + such as <c>type</c> or <c>named_table</c>.</p> + <p>For example:</p> + <code type="none"> mnesia:create_table(table, [{ram_copies, [node()]}, {disc_only_copies, nodes()}, - {storage_properties, - [{ets, [compressed]}, {dets, [{auto_save, 5000}]} ]}]) - </code> + {storage_properties, + [{ets, [compressed]}, {dets, [{auto_save, 5000}]} ]}])</code> </item> <item> <p><c>{type, Type}</c>, where <c>Type</c> must be - either of the atoms <c>set</c>, <c>ordered_set</c> or - <c>bag</c>. The default value is <c>set</c>. In a - <c>set</c> all records have unique keys and in a - <c>bag</c> several records may have the same key, but + either of the atoms <c>set</c>, <c>ordered_set</c>, or + <c>bag</c>. Default is <c>set</c>. In a + <c>set</c>, all records have unique keys. In a + <c>bag</c>, several records can have the same key, but the record content is unique. If a non-unique record is - stored the old, conflicting record(s) will simply be - overwritten. Note: currently 'ordered_set' - is not supported for 'disc_only_copies'. - </p> + stored, the old conflicting records are overwritten.</p> + <p>Notice that currently <c>ordered_set</c> is not + supported for <c>disc_only_copies</c>.</p> </item> <item> - <p><c>{local_content, Bool}</c>, where <c>Bool</c> must be - either <c>true</c> or <c>false</c>. The default value is <c>false</c>.</p> + <p><c>{local_content, Bool}</c>, where <c>Bool</c> is + <c>true</c> or <c>false</c>. Default is <c>false</c>.</p> </item> </list> - <p>For example, the following call creates the <c>person</c> table - previously defined and replicates it on 2 nodes: - </p> + <p>For example, the following call creates the <c>person</c> + table (defined earlier) and replicates it on two nodes:</p> <code type="none"> -mnesia:create_table(person, +mnesia:create_table(person, [{ram_copies, [N1, N2]}, - {attributes, record_info(fields,person)}]). - </code> - <p>If it was required that Mnesia build and maintain an extra index - table on the <c>address</c> attribute of all the <c>person</c> - records that are inserted in the table, the following code would be issued: - </p> + {attributes, record_info(fields, person)}]).</code> + <p>If it is required that <c>Mnesia</c> must build and + maintain an extra index table on attribute <c>address</c> + of all the <c>person</c> records that are inserted in the + table, the following code would be issued:</p> <code type="none"> mnesia:create_table(person, [{ram_copies, [N1, N2]}, {index, [address]}, - {attributes, record_info(fields,person)}]). - </code> - <p>The specification of <c>index</c> and <c>attributes</c> may be - hard coded as <c>{index, [4]}</c> and - <c>{attributes, [name, age, address, salary, children]}</c> - respectively. - </p> + {attributes, record_info(fields, person)}]). + </code> + <p>The specification of <c>index</c> and <c>attributes</c> + can be hard-coded as <c>{index, [2]}</c> and + <c>{attributes, [name, age, address, salary, children]}</c>, + respectively.</p> <p><c>mnesia:create_table/2</c> writes records into the - <c>schema</c> table. This function, as well as all other + table <c>schema</c>. This function, and all other schema manipulation functions, are implemented with the normal transaction management system. This guarantees that schema updates are performed on all nodes in an atomic @@ -877,163 +799,169 @@ mnesia:create_table(person, </func> <func> <name>deactivate_checkpoint(Name) -> ok | {error, Reason}</name> - <fsummary>Deactivate a checkpoint.</fsummary> + <fsummary>Deactivates a checkpoint.</fsummary> <desc> + <marker id="deactivate_checkpoint"></marker> <p>The checkpoint is automatically deactivated when some of - the tables involved have no retainer attached to them. This may - happen when nodes go down or when a replica is deleted. - Checkpoints will also be deactivated with this function. + the tables involved have no retainer attached to them. This + can occur when nodes go down or when a replica is deleted. + Checkpoints are also deactivated with this function. <c>Name</c> is the name of an active checkpoint.</p> </desc> </func> <func> <name>del_table_copy(Tab, Node) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Delete the replica of table <c>Tab</c>at node <c>Node</c>.</fsummary> + <fsummary>Deletes the replica of table <c>Tab</c> at node <c>Node</c>.</fsummary> <desc> + <marker id="del_table_copy"></marker> <p>Deletes the replica of table <c>Tab</c> at node <c>Node</c>. When the last replica is deleted with this - function, the table disappears entirely. - </p> - <p>This function may also be used to delete a replica of - the table named <c>schema</c>. Then the mnesia node will be removed. - Note: Mnesia must be stopped on the node first.</p> + function, the table disappears entirely.</p> + <p>This function can also be used to delete a replica of + the table named <c>schema</c>. The <c>Mnesia</c> node is + then removed. Notice that <c>Mnesia</c> must be + stopped on the node first.</p> </desc> </func> <func> <name>del_table_index(Tab, AttrName) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Delete an index in a table. </fsummary> + <fsummary>Deletes an index in a table.</fsummary> <desc> - <p>This function deletes the index on attribute with name + <marker id="del_table_index"></marker> + <p>Deletes the index on attribute with name <c>AttrName</c> in a table.</p> </desc> </func> <func> - <name>delete({Tab, Key}) -> transaction abort | ok </name> - <fsummary>Delete all records in table <c>Tab</c>with the key <c>Key</c>.</fsummary> + <name>delete({Tab, Key}) -> transaction abort | ok</name> + <fsummary>Deletes all records in table <c>Tab</c> with the key <c>Key</c>.</fsummary> <desc> - <p>Invokes <c>mnesia:delete(Tab, Key, write)</c></p> + <marker id="delete_2"></marker> + <p>Calls <c>mnesia:delete(Tab, Key, write)</c>.</p> </desc> </func> <func> - <name>delete(Tab, Key, LockKind) -> transaction abort | ok </name> - <fsummary>Delete all records in table <c>Tab</c>with the key <c>Key</c>.</fsummary> + <name>delete(Tab, Key, LockKind) -> transaction abort | ok</name> + <fsummary>Deletes all records in table <c>Tab</c>with the key <c>Key</c>.</fsummary> <desc> + <marker id="delete_3"></marker> <p>Deletes all records in table <c>Tab</c> with the key - <c>Key</c>. - </p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires a lock of type <c>LockKind</c> in the - record. Currently the lock types <c>write</c> and + <c>Key</c>.</p> + <p>The semantics of this function is context-sensitive. + For details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a lock of type + <c>LockKind</c> in the record. + Currently, the lock types <c>write</c> and <c>sticky_write</c> are supported.</p> </desc> </func> <func> - <name>delete_object(Record) -> transaction abort | ok </name> - <fsummary>Delete a record</fsummary> + <name>delete_object(Record) -> transaction abort | ok</name> + <fsummary>Delete a record.</fsummary> <desc> - <p>Invokes <c>mnesia:delete_object(Tab, Record, write)</c> where + <marker id="delete_object_1"></marker> + <p>Calls <c>mnesia:delete_object(Tab, Record, write)</c>, where <c>Tab</c> is <c>element(1, Record)</c>.</p> </desc> </func> <func> - <name>delete_object(Tab, Record, LockKind) -> transaction abort | ok </name> - <fsummary>Delete a record</fsummary> + <name>delete_object(Tab, Record, LockKind) -> transaction abort | ok</name> + <fsummary>Deletes a record.</fsummary> <desc> - <p>If a table is of type <c>bag</c>, we may sometimes - want to delete only some of the records with a certain - key. This can be done with the <c>delete_object/3</c> - function. A complete record must be supplied to this - function. - </p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires a lock of type <c>LockKind</c> on the - record. Currently the lock types <c>write</c> and + <marker id="delete_object_3"></marker> + <p>If a table is of type <c>bag</c>, it can sometimes be + needed to delete only some of the records with a certain + key. This can be done with the function <c>delete_object/3</c>. + A complete record must be supplied to this function.</p> + <p>The semantics of this function is context-sensitive. + For details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a lock of type + <c>LockKind</c> on the record. + Currently, the lock types <c>write</c> and <c>sticky_write</c> are supported.</p> </desc> </func> <func> <name>delete_schema(DiscNodes) -> ok | {error,Reason}</name> - <fsummary>Delete the schema on the given nodes</fsummary> + <fsummary>Deletes the schema on the given nodes.</fsummary> <desc> + <marker id="delete_schema"></marker> <p>Deletes a database created with <c>mnesia:create_schema/1</c>. <c>mnesia:delete_schema/1</c> fails if any of the Erlang - nodes given as <c>DiscNodes</c> is not alive, or if Mnesia - is running on any of the nodes. - </p> - <p>After the database has been deleted, it may still be - possible to start Mnesia as a disc-less node. This depends on - how the configuration parameter <c>schema_location</c> is set. - </p> + nodes given as <c>DiscNodes</c> are not alive, or if + <c>Mnesia</c> is running on any of the nodes.</p> + <p>After the database is deleted, it can still be possible + to start <c>Mnesia</c> as a disc-less node. This depends + on how configuration parameter <c>schema_location</c> is + set.</p> <warning> - <p>This function must be used with extreme - caution since it makes existing persistent data - obsolete. Think twice before using it. </p> + <p>Use this function with extreme caution, as it makes + existing persistent data obsolete. Think twice before + using it.</p> </warning> </desc> </func> <func> - <name>delete_table(Tab) -> {aborted, Reason} | {atomic, ok} </name> - <fsummary>Delete permanently all replicas of table <c>Tab</c>.</fsummary> + <name>delete_table(Tab) -> {aborted, Reason} | {atomic, ok}</name> + <fsummary>Deletes permanently all replicas of table <c>Tab</c>.</fsummary> <desc> + <marker id="delete_table"></marker> <p>Permanently deletes all replicas of table <c>Tab</c>.</p> </desc> </func> <func> - <name>dirty_all_keys(Tab) -> KeyList | exit({aborted, Reason}).</name> + <name>dirty_all_keys(Tab) -> KeyList | exit({aborted, Reason})</name> <fsummary>Dirty search for all record keys in table.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:all_keys/1</c> function.</p> + <marker id="delete_all_keys"></marker> + <p>Dirty equivalent of the function <c>mnesia:all_keys/1</c>.</p> </desc> </func> <func> - <name>dirty_delete({Tab, Key}) -> ok | exit({aborted, Reason}) </name> + <name>dirty_delete({Tab, Key}) -> ok | exit({aborted, Reason})</name> <fsummary>Dirty delete of a record.</fsummary> <desc> - <p>Invokes <c>mnesia:dirty_delete(Tab, Key)</c>.</p> + <marker id="dirty_delete"></marker> + <p>Calls <c>mnesia:dirty_delete(Tab, Key)</c>.</p> </desc> </func> <func> - <name>dirty_delete(Tab, Key) -> ok | exit({aborted, Reason}) </name> - <fsummary>Dirty delete of a record. </fsummary> + <name>dirty_delete(Tab, Key) -> ok | exit({aborted, Reason})</name> + <fsummary>Dirty delete of a record.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:delete/3</c> function.</p> + <p>Dirty equivalent of the function <c>mnesia:delete/3</c>.</p> </desc> </func> <func> - <name>dirty_delete_object(Record) </name> + <name>dirty_delete_object(Record)</name> <fsummary>Dirty delete of a record.</fsummary> <desc> - <p>Invokes <c>mnesia:dirty_delete_object(Tab, Record)</c> + <marker id="dirty_delete_object_1"></marker> + <p>Calls <c>mnesia:dirty_delete_object(Tab, Record)</c>, where <c>Tab</c> is <c>element(1, Record)</c>.</p> </desc> </func> <func> - <name>dirty_delete_object(Tab, Record) </name> - <fsummary>Dirty delete of a record. </fsummary> + <name>dirty_delete_object(Tab, Record)</name> + <fsummary>Dirty delete of a record.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:delete_object/3</c> function.</p> + <p>Dirty equivalent of the function <c>mnesia:delete_object/3</c>.</p> </desc> </func> <func> - <name>dirty_first(Tab) -> Key | exit({aborted, Reason}) </name> - <fsummary>Return the key for the first record in a table.</fsummary> + <name>dirty_first(Tab) -> Key | exit({aborted, Reason})</name> + <fsummary>Returns the key for the first record in a table.</fsummary> <desc> + <marker id="dirty_first"></marker> <p>Records in <c>set</c> or <c>bag</c> tables are not ordered. - However, there - is an ordering of the records which is not known - to the user. Accordingly, it is possible to traverse a table by means - of this function in conjunction with the <c>mnesia:dirty_next/2</c> - function. + However, there is an ordering of the records that is unknown + to the user. Therefore, a table can be traversed by this + function with the function <c>mnesia:dirty_next/2</c>. </p> - <p>If there are no records at all in the table, this function - returns the atom <c>'$end_of_table'</c>. For this reason, it - is highly undesirable, but not disallowed, to use this atom + <p>If there are no records in the table, this function + returns the atom <c>'$end_of_table'</c>. It is therefore + highly undesirable, but not disallowed, to use this atom as the key for any user records.</p> </desc> </func> @@ -1041,74 +969,82 @@ mnesia:create_table(person, <name>dirty_index_match_object(Pattern, Pos)</name> <fsummary>Dirty pattern match using index.</fsummary> <desc> - <p>Invokes <c>mnesia:dirty_index_match_object(Tab, Pattern, Pos)</c> where <c>Tab</c> is <c>element(1, Pattern)</c>.</p> + <marker id="dirty_index_match_object_2"></marker> + <p>Starts + <c>mnesia:dirty_index_match_object(Tab, Pattern, Pos)</c>, + where <c>Tab</c> is <c>element(1, Pattern)</c>.</p> </desc> </func> <func> <name>dirty_index_match_object(Tab, Pattern, Pos)</name> <fsummary>Dirty pattern match using index.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:index_match_object/4</c> function.</p> + <p>Dirty equivalent of the function + <c>mnesia:index_match_object/4</c>.</p> </desc> </func> <func> <name>dirty_index_read(Tab, SecondaryKey, Pos)</name> <fsummary>Dirty read using index.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:index_read/3</c> function.</p> + <marker id="dirty_index_read"></marker> + <p>Dirty equivalent of the function + <c>mnesia:index_read/3</c>.</p> </desc> </func> <func> - <name>dirty_last(Tab) -> Key | exit({aborted, Reason}) </name> - <fsummary>Return the key for the last record in a table.</fsummary> + <name>dirty_last(Tab) -> Key | exit({aborted, Reason})</name> + <fsummary>Returns the key for the last record in a table.</fsummary> <desc> - <p>This function works exactly like - <c>mnesia:dirty_first/1</c> but returns the last object in - Erlang term order for the <c>ordered_set</c> table type. For - all other table types, <c>mnesia:dirty_first/1</c> and + <marker id="dirty_last"></marker> + <p>Works exactly like <c>mnesia:dirty_first/1</c> but returns + the last object in Erlang term order for the <c>ordered_set</c> + table type. For all other table types, + <c>mnesia:dirty_first/1</c> and <c>mnesia:dirty_last/1</c> are synonyms.</p> </desc> </func> <func> - <name>dirty_match_object(Pattern) -> RecordList | exit({aborted, Reason}).</name> + <name>dirty_match_object(Pattern) -> RecordList | exit({aborted, Reason})</name> <fsummary>Dirty pattern match pattern.</fsummary> <desc> - <p>Invokes <c>mnesia:dirty_match_object(Tab, Pattern)</c> + <marker id="dirty_match_object_1"></marker> + <p>Calls <c>mnesia:dirty_match_object(Tab, Pattern)</c>, where <c>Tab</c> is <c>element(1, Pattern)</c>.</p> </desc> </func> <func> - <name>dirty_match_object(Tab, Pattern) -> RecordList | exit({aborted, Reason}).</name> + <name>dirty_match_object(Tab, Pattern) -> RecordList | exit({aborted, Reason})</name> <fsummary>Dirty pattern match pattern.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:match_object/3</c> function.</p> + <p>Dirty equivalent of the function + <c>mnesia:match_object/3</c>.</p> </desc> </func> <func> - <name>dirty_next(Tab, Key) -> Key | exit({aborted, Reason}) </name> - <fsummary>Return the next key in a table. </fsummary> + <name>dirty_next(Tab, Key) -> Key | exit({aborted, Reason})</name> + <fsummary>Return the next key in a table.</fsummary> <desc> - <p>This function makes it possible to traverse a table - and perform operations on all records in the table. When - the end of the table is reached, the special key + <marker id="dirty_next"></marker> + <p>Traverses a table and + performs operations on all records in the table. + When the end of the table is reached, the special key <c>'$end_of_table'</c> is returned. Otherwise, the function - returns a key which can be used to read the actual record.The + returns a key that can be used to read the actual record. The behavior is undefined if another Erlang process performs write operations on the table while it is being traversed with the - <c>mnesia:dirty_next/2</c> function.</p> + function <c>mnesia:dirty_next/2</c>.</p> </desc> </func> <func> - <name>dirty_prev(Tab, Key) -> Key | exit({aborted, Reason}) </name> - <fsummary>Return the previous key in a table. </fsummary> + <name>dirty_prev(Tab, Key) -> Key | exit({aborted, Reason})</name> + <fsummary>Returns the previous key in a table.</fsummary> <desc> - <p>This function works exactly like - <c>mnesia:dirty_next/2</c> but returns the previous object in - Erlang term order for the ordered_set table type. For - all other table types, <c>mnesia:dirty_next/2</c> and + <marker id="dirty_prev"></marker> + <p>Works exactly like <c>mnesia:dirty_next/2</c> but returns + the previous object in Erlang term order for the + <c>ordered_set</c> table type. For all other table types, + <c>mnesia:dirty_next/2</c> and <c>mnesia:dirty_prev/2</c> are synonyms.</p> </desc> </func> @@ -1116,33 +1052,34 @@ mnesia:create_table(person, <name>dirty_read({Tab, Key}) -> ValueList | exit({aborted, Reason}</name> <fsummary>Dirty read of records.</fsummary> <desc> - <p>Invokes <c>mnesia:dirty_read(Tab, Key)</c>.</p> + <marker id="dirty_read"></marker> + <p>Calls <c>mnesia:dirty_read(Tab, Key)</c>.</p> </desc> </func> <func> <name>dirty_read(Tab, Key) -> ValueList | exit({aborted, Reason}</name> <fsummary>Dirty read of records.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:read/3</c> function.</p> + <p>Dirty equivalent of the function <c>mnesia:read/3</c>.</p> </desc> </func> <func> <name>dirty_select(Tab, MatchSpec) -> ValueList | exit({aborted, Reason}</name> - <fsummary>Dirty match the objects in <c>Tab</c>against <c>MatchSpec</c>.</fsummary> + <fsummary>Dirty matches the objects in <c>Tab</c> against <c>MatchSpec</c>.</fsummary> <desc> - <p>This is the dirty equivalent of the - <c>mnesia:select/2</c> function.</p> + <marker id="dirty_select"></marker> + <p>Dirty equivalent of the function <c>mnesia:select/2</c>.</p> </desc> </func> <func> <name>dirty_slot(Tab, Slot) -> RecordList | exit({aborted, Reason})</name> - <fsummary>Return the list of records that are associated with Slot in a table.</fsummary> + <fsummary>Returns the list of records that are associated with <c>Slot</c> in a table.</fsummary> <desc> - <p>This function can be used to traverse a table in a - manner similar to the <c>mnesia:dirty_next/2</c> function. - A table has a number of slots which range from 0 (zero) to some - unknown upper bound. The function + <marker id="dirty_slot"></marker> + <p>Traverses a table in a + manner similar to the function <c>mnesia:dirty_next/2</c>. + A table has a number of slots that range from 0 (zero) to + an unknown upper bound. The function <c>mnesia:dirty_slot/2</c> returns the special atom <c>'$end_of_table'</c> when the end of the table is reached. The behavior of this function is undefined if a write @@ -1154,40 +1091,45 @@ mnesia:create_table(person, <name>dirty_update_counter({Tab, Key}, Incr) -> NewVal | exit({aborted, Reason})</name> <fsummary>Dirty update of a counter record.</fsummary> <desc> - <p>Invokes <c>mnesia:dirty_update_counter(Tab, Key, Incr)</c>.</p> + <marker id="dirty_update_counter"></marker> + <p>Calls <c>mnesia:dirty_update_counter(Tab, Key, Incr)</c>.</p> </desc> </func> <func> <name>dirty_update_counter(Tab, Key, Incr) -> NewVal | exit({aborted, Reason})</name> <fsummary>Dirty update of a counter record.</fsummary> <desc> - <p>There are no special counter records in Mnesia. However, + <p><c>Mnesia</c> has no special counter records. However, records of the form <c>{Tab, Key, Integer}</c> can be used - as (possibly disc resident) counters, when <c>Tab</c> is a - <c>set</c>. This function updates a counter with a - positive or negative number. However, counters can never become less + as (possibly disc-resident) counters when <c>Tab</c> is a + <c>set</c>. This function updates a counter with a positive + or negative number. However, counters can never become less than zero. There are two significant differences between this function and the action of first reading the record, performing the arithmetics, and then writing the record:</p> <list type="bulleted"> - <item>It is much more efficient</item> - <item><c>mnesia:dirty_update_counter/3</c> is - performed as an atomic operation despite the fact that it is not - protected by a transaction.</item> + <item>It is much more efficient. + </item> + <item><c>mnesia:dirty_update_counter/3</c> is performed + as an atomic operation although it is not protected + by a transaction. + </item> </list> <p>If two processes perform <c>mnesia:dirty_update_counter/3</c> - simultaneously, both updates will take effect without the + simultaneously, both updates take effect without the risk of losing one of the updates. The new value <c>NewVal</c> of the counter is returned.</p> - <p>If <c>Key</c> don't exits, a new record is created with the value - <c>Incr</c> if it is larger than 0, otherwise it is set to 0.</p> + <p>If <c>Key</c> do not exists, a new record is created with + value <c>Incr</c> if it is larger than 0, otherwise it is + set to 0.</p> </desc> </func> <func> <name>dirty_write(Record) -> ok | exit({aborted, Reason})</name> <fsummary>Dirty write of a record.</fsummary> <desc> - <p>Invokes <c>mnesia:dirty_write(Tab, Record)</c> + <marker id="dirty_write_1"></marker> + <p>Calls <c>mnesia:dirty_write(Tab, Record)</c>, where <c>Tab</c> is <c>element(1, Record)</c>.</p> </desc> </func> @@ -1195,623 +1137,607 @@ mnesia:create_table(person, <name>dirty_write(Tab, Record) -> ok | exit({aborted, Reason})</name> <fsummary>Dirty write of a record.</fsummary> <desc> - <p>This is the dirty equivalent of <c>mnesia:write/3</c>.</p> + <p>Dirty equivalent of the function <c>mnesia:write/3</c>.</p> </desc> </func> <func> <name>dump_log() -> dumped</name> - <fsummary>Perform a user initiated dump of the local log file.</fsummary> + <fsummary>Performs a user-initiated dump of the local log file.</fsummary> <desc> - <p>Performs a user initiated dump of the local log file. - This is usually not necessary since Mnesia, by default, - manages this automatically. - See configuration parameters - <seealso marker="#dump_log_time_threshold">dump_log_time_threshold</seealso> and - <seealso marker="#dump_log_write_threshold">dump_log_write_threshold</seealso>. - </p> + <marker id="dump_log"></marker> + <p>Performs a user-initiated dump of the local log file. + This is usually not necessary, as <c>Mnesia</c> by default + manages this automatically. See configuration parameters + <seealso marker="#dump_log_time_threshold">dump_log_time_threshold</seealso> + and + <seealso marker="#dump_log_write_threshold">dump_log_write_threshold</seealso>. + </p> </desc> </func> <func> <name>dump_tables(TabList) -> {atomic, ok} | {aborted, Reason}</name> - <fsummary>Dump all RAM tables to disc.</fsummary> + <fsummary>Dumps all RAM tables to disc.</fsummary> <desc> - <p>This function dumps a set of <c>ram_copies</c> tables + <marker id="dump_tables"></marker> + <p>Dumps a set of <c>ram_copies</c> tables to disc. The next time the system is started, these tables are initiated with the data found in the files that are the - result of this dump. None of the tables may have disc - resident replicas.</p> + result of this dump. None of the tables can have + disc-resident replicas.</p> </desc> </func> <func> - <name>dump_to_textfile(Filename) </name> - <fsummary>Dump local tables into a text file.</fsummary> + <name>dump_to_textfile(Filename)</name> + <fsummary>Dumps local tables into a text file.</fsummary> <desc> - <p>Dumps all local tables of a mnesia system into a text file - which can then be edited (by means of a normal text editor) - and then later be reloaded with + <marker id="dump_to_textfile"></marker> + <p>Dumps all local tables of a <c>Mnesia</c> system into a + text file, which can be edited (by a normal text editor) + and then be reloaded with <c>mnesia:load_textfile/1</c>. Only use this function for educational purposes. Use other functions to deal with real backups.</p> </desc> </func> <func> - <name>error_description(Error) -> String </name> - <fsummary>Return a string describing a particular Mnesia error.</fsummary> + <name>error_description(Error) -> String</name> + <fsummary>Returns a string describing a particular <c>Mnesia</c> error.</fsummary> <desc> - <p>All Mnesia transactions, including all the schema - update functions, either return the value <c>{atomic, Val}</c> or the tuple <c>{aborted, Reason}</c>. The - <c>Reason</c> can be either of the following atoms. The - <c>error_description/1</c> function returns a descriptive - string which describes the error. - </p> + <marker id="error_description"></marker> + <p>All <c>Mnesia</c> transactions, including all the schema + update functions, either return value <c>{atomic, Val}</c> + or the tuple <c>{aborted, Reason}</c>. <c>Reason</c> can + be either of the atoms in the following list. The + function <c>error_description/1</c> returns a descriptive + string that describes the error.</p> <list type="bulleted"> - <item> - <p><c>nested_transaction</c>. Nested transactions are - not allowed in this context. - </p> + <item><c>nested_transaction</c>. Nested transactions are + not allowed in this context. </item> - <item> - <p><c>badarg</c>. Bad or invalid argument, possibly - bad type. - </p> + <item><c>badarg</c>. Bad or invalid argument, possibly + bad type. </item> - <item> - <p><c>no_transaction</c>. Operation not allowed - outside transactions. - </p> + <item><c>no_transaction</c>. Operation not allowed + outside transactions. </item> - <item> - <p><c>combine_error</c>. Table options were illegally - combined. - </p> + <item><c>combine_error</c>. Table options illegally + combined. </item> - <item> - <p><c>bad_index</c>. Index already exists or was out - of bounds. - </p> + <item><c>bad_index</c>. Index already exists, or was out + of bounds. </item> - <item> - <p><c>already_exists</c>. Schema option is already set. - </p> + <item><c>already_exists</c>. Schema option to be activated + is already on. </item> - <item> - <p><c>index_exists</c>. Some operations cannot be performed on - tabs with index. - </p> + <item><c>index_exists</c>. Some operations cannot be + performed on tables with an index. </item> - <item> - <p><c>no_exists</c>. Tried to perform operation on - non-existing, or not alive, item. - </p> + <item><c>no_exists</c>. Tried to perform operation on + non-existing (not-alive) item. </item> - <item> - <p><c>system_limit</c>. Some system_limit was exhausted. - </p> + <item><c>system_limit</c>. A system limit was exhausted. </item> - <item> - <p><c>mnesia_down</c>. A transaction involving - records at some remote node which died while - transaction was executing. Record(s) are no longer - available elsewhere in the network. - </p> + <item><c>mnesia_down</c>. A transaction involves records + on a remote node, which became unavailable before the + transaction was completed. Records are no longer + available elsewhere in the network. </item> - <item> - <p><c>not_a_db_node</c>. A node which does not exist - in the schema was mentioned. - </p> + <item><c>not_a_db_node</c>. A node was mentioned that does + not exist in the schema. </item> - <item> - <p><c>bad_type</c>. Bad type on some arguments. - </p> + <item><c>bad_type</c>. Bad type specified in argument. </item> - <item> - <p><c>node_not_running</c>. Node not running. - </p> + <item><c>node_not_running</c>. Node is not running. </item> - <item> - <p><c>truncated_binary_file</c>. Truncated binary in file. - </p> + <item><c>truncated_binary_file</c>. Truncated binary in file. </item> - <item> - <p><c>active</c>. Some delete operations require that - all active records are removed. - </p> + <item><c>active</c>. Some delete operations require that + all active records are removed. </item> - <item> - <p><c>illegal</c>. Operation not supported on record. - </p> + <item><c>illegal</c>. Operation not supported on this + record. </item> </list> - <p>The <c>Error</c> may be <c>Reason</c>, - <c>{error, Reason}</c>, or <c>{aborted, Reason}</c>. The - <c>Reason</c> may be an atom or a tuple with <c>Reason</c> + <p><c>Error</c> can be <c>Reason</c>, + <c>{error, Reason}</c>, or <c>{aborted, Reason}</c>. + <c>Reason</c> can be an atom or a tuple with <c>Reason</c> as an atom in the first field.</p> + <p>The following examples illustrate a function that returns an error, + and the method to retrieve more detailed error information:</p> + <list type="bulleted"> + <item>The function + <seealso marker="#create_table/2">mnesia:create_table(bar, [{attributes, 3.14}])</seealso> + returns the tuple <c>{aborted,Reason}</c>, where <c>Reason</c> is + the tuple <c>{bad_type,bar,3.14000}</c>.</item> + <item>The function + <seealso marker="#error_description/1">mnesia:error_description(Reason)</seealso> + returns the term <c>{"Bad type on some provided + arguments",bar,3.14000}</c>, which is an error description + suitable for display.</item> + </list> </desc> </func> <func> <name>ets(Fun, [, Args]) -> ResultOfFun | exit(Reason)</name> - <fsummary>Call the Fun in a raw context which is not protected by a transaction.</fsummary> + <fsummary>Calls the <c>Fun</c> in a raw context that is not protected by a transaction.</fsummary> <desc> - <p>Call the <c>Fun</c> in a raw context which is not protected by - a transaction. The Mnesia function call is performed in the - <c>Fun</c> are performed directly on the local <c>ets</c> tables on - the assumption that the local storage type is + <marker id="ets"></marker> + <p>Calls the <c>Fun</c> in a raw context that is not protected by + a transaction. The <c>Mnesia</c> function call is performed in + the <c>Fun</c> and performed directly on the local <c>ets</c> + tables on the assumption that the local storage type is <c>ram_copies</c> and the tables are not replicated to other nodes. Subscriptions are not triggered and checkpoints are - not updated, but it is extremely fast. This function can + not updated, but it is extremely fast. This function can also be applied to <c>disc_copies</c> tables if all - operations are read only. See <c>mnesia:activity/4</c> - and the Mnesia User's Guide for more details.</p> - <p><em>Note:</em> Calling (nesting) a <c>mnesia:ets</c> - inside a transaction context will inherit the transaction semantics.</p> + operations are read only. For details, see + <c>mnesia:activity/4</c> and the User's Guide.</p> + <p>Notice that calling (nesting) a <c>mnesia:ets</c> inside a + transaction-context inherits the transaction semantics.</p> </desc> </func> <func> - <name>first(Tab) -> Key | transaction abort </name> - <fsummary>Return the key for the first record in a table.</fsummary> + <name>first(Tab) -> Key | transaction abort</name> + <fsummary>Returns the key for the first record in a table.</fsummary> <desc> - <p>Records in <c>set</c> or <c>bag</c> tables are not ordered. - However, there - is an ordering of the records which is not known - to the user. Accordingly, it is possible to traverse a table by means - of this function in conjunction with the <c>mnesia:next/2</c> - function. - </p> - <p>If there are no records at all in the table, this function - returns the atom <c>'$end_of_table'</c>. For this reason, it - is highly undesirable, but not disallowed, to use this atom + <marker id="first"></marker> + <p>Records in <c>set</c> or <c>bag</c> tables are not ordered. + However, there is an ordering of the records that is unknown + to the user. A table can therefore be traversed by this + function with the function <c>mnesia:next/2</c>.</p> + <p>If there are no records in the table, this function + returns the atom <c>'$end_of_table'</c>. It is therefore + highly undesirable, but not disallowed, to use this atom as the key for any user records.</p> </desc> </func> <func> - <name>foldl(Function, Acc, Table) -> NewAcc | transaction abort </name> - <fsummary>Call Function for each record in Table </fsummary> + <name>foldl(Function, Acc, Table) -> NewAcc | transaction abort</name> + <fsummary>Calls <c>Function</c> for each record in <c>Table</c>.</fsummary> <desc> - <p>Iterates over the table <c>Table</c> and calls - <c>Function(Record, NewAcc)</c> for each <c>Record</c> in the table. - The term returned from <c>Function</c> will be used as the second - argument in the next call to the <c>Function</c>. - </p> - <p><c>foldl</c> returns the same term as the last call to + <marker id="foldl"></marker> + <p>Iterates over the table <c>Table</c> and calls + <c>Function(Record, NewAcc)</c> for each <c>Record</c> in + the table. The term returned from <c>Function</c> is used + as the second argument in the next call to <c>Function</c>.</p> + <p><c>foldl</c> returns the same term as the last call to <c>Function</c> returned.</p> </desc> </func> <func> - <name>foldr(Function, Acc, Table) -> NewAcc | transaction abort </name> - <fsummary>Call Function for each record in Table </fsummary> + <name>foldr(Function, Acc, Table) -> NewAcc | transaction abort</name> + <fsummary>Calls <c>Function</c> for each record in <c>Table</c>.</fsummary> <desc> - <p>This function works exactly like - <c>foldl/3</c> but iterates the table in the opposite order - for the <c>ordered_set</c> table type. For - all other table types, <c>foldr/3</c> and + <marker id="foldr"></marker> + <p>Works exactly like <c>foldl/3</c> but iterates the table + in the opposite order for the <c>ordered_set</c> table type. + For all other table types, <c>foldr/3</c> and <c>foldl/3</c> are synonyms.</p> </desc> </func> <func> - <name>force_load_table(Tab) -> yes | ErrorDescription </name> - <fsummary>Force a table to be loaded into the system </fsummary> + <name>force_load_table(Tab) -> yes | ErrorDescription</name> + <fsummary>Forces a table to be loaded into the system.</fsummary> <desc> - <p>The Mnesia algorithm for table load might lead to a + <marker id="force_load_table"></marker> + <p>The <c>Mnesia</c> algorithm for table load can lead to a situation where a table cannot be loaded. This situation - occurs when a node is started and Mnesia concludes, or + occurs when a node is started and <c>Mnesia</c> concludes, or suspects, that another copy of the table was active after - this local copy became inactive due to a system crash. - </p> + this local copy became inactive because of a system crash.</p> <p>If this situation is not acceptable, this function can be - used to override the strategy of the Mnesia table load - algorithm. This could lead to a situation where some - transaction effects are lost with a inconsistent database as + used to override the strategy of the <c>Mnesia</c> table + load algorithm. This can lead to a situation where some + transaction effects are lost with an inconsistent database as result, but for some applications high availability is more important than consistent data.</p> </desc> </func> <func> <name>index_match_object(Pattern, Pos) -> transaction abort | ObjList</name> - <fsummary>Match records and utilizes index information.</fsummary> + <fsummary>Matches records and uses index information.</fsummary> <desc> - <p>Invokes <c>mnesia:index_match_object(Tab, Pattern, Pos, read)</c> where <c>Tab</c> is <c>element(1, Pattern)</c>.</p> + <marker id="index_match_object_2"></marker> + <p>Starts + <c>mnesia:index_match_object(Tab, Pattern, Pos, read)</c>, + where <c>Tab</c> is <c>element(1, Pattern)</c>.</p> </desc> </func> <func> <name>index_match_object(Tab, Pattern, Pos, LockKind) -> transaction abort | ObjList</name> - <fsummary>Match records and utilizes index information.</fsummary> - <desc> - <p>In a manner similar to the <c>mnesia:index_read/3</c> - function, we can also utilize any index information when we - try to match records. This function takes a pattern which - obeys the same rules as the <c>mnesia:match_object/3</c> - function with the exception that this function requires the - following conditions: - </p> + <fsummary>Matches records and uses index information.</fsummary> + <desc> + <marker id="index_match_object_4"></marker> + <p>In a manner similar to the function <c>mnesia:index_read/3</c>, + any index information can be used when trying to match records. + This function takes a pattern that obeys the same rules as the + function <c>mnesia:match_object/3</c>, except that this function + requires the following conditions:</p> <list type="bulleted"> <item> <p>The table <c>Tab</c> must have an index on - position <c>Pos</c>. - </p> + position <c>Pos</c>.</p> </item> <item> <p>The element in position <c>Pos</c> in - <c>Pattern</c> must be bound. <c>Pos</c> may either be - an integer (#record.Field), or an attribute name.</p> + <c>Pattern</c> must be bound. <c>Pos</c> is + an integer (<c>#record.Field</c>) or an attribute name.</p> </item> </list> <p>The two index search functions described here are - automatically invoked when searching tables with <c>qlc</c> - list comprehensions and also when using the low level - <c>mnesia:[dirty_]match_object</c> functions. - </p> - <p></p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires a lock of type <c>LockKind</c> on the - entire table or on a single record. Currently, the lock type - <c>read</c> is supported. - </p> + automatically started when searching tables with <c>qlc</c> + list comprehensions and also when using the low-level + <c>mnesia:[dirty_]match_object</c> functions.</p> + <p>The semantics of this function is context-sensitive. + For details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a lock of type + <c>LockKind</c> on the entire table or on a single record. + Currently, the lock type <c>read</c> is supported.</p> </desc> </func> <func> - <name>index_read(Tab, SecondaryKey, Pos) -> transaction abort | RecordList </name> - <fsummary>Read records via index table. </fsummary> + <name>index_read(Tab, SecondaryKey, Pos) -> transaction abort | RecordList</name> + <fsummary>Reads records through index table.</fsummary> <desc> - <p>Assume there is an index on position <c>Pos</c> for a + <marker id="index_read"></marker> + <p>Assume that there is an index on position <c>Pos</c> for a certain record type. This function can be used to read the records without knowing the actual key for the record. For - example, with an index in position 1 of the <c>person</c> - table, the call <c>mnesia:index_read(person, 36, #person.age)</c> returns a list of all persons with age - equal to 36. <c>Pos</c> may also be an attribute name - (atom), but if the notation <c>mnesia:index_read(person, 36, age)</c> is used, the field position will be searched for in - runtime, for each call. - </p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires a read lock on the entire table.</p> + example, with an index in position 1 of table <c>person</c>, + the call <c>mnesia:index_read(person, 36, #person.age)</c> + returns a list of all persons with age 36. <c>Pos</c> can + also be an attribute name (atom), but if the notation + <c>mnesia:index_read(person, 36, age)</c> is used, the + field position is searched for in runtime, for each call.</p> + <p>The semantics of this function is context-sensitive. + For details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a read lock on the entire + table.</p> </desc> </func> <func> - <name>info() -> ok </name> - <fsummary>Print some information about the system on the tty.</fsummary> + <name>info() -> ok</name> + <fsummary>Prints system information on the terminal.</fsummary> <desc> - <p>Prints some information about the system on the tty. - This function may be used even if Mnesia is not started. - However, more information will be displayed if Mnesia is - started.</p> + <marker id="info"></marker> + <p>Prints system information on the terminal. + This function can be used even if <c>Mnesia</c> is not + started. However, more information is displayed if + <c>Mnesia</c> is started.</p> </desc> </func> <func> <name>install_fallback(Opaque) -> ok | {error,Reason}</name> - <fsummary>Install a backup as fallback.</fsummary> + <fsummary>Installs a backup as fallback.</fsummary> <desc> - <p>Invokes <c>mnesia:install_fallback(Opaque, Args)</c> where + <marker id="install_fallback_1"></marker> + <p>Calls <c>mnesia:install_fallback(Opaque, Args)</c>, where <c>Args</c> is <c>[{scope, global}]</c>.</p> </desc> </func> <func> <name>install_fallback(Opaque), BackupMod) -> ok | {error,Reason}</name> - <fsummary>Install a backup as fallback.</fsummary> + <fsummary>Installs a backup as fallback.</fsummary> <desc> - <p>Invokes <c>mnesia:install_fallback(Opaque, Args)</c> where + <p>Calls <c>mnesia:install_fallback(Opaque, Args)</c>, where <c>Args</c> is <c>[{scope, global}, {module, BackupMod}]</c>.</p> </desc> </func> <func> <name>install_fallback(Opaque, Args) -> ok | {error,Reason}</name> - <fsummary>Install a backup as fallback.</fsummary> - <desc> - <p>This function is used to install a backup as fallback. The - fallback will be used to restore the database at the next - start-up. Installation of fallbacks requires Erlang to be up - and running on all the involved nodes, but it does not - matter if Mnesia is running or not. The installation of the - fallback will fail if the local node is not one of the disc - resident nodes in the backup. - </p> - <p><c>Args</c> is a list of the following tuples: - </p> + <fsummary>Installs a backup as fallback.</fsummary> + <desc> + <p>Installs a backup as fallback. The fallback is used to + restore the database at the next startup. Installation of + fallbacks requires Erlang to be operational on all the + involved nodes, but it does not matter if <c>Mnesia</c> + is running or not. The installation of the fallback fails + if the local node is not one of the disc-resident nodes + in the backup.</p> + <p><c>Args</c> is a list of the following tuples:</p> <list type="bulleted"> <item> - <p><c>{module, BackupMod}</c>. - All accesses of the backup media is performed via a - callback module named <c>BackupMod</c>. The - <c>Opaque</c> argument is forwarded to the callback - module which may interpret it as it wish. The default + <p><c>{module, BackupMod}</c>. + All accesses of the backup media are performed through + a callback module named <c>BackupMod</c>. Argument + <c>Opaque</c> is forwarded to the callback module, + which can interpret it as it wishes. The default callback module is called <c>mnesia_backup</c> and it - interprets the <c>Opaque</c> argument as a local + interprets argument <c>Opaque</c> as a local filename. The default for this module is also - configurable via the <c>-mnesia mnesia_backup</c> - configuration parameter. </p> + configurable through configuration parameter + <c>-mnesia mnesia_backup</c>.</p> </item> <item> - <p><c>{scope, Scope}</c> - The <c>Scope</c> of a fallback may either be + <p><c>{scope, Scope}</c>. + The <c>Scope</c> of a fallback is either <c>global</c> for the entire database or <c>local</c> for one node. By default, the installation of a fallback - is a global operation which either is performed all - nodes with disc resident schema or none. Which nodes - that are disc resident or not, is determined from the - schema info in the backup.</p> - <p>If the <c>Scope</c> of the operation is <c>local</c> - the fallback will only be installed on the local node.</p> + is a global operation, which either is performed on all + nodes with a disc-resident schema or none. Which nodes + that are disc-resident is determined from the + schema information in the backup.</p> + <p>If <c>Scope</c> of the operation is <c>local</c>, + the fallback is only installed on the local node.</p> </item> <item> - <p><c>{mnesia_dir, AlternateDir}</c> + <p><c>{mnesia_dir, AlternateDir}</c>. This argument is only valid if the scope of the installation is <c>local</c>. Normally the installation - of a fallback is targeted towards the Mnesia directory - as configured with the <c>-mnesia dir</c> configuration - parameter. But by explicitly supplying an - <c>AlternateDir</c> the fallback will be installed there - regardless of the Mnesia directory configuration + of a fallback is targeted to the <c>Mnesia</c> directory, + as configured with configuration parameter + <c>-mnesia dir</c>. But by explicitly supplying an + <c>AlternateDir</c>, the fallback is installed there + regardless of the <c>Mnesia</c> directory configuration parameter setting. After installation of a fallback on - an alternate Mnesia directory that directory is fully - prepared for usage as an active Mnesia directory. - </p> - <p>This is a somewhat dangerous feature which must be - used with care. By unintentional mixing of directories - you may easily end up with a inconsistent database, if + an alternative <c>Mnesia</c> directory, that directory + is fully prepared for use as an active <c>Mnesia</c> + directory.</p> + <p>This is a dangerous feature that must be + used with care. By unintentional mixing of directories, + you can easily end up with an inconsistent database, if the same backup is installed on more than one directory.</p> </item> </list> </desc> </func> <func> - <name>is_transaction() -> boolean </name> - <fsummary>Check if code is running in a transaction.</fsummary> + <name>is_transaction() -> boolean</name> + <fsummary>Checks if code is running in a transaction.</fsummary> <desc> - <p>When this function is executed inside a transaction context + <marker id="is_transaction"></marker> + <p>When this function is executed inside a transaction-context, it returns <c>true</c>, otherwise <c>false</c>.</p> </desc> </func> <func> - <name>last(Tab) -> Key | transaction abort </name> - <fsummary>Return the key for the last record in a table.</fsummary> + <name>last(Tab) -> Key | transaction abort</name> + <fsummary>Returns the key for the last record in a table.</fsummary> <desc> - <p>This function works exactly like - <c>mnesia:first/1</c> but returns the last object in - Erlang term order for the <c>ordered_set</c> table type. For - all other table types, <c>mnesia:first/1</c> and + <p>Works exactly like + <c>mnesia:first/1</c>, but returns the last object in + Erlang term order for the <c>ordered_set</c> table type. + For all other table types, <c>mnesia:first/1</c> and <c>mnesia:last/1</c> are synonyms.</p> </desc> </func> <func> <name>load_textfile(Filename)</name> - <fsummary>Load tables from a text file.</fsummary> + <fsummary>Loads tables from a text file.</fsummary> <desc> + <marker id="load_textfile"></marker> <p>Loads a series of definitions and data found in the text file (generated with <c>mnesia:dump_to_textfile/1</c>) - into Mnesia. This function also starts Mnesia and possibly - creates a new schema. This function is intended for - educational purposes only and using other functions to deal - with real backups, is recommended.</p> + into <c>Mnesia</c>. This function also starts <c>Mnesia</c> + and possibly creates a new schema. This function is + intended for educational purposes only. It is recommended + to use other functions to deal with real backups.</p> </desc> </func> <func> <name>lock(LockItem, LockKind) -> Nodes | ok | transaction abort</name> <fsummary>Explicit grab lock.</fsummary> <desc> + <marker id="lock"></marker> <p>Write locks are normally acquired on all nodes where a - replica of the table resides (and is active). Read locks are - acquired on one node (the local node if a local - replica exists). Most of the context sensitive access functions - acquire an implicit lock if they are invoked in a - transaction context. The granularity of a lock may either - be a single record or an entire table. - </p> - <p>The normal usage is to call the function without checking - the return value since it exits if it fails and the - transaction is restarted by the transaction manager. It - returns all the locked nodes if a write lock is acquired, and - <c>ok</c> if it was a read lock. - </p> - <p>This function <c>mnesia:lock/2</c> is intended to support - explicit locking on tables but also intended for situations - when locks need to be acquired regardless of how tables are - replicated. Currently, two <c>LockKind</c>'s are supported: - </p> + replica of the table resides (and is active). Read locks + are acquired on one node (the local node if a local + replica exists). Most of the context-sensitive access + functions acquire an implicit lock if they are started in a + transaction-context. The granularity of a lock can either + be a single record or an entire table.</p> + <p>The normal use is to call the function without checking + the return value, as it exits if it fails and the + transaction is restarted by the transaction manager. It + returns all the locked nodes if a write lock is acquired + and <c>ok</c> if it was a read lock.</p> + <p>The function <c>mnesia:lock/2</c> is intended to support + explicit locking on tables, but is also intended for + situations when locks need to be acquired regardless of + how tables are replicated. Currently, two kinds of + <c>LockKind</c> are supported:</p> <taglist> <tag><c>write</c></tag> <item> - <p>Write locks are exclusive, which means that if one + <p>Write locks are exclusive. This means that if one transaction manages to acquire a write lock on an item, - no other transaction may acquire any kind of lock on the - same item. - </p> + no other transaction can acquire any kind of lock on + the same item.</p> </item> <tag><c>read</c></tag> <item> - <p>Read locks may be shared, which means that if one + <p>Read locks can be shared. This means that if one transaction manages to acquire a read lock on an item, - other transactions may also acquire a read lock on the - same item. However, if someone has a read lock no one can - acquire a write lock at the same item. If some one has a - write lock no one can acquire a read lock nor - a write lock at the same item.</p> + other transactions can also acquire a read lock on the + same item. However, if someone has a read lock, no one + can acquire a write lock at the same item. If someone + has a write lock, no one can acquire either a read lock + or a write lock at the same item.</p> </item> </taglist> <p>Conflicting lock requests are automatically queued if there is no risk of a deadlock. Otherwise the transaction must be - aborted and executed again. Mnesia does this automatically - as long as the upper limit of maximum <c>retries</c> is not - reached. See <c>mnesia:transaction/3</c> for the details. - </p> - <p>For the sake of completeness sticky write locks will also - be described here even if a sticky write lock is not - supported by this particular function: - </p> + terminated and executed again. <c>Mnesia</c> does this + automatically as long as the upper limit of the maximum + <c>retries</c> is not reached. For details, see + <c>mnesia:transaction/3</c>.</p> + <p>For the sake of completeness, sticky write locks are also + described here even if a sticky write lock is not + supported by this function:</p> <taglist> <tag><c>sticky_write</c></tag> <item> - <p>Sticky write locks are a mechanism which can be used + <p>Sticky write locks are a mechanism that can be used to optimize write lock acquisition. If your application uses replicated tables mainly for fault tolerance (as opposed to read access optimization purpose), sticky - locks may be the best option available. - </p> - <p>When a sticky write lock is acquired, all nodes will be - informed which node is locked. Subsequently, - sticky lock requests from the same node will be - performed as a local operation without any + locks can be the best option available.</p> + <p>When a sticky write lock is acquired, all nodes are + informed which node is locked. Then, + sticky lock requests from the same node are + performed as a local operation without any communication with other nodes. The sticky lock - lingers on the node even after the transaction has - ended. See the Mnesia User's Guide for more information.</p> + lingers on the node even after the transaction + ends. For details, see the User's Guide.</p> </item> </taglist> - <p>Currently, two kinds of <c>LockItem</c>'s are supported by - this function: - </p> + <p>Currently, this function supports two kinds of + <c>LockItem</c>:</p> <taglist> <tag><c>{table, Tab}</c></tag> <item> <p>This acquires a lock of type <c>LockKind</c> on the - entire table <c>Tab</c>. - </p> + entire table <c>Tab</c>.</p> </item> <tag><c>{global, GlobalKey, Nodes}</c></tag> <item> <p>This acquires a lock of type <c>LockKind</c> on the global resource <c>GlobalKey</c>. The lock is acquired - on all active nodes in the <c>Nodes</c> list. </p> + on all active nodes in the <c>Nodes</c> list.</p> </item> </taglist> - <p>Locks are released when the outermost transaction ends. - </p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires locks otherwise it just ignores the - request.</p> + <p>Locks are released when the outermost transaction ends.</p> + <p>The semantics of this function is context-sensitive. + For details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires locks, otherwise it + ignores the request.</p> </desc> </func> <func> - <name>match_object(Pattern) ->transaction abort | RecList </name> - <fsummary>Match <c>Pattern</c>for records. </fsummary> + <name>match_object(Pattern) -> transaction abort | RecList</name> + <fsummary>Matches <c>Pattern</c> for records.</fsummary> <desc> - <p>Invokes <c>mnesia:match_object(Tab, Pattern, read)</c> where + <marker id="match_object_1"></marker> + <p>Calls <c>mnesia:match_object(Tab, Pattern, read)</c>, where <c>Tab</c> is <c>element(1, Pattern)</c>.</p> </desc> </func> <func> - <name>match_object(Tab, Pattern, LockKind) ->transaction abort | RecList </name> - <fsummary>Match <c>Pattern</c>for records. </fsummary> + <name>match_object(Tab, Pattern, LockKind) -> transaction abort | RecList</name> + <fsummary>Matches <c>Pattern</c> for records.</fsummary> <desc> - <p>This function takes a pattern with 'don't care' variables - denoted as a '_' parameter. This function returns a list of - records which matched the pattern. Since the second element + <marker id="match_object_3"></marker> + <p>Takes a pattern with "don't care" variables + denoted as a <c>'_'</c> parameter. This function returns + a list of records that matched the pattern. + Since the second element of a record in a table is considered to be the key for the record, the performance of this function depends on whether - this key is bound or not. - </p> - <p>For example, the call <c>mnesia:match_object(person, {person, '_', 36, '_', '_'}, read)</c> returns a list of all person records with an - age field of thirty-six (36). - </p> + this key is bound or not.</p> + <p>For example, the call <c>mnesia:match_object(person, + {person, '_', 36, '_', '_'}, read)</c> returns a list of + all person records with an <c>age</c> field of 36.</p> <p>The function <c>mnesia:match_object/3</c> - automatically uses indices if these exist. However, no - heuristics are performed in order to select the best - index. - </p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires a lock of type <c>LockKind</c> on the - entire table or a single record. Currently, the lock type - <c>read</c> is supported.</p> + automatically uses indexes if these exist. However, no + heuristics are performed to select the best index.</p> + <p>The semantics of this function is context-sensitive. + For details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a lock of type + <c>LockKind</c> on the entire table or a single record. + Currently, the lock type <c>read</c> is supported.</p> </desc> </func> <func> <name>move_table_copy(Tab, From, To) -> {aborted, Reason} | {atomic, ok}</name> - <fsummary>Move the copy of table <c>Tab</c>from node<c>From</c>to node <c>To</c>.</fsummary> + <fsummary>Moves the copy of table <c>Tab</c> from node <c>From</c> to node <c>To</c>.</fsummary> <desc> + <marker id="move_table_copy"></marker> <p>Moves the copy of table <c>Tab</c> from node - <c>From</c> to node <c>To</c>. - </p> + <c>From</c> to node <c>To</c>.</p> <p>The storage type is preserved. For example, a RAM table - moved from one node remains a RAM on the new node. It is - still possible for other transactions to read and write in - the table while it is being moved. - </p> + moved from one node remains a RAM on the new node. Other + transactions can still read and write in + the table while it is being moved.</p> <p>This function cannot be used on <c>local_content</c> tables.</p> </desc> </func> <func> - <name>next(Tab, Key) -> Key | transaction abort </name> - <fsummary>Return the next key in a table. </fsummary> + <name>next(Tab, Key) -> Key | transaction abort</name> + <fsummary>Returns the next key in a table.</fsummary> <desc> - <p>This function makes it possible to traverse a table - and perform operations on all records in the table. When + <marker id="next"></marker> + <p>Traverses a table and + performs operations on all records in the table. When the end of the table is reached, the special key - <c>'$end_of_table'</c> is returned. Otherwise, the function - returns a key which can be used to read the actual record.</p> + <c>'$end_of_table'</c> is returned. Otherwise the function + returns a key that can be used to read the actual record.</p> </desc> </func> <func> - <name>prev(Tab, Key) -> Key | transaction abort </name> - <fsummary>Return the previous key in a table. </fsummary> + <name>prev(Tab, Key) -> Key | transaction abort</name> + <fsummary>Returns the previous key in a table.</fsummary> <desc> - <p>This function works exactly like - <c>mnesia:next/2</c> but returns the previous object in - Erlang term order for the ordered_set table type. For - all other table types, <c>mnesia:next/2</c> and + <p>Works exactly like + <c>mnesia:next/2</c>, but returns the previous object in + Erlang term order for the <c>ordered_set</c> table type. + For all other table types, <c>mnesia:next/2</c> and <c>mnesia:prev/2</c> are synonyms.</p> </desc> </func> <func> - <name>read({Tab, Key}) -> transaction abort | RecordList </name> - <fsummary>Read records(s) with a given key. </fsummary> + <name>read({Tab, Key}) -> transaction abort | RecordList</name> + <fsummary>Reads records(s) with a given key.</fsummary> <desc> - <p>Invokes <c>mnesia:read(Tab, Key, read)</c>.</p> + <marker id="read_2"></marker> + <p>Calls function <c>mnesia:read(Tab, Key, read)</c>.</p> </desc> </func> <func> - <name>read(Tab, Key) -> transaction abort | RecordList </name> - <fsummary>Read records(s) with a given key. </fsummary> + <name>read(Tab, Key) -> transaction abort | RecordList</name> + <fsummary>Reads records(s) with a given key.</fsummary> <desc> - <p>Invokes <c>mnesia:read(Tab, Key, read)</c>.</p> + <p>Calls function <c>mnesia:read(Tab, Key, read)</c>.</p> </desc> </func> <func> - <name>read(Tab, Key, LockKind) -> transaction abort | RecordList </name> - <fsummary>Read records(s) with a given key. </fsummary> + <name>read(Tab, Key, LockKind) -> transaction abort | RecordList</name> + <fsummary>Reads records(s) with a given key.</fsummary> <desc> - <p>This function reads all records from table <c>Tab</c> with + <marker id="read_3"></marker> + <p>Reads all records from table <c>Tab</c> with key <c>Key</c>. This function has the same semantics regardless of the location of <c>Tab</c>. If the table is - of type <c>bag</c>, the <c>mnesia:read(Tab, Key)</c> can + of type <c>bag</c>, the function + <c>mnesia:read(Tab, Key)</c> can return an arbitrarily long list. If the table is of type - <c>set</c>, the list is either of length 1, or <c>[]</c>. - </p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires a lock of type + <c>set</c>, the list is either of length 1, or <c>[]</c>.</p> + <p>The semantics of this function is context-sensitive. + For details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a lock of type <c>LockKind</c>. Currently, the lock types <c>read</c>, - <c>write</c> and <c>sticky_write</c> are supported. - </p> - <p>If the user wants to update the record it is more efficient to - use <c>write/sticky_write</c> as the LockKind. If majority checking - is active on the table, it will be checked as soon as a write lock is - attempted. This can be used to quickly abort if the majority condition - isn't met. - </p> + <c>write</c>, and <c>sticky_write</c> are supported.</p> + <p>If the user wants to update the record, it is more + efficient to use <c>write/sticky_write</c> as the + <c>LockKind</c>. If majority checking is active on the + table, it is checked as soon as a write lock is + attempted. This can be used to end quickly if the + majority condition is not met.</p> </desc> </func> <func> <name>read_lock_table(Tab) -> ok | transaction abort</name> - <fsummary>Set a read lock on an entire table.</fsummary> + <fsummary>Sets a read lock on an entire table.</fsummary> <desc> - <p>Invokes <c>mnesia:lock({table, Tab}, read)</c>.</p> + <marker id="read_lock_table"></marker> + <p>Calls the function + <c>mnesia:lock({table, Tab}, read)</c>.</p> </desc> </func> <func> <name>report_event(Event) -> ok</name> - <fsummary>Report a user event to Mnesia's event handler.</fsummary> + <fsummary>Reports a user event to the <c>Mnesia</c> event handler.</fsummary> <desc> - <p>When tracing a system of Mnesia applications it is useful - to be able to interleave Mnesia's own events with - application related events that give information about the - application context. - </p> + <marker id="report_event"></marker> + <p>When tracing a system of <c>Mnesia</c> applications it is + useful to be able to interleave <c>Mnesia</c> own events with + application-related events that give information about the + application context.</p> <p>Whenever the application begins a - new and demanding Mnesia task, or if it is entering a new - interesting phase in its execution, it may be a good idea to - use <c>mnesia:report_event/1</c>. The <c>Event</c> may be + new and demanding <c>Mnesia</c> task, or if it enters a new + interesting phase in its execution, it can be a good idea to + use <c>mnesia:report_event/1</c>. <c>Event</c> can be any term and generates a <c>{mnesia_user, Event}</c> event - for any processes that subscribe to Mnesia system + for any processes that subscribe to <c>Mnesia</c> system events.</p> </desc> </func> @@ -1819,221 +1745,237 @@ mnesia:create_table(person, <name>restore(Opaque, Args) -> {atomic, RestoredTabs} |{aborted, Reason}</name> <fsummary>Online restore of backup.</fsummary> <desc> - <p>With this function, tables may be restored online from a - backup without restarting Mnesia. <c>Opaque</c> is forwarded - to the backup module. <c>Args</c> is a list of the following - tuples: - </p> + <marker id="restore"></marker> + <p>With this function, tables can be restored online from a + backup without restarting <c>Mnesia</c>. + <c>Opaque</c> is forwarded to the backup module. + <c>Args</c> is a list of the following tuples:</p> <list type="bulleted"> <item> - <p><c>{module,BackupMod}</c> The backup module - <c>BackupMod</c> will be used to access the backup - media. If omitted, the default backup module will be - used. - </p> + <c>{module,BackupMod}</c>. The backup module + <c>BackupMod</c> is used to access the backup media. + If omitted, the default backup module is used. </item> - <item><c>{skip_tables, TabList}</c> Where <c>TabList</c> - is a list of tables which should not be read from the + <item><c>{skip_tables, TabList}</c>, where <c>TabList</c> + is a list of tables that is not to be read from the backup. </item> - <item><c>{clear_tables, TabList}</c> Where - <c>TabList</c> is a list of tables which should be - cleared, before the records from the backup are inserted, - ie. all records in the tables are deleted before the - tables are restored. Schema information about the tables - is not cleared or read from backup. - </item> - <item><c>{keep_tables, TabList}</c> Where <c>TabList</c> - is a list of tables which should be not be cleared, before - the records from the backup are inserted, i.e. the records - in the backup will be added to the records in the table. + <item><c>{clear_tables, TabList}</c>, where + <c>TabList</c> is a list of tables that is to be + cleared before the records from the backup are inserted. + That is, all records in the tables are deleted before the + tables are restored. Schema information about the tables + is not cleared or read from the backup. + </item> + <item><c>{keep_tables, TabList}</c>, where <c>TabList</c> + is a list of tables that is not to be cleared before the + records from the backup are inserted. That is, the records + in the backup are added to the records in the table. Schema information about the tables is not cleared or read - from backup. - </item> - <item><c>{recreate_tables, TabList}</c> Where - <c>TabList</c> is a list of tables which should be - re-created, before the records from the backup are - inserted. The tables are first deleted and then created with - the schema information from the backup. All the nodes in the - backup needs to be up and running. - </item> - <item><c>{default_op, Operation}</c> Where <c>Operation</c> is - one of the following operations <c>skip_tables</c>, - <c>clear_tables</c>, <c>keep_tables</c> or - <c>recreate_tables</c>. The default operation specifies - which operation should be used on tables from the backup - which are not specified in any of the lists above. If - omitted, the operation <c>clear_tables</c> will be used. + from the backup. + </item> + <item><c>{recreate_tables, TabList}</c>, where + <c>TabList</c> is a list of tables that is to be + recreated before the records from the backup are inserted. + The tables are first deleted and then created with the + schema information from the backup. All the nodes in the + backup need to be operational. + </item> + <item><c>{default_op, Operation}</c>, where <c>Operation</c> + is either of the operations <c>skip_tables</c>, + <c>clear_tables</c>, <c>keep_tables</c>, or + <c>recreate_tables</c>. The default operation specifies + which operation that is to be used on tables from the backup + that is not specified in any of the mentioned lists. If + omitted, operation <c>clear_tables</c> is used. </item> </list> - <p>The affected tables are write locked during the - restoration, but regardless of the lock conflicts caused by - this, the applications can continue to do their work while + <p>The affected tables are write-locked during the + restoration. However, regardless of the lock conflicts caused + by this, the applications can continue to do their work while the restoration is being performed. The restoration is - performed as one single transaction. - </p> - <p>If the database is - huge, it may not be possible to restore it online. In such - cases, the old database must be restored by installing a + performed as one single transaction.</p> + <p>If the database is huge, + it it not always possible to restore it online. In such + cases, restore the old database by installing a fallback and then restart.</p> </desc> </func> <func> - <name>s_delete({Tab, Key}) -> ok | transaction abort </name> - <fsummary>Set sticky lock and delete records.</fsummary> + <name>s_delete({Tab, Key}) -> ok | transaction abort</name> + <fsummary>Sets sticky lock and delete records.</fsummary> <desc> - <p>Invokes <c>mnesia:delete(Tab, Key, sticky_write)</c></p> + <marker id="s_delete"></marker> + <p>Calls the function + <c>mnesia:delete(Tab, Key, sticky_write)</c></p> </desc> </func> <func> - <name>s_delete_object(Record) -> ok | transaction abort </name> - <fsummary>Set sticky lock and delete record.</fsummary> + <name>s_delete_object(Record) -> ok | transaction abort</name> + <fsummary>Sets sticky lock and delete record.</fsummary> <desc> - <p>Invokes <c>mnesia:delete_object(Tab, Record, sticky_write)</c> where <c>Tab</c> is <c>element(1, Record)</c>.</p> + <marker id="s_delete_object"></marker> + <p>Calls the function + <c>mnesia:delete_object(Tab, Record, sticky_write)</c>, + where <c>Tab</c> is <c>element(1, Record)</c>.</p> </desc> </func> <func> - <name>s_write(Record) -> ok | transaction abort </name> - <fsummary>Write <c>Record</c>and sets stick lock.</fsummary> + <name>s_write(Record) -> ok | transaction abort</name> + <fsummary>Writes <c>Record</c> and sets sticky lock.</fsummary> <desc> - <p>Invokes <c>mnesia:write(Tab, Record, sticky_write)</c> + <marker id="s_write"></marker> + <p>Calls the function + <c>mnesia:write(Tab, Record, sticky_write)</c>, where <c>Tab</c> is <c>element(1, Record)</c>.</p> </desc> </func> <func> - <name>schema() -> ok </name> - <fsummary>Print information about all table definitions on the tty. </fsummary> + <name>schema() -> ok</name> + <fsummary>Prints information about all table definitions on the terminal.</fsummary> <desc> - <p>Prints information about all table definitions on the tty.</p> + <p>Prints information about all table definitions on the terminal.</p> </desc> </func> <func> - <name>schema(Tab) -> ok </name> - <fsummary>Print information about one table definition on the tty.</fsummary> + <name>schema(Tab) -> ok</name> + <fsummary>Prints information about one table definition on the terminal.</fsummary> <desc> - <p>Prints information about one table definition on the tty.</p> + <p>Prints information about one table definition on the terminal.</p> </desc> </func> <func> - <name>select(Tab, MatchSpec [, Lock]) -> transaction abort | [Object] </name> - <fsummary>Match the objects in <c>Tab</c>against <c>MatchSpec</c>.</fsummary> + <name>select(Tab, MatchSpec [, Lock]) -> transaction abort | [Object]</name> + <fsummary>Matches the objects in <c>Tab</c> against <c>MatchSpec</c>.</fsummary> <desc> - <p>Matches the objects in the table <c>Tab</c> using a - match_spec as described in the ERTS Users Guide. Optionally a lock + <marker id="select_2_3"></marker> + <p>Matches the objects in table <c>Tab</c> using a + <c>match_spec</c> as described in the + <seealso marker="stdlib:ets#select/3">ets:select/3</seealso>. + Optionally a lock <c>read</c> or <c>write</c> can be given as the third - argument, default is <c>read</c>. The return value depends - on the <c>MatchSpec</c>.</p> - <p><em>Note:</em> for best performance <c>select</c> should - be used before any modifying operations are done on that table - in the same transaction, i.e. don't use <c>write</c> or <c>delete</c> - before a <c>select</c>.</p> - <p>In its simplest forms the match_spec's look like this:</p> + argument. Default is <c>read</c>. The return value depends + on <c>MatchSpec</c>.</p> + <p>Notice that for best performance, <c>select</c> is to be + used before any modifying operations are done on that table + in the same transaction. That is, do not use <c>write</c> + or <c>delete</c> before a <c>select</c>.</p> + <p>In its simplest forms, the <c>match_spec</c> look as + follows:</p> <list type="bulleted"> - <item>MatchSpec = [MatchFunction]</item> - <item>MatchFunction = {MatchHead, [Guard], [Result]}</item> - <item>MatchHead = tuple() | record()</item> - <item>Guard = {"Guardtest name", ...}</item> - <item>Result = "Term construct"</item> + <item><c>MatchSpec = [MatchFunction]</c></item> + <item><c>MatchFunction = {MatchHead, [Guard], [Result]}</c></item> + <item><c>MatchHead = tuple() | record()</c></item> + <item><c>Guard = {"Guardtest name", ...}</c></item> + <item><c>Result = "Term construct"</c></item> </list> - <p>See the ERTS Users Guide and <c>ets</c> documentation for a - complete description of the select.</p> - <p>For example to find the names of all male persons with an age over 30 in table - Tab do:</p> + <p>For a complete description of <c>select</c>, see the + <seealso marker="erts:index">ERTS</seealso> User's Guide and the + <seealso marker="stdlib:ets">ets</seealso> manual page in + <c>STDLIB</c>.</p> + <p>For example, to find the names of all male persons older + than 30 in table <c>Tab</c>:</p> <code type="none"> MatchHead = #person{name='$1', sex=male, age='$2', _='_'}, Guard = {'>', '$2', 30}, Result = '$1', -mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]), - </code> +mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]),</code> </desc> </func> <func> - <name>select(Tab, MatchSpec, NObjects, Lock) -> transaction abort | {[Object],Cont} | '$end_of_table'</name> - <fsummary>Match the objects in <c>Tab</c>against <c>MatchSpec</c>.</fsummary> + <name>select(Tab, MatchSpec, NObjects, Lock) -> transaction abort | {[Object],Cont} | '$end_of_table'</name> + <fsummary>Matches the objects in <c>Tab</c> against <c>MatchSpec</c>.</fsummary> <desc> - <p>Matches the objects in the table <c>Tab</c> using a - match_spec as described in ERTS users guide, and returns - a chunk of terms and a continuation, the wanted number - of returned terms is specified by the <c>NObjects</c> argument. - The lock argument can be <c>read</c> or <c>write</c>. - The continuation should be used as argument to <c>mnesia:select/1</c>, + <marker id="select_4"></marker> + <p>Matches the objects in table <c>Tab</c> using a + <c>match_spec</c> as described in the + <seealso marker="erts:index">ERTS</seealso> User's Guide, + and returns a chunk of terms and a continuation. + The wanted number of returned terms is specified by + argument <c>NObjects</c>. The lock argument can be + <c>read</c> or <c>write</c>. The continuation is to be + used as argument to <c>mnesia:select/1</c>, if more or all answers are needed.</p> - <p><em>Note:</em> for best performance <c>select</c> should - be used before any modifying operations are done on that - table in the same transaction, i.e. don't use + <p>Notice that for best performance, <c>select</c> is to be + used before any modifying operations are done on that table + in the same transaction. That is, do not use <c>mnesia:write</c> or <c>mnesia:delete</c> before a - <c>mnesia:select</c>. For efficiency the <c>NObjects</c> is - a recommendation only and the result may contain anything - from an empty list to all available results. </p> + <c>mnesia:select</c>. For efficiency, <c>NObjects</c> is + a recommendation only and the result can contain anything + from an empty list to all available results.</p> </desc> </func> <func> - <name>select(Cont) -> transaction abort | {[Object],Cont} | '$end_of_table'</name> - <fsummary>Continues selecting objects. </fsummary> + <name>select(Cont) -> transaction abort | {[Object],Cont} | '$end_of_table'</name> + <fsummary>Continues selecting objects.</fsummary> <desc> <p>Selects more objects with the match specification initiated - by <c>mnesia:select/4</c>. - </p> - <p><em>Note:</em> Any modifying operations, i.e. <c>mnesia:write</c> - or <c>mnesia:delete</c>, that are done between the <c>mnesia:select/4</c> - and <c>mnesia:select/1</c> calls will not be visible in the result.</p> + by <c>mnesia:select/4</c>.</p> + <p>Notice that any modifying operations, that is, + <c>mnesia:write</c> or <c>mnesia:delete</c>, that are done + between the <c>mnesia:select/4</c> and <c>mnesia:select/1</c> + calls are not visible in the result.</p> </desc> </func> <func> <name>set_debug_level(Level) -> OldLevel</name> - <fsummary>Change the internal debug level of Mnesia</fsummary> + <fsummary>Changes the internal debug level of <c>Mnesia</c>.</fsummary> <desc> - <p>Changes the internal debug level of Mnesia. See the - chapter about configuration parameters for details.</p> + <marker id="set_debug_level"></marker> + <p>Changes the internal debug level of <c>Mnesia</c>. + For details, see + <seealso marker="#configuration_parameters">Section + Configuration Parameters</seealso>.</p> </desc> </func> <func> - <name>set_master_nodes(MasterNodes) -> ok | {error, Reason} </name> - <fsummary>Set the master nodes for all tables</fsummary> + <name>set_master_nodes(MasterNodes) -> ok | {error, Reason}</name> + <fsummary>Sets the master nodes for all tables.</fsummary> <desc> - <p>For each table Mnesia will determine its replica nodes - (<c>TabNodes</c>) and invoke <c>mnesia:set_master_nodes(Tab, TabMasterNodes)</c> where <c>TabMasterNodes</c> is the - intersection of <c>MasterNodes</c> and <c>TabNodes</c>. See - <c>mnesia:set_master_nodes/2</c> about the semantics.</p> + <marker id="set_master_nodes_1"></marker> + <p>For each table <c>Mnesia</c> determines its replica nodes + (<c>TabNodes</c>) and starts + <c>mnesia:set_master_nodes(Tab, TabMasterNodes)</c>. where + <c>TabMasterNodes</c> is the intersection of + <c>MasterNodes</c> and <c>TabNodes</c>. For semantics, see + <c>mnesia:set_master_nodes/2</c>.</p> </desc> </func> <func> - <name>set_master_nodes(Tab, MasterNodes) -> ok | {error, Reason} </name> - <fsummary>Set the master nodes for a table</fsummary> + <name>set_master_nodes(Tab, MasterNodes) -> ok | {error, Reason}</name> + <fsummary>Sets the master nodes for a table.</fsummary> <desc> - <p>If the application detects that there has been a - communication failure (in a potentially partitioned network) which - may have caused an inconsistent database, it may use the + <marker id="set_master_nodes_2"></marker> + <p>If the application detects a + communication failure (in a potentially partitioned network) + that can have caused an inconsistent database, it can use the function <c>mnesia:set_master_nodes(Tab, MasterNodes)</c> to - define from which nodes each table will be loaded. - At startup Mnesia's normal table load algorithm will be - bypassed and the table will be loaded from one of the master - nodes defined for the table, regardless of when and if Mnesia - was terminated on other nodes. The <c>MasterNodes</c> may only - contain nodes where the table has a replica and if the + define from which nodes each table is to be loaded. + At startup, the <c>Mnesia</c> normal table load algorithm is + bypassed and the table is loaded from one of the master nodes + defined for the table, regardless of when and if <c>Mnesia</c> + terminated on other nodes. <c>MasterNodes</c> can only + contain nodes where the table has a replica. If the <c>MasterNodes</c> list is empty, the master node recovery - mechanism for the particular table will be reset and the - normal load mechanism will be used at next restart. - </p> - <p>The master node setting is always local and it may be - changed regardless of whether Mnesia is started or not. - </p> - <p>The database may also become inconsistent if the - <c>max_wait_for_decision</c> configuration parameter is used + mechanism for the particular table is reset, and the + normal load mechanism is used at the next restart.</p> + <p>The master node setting is always local. It can be + changed regardless if <c>Mnesia</c> is started or not.</p> + <p>The database can also become inconsistent if + configuration parameter <c>max_wait_for_decision</c> is used or if <c>mnesia:force_load_table/1</c> is used.</p> </desc> </func> <func> <name>snmp_close_table(Tab) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Remove the possibility for SNMP to manipulate the table.</fsummary> + <fsummary>Removes the possibility for SNMP to manipulate the table.</fsummary> <desc> - <p>Removes the possibility for SNMP to manipulate the - table.</p> + <p>Removes the possibility for SNMP to manipulate the table.</p> </desc> </func> <func> <name>snmp_get_mnesia_key(Tab, RowIndex) -> {ok, Key} | undefined</name> - <fsummary>Get the corresponding Mnesia key from an SNMP index.</fsummary> + <fsummary>Gets the corresponding <c>Mnesia</c> key from an SNMP index.</fsummary> <type> <v>Tab ::= atom()</v> <v>RowIndex ::= [integer()]</v> @@ -2041,44 +1983,43 @@ mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]), <v>key() ::= integer() | string() | [integer()]</v> </type> <desc> - <p>Transforms an SNMP index to the corresponding Mnesia key. - If the SNMP table has multiple keys, the key is a tuple of - the key columns.</p> + <p>Transforms an SNMP index to the corresponding <c>Mnesia</c> + key. If the SNMP table has multiple keys, the key is a tuple + of the key columns.</p> </desc> </func> <func> <name>snmp_get_next_index(Tab, RowIndex) -> {ok, NextIndex} | endOfTable</name> - <fsummary>Get the index of the next lexicographical row.</fsummary> + <fsummary>Gets the index of the next lexicographical row.</fsummary> <type> <v>Tab ::= atom()</v> <v>RowIndex ::= [integer()]</v> <v>NextIndex ::= [integer()]</v> </type> <desc> - <p>The <c>RowIndex</c> may specify a non-existing row. - Specifically, it might be the empty list. Returns the index + <p><c>RowIndex</c> can specify a non-existing row. + Specifically, it can be the empty list. Returns the index of the next lexicographical row. If <c>RowIndex</c> is the - empty list, this function will return the index of the first row + empty list, this function returns the index of the first row in the table.</p> </desc> </func> <func> <name>snmp_get_row(Tab, RowIndex) -> {ok, Row} | undefined</name> - <fsummary>Retrieve a row indexed by an SNMP index.</fsummary> + <fsummary>Retrieves a row indexed by an SNMP index.</fsummary> <type> <v>Tab ::= atom()</v> <v>RowIndex ::= [integer()]</v> <v>Row ::= record(Tab)</v> </type> <desc> - <p>Makes it possible to read a row by its SNMP index. This - index is specified as an SNMP OBJECT IDENTIFIER, a list of - integers.</p> + <p>Reads a row by its SNMP index. This index is specified as + an SNMP Object Identifier, a list of integers.</p> </desc> </func> <func> <name>snmp_open_table(Tab, SnmpStruct) -> {aborted, R} | {atomic, ok}</name> - <fsummary>Organize a Mnesia table as an SNMP table.</fsummary> + <fsummary>Organizes a <c>Mnesia</c> table as an SNMP table.</fsummary> <type> <v>Tab ::= atom()</v> <v>SnmpStruct ::= [{key, type()}]</v> @@ -2086,624 +2027,558 @@ mnesia:select(Tab,[{MatchHead, [Guard], [Result]}]), <v>type_spec() ::= fix_string | string | integer</v> </type> <desc> - <p>It is possible to establish a direct one to one mapping - between Mnesia tables and SNMP tables. Many - telecommunication applications are controlled and monitored - by the SNMP protocol. This connection between Mnesia and - SNMP makes it simple and convenient to achieve this. - </p> - <p>The <c>SnmpStruct</c> argument is a list of SNMP + <p>A direct one-to-one mapping can be established between + <c>Mnesia</c> tables and SNMP tables. Many telecommunication + applications are controlled and monitored by the SNMP + protocol. This connection between <c>Mnesia</c> and SNMP + makes it simple and convenient to achieve this mapping.</p> + <p>Argument <c>SnmpStruct</c> is a list of SNMP information. Currently, the only information needed is - information about the key types in the table. It is not - possible to handle multiple keys in Mnesia, but many SNMP + information about the key types in the table. Multiple + keys cannot be handled in <c>Mnesia</c>, but many SNMP tables have multiple keys. Therefore, the following convention is used: if a table has multiple keys, these must - always be stored as a tuple of the keys. Information about + always be stored as a tuple of the keys. Information about the key types is specified as a tuple of atoms describing - the types. The only significant type is - <c>fix_string</c>. This means that a string has fixed - size. For example: - </p> + the types. The only significant type is <c>fix_string</c>. + This means that a string has a fixed size.</p> + <p>For example, the following causes table <c>person</c> + to be ordered as an SNMP table:</p> <code type="none"> -mnesia:snmp_open_table(person, [{key, string}]) - </code> - <p>causes the <c>person</c> table to be ordered as an SNMP - table. - </p> +mnesia:snmp_open_table(person, [{key, string}])</code> <p>Consider the following schema for a table of company employees. Each employee is identified by department number - and name. The other table column stores the telephone number: - </p> + and name. The other table column stores the telephone + number:</p> <code type="none"> mnesia:create_table(employee, [{snmp, [{key, {integer, string}}]}, - {attributes, record_info(fields, employees)}]), - </code> - <p>The corresponding SNMP table would have three columns; - <c>department</c>, <c>name</c> and <c>telno</c>. - </p> - <p>It is possible to have table columns that are not visible + {attributes, record_info(fields, employees)}]),</code> + <p>The corresponding SNMP table would have three columns: + <c>department</c>, <c>name</c>, and <c>telno</c>.</p> + <p>An option is to have table columns that are not visible through the SNMP protocol. These columns must be the last columns of the table. In the previous example, the SNMP table could have columns <c>department</c> and <c>name</c> - only. The application could then use the <c>telno</c> column + only. The application could then use column <c>telno</c> internally, but it would not be visible to the SNMP - managers. - </p> + managers.</p> <p>In a table monitored by SNMP, all elements must be - integers, strings, or lists of integers. - </p> + integers, strings, or lists of integers.</p> <p>When a table is SNMP ordered, modifications are more - expensive than usual, O(logN). And more memory is used. - </p> - <p><em>Note:</em>Only the lexicographical SNMP ordering is - implemented in Mnesia, not the actual SNMP monitoring.</p> + expensive than usual, O(logN). Also, more memory is used.</p> + <p>Notice that only the lexicographical SNMP ordering is + implemented in <c>Mnesia</c>, not the actual SNMP monitoring.</p> </desc> </func> <func> - <name>start() -> ok | {error, Reason} </name> - <fsummary>Start a local Mnesia system.</fsummary> + <name>start() -> ok | {error, Reason}</name> + <fsummary>Starts a local <c>Mnesia</c> system.</fsummary> <desc> - <p>The start-up procedure for a set of Mnesia nodes is a - fairly complicated operation. A Mnesia system consists of a set - of nodes, with Mnesia started locally on all + <marker id="start"></marker> + <p>The startup procedure for a set of <c>Mnesia</c> nodes is a + fairly complicated operation. A <c>Mnesia</c> system consists + of a set of nodes, with <c>Mnesia</c> started locally on all participating nodes. Normally, each node has a directory where - all the Mnesia files are written. This directory will be - referred to as the Mnesia directory. Mnesia may also be - started on disc-less nodes. See <c>mnesia:create_schema/1</c> - and the Mnesia User's Guide for more information about disc-less - nodes. - </p> - <p>The set of nodes which makes up a Mnesia system is kept in - a schema and it is possible to add and remove Mnesia nodes + all the <c>Mnesia</c> files are written. This directory is + referred to as the <c>Mnesia</c> directory. <c>Mnesia</c> can + also be started on disc-less nodes. For more information + about disc-less nodes, see <c>mnesia:create_schema/1</c> + and the User's Guide.</p> + <p>The set of nodes that makes up a <c>Mnesia</c> system is kept + in a schema. <c>Mnesia</c> nodes can be added to or removed from the schema. The initial schema is normally created on disc with the function <c>mnesia:create_schema/1</c>. On disc-less nodes, a tiny default schema is generated each time - Mnesia is started. During the start-up procedure, Mnesia - will exchange schema information between the nodes in order - to verify that the table definitions are compatible. - </p> - <p>Each schema has a unique cookie which may be regarded as a + <c>Mnesia</c> is started. During the startup procedure, + <c>Mnesia</c> exchanges schema information between the nodes + to verify that the table definitions are compatible.</p> + <p>Each schema has a unique cookie, which can be regarded as a unique schema identifier. The cookie must be the same on all - nodes where Mnesia is supposed to run. See the Mnesia - User's Guide for more information about these details. - </p> - <p>The schema file, as well as all other files which Mnesia - needs, are kept in the Mnesia directory. The command line - option <c>-mnesia dir Dir</c> can be used to specify the - location of this directory to the Mnesia system. If no such - command line option is found, the name of the directory - defaults to <c>Mnesia.Node</c>. - </p> - <p><c>application:start(mnesia)</c> may also be used.</p> + nodes where <c>Mnesia</c> is supposed to run. For details, + see the User's Guide.</p> + <p>The schema file and all other files that <c>Mnesia</c> + needs are kept in the <c>Mnesia</c> directory. The + command-line option <c>-mnesia dir Dir</c> can be used to + specify the location of this directory to the <c>Mnesia</c> + system. If no such command-line option is found, the name + of the directory defaults to <c>Mnesia.Node</c>.</p> + <p><c>application:start(mnesia)</c> can also be used.</p> </desc> </func> <func> - <name>stop() -> stopped </name> - <fsummary>Stop Mnesia locally.</fsummary> + <name>stop() -> stopped</name> + <fsummary>Stops <c>Mnesia</c> locally.</fsummary> <desc> - <p>Stops Mnesia locally on the current node. - </p> - <p><c>application:stop(mnesia)</c> may also be used.</p> + <marker id="stop"></marker> + <p>Stops <c>Mnesia</c> locally on the current node.</p> + <p><c>application:stop(mnesia)</c> can also be used.</p> </desc> </func> <func> - <name>subscribe(EventCategory) -> {ok, Node} | {error, Reason} </name> - <fsummary>Subscribe to events of type <c>EventCategory</c>.</fsummary> + <name>subscribe(EventCategory) -> {ok, Node} | {error, Reason}</name> + <fsummary>Subscribes to events of type <c>EventCategory</c>.</fsummary> <desc> + <marker id="subscribe"></marker> <p>Ensures that a copy of all events of type - <c>EventCategory</c> are sent to the caller. The event - types available are described in the Mnesia User's Guide at <seealso marker="Mnesia_chap5#event_handling">Mnesia Event Handling</seealso>.</p> - <p><c>Node</c> is the local node. For table events to be subscribed, mnesia must have a readable local copy of the table on the node.</p> + <c>EventCategory</c> is sent to the caller. The available + event types are described in the <seealso marker="Mnesia_chap5#event_handling">User's Guide</seealso>.</p> </desc> </func> <func> - <name>sync_dirty(Fun, [, Args]) -> ResultOfFun | exit(Reason) </name> - <fsummary>Call the Fun in a context which is not protected by a transaction.</fsummary> + <name>sync_dirty(Fun, [, Args]) -> ResultOfFun | exit(Reason)</name> + <fsummary>Calls the <c>Fun</c> in a context that is not protected by a transaction.</fsummary> <desc> - <p>Call the <c>Fun</c> in a context which is not protected - by a transaction. The Mnesia function calls performed in the - <c>Fun</c> are mapped to the corresponding dirty functions. + <marker id="sync_dirty"></marker> + <p>Calls the <c>Fun</c> in a context that is not protected by + a transaction. The <c>Mnesia</c> function calls performed in + the <c>Fun</c> are mapped to the corresponding dirty functions. It is performed in almost the same context as <c>mnesia:async_dirty/1,2</c>. The difference is that the operations are performed synchronously. The caller waits for the updates to be performed on all active replicas before - the <c>Fun</c> returns. See <c>mnesia:activity/4</c> and the - Mnesia User's Guide for more details.</p> + the <c>Fun</c> returns. For details, see + <c>mnesia:activity/4</c> and the User's Guide.</p> </desc> </func> <func> - <name>sync_log() -> ok | {error, Reason} </name> - <fsummary>Perform a file sync of the local log file.</fsummary> + <name>sync_log() -> ok | {error, Reason}</name> + <fsummary>Performs a file sync of the local log file.</fsummary> <desc> <p>Ensures that the local transaction log file is synced to disk. - On a single node system data written to disk tables, since the last dump, - can be lost in case of a power outage. - See <seealso marker="#dump_log/0">dump_log/0</seealso>. - </p> + On a single node system, data written to disk tables since the + last dump can be lost if there is a power outage. + See <seealso marker="#dump_log/0">dump_log/0</seealso>.</p> </desc> </func> - <func> - <name>sync_transaction(Fun, [[, Args], Retries]) -> {aborted, Reason} | {atomic, ResultOfFun} </name> - <fsummary>Synchronously execute a transaction.</fsummary> + <name>sync_transaction(Fun, [[, Args], Retries]) -> {aborted, Reason} | {atomic, ResultOfFun}</name> + <fsummary>Synchronously executes a transaction.</fsummary> <desc> - <p>This function waits until data have been committed and + <marker id="sync_transaction"></marker> + <p>Waits until data have been committed and logged to disk (if disk is used) on every involved node before - it returns, otherwise it behaves as + it returns, otherwise it behaves as <c>mnesia:transaction/[1,2,3]</c>.</p> - <p>This functionality can be used to avoid that one process may overload - a database on another node.</p> + <p>This functionality can be used to avoid that one process + overloads a database on another node.</p> </desc> </func> <func> <name>system_info(InfoKey) -> Info | exit({aborted, Reason})</name> - <fsummary>Return information about the Mnesia system</fsummary> + <fsummary>Returns information about the <c>Mnesia</c> system.</fsummary> <desc> - <p>Returns information about the Mnesia system, such as - transaction statistics, db_nodes, and configuration parameters. - Valid keys are:</p> + <marker id="system_info"></marker> + <p>Returns information about the <c>Mnesia</c> system, such as + transaction statistics, <c>db_nodes</c>, and configuration + parameters. The valid keys are as follows:</p> <list type="bulleted"> <item> - <p><c>all</c>. This argument returns a list of all - local system information. Each element is a - <c>{InfoKey, InfoVal}</c> tuples.<em>Note:</em> New <c>InfoKey</c>'s may - be added and old undocumented <c>InfoKey</c>'s may be removed without + <p><c>all</c>. Returns a list of all local system + information. Each element is a <c>{InfoKey, InfoVal}</c> + tuple.</p> + <p>New <c>InfoKey</c>s can be added and old + undocumented <c>InfoKey</c>s can be removed without notice.</p> </item> <item> - <p><c>access_module</c>. This argument returns the name of - the module which is configured to be the activity access - callback module. - </p> + <p><c>access_module</c>. Returns the name of module that is + configured to be the activity access callback module.</p> </item> <item> - <p><c>auto_repair</c>. This argument returns - <c>true</c> or <c>false</c> to indicate if Mnesia is - configured to invoke the auto repair facility on corrupted - disc files. - </p> + <p><c>auto_repair</c>. Returns <c>true</c> or <c>false</c> + to indicate if <c>Mnesia</c> is configured to start the + auto-repair facility on corrupted disc files.</p> </item> <item> - <p><c>backup_module</c>. This argument returns the name of - the module which is configured to be the backup - callback module. - </p> + <p><c>backup_module</c>. Returns the name of the module + that is configured to be the backup callback module.</p> </item> <item> - <p><c>checkpoints</c>. This argument - returns a list of the names of the - checkpoints currently active on this node. - </p> + <p><c>checkpoints</c>. Returns a list of the names of the + checkpoints currently active on this node.</p> </item> <item> - <p><c>event_module</c>. This argument returns the name of - the module which is the event handler callback module. - </p> + <p><c>event_module</c>. Returns the name of the module + that is the event handler callback module.</p> </item> <item> - <p><c>db_nodes</c>. This argument returns - the nodes which make up the persistent database. Disc - less nodes will only be included in the list of nodes if - they explicitly has been added to the schema, e.g. with + <p><c>db_nodes</c>. Returns the nodes that make up the + persistent database. Disc-less nodes are only included + in the list of nodes if they explicitly have been added + to the schema, for example, with <c>mnesia:add_table_copy/3</c>. The function can be - invoked even if Mnesia is not yet running. - </p> + started even if <c>Mnesia</c> is not yet running.</p> </item> <item> - <p><c>debug</c>. This argument returns the current - debug level of Mnesia. - </p> + <p><c>debug</c>. Returns the current debug level of + <c>Mnesia</c>.</p> </item> <item> - <p><c>directory</c>. This argument returns the name of - the Mnesia directory. It can be invoked even if Mnesia is - not yet running. - </p> + <p><c>directory</c>. Returns the name of the <c>Mnesia</c> + directory. It can be called even if <c>Mnesia</c> is + not yet running.</p> </item> <item> - <p><c>dump_log_load_regulation</c>. This argument - returns a boolean which tells whether Mnesia is - configured to load regulate the dumper process or not. - This feature is temporary and will disappear in future - releases. - </p> + <p><c>dump_log_load_regulation</c>. Returns a boolean that + tells if <c>Mnesia</c> is configured to regulate the + dumper process load.</p> + <p>This feature is temporary and will be removed in future + releases.</p> </item> <item> - <p><c>dump_log_time_threshold</c>. This argument - returns the time threshold for transaction log dumps in - milliseconds. - </p> + <p><c>dump_log_time_threshold</c>. Returns the time + threshold for transaction log dumps in milliseconds.</p> </item> <item> - <p><c>dump_log_update_in_place</c>. This argument - returns a boolean which tells whether Mnesia is - configured to perform the updates in the dets files - directly or if the updates should be performed in a copy - of the dets files. - </p> + <p><c>dump_log_update_in_place</c>. Returns a boolean that + tells if <c>Mnesia</c> is configured to perform the + updates in the <c>dets</c> files directly, or if the + updates are to be performed in a copy of the <c>dets</c> + files.</p> </item> <item> - <p><c>dump_log_write_threshold</c>. This argument - returns the write threshold for transaction log dumps as - the number of writes to the transaction log. - </p> + <p><c>dump_log_write_threshold</c>. + Returns the write threshold for transaction log dumps as + the number of writes to the transaction log.</p> </item> <item> - <p><c>extra_db_nodes</c>. This argument returns a list - of extra db_nodes to be contacted at start-up. - </p> + <p><c>extra_db_nodes</c>. Returns a list + of extra <c>db_nodes</c> to be contacted at startup.</p> </item> <item> - <p><c>fallback_activated</c>. This argument returns - true if a fallback is activated, otherwise false. - </p> + <p><c>fallback_activated</c>. Returns <c>true</c> + if a fallback is activated, otherwise <c>false</c>.</p> </item> <item> - <p><c>held_locks</c>. This argument returns a list of - all locks held by the local Mnesia lock manager. - </p> + <p><c>held_locks</c>. Returns a list of all + locks held by the local <c>Mnesia</c> lock manager.</p> </item> <item> - <p><c>is_running</c>. This argument returns <c>yes</c> - or <c>no</c> to indicate if Mnesia is running. It may - also return <c>starting</c> or <c>stopping</c>. Can be - invoked even if Mnesia is not yet running. - </p> + <p><c>is_running</c>. Returns <c>yes</c> or <c>no</c> to + indicate if <c>Mnesia</c> is running. It can + also return <c>starting</c> or <c>stopping</c>. Can be + called even if <c>Mnesia</c> is not yet running.</p> </item> <item> - <p><c>local_tables</c>. This argument returns a list - of all tables which are configured to reside locally. - </p> + <p><c>local_tables</c>. Returns a list + of all tables that are configured to reside locally.</p> </item> <item> - <p><c>lock_queue</c>. This argument returns a list of + <p><c>lock_queue</c>. Returns a list of all transactions that are queued for execution by the - local lock manager. - </p> + local lock manager.</p> </item> <item> - <p><c>log_version</c>. This argument returns the - version number of the Mnesia transaction log format. - </p> + <p><c>log_version</c>. Returns the version + number of the <c>Mnesia</c> transaction log format.</p> </item> <item> - <p><c>master_node_tables</c>. This argument returns a - list of all tables with at least one master node. - </p> + <p><c>master_node_tables</c>. Returns a + list of all tables with at least one master node.</p> </item> <item> - <p><c>protocol_version</c>. This argument - returns the version number - of the Mnesia inter-process communication protocol. - </p> + <p><c>protocol_version</c>. Returns the version number of + the <c>Mnesia</c> inter-process communication protocol.</p> </item> <item> - <p><c>running_db_nodes</c>. This argument returns a - list of nodes where Mnesia currently is running. This - function can be invoked even if Mnesia is not yet - running, but it will then have slightly different - semantics. If Mnesia is down on the local node, the - function will return those other <c>db_nodes</c> and - <c>extra_db_nodes</c> that for the moment are up and - running. If Mnesia is started, the function will return - those nodes that Mnesia on the local node is fully - connected to. Only those nodes that Mnesia has exchanged - schema information with are included as + <p><c>running_db_nodes</c>. Returns a list of nodes where + <c>Mnesia</c> currently is running. This function can be + called even if <c>Mnesia</c> is not yet running, but it + then has slightly different semantics.</p> + <p>If <c>Mnesia</c> is down on the local node, the function + returns those other <c>db_nodes</c> and + <c>extra_db_nodes</c> that for the moment are + operational.</p> + <p>If <c>Mnesia</c> is started, the function returns + those nodes that <c>Mnesia</c> on the local node is fully + connected to. Only those nodes that <c>Mnesia</c> has + exchanged schema information with are included as <c>running_db_nodes</c>. After the merge of schemas, the - local Mnesia system is fully operable and applications - may perform access of remote replicas. Before the schema - merge Mnesia will only operate locally. Sometimes there - may be more nodes included in the + local <c>Mnesia</c> system is fully operable and + applications can perform access of remote replicas. + Before the schema merge, <c>Mnesia</c> only operates + locally. Sometimes there are more nodes included in the <c>running_db_nodes</c> list than all <c>db_nodes</c> - and <c>extra_db_nodes</c> together. - </p> + and <c>extra_db_nodes</c> together.</p> </item> <item> - <p><c>schema_location</c>. This argument returns the - initial schema location. - </p> + <p><c>schema_location</c>. Returns the + initial schema location.</p> </item> <item> - <p><c>subscribers</c>. This argument returns a list of - local processes currently subscribing to system events. - </p> + <p><c>subscribers</c>. Returns a list of + local processes currently subscribing to system events.</p> </item> <item> - <p><c>tables</c>. This argument returns a list of all - locally known tables. - </p> + <p><c>tables</c>. Returns a list of all + locally known tables.</p> </item> <item> - <p><c>transactions</c>. This argument returns a list - of all currently active local transactions. - </p> + <p><c>transactions</c>. Returns a list + of all currently active local transactions.</p> </item> <item> - <p><c>transaction_failures</c>. This argument returns - a number which indicates how many transactions have - failed since Mnesia was started. - </p> + <p><c>transaction_failures</c>. Returns a + number that indicates how many transactions have + failed since <c>Mnesia</c> was started.</p> </item> <item> - <p><c>transaction_commits</c>. This argument returns a - number which indicates how many transactions have - terminated successfully since Mnesia was started. - </p> + <p><c>transaction_commits</c>. Returns a + number that indicates how many transactions have + terminated successfully since <c>Mnesia</c> was started.</p> </item> <item> - <p><c>transaction_restarts</c>. This argument returns - a number which indicates how many transactions have been - restarted since Mnesia was started. - </p> + <p><c>transaction_restarts</c>. Returns a + number that indicates how many transactions have been + restarted since <c>Mnesia</c> was started.</p> </item> <item> - <p><c>transaction_log_writes</c>. This argument - returns a number which indicates the number of write - operation that have been performed to the transaction - log since start-up. - </p> + <p><c>transaction_log_writes</c>. + Returns a number that indicates how many write + operations that have been performed to the transaction + log since startup.</p> </item> <item> - <p><c>use_dir</c>. This argument returns a boolean - which indicates whether the Mnesia directory is used or - not. Can be invoked even if Mnesia is not yet running. - </p> + <p><c>use_dir</c>. Returns a boolean that indicates if + the <c>Mnesia</c> directory is used or not. Can be + started even if <c>Mnesia</c> is not yet running.</p> </item> <item> - <p><c>version</c>. This argument returns the current - version number of Mnesia. - </p> + <p><c>version</c>. Returns the current + version number of <c>Mnesia</c>.</p> </item> </list> </desc> </func> <func> - <name>table(Tab [,[Option]]) -> QueryHandle </name> + <name>table(Tab [,[Option]]) -> QueryHandle</name> <fsummary>Return a QLC query handle.</fsummary> <desc> - <p><marker id="qlc_table"></marker> -Returns a QLC (Query List Comprehension) query handle, see - <seealso marker="stdlib:qlc">qlc(3)</seealso>.The module <c>qlc</c> implements a query language, it - can use mnesia tables as sources of data. Calling - <c>mnesia:table/1,2</c> is the means to make the <c>mnesia</c> - table <c>Tab</c> usable to QLC.</p> - <p>The list of Options may contain mnesia options or QLC - options, the following options are recognized by Mnesia: - <c>{traverse, SelectMethod},{lock, Lock},{n_objects,Number}</c>, any other option is forwarded - to QLC. The <c>lock</c> option may be <c>read</c> or - <c>write</c>, default is <c>read</c>. The option - <c>n_objects</c> specifies (roughly) the number of objects - returned from mnesia to QLC. Queries to remote tables may - need a larger chunks to reduce network overhead, default - <c>100</c> objects at a time are returned. The option - <c>traverse</c> determines the method to traverse the whole - table (if needed), the default method is <c>select</c>:</p> + <marker id="qlc_table"></marker> + <marker id="table"></marker> + <p>Returns a Query List Comprehension (QLC) query handle, + see the <seealso marker="stdlib:qlc">qlc(3)</seealso> + manual page in <c>STDLIB</c>. The module <c>qlc</c> + implements a query language that can use <c>Mnesia</c> + tables as sources of data. Calling + <c>mnesia:table/1,2</c> is the means to make the + <c>mnesia</c> table <c>Tab</c> usable to QLC.</p> + <p><c>Option</c> can contain <c>Mnesia</c> + options or QLC options. <c>Mnesia</c> recognizes the + following options (any other option is forwarded to + QLC).</p> + <list type="bulleted"> + <item><c>{lock, Lock}</c>, where <c>lock</c> can be + <c>read</c> or <c>write</c>. Default is <c>read</c>. + </item> + <item><c>{n_objects,Number}</c>, where <c>n_objects</c> + specifies (roughly) the number of objects returned + from <c>Mnesia</c> to QLC. Queries to remote tables + can need a larger chunk to reduce network overhead. + By default, <c>100</c> objects at a time are returned. + </item> + <item><c>{traverse, SelectMethod}</c>, where + <c>traverse</c> determines the method to traverse + the whole table (if needed). The default method is + <c>select</c>. + </item> + </list> + <p>There are two alternatives for <c>select</c>:</p> <list type="bulleted"> <item> <p><c>select</c>. The table is traversed by calling - <c>mnesia:select/4</c> and <c>mnesia:select/1</c>. The - match specification (the second argument of <c>select/3</c>) - is assembled by QLC: simple filters are - translated into equivalent match specifications while - more complicated filters have to be applied to all + <c>mnesia:select/4</c> and <c>mnesia:select/1</c>. + The match specification (the second argument of + <c>select/3</c>) is assembled by QLC: simple filters + are translated into equivalent match specifications. + More complicated filters need to be applied to all objects returned by <c>select/3</c> given a match specification that matches all objects.</p> </item> <item> - <p><c>{select, MatchSpec}</c>. As for <c>select</c> - the table is traversed by calling <c>mnesia:select/3</c> and - <c>mnesia:select/1</c>. The difference is that the match - specification is explicitly given. This is how to state - match specifications that cannot easily be expressed - within the syntax provided by QLC.</p> + <p><c>{select, MatchSpec}</c>. As for <c>select</c>, + the table is traversed by calling <c>mnesia:select/3</c> + and <c>mnesia:select/1</c>. The difference is that the + match specification is explicitly given. This is how to + state match specifications that cannot easily be + expressed within the syntax provided by QLC.</p> </item> </list> </desc> </func> <func> <name>table_info(Tab, InfoKey) -> Info | exit({aborted, Reason})</name> - <fsummary>Return local information about table.</fsummary> + <fsummary>Returns local information about table.</fsummary> <desc> + <marker id="table_info"></marker> <p>The <c>table_info/2</c> function takes two arguments. - The first is the name of a Mnesia table, the second is one of - the following keys: - </p> + The first is the name of a <c>Mnesia</c> table. + The second is one of the following keys:</p> <list type="bulleted"> <item> - <p><c>all</c>. This argument returns a list of all - local table information. Each element is a <c>{InfoKey, ItemVal}</c> tuples. <em>Note:</em> New <c>InfoItem</c>'s may be - added and old undocumented <c>InfoItem</c>'s may be removed without - notice.</p> + <p><c>all</c>. Returns a list of all local table + information. Each element is a + <c>{InfoKey, ItemVal}</c> tuple.</p> + <p>New <c>InfoItem</c>s can be added and old undocumented + <c>InfoItem</c>s can be removed without notice.</p> </item> <item> - <p><c>access_mode</c>. This argument returns the - access mode of the table. The access mode may either be - read_only or read_write. - </p> + <p><c>access_mode</c>. Returns the + access mode of the table. The access mode can be + <c>read_only</c> or <c>read_write</c>.</p> </item> <item> - <p><c>arity</c>. This argument returns the arity of - records in the table as specified in the schema. - </p> + <p><c>arity</c>. Returns the arity of + records in the table as specified in the schema.</p> </item> <item> - <p><c>attributes</c>. This argument returns the table - attribute names which are specified in the schema. - </p> + <p><c>attributes</c>. Returns the table + attribute names that are specified in the schema.</p> </item> <item> - <p><c>checkpoints</c>. This argument returns the names - of the currently active checkpoints which involves this - table on this node. - </p> + <p><c>checkpoints</c>. Returns the names + of the currently active checkpoints, which involve this + table on this node.</p> </item> <item> - <p><c>cookie</c>. This argument returns a table cookie - which is a unique system generated identifier for the + <p><c>cookie</c>. Returns a table cookie, + which is a unique system-generated identifier for the table. The cookie is used internally to ensure that two different table definitions using the same table name cannot accidentally be intermixed. The cookie is - generated when the table is initially created. - </p> + generated when the table is created initially.</p> </item> <item> - <p><c>disc_copies</c>. This argument returns the nodes - where a disc_copy of the table resides according to the - schema. - </p> + <p><c>disc_copies</c>. Returns the nodes where a + <c>disc_copy</c> of the table resides according to the + schema.</p> </item> <item> - <p><c>disc_only_copies </c>. This argument returns the - nodes where a disc_only_copy of the table resides - according to the schema. - </p> + <p><c>disc_only_copies</c>. Returns the nodes where a + <c>disc_only_copy</c> of the table resides + according to the schema.</p> </item> <item> - <p><c>index</c>. This argument returns the list of - index position integers for the table. - </p> + <p><c>index</c>. Returns the list of + index position integers for the table.</p> </item> <item> - <p><c>load_node</c>. This argument returns the name of - the node that Mnesia loaded the table from. The - structure of the returned value is unspecified but may - be useful for debugging purposes. - </p> + <p><c>load_node</c>. Returns the name of + the node that <c>Mnesia</c> loaded the table from. The + structure of the returned value is unspecified, but + can be useful for debugging purposes.</p> </item> <item> - <p><c>load_order</c>. This argument returns the load + <p><c>load_order</c>. Returns the load order priority of the table. It is an integer and - defaults to <c>0</c> (zero). - </p> + defaults to <c>0</c> (zero).</p> </item> <item> - <p><c>load_reason</c>. This argument returns the - reason of why Mnesia decided to load the table. The - structure of the returned value is unspecified but may - be useful for debugging purposes. - </p> + <p><c>load_reason</c>. Returns the + reason of why <c>Mnesia</c> decided to load the table. + The structure of the returned value is unspecified, but + can be useful for debugging purposes.</p> </item> <item> - <p><c>local_content</c>. This argument returns - <c>true</c> or <c>false</c> to indicate whether the - table is configured to have locally unique content on - each node. - </p> + <p><c>local_content</c>. Returns <c>true</c> or + <c>false</c> to indicate if the table is configured to + have locally unique content on each node.</p> </item> <item> - <p><c>master_nodes</c>. This argument returns the - master nodes of a table. - </p> + <p><c>master_nodes</c>. Returns the master nodes of a + table.</p> </item> <item> - <p><c>memory</c>. This argument returns the number of - words allocated to the table on this node. - </p> + <p><c>memory</c>. Returns the number of + words allocated to the table on this node.</p> </item> <item> - <p><c>ram_copies</c>. This argument returns the nodes - where a ram_copy of the table resides according to the - schema. - </p> + <p><c>ram_copies</c>. Returns the nodes where a + <c>ram_copy</c> of the table resides according to the + schema.</p> </item> <item> - <p><c>record_name</c>. This argument returns the - record name, common for all records in the table - </p> + <p><c>record_name</c>. Returns the + record name, common for all records in the table.</p> </item> <item> - <p><c>size</c>. This argument returns the number of - records inserted in the table. - </p> + <p><c>size</c>. Returns the number of + records inserted in the table.</p> </item> <item> - <p><c>snmp</c>. This argument returns the SNMP struct. - <c>[]</c>meaning that the table currently has no SNMP - properties. - </p> + <p><c>snmp</c>. Returns the SNMP struct. <c>[]</c> means + that the table currently has no SNMP properties.</p> </item> <item> - <p><c>storage_type</c>.This argument returns the local + <p><c>storage_type</c>. Returns the local storage type of the table. It can be <c>disc_copies</c>, <c>ram_copies</c>, <c>disc_only_copies</c>, or the atom <c>unknown</c>. <c>unknown</c> is returned for all - tables which only reside remotely. - </p> + tables that only reside remotely.</p> </item> <item> - <p><c>subscribers</c>. This argument returns a list + <p><c>subscribers</c>. Returns a list of local processes currently subscribing to local table - events which involve this table on this node. - </p> + events that involve this table on this node.</p> </item> <item> - <p><c>type</c>. This argument returns the table type, - which is either <c>bag</c>, <c>set</c> or <c>ordered_set</c>.. - </p> + <p><c>type</c>. Returns the table type, which is + <c>bag</c>, <c>set</c>, or <c>ordered_set</c>.</p> </item> <item> - <p><c>user_properties</c>. This argument returns the - user associated table properties of the table. It is a - list of the stored property records. - </p> + <p><c>user_properties</c>. Returns the + user-associated table properties of the table. It is a + list of the stored property records.</p> </item> <item> - <p><c>version</c>. This argument returns the current + <p><c>version</c>. Returns the current version of the table definition. The table version is incremented when the table definition is changed. The - table definition may be incremented directly when the - table definition has been changed in a schema - transaction, or when a committed table definition is - merged with table definitions from other nodes during - start-up. - </p> + table definition can be incremented directly when it + has been changed in a schema transaction, or + when a committed table definition is merged with + table definitions from other nodes during startup.</p> </item> <item> - <p><c>where_to_read</c>.This argument returns the node - where the table can be read. If the value <c>nowhere</c> - is returned, the table is not loaded, or it resides at a - remote node which is not running. - </p> + <p><c>where_to_read</c>. Returns the node + where the table can be read. If value <c>nowhere</c> + is returned, either the table is not loaded or it + resides at a remote node that is not running.</p> </item> <item> - <p><c>where_to_write</c>. This argument returns a list - of the nodes that currently hold an active replica of - the table. - </p> + <p><c>where_to_write</c>. Returns a list of the nodes + that currently hold an active replica of the table.</p> </item> <item> - <p><c>wild_pattern</c>. This argument returns a - structure which can be given to the various match - functions for a certain table. A record tuple is where all - record fields have the value <c>'_'</c>. - </p> + <p><c>wild_pattern</c>. Returns a + structure that can be given to the various match + functions for a certain table. A record tuple is where + all record fields have value <c>'_'</c>.</p> </item> </list> </desc> </func> <func> <name>transaction(Fun [[, Args], Retries]) -> {aborted, Reason} | {atomic, ResultOfFun}</name> - <fsummary>Execute a transaction.</fsummary> + <fsummary>Executes a transaction.</fsummary> <desc> - <p>This function executes the functional object <c>Fun</c> - with arguments <c>Args</c> as a transaction. - </p> - <p>The code which executes inside the transaction + <marker id="transaction"></marker> + <p>Executes the functional object <c>Fun</c> + with arguments <c>Args</c> as a transaction.</p> + <p>The code that executes inside the transaction can consist of a series of table manipulation functions. - If something goes wrong inside the transaction as a result of a - user error or a certain table not being available, the - entire transaction is aborted and the function - <c>transaction/1</c> returns the tuple - <c>{aborted, Reason}</c>. - </p> - <p>If all is well, <c>{atomic, ResultOfFun}</c> is returned where - <c>ResultOfFun</c> is the value of the last expression in - <c>Fun</c>. - </p> - <p>A function which adds a family to the database can be - written as follows if we have a structure <c>{family, Father, Mother, ChildrenList}</c>: - </p> + If something goes wrong inside the transaction as a result + of a user error or a certain table not being available, the + entire transaction is terminated and the function + <c>transaction/1</c> returns the tuple + <c>{aborted, Reason}</c>.</p> + <p>If all is going well, <c>{atomic, ResultOfFun}</c> is + returned, where <c>ResultOfFun</c> is the value of the + last expression in <c>Fun</c>.</p> + <p>A function that adds a family to the database can be + written as follows if there is a structure + <c>{family, Father, Mother, ChildrenList}</c>:</p> <code type="none"> add_family({family, F, M, Children}) -> ChildOids = lists:map(fun oid/1, Children), @@ -2715,23 +2590,20 @@ add_family({family, F, M, Children}) -> end, mnesia:transaction(Trans). -oid(Rec) -> {element(1, Rec), element(2, Rec)}. - </code> - <p>This code adds a set of people to the database. Running this code - within one transaction will ensure that either the whole - family is added to the database, or the whole transaction - aborts. For example, if the last child is badly formatted, - or the executing process terminates due to an +oid(Rec) -> {element(1, Rec), element(2, Rec)}.</code> + <p>This code adds a set of people to the database. Running + this code within one transaction ensures that either the whole + family is added to the database, or the whole transaction + terminates. For example, if the last child is badly formatted, + or the executing process terminates because of an <c>'EXIT'</c> signal while executing the family code, the - transaction aborts. Accordingly, the situation where half a - family is added can never occur. - </p> + transaction terminates. Thus, the situation where half a + family is added can never occur.</p> <p>It is also useful to update the database within a transaction if several processes concurrently update the same records. - For example, the function <c>raise(Name, Amount)</c>, which - adds <c>Amount</c> to the salary field of a person, should - be implemented as follows: - </p> + For example, the function <c>raise(Name, Amount)</c>, which + adds <c>Amount</c> to the salary field of a person, is to + be implemented as follows:</p> <code type="none"> raise(Name, Amount) -> mnesia:transaction(fun() -> @@ -2743,71 +2615,68 @@ raise(Name, Amount) -> _ -> mnesia:abort("No such person") end - end). - </code> - <p>When this function executes within a transaction, + end).</code> + <p>When this function executes within a transaction, several processes running on different nodes can concurrently - execute the <c>raise/2</c> function without interfering - with each other. - </p> - <p>Since Mnesia detects deadlocks, a transaction can be - restarted any number of times. This function will attempt a restart as specified in - <c>Retries</c>. <c>Retries</c> must - be an integer greater than 0 or the atom <c>infinity</c>. Default is - <c>infinity</c>.</p> + execute the function <c>raise/2</c> without interfering + with each other.</p> + <p>Since <c>Mnesia</c> detects deadlocks, a transaction can be + restarted any number of times. This function attempts a + restart as specified in <c>Retries</c>. <c>Retries</c> must + be an integer greater than 0 or the atom <c>infinity</c>. + Default is <c>infinity</c>.</p> </desc> </func> <func> - <name>transform_table(Tab, Fun, NewAttributeList, NewRecordName) -> {aborted, R} | {atomic, ok} </name> - <fsummary>Change format on all records in table. <c>Tab</c></fsummary> + <name>transform_table(Tab, Fun, NewAttributeList, NewRecordName) -> {aborted, R} | {atomic, ok}</name> + <fsummary>Changes format on all records in table <c>Tab</c>.</fsummary> <desc> - <p>This function applies the argument <c>Fun</c> to all - records in the table. <c>Fun</c> is a function which takes a - record of the old type and returns a transformed record of the - new type. The <c>Fun</c> argument can also be the atom - <c>ignore</c>, it indicates that only the meta data about the table will - be updated. Usage of <c>ignore</c> is not recommended but included - as a possibility for the user to do his own transform. - <c>NewAttributeList</c> and <c>NewRecordName</c> - specifies the attributes and the new record type of converted - table. Table name will always remain unchanged, if the - record_name is changed only the mnesia functions which - uses table identifiers will work, e.g. <c>mnesia:write/3</c> - will work but <c>mnesia:write/1</c> will not.</p> + <marker id="transform_table_4"></marker> + <p>Applies argument <c>Fun</c> to all + records in the table. <c>Fun</c> is a function that takes a + record of the old type and returns a transformed record of + the new type. Argument <c>Fun</c> can also be the atom + <c>ignore</c>, which indicates that only the metadata + about the table is updated. Use of + <c>ignore</c> is not recommended, but included + as a possibility for the user do to an own transformation.</p> + <p><c>NewAttributeList</c> and <c>NewRecordName</c> + specify the attributes and the new record type of the + converted table. Table name always remains unchanged. If + <c>record_name</c> is changed, only the <c>Mnesia</c> + functions that use table identifiers work, for example, + <c>mnesia:write/3</c> works, but not <c>mnesia:write/1</c>.</p> </desc> </func> <func> - <name>transform_table(Tab, Fun, NewAttributeList) -> {aborted, R} | {atomic, ok} </name> - <fsummary>Change format on all records in table. <c>Tab</c></fsummary> + <name>transform_table(Tab, Fun, NewAttributeList) -> {aborted, R} | {atomic, ok}</name> + <fsummary>Changes format on all records in table <c>Tab</c>.</fsummary> <desc> - <p>Invokes <c>mnesia:transform_table(Tab, Fun, NewAttributeList, RecName)</c> - where <c>RecName</c> is <c>mnesia:table_info(Tab, record_name)</c>.</p> + <p>Calls <c>mnesia:transform_table(Tab, Fun, + NewAttributeList, RecName)</c>, where <c>RecName</c> is + <c>mnesia:table_info(Tab, record_name)</c>.</p> </desc> </func> <func> <name>traverse_backup(Source, [SourceMod,] Target, [TargetMod,] Fun, Acc) -> {ok, LastAcc} | {error, Reason}</name> <fsummary>Traversal of a backup.</fsummary> <desc> - <p>With this function it is possible to iterate over a backup, - either for the purpose of transforming it into a new backup, - or just reading it. The arguments are explained briefly - below. See the Mnesia User's Guide for additional - details. - </p> + <marker id="traverse_backup"></marker> + <p>Iterates over a backup, either to transform it into a + new backup, or read it. The arguments are explained briefly + here. For details, see the User's Guide.</p> <list type="bulleted"> <item><c>SourceMod</c> and <c>TargetMod</c> are the names of - the modules which actually access the backup - media. + the modules that actually access the backup media. </item> <item><c>Source</c> and <c>Target</c> are opaque data used - exclusively by the modules <c>SourceMod</c> and - <c>TargetMod</c> for the purpose of initializing the - backup media. + exclusively by modules <c>SourceMod</c> and <c>TargetMod</c> + to initialize the backup media. </item> <item><c>Acc</c> is an initial accumulator value. </item> <item><c>Fun(BackupItems, Acc)</c> is applied to each item in - the backup. The Fun must return a tuple + the backup. The <c>Fun</c> must return a tuple <c>{BackupItems,NewAcc}</c>, where <c>BackupItems</c> is a list of valid backup items, and <c>NewAcc</c> is a new accumulator value. The returned backup items are written @@ -2821,202 +2690,190 @@ raise(Name, Amount) -> </func> <func> <name>uninstall_fallback() -> ok | {error,Reason}</name> - <fsummary>Uninstall a fallback.</fsummary> + <fsummary>Uninstalls a fallback.</fsummary> <desc> - <p>Invokes <c>mnesia:uninstall_fallback([{scope, global}])</c>.</p> + <marker id="uninstall_fallback_0"></marker> + <p>Calls the function + <c>mnesia:uninstall_fallback([{scope, global}])</c>.</p> </desc> </func> <func> <name>uninstall_fallback(Args) -> ok | {error,Reason}</name> - <fsummary>Uninstall a fallback.</fsummary> + <fsummary>Uninstalls a fallback.</fsummary> <desc> - <p>This function is used to de-install a fallback before it + <p>Deinstalls a fallback before it has been used to restore the database. This is normally a distributed operation that is either performed on all - nodes with disc resident schema or none. Uninstallation of - fallbacks requires Erlang to be up and running on all - involved nodes, but it does not matter if Mnesia is running - or not. Which nodes that are considered as disc-resident - nodes is determined from the schema info in the local - fallback. - </p> - <p><c>Args</c> is a list of the following tuples: - </p> + nodes with disc resident schema, or none. Uninstallation of + fallbacks requires Erlang to be operational on all + involved nodes, but it does not matter if <c>Mnesia</c> is + running or not. Which nodes that are considered as + disc-resident nodes is determined from the schema + information in the local fallback.</p> + <p><c>Args</c> is a list of the following tuples:</p> <list type="bulleted"> - <item> - <p><c>{module, BackupMod}</c>. - See <c>mnesia:install_fallback/2</c> about the - semantics.</p> + <item><c>{module, BackupMod}</c>. For semantics, + see <c>mnesia:install_fallback/2</c>. </item> - <item> - <p><c>{scope, Scope}</c> - See <c>mnesia:install_fallback/2</c> about the - semantics.</p> + <item><c>{scope, Scope}</c>. For semantics, + see <c>mnesia:install_fallback/2</c>. </item> - <item> - <p><c>{mnesia_dir, AlternateDir}</c> - See <c>mnesia:install_fallback/2</c> about the - semantics.</p> + <item><c>{mnesia_dir, AlternateDir}</c>. For semantics, + see <c>mnesia:install_fallback/2</c>. </item> </list> </desc> </func> <func> - <name>unsubscribe(EventCategory) -> {ok, Node} | {error, Reason} </name> - <fsummary>Subscribe to events of type <c>EventCategory</c>.</fsummary> + <name>unsubscribe(EventCategory) -> {ok, Node} | {error, Reason}</name> + <fsummary>Subscribes to events of type <c>EventCategory</c>.</fsummary> <desc> + <marker id="unsubscribe"></marker> <p>Stops sending events of type <c>EventCategory</c> to the caller.</p> <p><c>Node</c> is the local node.</p> </desc> </func> <func> - <name>wait_for_tables(TabList,Timeout) -> ok | {timeout, BadTabList} | {error, Reason} </name> - <fsummary>Wait for tables to be accessible.</fsummary> + <name>wait_for_tables(TabList, Timeout) -> ok | {timeout, BadTabList} | {error, Reason}</name> + <fsummary>Waits for tables to be accessible.</fsummary> <desc> - <p>Some applications need to wait for certain tables to - be accessible in order to do useful work. - <c>mnesia:wait_for_tables/2</c> hangs until all tables in the - <c>TabList</c> are accessible, or until <c>timeout</c> is - reached.</p> + <marker id="wait_for_tables"></marker> + <p>Some applications need to wait for certain tables to be + accessible to do useful work. <c>mnesia:wait_for_tables/2</c> + either hangs until all tables in <c>TabList</c> are accessible, + or until <c>timeout</c> is reached.</p> </desc> </func> <func> - <name>wread({Tab, Key}) -> transaction abort | RecordList </name> - <fsummary>Read records with given key.</fsummary> + <name>wread({Tab, Key}) -> transaction abort | RecordList</name> + <fsummary>Reads records with given key.</fsummary> <desc> - <p>Invoke <c>mnesia:read(Tab, Key, write)</c>.</p> + <marker id="wread"></marker> + <p>Calls the function <c>mnesia:read(Tab, Key, write)</c>.</p> </desc> </func> <func> - <name>write(Record) -> transaction abort | ok </name> + <name>write(Record) -> transaction abort | ok</name> <fsummary>Writes a record into the database.</fsummary> <desc> - <p>Invoke <c>mnesia:write(Tab, Record, write)</c> where - <c>Tab</c> is <c>element(1, Record)</c>.</p> + <marker id="write_1"></marker> + <p>Calls the function <c>mnesia:write(Tab, Record, write)</c>, + where <c>Tab</c> is <c>element(1, Record)</c>.</p> </desc> </func> <func> - <name>write(Tab, Record, LockKind) -> transaction abort | ok </name> - <fsummary>Write a record into the database.</fsummary> + <name>write(Tab, Record, LockKind) -> transaction abort | ok</name> + <fsummary>Writes a record into the database.</fsummary> <desc> - <p>Writes the record <c>Record</c> to the table <c>Tab</c>. - </p> - <p>The function returns <c>ok</c>, or aborts if an error - occurs. For example, the transaction aborts if no - <c>person</c> table exists. - </p> - <p>The semantics of this function is context sensitive. See - <c>mnesia:activity/4</c> for more information. In transaction - context it acquires a lock of type <c>LockKind</c>. The - following lock types are supported: <c>write</c> and - <c>sticky_write</c>.</p> + <marker id="write_3"></marker> + <p>Writes record <c>Record</c> to table <c>Tab</c>.</p> + <p>The function returns <c>ok</c>, or terminates if an error + occurs. For example, the transaction terminates if no + <c>person</c> table exists.</p> + <p>The semantics of this function is context-sensitive. For + details, see <c>mnesia:activity/4</c>. In + transaction-context, it acquires a lock of type + <c>LockKind</c>. The lock types <c>write</c> and + <c>sticky_write</c> are supported.</p> </desc> </func> <func> <name>write_lock_table(Tab) -> ok | transaction abort</name> - <fsummary>Set write lock on an entire table.</fsummary> + <fsummary>Sets write lock on an entire table.</fsummary> <desc> - <p>Invokes <c>mnesia:lock({table, Tab}, write)</c>.</p> + <marker id="write_lock_table"></marker> + <p>Calls the function + <c>mnesia:lock({table, Tab}, write)</c>.</p> </desc> </func> </funcs> <section> <title>Configuration Parameters</title> - <p>Mnesia reads the following application configuration + <marker id="configuration_parameters"></marker> + <p><c>Mnesia</c> reads the following application configuration parameters:</p> <list type="bulleted"> <item> - <p><c>-mnesia access_module Module</c>. The - name of the Mnesia activity access callback module. The default is - <c>mnesia</c>. - </p> + <p><c>-mnesia access_module Module</c>. The name of the + <c>Mnesia</c> activity access callback module. Default is + <c>mnesia</c>.</p> </item> <item> - <p><c>-mnesia auto_repair true | false</c>. This flag controls - whether Mnesia will try to automatically repair - files that have not been properly closed. The default is - <c>true</c>. - </p> + <p><c>-mnesia auto_repair true | false</c>. This flag + controls if <c>Mnesia</c> automatically tries to repair + files that have not been properly closed. Default is + <c>true</c>.</p> </item> <item> - <p><c>-mnesia backup_module Module</c>. The - name of the Mnesia backup callback module. The default is - <c>mnesia_backup</c>. - </p> + <p><c>-mnesia backup_module Module</c>. The name of the + <c>Mnesia</c> backup callback module. Default is + <c>mnesia_backup</c>.</p> </item> <item> - <p><c>-mnesia debug Level</c> - Controls the debug level of Mnesia. - Possible values are:</p> + <p><c>-mnesia debug Level</c>. Controls the debug level + of <c>Mnesia</c>. The possible values are as follows:</p> <taglist> <tag><c>none</c></tag> <item> - <p>No trace outputs at all. This is the default setting. - </p> + <p>No trace outputs. This is the default.</p> </item> <tag><c>verbose</c></tag> <item> <p>Activates tracing of important debug events. These - debug events generate <c>{mnesia_info, Format, Args}</c> - system events. Processes may subscribe to these events with - <c>mnesia:subscribe/1</c>. The events are always sent to Mnesia's - event handler. - </p> + events generate <c>{mnesia_info, Format, Args}</c> + system events. Processes can subscribe to these events with + <c>mnesia:subscribe/1</c>. The events are always sent to + the <c>Mnesia</c> event handler.</p> </item> <tag><c>debug</c></tag> <item> <p>Activates all events at the verbose level plus full trace of all debug events. These debug events generate - <c>{mnesia_info, Format, Args}</c> system events. Processes may - subscribe to these events with <c>mnesia:subscribe/1</c>. The - events are always sent to the Mnesia event handler. On this - debug level, the Mnesia event handler starts subscribing to - updates in the schema table. - </p> + <c>{mnesia_info, Format, Args}</c> system events. + Processes can subscribe to these events with + <c>mnesia:subscribe/1</c>. The events are always sent to + the <c>Mnesia</c> event handler. On this debug level, + the <c>Mnesia</c> event handler starts subscribing to + updates in the schema table.</p> </item> <tag><c>trace</c></tag> <item> - <p>Activates all events at the level debug. On this - debug level, the Mnesia event handler starts subscribing to - updates on all Mnesia tables. This level is only intended - for debugging small toy systems since many large - events may be generated. - </p> + <p>Activates all events at the debug level. On this + level, the <c>Mnesia</c> event handler starts subscribing + to updates on all <c>Mnesia</c> tables. This level is + intended only for debugging small toy systems, as many + large events can be generated.</p> </item> <tag><c>false</c></tag> - <item> - <p>An alias for none. - </p> + <item>An alias for none. </item> <tag><c>true</c></tag> - <item> - <p>An alias for debug. - </p> + <item>An alias for debug. </item> </taglist> </item> <item> <p><c>-mnesia core_dir Directory</c>. The name of the - directory where Mnesia core files is stored or - false. Setting it implies that also ram only nodes, will - generate a core file if a crash occurs. </p> + directory where <c>Mnesia</c> core files is stored, or + false. Setting it implies that also RAM-only nodes + generate a core file if a crash occurs.</p> </item> <item> - <p><c>-mnesia dc_dump_limit Number</c>. - Controls how often <c>disc_copies</c> tables are dumped from memory. + <p><c>-mnesia dc_dump_limit Number</c>. Controls how often + <c>disc_copies</c> tables are dumped from memory. Tables are dumped when <c>filesize(Log) > (filesize(Tab)/Dc_dump_limit)</c>. - Lower values reduces cpu overhead but increases disk space and - startup times. The default is 4.</p> + Lower values reduce CPU overhead but increase disk space + and startup times. Default is 4.</p> </item> <item> <p><c>-mnesia dir Directory</c>. The name of the directory - where all Mnesia data is stored. The name of the directory must - be unique for the current node. Two nodes may, under no - circumstances, share the same Mnesia directory. The results are - totally unpredictable.</p> + where all <c>Mnesia</c> data is stored. The directory name + must be unique for the current node. Two nodes must never + share the the same <c>Mnesia</c> directory. The results + are unpredictable.</p> </item> <item> <p><c>-mnesia dump_disc_copies_at_startup true | false</c>. @@ -3026,156 +2883,148 @@ raise(Name, Amount) -> </item> <item> <p><c>-mnesia dump_log_load_regulation true | false</c>. - Controls if the log dumps should be performed as fast as - possible or if the dumper should do its own load - regulation. This feature is temporary and will disappear in a - future release. The default is <c>false</c>. - </p> + Controls if log dumps are to be performed as fast as + possible, or if the dumper is to do its own load + regulation. Default is <c>false</c>.</p> + <p>This feature is temporary and will be removed in a + future release</p> </item> <item> <p><c>-mnesia dump_log_update_in_place true | false</c>. - Controls if log dumps are performed on a copy of - the original data file, or if the log dump is - performed on the original data file. The default is <c>true</c></p> + Controls if log dumps are performed on a copy of the + original data file, or if the log dump is performed + on the original data file. Default is <c>true</c></p> </item> <item> - <marker id=" dump_log_write_threshold"></marker> - <p><c>-mnesia dump_log_write_threshold Max</c>, where - <c>Max</c> is an integer which specifies the maximum number of writes - allowed to the transaction log before a new dump of the log - is performed. It defaults to 100 log writes. - </p> + <marker id=" dump_log_write_threshold"></marker> + <p><c>-mnesia dump_log_write_threshold Max</c>. + <c>Max</c> is an integer that specifies the maximum + number of writes allowed to the transaction log before + a new dump of the log is performed. Default is <c>100</c> + log writes.</p> </item> <item> - <marker id=" dump_log_time_threshold"></marker> - <p><c>-mnesia dump_log_time_threshold Max</c>, - where <c>Max</c> is an integer which - specifies the dump log interval in milliseconds. It defaults - to 3 minutes. If a dump has not been performed within - <c>dump_log_time_threshold</c> milliseconds, then a new dump is - performed regardless of how many writes have been - performed. - </p> + <marker id=" dump_log_time_threshold"></marker> + <p><c>-mnesia dump_log_time_threshold Max</c>. + <c>Max</c> is an integer that specifies the dump log + interval in milliseconds. Default is 3 minutes. If a + dump has not been performed within + <c>dump_log_time_threshold</c> milliseconds, a new dump + is performed regardless of the number of writes + performed.</p> </item> <item> - <p><c>-mnesia event_module Module</c>. The - name of the Mnesia event handler callback module. The default is - <c>mnesia_event</c>. - </p> + <p><c>-mnesia event_module Module</c>. The name of the + <c>Mnesia</c> event handler callback module. Default is + <c>mnesia_event</c>.</p> </item> <item> <p><c>-mnesia extra_db_nodes Nodes</c> specifies a list of - nodes, in addition to the ones found in the schema, with which - Mnesia should also establish contact. The default value - is the empty list <c>[]</c>. - </p> + nodes, in addition to the ones found in the schema, with + which <c>Mnesia</c> is also to establish contact. Default + is <c>[]</c> (empty list).</p> </item> <item> - <p><c>-mnesia fallback_error_function {UserModule, UserFunc}</c> - specifies a user supplied callback function - which will be called if a fallback is installed and mnesia - goes down on another node. Mnesia will call the function - with one argument the name of the dying node, e.g. - <c>UserModule:UserFunc(DyingNode)</c>. - Mnesia should be restarted or else - the database could be inconsistent. - The default behaviour is to terminate mnesia. - </p> + <p><c>-mnesia fallback_error_function {UserModule, UserFunc}</c>. + Specifies a user-supplied callback function, which is + called if a fallback is installed and <c>Mnesia</c> goes + down on another node. <c>Mnesia</c> calls the function + with one argument, the name of the dying node, for example, + <c>UserModule:UserFunc(DyingNode)</c>. <c>Mnesia</c> must + be restarted, otherwise the database can be inconsistent. + The default behavior is to terminate <c>Mnesia</c>.</p> </item> <item> <p><c>-mnesia max_wait_for_decision Timeout</c>. Specifies - how long Mnesia will wait for other nodes to share their - knowledge regarding the outcome of an unclear transaction. By - default the <c>Timeout</c> is set to the atom - <c>infinity</c>, which implies that if Mnesia upon startup - encounters a "heavyweight transaction" whose outcome is - unclear, the local Mnesia will wait until Mnesia is started - on some (in worst cases all) of the other nodes that were - involved in the interrupted transaction. This is a very rare - situation, but when/if it happens, Mnesia does not guess if - the transaction on the other nodes was committed or aborted. - Mnesia will wait until it knows the outcome and then act - accordingly. - </p> + how long <c>Mnesia</c> waits for other nodes to share their + knowledge about the outcome of an unclear transaction. By + default, <c>Timeout</c> is set to the atom <c>infinity</c>. + This implies that if <c>Mnesia</c> upon startup detects + a "heavyweight transaction" whose outcome is unclear, the + local <c>Mnesia</c> waits until <c>Mnesia</c> is started + on some (in the worst case all) of the other nodes that were + involved in the interrupted transaction. This is a rare + situation, but if it occurs, <c>Mnesia</c> does not guess if + the transaction on the other nodes was committed or + terminated. <c>Mnesia</c> waits until it knows the outcome + and then acts accordingly.</p> <p>If <c>Timeout</c> is set to an integer value in - milliseconds, Mnesia will force "heavyweight transactions" + milliseconds, <c>Mnesia</c> forces "heavyweight transactions" to be finished, even if the outcome of the transaction for the moment is unclear. After <c>Timeout</c> milliseconds, - Mnesia will commit/abort the transaction and continue with - the startup. This may lead to a situation where the - transaction is committed on some nodes and aborted on other - nodes. If the transaction was a schema transaction, the - inconsistency may be fatal. - </p> + <c>Mnesia</c> commits or terminates the transaction and + continues with the startup. This can lead to a situation + where the transaction is committed on some nodes and + terminated on other nodes. If the transaction is a + schema transaction, the inconsistency can be fatal.</p> </item> <item> - <p><c>-mnesia no_table_loaders NUMBER</c> specifies the number of - parallel table loaders during start. More loaders can be good if the - network latency is high or if many tables contains few records. - The default value is <c>2</c>. - </p> + <p><c>-mnesia no_table_loaders NUMBER</c>. Specifies the number + of parallel table loaders during start. More loaders can be + good if the network latency is high or if many tables + contain few records. Default is <c>2</c>.</p> </item> <item> - <p><c>-mnesia send_compressed Level</c> specifies the level of - compression to be used when copying a table from the local node to - another one. The default level is 0. - </p> - <p><c>Level</c> must be an integer in the interval [0, 9], with 0 - representing no compression and 9 representing maximum compression. - Before setting it to a non-zero value, make sure the remote nodes - understand this configuration. - </p> + <p><c>-mnesia send_compressed Level</c>. Specifies the level of + compression to be used when copying a table from the local + node to another one. Default is <c>0</c>.</p> + <p><c>Level</c> must be an integer in the interval + <c>[0, 9]</c>, where <c>0</c> means no compression and + <c>9</c> means maximum compression. Before setting it to a + non-zero value, ensure that the remote nodes + understand this configuration.</p> </item> <item> - <p><c>-mnesia schema_location Loc</c> controls where - Mnesia will look for its schema. The parameter - <c>Loc</c> may be one of the following atoms: </p> + <p><c>-mnesia schema_location Loc</c>. Controls where + <c>Mnesia</c> looks for its schema. Parameter + <c>Loc</c> can be one of the following atoms:</p> <taglist> <tag><c>disc</c></tag> <item> <p>Mandatory disc. The schema is assumed to be located - in the Mnesia directory. If the schema cannot be found, - Mnesia refuses to start. This is the old behavior. - </p> + in the <c>Mnesia</c> directory. If the schema cannot + be found, <c>Mnesia</c> refuses to start. This is the + old behavior.</p> </item> <tag><c>ram</c></tag> <item> <p>Mandatory RAM. The schema resides in RAM - only. At start-up, a tiny new schema is generated. This - default schema just contains the definition of the schema - table and only resides on the local node. Since no other - nodes are found in the default schema, the configuration - parameter <c>extra_db_nodes</c> must be used in - order to let the - node share its table definitions with other nodes. (The - <c>extra_db_nodes</c> parameter may also be used on disc based nodes.) - </p> + only. At startup, a tiny new schema is generated. This + default schema only contains the definition of the schema + table and only resides on the local node. Since no other + nodes are found in the default schema, configuration + parameter <c>extra_db_nodes</c> must be used to let the + node share its table definitions with other nodes.</p> + <p>Parameter <c>extra_db_nodes</c> can also be + used on disc based nodes.</p> </item> <tag><c>opt_disc</c></tag> <item> - <p>Optional disc. The schema may reside either on disc - or in RAM. If the schema is found on disc, Mnesia starts as a - disc based node and the storage type of the schema table is - <c>disc_copies</c>. If no schema is found on disc, Mnesia starts - as a disc-less node and the storage type of the schema table is - <c>ram_copies</c>. The default value for the application parameter - is <c>opt_disc</c>. - </p> + <p>Optional disc. The schema can reside on disc or in + RAM. If the schema is found on disc, <c>Mnesia</c> + starts as a disc-based node and the storage type of + the schema table is <c>disc_copies</c>. If no schema is + found on disc, <c>Mnesia</c> starts as a disc-less node + and the storage type of the schema table is + <c>ram_copies</c>. Default value for the application + parameter is <c>opt_disc</c>.</p> </item> </taglist> </item> </list> - <p>First the SASL application parameters are checked, then - the command line flags are checked, and finally, the default - value is chosen. - </p> + <p>First, the <c>SASL</c> application parameters are checked, + then the command-line flags are checked, and finally, the + default value is chosen.</p> </section> <section> <title>See Also</title> - <p>mnesia_registry(3), mnesia_session(3), qlc(3), - dets(3), ets(3), disk_log(3), application(3) - </p> + <p><seealso marker="kernel:application">application(3)</seealso>, + <seealso marker="stdlib:dets">dets(3)</seealso>, + <seealso marker="kernel:disk_log">disk_log(3)</seealso>, + <seealso marker="stdlib:ets">ets(3)</seealso>, + <seealso marker="mnesia:mnesia_registry">mnesia_registry(3)</seealso>, + <seealso marker="stdlib:qlc">qlc(3)</seealso></p> </section> </erlref> diff --git a/lib/mnesia/doc/src/mnesia_frag_hash.xml b/lib/mnesia/doc/src/mnesia_frag_hash.xml index 0d660925e7..fe6d847655 100644 --- a/lib/mnesia/doc/src/mnesia_frag_hash.xml +++ b/lib/mnesia/doc/src/mnesia_frag_hash.xml @@ -34,22 +34,23 @@ <file>mnesia_frag_hash.sgml</file> </header> <module>mnesia_frag_hash</module> - <modulesummary>Defines mnesia_frag_hash callback behaviour</modulesummary> + <modulesummary>Defines mnesia_frag_hash callback behavior</modulesummary> <description> - <p>The module <c>mnesia_frag_hash</c> defines a callback - behaviour for user defined hash functions of fragmented tables.</p> + <p>This module defines a callback behavior for user-defined hash + functions of fragmented tables.</p> <p>Which module that is selected to implement the <c>mnesia_frag_hash</c> - behaviour for a particular fragmented table is specified together + behavior for a particular fragmented table is specified together with the other <c>frag_properties</c>. The <c>hash_module</c> defines the module name. The <c>hash_state</c> defines the initial hash state.</p> - <p>It implements dynamic hashing which is a kind of hashing + <p>This module implements dynamic hashing, which is a kind of hashing that grows nicely when new fragments are added. It is well - suited for scalable hash tables</p> + suited for scalable hash tables.</p> </description> + <funcs> <func> <name>init_state(Tab, State) -> NewState | abort(Reason)</name> - <fsummary>Initiate the hash state for a new table</fsummary> + <fsummary>Initiates the hash state for a new table.</fsummary> <type> <v>Tab = atom()</v> <v>State = term()</v> @@ -57,21 +58,21 @@ <v>Reason = term()</v> </type> <desc> - <p>This function is invoked when a fragmented table is - created with <c>mnesia:create_table/2</c> or when a - normal (un-fragmented) table is converted to be a + <p>Starts when a fragmented table is + created with the function <c>mnesia:create_table/2</c> or + when a normal (unfragmented) table is converted to be a fragmented table with <c>mnesia:change_table_frag/2</c>.</p> - <p>Note that the <c>add_frag/2</c> function will be invoked - one time each for the rest of the fragments (all but number 1) + <p>Notice that the function <c>add_frag/2</c> is started + one time for each of the other fragments (except number 1) as a part of the table creation procedure.</p> - <p><c>State</c> is the initial value of the <c>hash_state</c> <c>frag_property</c>. The <c>NewState</c> will be stored as - <c>hash_state</c> among the other <c>frag_properties</c>. - </p> + <p><c>State</c> is the initial value of the <c>hash_state</c> + <c>frag_property</c>. <c>NewState</c> is stored as + <c>hash_state</c> among the other <c>frag_properties</c>.</p> </desc> </func> <func> <name>add_frag(State) -> {NewState, IterFrags, AdditionalLockFrags} | abort(Reason)</name> - <fsummary>This function is invoked when a new fragment is added to a fragmented table</fsummary> + <fsummary>Starts when a new fragment is added to a fragmented table.</fsummary> <type> <v>State = term()</v> <v>NewState = term()</v> @@ -80,27 +81,26 @@ <v>Reason = term()</v> </type> <desc> - <p>In order to scale well, it is a good idea ensure that the - records are evenly distributed over all fragments including + <p>To scale well, it is a good idea to ensure that the + records are evenly distributed over all fragments, including the new one.</p> - <p>The <c>NewState</c> will be stored as <c>hash_state</c> among the - other <c>frag_properties</c>. - </p> - <p>As a part of the <c>add_frag</c> procedure, Mnesia will iterate + <p><c>NewState</c> is stored as <c>hash_state</c> among the + other <c>frag_properties</c>.</p> + <p>As a part of the <c>add_frag</c> procedure, <c>Mnesia</c> iterates over all fragments corresponding to the <c>IterFrags</c> numbers - and invoke <c>key_to_frag_number(NewState,RecordKey)</c> for + and starts <c>key_to_frag_number(NewState,RecordKey)</c> for each record. If the new fragment differs from the old - fragment, the record will be moved to the new fragment.</p> + fragment, the record is moved to the new fragment.</p> <p>As the <c>add_frag</c> procedure is a part of a schema - transaction Mnesia will acquire a write locks on the - affected tables. That is both the fragments corresponding + transaction, <c>Mnesia</c> acquires write locks on the + affected tables. That is, both the fragments corresponding to <c>IterFrags</c> and those corresponding to <c>AdditionalLockFrags</c>.</p> </desc> </func> <func> <name>del_frag(State) -> {NewState, IterFrags, AdditionalLockFrags} | abort(Reason)</name> - <fsummary>This function is invoked when a fragment is deleted from a fragmented table</fsummary> + <fsummary>Starts when a fragment is deleted from a fragmented table.</fsummary> <type> <v>State = term()</v> <v>NewState = term()</v> @@ -109,39 +109,38 @@ <v>Reason = term()</v> </type> <desc> - <p>The <c>NewState</c> will be stored as <c>hash_state</c> among the - other <c>frag_properties</c>. - </p> - <p>As a part of the <c>del_frag</c> procedure, Mnesia will iterate + <p><c>NewState</c> is stored as <c>hash_state</c> among the + other <c>frag_properties</c>.</p> + <p>As a part of the <c>del_frag</c> procedure, <c>Mnesia</c> iterates over all fragments corresponding to the <c>IterFrags</c> numbers - and invoke <c>key_to_frag_number(NewState,RecordKey)</c> for + and starts <c>key_to_frag_number(NewState,RecordKey)</c> for each record. If the new fragment differs from the old - fragment, the record will be moved to the new fragment.</p> - <p>Note that all records in the last fragment must be moved to - another fragment as the entire fragment will be deleted.</p> + fragment, the record is moved to the new fragment.</p> + <p>Notice that all records in the last fragment must be moved to + another fragment, as the entire fragment is deleted.</p> <p>As the <c>del_frag</c> procedure is a part of a schema - transaction Mnesia will acquire a write locks on the - affected tables. That is both the fragments corresponding + transaction, <c>Mnesia</c> acquires write locks on the + affected tables. That is, both the fragments corresponding to <c>IterFrags</c> and those corresponding to <c>AdditionalLockFrags</c>.</p> </desc> </func> <func> <name>key_to_frag_number(State, Key) -> FragNum | abort(Reason)</name> - <fsummary>Resolves the key of a record into a fragment number</fsummary> + <fsummary>Resolves the key of a record into a fragment number.</fsummary> <type> <v>FragNum = integer()()</v> <v>Reason = term()</v> </type> <desc> - <p>This function is invoked whenever Mnesia needs to determine + <p>Starts whenever <c>Mnesia</c> needs to determine which fragment a certain record belongs to. It is typically - invoked at read, write and delete.</p> + started at <c>read</c>, <c>write</c>, and <c>delete</c>.</p> </desc> </func> <func> <name>match_spec_to_frag_numbers(State, MatchSpec) -> FragNums | abort(Reason)</name> - <fsummary>Resolves a MatchSpec into a list of fragment numbers</fsummary> + <fsummary>Resolves a <c>MatchSpec</c> into a list of fragment numbers.</fsummary> <type> <v>MatcSpec = ets_select_match_spec()</v> <v>FragNums = [FragNum]</v> @@ -149,17 +148,17 @@ <v>Reason = term()</v> </type> <desc> - <p>This function is invoked whenever Mnesia needs to determine - which fragments that needs to be searched for a MatchSpec. - It is typically invoked at select and match_object.</p> + <p>This function is called whenever <c>Mnesia</c> needs to determine + which fragments that need to be searched for a <c>MatchSpec</c>. + It is typically called by <c>select</c> and + <c>match_object</c>.</p> </desc> </func> </funcs> <section> <title>See Also</title> - <p>mnesia(3) - </p> + <p><seealso marker="mnesia:mnesia">mnesia(3)</seealso></p> </section> </erlref> diff --git a/lib/mnesia/doc/src/mnesia_registry.xml b/lib/mnesia/doc/src/mnesia_registry.xml index ad2b927315..c4b5bdc59a 100644 --- a/lib/mnesia/doc/src/mnesia_registry.xml +++ b/lib/mnesia/doc/src/mnesia_registry.xml @@ -34,71 +34,66 @@ <file>mnesia_registry.sgml</file> </header> <module>mnesia_registry</module> - <modulesummary>Dump support for registries in erl_interface. </modulesummary> + <modulesummary>Dump support for registries in erl_interface.</modulesummary> <description> - <p>The module <c>mnesia_registry</c> is usually part of - <c>erl_interface</c>, but for the time being, it is a part of the - Mnesia application. - </p> - <p><c>mnesia_registry</c> is mainly an module intended for - internal usage within OTP, but it has two functions that - are exported for public use. - </p> - <p>On C-nodes <c>erl_interface</c> has support for registry - tables. These reside in RAM on the C-node but they may also be - dumped into Mnesia tables. By default, the dumping of registry - tables via <c>erl_interface</c> causes a corresponding Mnesia - table to be created with <c>mnesia_registry:create_table/1</c> - if necessary. - </p> - <p>The tables that are created with these functions can be - administered as all other Mnesia tables. They may be included in - backups or replicas may be added etc. The tables are in fact - normal Mnesia tables owned by the user of the corresponding - <c>erl_interface</c> registries. - </p> + <p>This module is usually part of the <c>erl_interface</c> + application, but is currently part of the <c>Mnesia</c> + application.</p> + <p>This module is mainly intended for internal use within OTP, + but it has two functions that are exported for public use.</p> + <p>On C-nodes, <c>erl_interface</c> has support for registry + tables. These tables reside in RAM on the C-node, but can also + be dumped into <c>Mnesia</c> tables. By default, the dumping + of registry tables through <c>erl_interface</c> causes a + corresponding <c>Mnesia</c> table to be created with + <c>mnesia_registry:create_table/1</c>, if necessary.</p> + <p>Tables that are created with these functions can be + administered as all other <c>Mnesia</c> tables. They can be + included in backups, replicas can be added, and so on. + The tables are normal <c>Mnesia</c> tables owned by the user + of the corresponding <c>erl_interface</c> registries.</p> </description> + <funcs> <func> <name>create_table(Tab) -> ok | exit(Reason)</name> <fsummary>Creates a registry table in Mnesia.</fsummary> <desc> - <p>This is a wrapper function for - <c>mnesia:create_table/2</c> which creates a table (if there is no existing table) - with an appropriate set of <c>attributes</c>. The table will - only reside on the local node and its storage type will be - the same as the <c>schema</c> table on the local - node, ie. <c>{ram_copies,[node()]}</c> or - <c>{disc_copies,[node()]}</c>. - </p> - <p>It is this function that is used by <c>erl_interface</c> to - create the Mnesia table if it did not already exist.</p> + <p>A wrapper function for <c>mnesia:create_table/2</c>, + which creates a table (if there is no existing table) + with an appropriate set of <c>attributes</c>. The table + only resides on the local node and its storage type is + the same as the <c>schema</c> table on the local node, + that is, <c>{ram_copies,[node()]}</c> or + <c>{disc_copies,[node()]}</c>.</p> + <p>This function is used by <c>erl_interface</c> to + create the <c>Mnesia</c> table if it does not already + exist.</p> </desc> </func> <func> <name>create_table(Tab, TabDef) -> ok | exit(Reason)</name> - <fsummary>Creates a customized registry table in Mnesia. </fsummary> + <fsummary>Creates a customized registry table in Mnesia.</fsummary> <desc> - <p>This is a wrapper function for - <c>mnesia:create_table/2</c> which creates a table (if there is no existing table) - with an appropriate set of <c>attributes</c>. The attributes - and <c>TabDef</c> are forwarded to - <c>mnesia:create_table/2</c>. For example, if the table should - reside as <c>disc_only_copies</c> on all nodes a call would - look like:</p> + <p>A wrapper function for <c>mnesia:create_table/2</c>, + which creates a table (if there is no existing table) + with an appropriate set of <c>attributes</c>. The + attributes and <c>TabDef</c> are forwarded to + <c>mnesia:create_table/2</c>. For example, if the table + is to reside as <c>disc_only_copies</c> on all nodes, + a call looks as follows:</p> <code type="none"> TabDef = [{{disc_only_copies, node()|nodes()]}], - mnesia_registry:create_table(my_reg, TabDef) - </code> + mnesia_registry:create_table(my_reg, TabDef)</code> </desc> </func> </funcs> <section> <title>See Also</title> - <p>mnesia(3), erl_interface(3) - </p> + <p><seealso marker="erl_interface:index">erl_interface(3)</seealso>, + <seealso marker="mnesia:mnesia">mnesia(3)</seealso></p> </section> - + </erlref> diff --git a/lib/mnesia/doc/src/notes.xml b/lib/mnesia/doc/src/notes.xml index 7f6ff1e655..dc98efbff3 100644 --- a/lib/mnesia/doc/src/notes.xml +++ b/lib/mnesia/doc/src/notes.xml @@ -38,37 +38,7 @@ thus constitutes one section in this document. The title of each section is the version number of Mnesia.</p> - <section><title>Mnesia 4.13</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Mnesia's dirty functions did not always exit with - <c>{aborted, Reason}</c> as documented when an error - occurred.</p> - <p> - Own Id: OTP-12714</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Make Mnesia DCD dump behavior at start up optional, when - turned off mnesia loads large disc_copies tables faster.</p> - <p> - Own Id: OTP-12481</p> - </item> - </list> - </section> - -</section> - -<section><title>Mnesia 4.12.5</title> + <section><title>Mnesia 4.12.5</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/mnesia/doc/src/part.xml b/lib/mnesia/doc/src/part.xml index 2a16b0a791..ee458dcba0 100644 --- a/lib/mnesia/doc/src/part.xml +++ b/lib/mnesia/doc/src/part.xml @@ -29,12 +29,13 @@ <file>part.sgml</file> </header> <description> - <p><em>Mnesia</em> is a distributed DataBase Management - System(DBMS), appropriate for telecommunications applications and other - Erlang applications which require continuous operation and exhibit soft + <p>The <c>Mnesia</c> application is a distributed Database Management + System (DBMS), appropriate for telecommunications applications and other + Erlang applications, which require continuous operation and exhibit soft real-time properties.</p> </description> <xi:include href="Mnesia_chap1.xml"/> + <xi:include href="Mnesia_overview.xml"/> <xi:include href="Mnesia_chap2.xml"/> <xi:include href="Mnesia_chap3.xml"/> <xi:include href="Mnesia_chap4.xml"/> @@ -44,6 +45,5 @@ <xi:include href="Mnesia_App_A.xml"/> <xi:include href="Mnesia_App_B.xml"/> <xi:include href="Mnesia_App_C.xml"/> - <xi:include href="Mnesia_App_D.xml"/> </part> diff --git a/lib/mnesia/doc/src/ref_man.xml b/lib/mnesia/doc/src/ref_man.xml index e3c75be6e1..6cb91a5c40 100644 --- a/lib/mnesia/doc/src/ref_man.xml +++ b/lib/mnesia/doc/src/ref_man.xml @@ -32,10 +32,10 @@ <file>refman.sgml</file> </header> <description> - <p><em>Mnesia</em> is a distributed DataBase Management + <p>The <c>Mnesia</c> application is a distributed Database Management System (DBMS), appropriate for telecommunications applications and other - Erlang applications which require continuous operation and exhibit soft - real-time properties. </p> + Erlang applications, which require continuous operation and exhibit soft + real-time properties.</p> </description> <xi:include href="mnesia.xml"/> <xi:include href="mnesia_frag_hash.xml"/> diff --git a/lib/mnesia/src/mnesia_log.erl b/lib/mnesia/src/mnesia_log.erl index 21ad0ffdb6..8620949dc0 100644 --- a/lib/mnesia/src/mnesia_log.erl +++ b/lib/mnesia/src/mnesia_log.erl @@ -349,6 +349,8 @@ open_log(Name, Header, Fname, Exists, Repair, Mode) -> mnesia_lib:important("Data may be missing, log ~p repaired: Lost ~p bytes~n", [Fname, BadBytes]), Log; + {error, Reason = {file_error, _Fname, emfile}} -> + fatal("Cannot open log file ~p: ~p~n", [Fname, Reason]); {error, Reason} when Repair == true -> file:delete(Fname), mnesia_lib:important("Data may be missing, Corrupt logfile deleted: ~p, ~p ~n", diff --git a/lib/observer/doc/src/crashdump_ug.xml b/lib/observer/doc/src/crashdump_ug.xml index d22fb4cc40..ccd4d8a5b3 100644 --- a/lib/observer/doc/src/crashdump_ug.xml +++ b/lib/observer/doc/src/crashdump_ug.xml @@ -228,20 +228,17 @@ <p>The <em>ETS Tables</em> panel shows all ETS table information found in the dump. The 'Id' is the same as the 'Table' field found in the raw crashdump, and 'Memory' is the 'Words' field from the - raw crashdump translated into bytes. 'Type' is the type of table, - and it can be either "hash" or "tree". For tree tables there will - be no value in the 'Bucket' field.</p> + raw crashdump translated into bytes. For tree tables there will + be no value in the 'Objects' field.</p> + + <p>To open the detailed information page about the table, double + click or right click the row and select "Properties for + 'Identifier'".</p> <p>To open the detailed information page about the owner process of an ETS table, right click the row and select "Properties for <pid>".</p> - <p>Double clicking a row in the ETS Tables panel has no - effect.</p> - - <p>From the left hand menu you can also select to see internal ETS - tables.</p> - <p> <seealso marker="erts:crash_dump#ets_tables"> More...</seealso> @@ -267,6 +264,22 @@ </section> <section> + <marker id="schedulers"/> + <title>Schedulers</title> + + <p>The <em>Schedulers</em> panel shows all scheduler information + found in the dump.</p> + + <p>To open the detailed information page about the scheduler, + double click or right click the row and select "Properties for + 'Identifier'".</p> + + <p> + <seealso marker="erts:crash_dump">More...</seealso> + </p> + </section> + + <section> <marker id="funs"/> <title>Funs</title> diff --git a/lib/observer/doc/src/notes.xml b/lib/observer/doc/src/notes.xml index 338708db43..a9ec68fc9e 100644 --- a/lib/observer/doc/src/notes.xml +++ b/lib/observer/doc/src/notes.xml @@ -31,28 +31,6 @@ <p>This document describes the changes made to the Observer application.</p> -<section><title>Observer 2.1</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Added the possibility to view sasl log entries for - processes.</p> - <p> - Own Id: OTP-12504</p> - </item> - <item> - <p> - Add memory allocator usage and utilization graphs.</p> - <p> - Own Id: OTP-12631</p> - </item> - </list> - </section> - -</section> - <section><title>Observer 2.0.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/observer/src/Makefile b/lib/observer/src/Makefile index a42967644a..8c6606d0a6 100644 --- a/lib/observer/src/Makefile +++ b/lib/observer/src/Makefile @@ -51,6 +51,7 @@ MODULES= \ cdv_multi_wx \ cdv_port_cb \ cdv_proc_cb \ + cdv_sched_cb \ cdv_table_wx \ cdv_term_cb \ cdv_timer_cb \ diff --git a/lib/observer/src/cdv_bin_cb.erl b/lib/observer/src/cdv_bin_cb.erl index d5fbceff1e..8b427e92b7 100644 --- a/lib/observer/src/cdv_bin_cb.erl +++ b/lib/observer/src/cdv_bin_cb.erl @@ -17,14 +17,14 @@ %% %CopyrightEnd% -module(cdv_bin_cb). --export([get_details/1, +-export([get_details/2, detail_pages/0]). %% Callbacks for cdv_detail_wx -get_details({Type, {T,Key}}) -> +get_details({Type, {T,Key}}, _) -> [{Key,Term}] = ets:lookup(T,Key), {ok,{"Expanded Binary", {Type, Term}, []}}; -get_details({cdv, Id}) -> +get_details({cdv, Id}, _) -> {ok,Bin} = crashdump_viewer:expand_binary(Id), {ok,{"Expanded Binary", {cvd, Bin}, []}}. diff --git a/lib/observer/src/cdv_detail_wx.erl b/lib/observer/src/cdv_detail_wx.erl index dc93507a36..ec0d877d87 100644 --- a/lib/observer/src/cdv_detail_wx.erl +++ b/lib/observer/src/cdv_detail_wx.erl @@ -19,7 +19,7 @@ -behaviour(wx_object). --export([start_link/3]). +-export([start_link/4]). -export([init/1, handle_event/2, handle_cast/2, terminate/2, code_change/3, handle_call/3, handle_info/2]). @@ -38,13 +38,13 @@ -define(ID_NOTEBOOK, 604). %% Detail view -start_link(Id, ParentFrame, Callback) -> - wx_object:start_link(?MODULE, [Id, ParentFrame, Callback, self()], []). +start_link(Id, Data, ParentFrame, Callback) -> + wx_object:start_link(?MODULE, [Id, Data, ParentFrame, Callback, self()], []). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -init([Id, ParentFrame, Callback, Parent]) -> - case Callback:get_details(Id) of +init([Id, Data, ParentFrame, Callback, Parent]) -> + case Callback:get_details(Id, Data) of {ok,Details} -> init(Id,ParentFrame,Callback,Parent,Details); {yes_no, Info, Fun} -> diff --git a/lib/observer/src/cdv_dist_cb.erl b/lib/observer/src/cdv_dist_cb.erl index f7e6c9aded..f45fb1f524 100644 --- a/lib/observer/src/cdv_dist_cb.erl +++ b/lib/observer/src/cdv_dist_cb.erl @@ -21,7 +21,7 @@ col_spec/0, get_info/1, get_detail_cols/1, - get_details/1, + get_details/2, detail_pages/0, format/1]). @@ -55,10 +55,10 @@ get_info(_) -> {Info,TW}. get_detail_cols(_) -> - {[?COL_CH,?COL_CTRL],true}. + {[{node, ?COL_CH},{port,?COL_CTRL}],true}. %% Callbacks for cdv_detail_wx -get_details(Id) -> +get_details(Id, _) -> case crashdump_viewer:node_info(Id) of {ok,Info,TW} -> Proplist = crashdump_viewer:to_proplist(record_info(fields,nod),Info), diff --git a/lib/observer/src/cdv_ets_cb.erl b/lib/observer/src/cdv_ets_cb.erl index 2a5c170e58..371c7f0b32 100644 --- a/lib/observer/src/cdv_ets_cb.erl +++ b/lib/observer/src/cdv_ets_cb.erl @@ -20,7 +20,10 @@ -export([col_to_elem/1, col_spec/0, get_info/1, - get_detail_cols/1]). + get_details/2, + get_detail_cols/1, + detail_pages/0 + ]). -include_lib("wx/include/wx.hrl"). -include("crashdump_viewer.hrl"). @@ -41,7 +44,7 @@ col_to_elem(?COL_ID) -> #ets_table.id; col_to_elem(?COL_NAME) -> #ets_table.name; col_to_elem(?COL_SLOT) -> #ets_table.slot; col_to_elem(?COL_OWNER) -> #ets_table.pid; -col_to_elem(?COL_TYPE) -> #ets_table.type; +col_to_elem(?COL_TYPE) -> #ets_table.data_type; col_to_elem(?COL_BUCK) -> #ets_table.buckets; col_to_elem(?COL_OBJ) -> #ets_table.size; col_to_elem(?COL_MEM) -> #ets_table.memory. @@ -50,18 +53,68 @@ col_spec() -> [{"Id", ?wxLIST_FORMAT_LEFT, 200}, {"Name", ?wxLIST_FORMAT_LEFT, 200}, {"Slot", ?wxLIST_FORMAT_RIGHT, 50}, - {"Owner", ?wxLIST_FORMAT_CENTRE, 90}, - {"Buckets", ?wxLIST_FORMAT_RIGHT, 50}, - {"Objects", ?wxLIST_FORMAT_RIGHT, 50}, - {"Memory", ?wxLIST_FORMAT_RIGHT, 80}, - {"Type", ?wxLIST_FORMAT_LEFT, 50} + {"Owner", ?wxLIST_FORMAT_CENTRE, 120}, + {"Objects", ?wxLIST_FORMAT_RIGHT, 80}, + {"Memory", ?wxLIST_FORMAT_RIGHT, 80} +% {"Type", ?wxLIST_FORMAT_LEFT, 50} ]. get_info(Owner) -> {ok,Info,TW} = crashdump_viewer:ets_tables(Owner), {Info,TW}. +%% Callbacks for cdv_detail_wx +get_details(_Id, not_found) -> + Info = "The table you are searching for could not be found.", + {info,Info}; +get_details(Id, Data) -> + Proplist = crashdump_viewer:to_proplist(record_info(fields,ets_table),Data), + {ok,{"Table:" ++ Id,Proplist,""}}. + get_detail_cols(all) -> - {[?COL_OWNER],false}; -get_detail_cols(_) -> - {[],false}. + {[{ets, ?COL_ID}, {process, ?COL_OWNER}],true}; +get_detail_cols(_W) -> + {[],true}. + + +%%%%%%%%%%%%%%%%%%%%%%%% + +detail_pages() -> + [{"Table Information", fun init_gen_page/2}]. + +init_gen_page(Parent, Info0) -> + Fields = info_fields(), + Details = proplists:get_value(details, Info0), + Info = if is_map(Details) -> Info0 ++ maps:to_list(Details); + true -> Info0 + end, + cdv_info_wx:start_link(Parent,{Fields,Info,[]}). + +%%% Internal +info_fields() -> + [{"Overview", + [{"Id", id}, + {"Name", name}, + {"Slot", slot}, + {"Owner", owner}, + {"Data Structure", data_type} + ]}, + {"Settings", + [{"Type", type}, + {"Protection", protection}, + {"Compressed", compressed}, + {"Fixed", fixed}, + {"Lock write concurrency", write_c}, + {"Lock read concurrency", read_c} + ]}, + {"Memory Usage", + [{"Buckets", buckets}, + {"Size", size}, + {"Memory", memory}, + {"Min Chain Length", chain_min}, + {"Avg Chain Length", chain_avg}, + {"Max Chain Length", chain_max}, + {"Chain Length Std Dev", chain_stddev}, + {"Chain Length Expected Std Dev", chain_exp_stddev} + ]} + ]. diff --git a/lib/observer/src/cdv_fun_cb.erl b/lib/observer/src/cdv_fun_cb.erl index 689ef0e3bb..067377254a 100644 --- a/lib/observer/src/cdv_fun_cb.erl +++ b/lib/observer/src/cdv_fun_cb.erl @@ -55,4 +55,4 @@ get_info(_) -> {Info,TW}. get_detail_cols(_) -> - {[?COL_MOD],false}. + {[{module, ?COL_MOD}],false}. diff --git a/lib/observer/src/cdv_gen_cb.erl b/lib/observer/src/cdv_gen_cb.erl index 6be717d76d..aa5e7c5182 100644 --- a/lib/observer/src/cdv_gen_cb.erl +++ b/lib/observer/src/cdv_gen_cb.erl @@ -42,4 +42,6 @@ info_fields() -> {"Processes",num_procs}, {"ETS tables",num_ets}, {"Timers",num_timers}, - {"Funs",num_fun}]}]. + {"Funs",num_fun}, + {"Calling Thread", thread} + ]}]. diff --git a/lib/observer/src/cdv_html_wx.erl b/lib/observer/src/cdv_html_wx.erl index b79c647f63..6d19589f5d 100644 --- a/lib/observer/src/cdv_html_wx.erl +++ b/lib/observer/src/cdv_html_wx.erl @@ -126,7 +126,7 @@ expand(Id,Callback,#state{expand_wins=Opened0}=State) -> Opened = case lists:keyfind(Id,1,Opened0) of false -> - EW = cdv_detail_wx:start_link(Id,State#state.panel,Callback), + EW = cdv_detail_wx:start_link(Id,[],State#state.panel,Callback), wx_object:get_pid(EW) ! active, [{Id,EW}|Opened0]; {_,EW} -> diff --git a/lib/observer/src/cdv_mod_cb.erl b/lib/observer/src/cdv_mod_cb.erl index e829ff4fca..8d33f9da9d 100644 --- a/lib/observer/src/cdv_mod_cb.erl +++ b/lib/observer/src/cdv_mod_cb.erl @@ -21,7 +21,7 @@ col_spec/0, get_info/1, get_detail_cols/1, - get_details/1, + get_details/2, detail_pages/0, format/1]). @@ -49,10 +49,10 @@ get_info(_) -> {Info,TW}. get_detail_cols(_) -> - {[?COL_ID],true}. + {[{module, ?COL_ID}],true}. %% Callbacks for cdv_detail_wx -get_details(Id) -> +get_details(Id, _) -> {ok,Info,TW} = crashdump_viewer:loaded_mod_details(Id), Proplist = crashdump_viewer:to_proplist(record_info(fields,loaded_mod),Info), Title = io_lib:format("~s",[Info#loaded_mod.mod]), diff --git a/lib/observer/src/cdv_port_cb.erl b/lib/observer/src/cdv_port_cb.erl index 08488d3e34..409431218b 100644 --- a/lib/observer/src/cdv_port_cb.erl +++ b/lib/observer/src/cdv_port_cb.erl @@ -21,7 +21,7 @@ col_spec/0, get_info/1, get_detail_cols/1, - get_details/1, + get_details/2, detail_pages/0, format/1]). @@ -57,10 +57,10 @@ get_info(_) -> {Info,TW}. get_detail_cols(_) -> - {[?COL_ID,?COL_CONN],true}. + {[{port, ?COL_ID},{process, ?COL_CONN}],true}. %% Callbacks for cdv_detail_wx -get_details(Id) -> +get_details(Id, _Data) -> case crashdump_viewer:port(Id) of {ok,Info,TW} -> Proplist = @@ -70,7 +70,7 @@ get_details(Id) -> Info = "The port you are searching for was residing on " "a remote node. No port information is available. " "Show information about the remote node?", - Fun = fun() -> cdv_virtual_list_wx:start_detail_win(NodeId) end, + Fun = fun() -> cdv_virtual_list_wx:start_detail_win(NodeId, node) end, {yes_no, Info, Fun}; {error,not_found} -> Info = "The port you are searching for could not be found.", diff --git a/lib/observer/src/cdv_proc_cb.erl b/lib/observer/src/cdv_proc_cb.erl index d1549f79eb..0af6a9c235 100644 --- a/lib/observer/src/cdv_proc_cb.erl +++ b/lib/observer/src/cdv_proc_cb.erl @@ -21,7 +21,7 @@ col_spec/0, get_info/1, get_detail_cols/1, - get_details/1, + get_details/2, detail_pages/0]). -include_lib("wx/include/wx.hrl"). @@ -57,10 +57,10 @@ get_info(_) -> {Info,TW}. get_detail_cols(_) -> - {[?COL_ID],true}. + {[{process, ?COL_ID}],true}. %% Callbacks for cdv_detail_wx -get_details(Id) -> +get_details(Id, _) -> case crashdump_viewer:proc_details(Id) of {ok,Info,TW} -> %% The following table is used by observer_html_lib @@ -76,7 +76,7 @@ get_details(Id) -> Info = "The process you are searching for was residing on " "a remote node. No process information is available. " "Show information about the remote node?", - Fun = fun() -> cdv_virtual_list_wx:start_detail_win(NodeId) end, + Fun = fun() -> cdv_virtual_list_wx:start_detail_win(NodeId, port) end, {yes_no, Info, Fun}; {error,not_found} -> Info = "The process you are searching for could not be found.", @@ -126,11 +126,13 @@ info_fields() -> {dynamic, current_func}, {"Registered Name", name}, {"Status", state}, + {"Internal State", int_state}, {"Started", start_time}, {"Parent", {click,parent}}, {"Message Queue Len",msg_q_len}, {"Run queue", run_queue}, {"Reductions", reds}, + {"Program counter", prog_count}, {"Continuation pointer",cp}, {"Arity",arity}]}, diff --git a/lib/observer/src/cdv_sched_cb.erl b/lib/observer/src/cdv_sched_cb.erl new file mode 100644 index 0000000000..6ef4886c5e --- /dev/null +++ b/lib/observer/src/cdv_sched_cb.erl @@ -0,0 +1,117 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2013-2014. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +-module(cdv_sched_cb). + +-export([col_to_elem/1, + col_spec/0, + get_info/1, + get_details/2, + get_detail_cols/1, + detail_pages/0 + ]). + +-include_lib("wx/include/wx.hrl"). +-include("crashdump_viewer.hrl"). + +%% Columns +-define(COL_ID, 0). +-define(COL_PROC, ?COL_ID+1). +-define(COL_PORT, ?COL_PROC+1). +-define(COL_RQL, ?COL_PORT+1). +-define(COL_PQL, ?COL_RQL+1). + +%% Callbacks for cdv_virtual_list_wx +col_to_elem(id) -> col_to_elem(?COL_ID); +col_to_elem(?COL_ID) -> #sched.name; +col_to_elem(?COL_PROC) -> #sched.process; +col_to_elem(?COL_PORT) -> #sched.port; +col_to_elem(?COL_RQL) -> #sched.run_q; +col_to_elem(?COL_PQL) -> #sched.port_q. + +col_spec() -> + [{"Id", ?wxLIST_FORMAT_RIGHT, 50}, + {"Current Process", ?wxLIST_FORMAT_CENTER, 130}, + {"Current Port", ?wxLIST_FORMAT_CENTER, 130}, + {"Run Queue Length", ?wxLIST_FORMAT_RIGHT, 180}, + {"Port Queue Length", ?wxLIST_FORMAT_RIGHT, 180}]. + +get_info(_) -> + {ok,Info,TW} = crashdump_viewer:schedulers(), + {Info,TW}. + +get_details(_Id, not_found) -> + Info = "The scheduler you are searching for could not be found.", + {info,Info}; +get_details(Id, Data) -> + Proplist = crashdump_viewer:to_proplist(record_info(fields,sched),Data), + {ok,{"Scheduler: " ++ Id,Proplist,""}}. + +get_detail_cols(all) -> + {[{sched, ?COL_ID}, {process, ?COL_PROC}, {process, ?COL_PORT}],true}; +get_detail_cols(_) -> + {[],false}. + +%%%%%%%%%%%%%%%%%%%%%%%% + +detail_pages() -> + [{"Scheduler Information", fun init_gen_page/2}]. + +init_gen_page(Parent, Info0) -> + Fields = info_fields(), + Details = proplists:get_value(details, Info0), + Info = if is_map(Details) -> Info0 ++ maps:to_list(Details); + true -> Info0 + end, + cdv_info_wx:start_link(Parent,{Fields,Info,[]}). + +%%% Internal +info_fields() -> + [{"Scheduler Overview", + [{"Id", id}, + {"Current Process",process}, + {"Current Port", port}, + {"Sleep Info Flags", sleep_info}, + {"Sleep Aux Work", sleep_aux} + ]}, + {"Run Queues", + [{"Flags", runq_flags}, + {"Priority Max Length", runq_max}, + {"Priority High Length", runq_high}, + {"Priority Normal Length", runq_norm}, + {"Priority Low Length", runq_low}, + {"Port Length", port_q} + ]}, + {"Current Process", + [{"State", currp_state}, + {"Internal State", currp_int_state}, + {"Program Counter", currp_prg_cnt}, + {"CP", currp_cp}, + {"Stack", {currp_stack, 0}}, + {" ", {currp_stack, 1}}, + {" ", {currp_stack, 2}}, + {" ", {currp_stack, 3}}, + {" ", {currp_stack, 4}}, + {" ", {currp_stack, 5}}, + {" ", {currp_stack, 6}}, + {" ", {currp_stack, 7}}, + {" ", {currp_stack, 8}}, + {" ", {currp_stack, 9}}, + {" ", {currp_stack, 10}}, + {" ", {currp_stack, 11}} + ]} + ]. diff --git a/lib/observer/src/cdv_term_cb.erl b/lib/observer/src/cdv_term_cb.erl index 4451045012..6db6d54514 100644 --- a/lib/observer/src/cdv_term_cb.erl +++ b/lib/observer/src/cdv_term_cb.erl @@ -17,11 +17,11 @@ %% %CopyrightEnd% -module(cdv_term_cb). --export([get_details/1, +-export([get_details/2, detail_pages/0]). %% Callbacks for cdv_detail_wx -get_details({Type, {T,Key}}) -> +get_details({Type, {T,Key}}, _) -> [{Key,Term}] = ets:lookup(T,Key), {ok,{"Expanded Term", {Type,[Term, T]}, []}}. diff --git a/lib/observer/src/cdv_timer_cb.erl b/lib/observer/src/cdv_timer_cb.erl index d44592cf18..b4564941ea 100644 --- a/lib/observer/src/cdv_timer_cb.erl +++ b/lib/observer/src/cdv_timer_cb.erl @@ -49,6 +49,6 @@ get_info(Owner) -> {Info,TW}. get_detail_cols(all) -> - {[?COL_OWNER],false}; + {[{process, ?COL_OWNER}],false}; get_detail_cols(_) -> {[],false}. diff --git a/lib/observer/src/cdv_virtual_list_wx.erl b/lib/observer/src/cdv_virtual_list_wx.erl index bfe115a42e..c0bc7018cb 100644 --- a/lib/observer/src/cdv_virtual_list_wx.erl +++ b/lib/observer/src/cdv_virtual_list_wx.erl @@ -19,7 +19,8 @@ -behaviour(wx_object). --export([start_link/2, start_link/3, start_detail_win/1]). +-export([start_link/2, start_link/3, + start_detail_win/1, start_detail_win/2]). %% wx_object callbacks -export([init/1, handle_info/2, terminate/2, code_change/3, handle_call/3, @@ -65,22 +66,31 @@ start_link(ParentWin, Callback, Owner) -> wx_object:start_link(?MODULE, [ParentWin, Callback, Owner], []). start_detail_win(Id) -> - Callback = - case Id of - "<"++_ -> - cdv_proc_cb; - "#Port"++_ -> - cdv_port_cb; - _ -> - case catch list_to_integer(Id) of - NodeId when is_integer(NodeId) -> - cdv_dist_cb; - _ -> - cdv_mod_cb - end - end, - start_detail_win(Callback,Id). -start_detail_win(Callback,Id) -> + case Id of + "<"++_ -> + start_detail_win(Id, process); + "#Port"++_ -> + start_detail_win(Id, port); + _ -> + io:format("cdv: unknown identifier: ~p~n",[Id]), + ignore + end. + +start_detail_win(Id, process) -> + start_detail_win_2(cdv_proc_cb, Id); +start_detail_win(Id, port) -> + start_detail_win_2(cdv_port_cb, Id); +start_detail_win(Id, node) -> + start_detail_win_2(cdv_dist_cb, Id); +start_detail_win(Id, module) -> + start_detail_win_2(cdv_mod_cb, Id); +start_detail_win(Id, ets) -> + start_detail_win_2(cdv_ets_cb, Id); +start_detail_win(Id, sched) -> + start_detail_win_2(cdv_sched_cb, Id). + + +start_detail_win_2(Callback,Id) -> wx_object:cast(Callback,{start_detail_win,Id}). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -158,15 +168,14 @@ create_list_box(Panel, Holder, Callback, Owner) -> do_start_detail_win(undefined, State) -> State; do_start_detail_win(Id, #state{panel=Panel,detail_wins=Opened, - callback=Callback}=State) -> + holder=Holder,callback=Callback}=State) -> NewOpened = case lists:keyfind(Id, 1, Opened) of false -> - case cdv_detail_wx:start_link(Id, Panel, Callback) of - {error, _} -> - Opened; - IW -> - [{Id, IW} | Opened] + Data = call(Holder, {get_data, self(), Id}), + case cdv_detail_wx:start_link(Id, Data, Panel, Callback) of + {error, _} -> Opened; + IW -> [{Id, IW} | Opened] end; {_, IW} -> wxFrame:raise(IW), @@ -247,8 +256,8 @@ handle_event(#wx{id=MenuId, event=#wxCommand{type = command_menu_selected}}, #state{menu_items=MenuItems} = State) -> case lists:keyfind(MenuId,1,MenuItems) of - {MenuId,Id} -> - start_detail_win(Id); + {MenuId,Type,Id} -> + start_detail_win(Id, Type); false -> ok end, @@ -265,7 +274,7 @@ handle_event(#wx{event=#wxList{type=command_list_item_right_click, Menu = wxMenu:new(), MenuItems = lists:flatmap( - fun(Col) -> + fun({Type, Col}) -> MenuId = ?ID_DETAILS + Col, ColText = call(Holder, {get_row, self(), Row, Col}), case ColText of @@ -273,14 +282,15 @@ handle_event(#wx{event=#wxList{type=command_list_item_right_click, _ -> What = case catch list_to_integer(ColText) of - NodeId when is_integer(NodeId) -> + NodeId when is_integer(NodeId), + Type =:= node -> "node " ++ ColText; _ -> ColText end, Text = "Properties for " ++ What, wxMenu:append(Menu, MenuId, Text), - [{MenuId,ColText}] + [{MenuId,Type,ColText}] end end, MenuCols), @@ -300,9 +310,14 @@ handle_event(#wx{event=#wxList{type=command_list_col_click, col=Col}}, handle_event(#wx{event=#wxList{type=command_list_item_activated, itemIndex=Row}}, - #state{holder=Holder} = State) -> - Id = call(Holder, {get_row, self(), Row, id}), - start_detail_win(Id), + #state{holder=Holder, menu_cols=MenuCols} = State) -> + case MenuCols of + [{Type, _}|_] -> + Id = call(Holder, {get_row, self(), Row, id}), + start_detail_win(Id, Type); + _ -> + ignore + end, {noreply, State}; handle_event(Event, State) -> @@ -346,7 +361,7 @@ init_table_holder(Parent, Attrs, Callback, InfoList0) -> attrs=Attrs, callback=Callback}). -table_holder(#holder{callback=Callback, attrs=Attrs}=S0) -> +table_holder(#holder{callback=Callback, attrs=Attrs, info=Info}=S0) -> receive _M={get_row, From, Row, Col} -> %% erlang:display(_M), @@ -360,6 +375,9 @@ table_holder(#holder{callback=Callback, attrs=Attrs}=S0) -> %% erlang:display(_M), State = change_sort(Callback:col_to_elem(Col), S0), table_holder(State); + _M={get_data, From, Id} -> + search_id(From, Id, Callback, Info), + table_holder(S0); stop -> ok; What -> @@ -367,6 +385,21 @@ table_holder(#holder{callback=Callback, attrs=Attrs}=S0) -> table_holder(S0) end. +search_id(From, Id, Callback, Info) -> + Find = fun(_, RowInfo, _) -> + search_id(Callback, RowInfo, Id) + end, + Res = try array:foldl(Find, not_found, Info) + catch Data -> Data end, + From ! {self(), Res}, + ok. + +search_id(Callback, RowInfo, Id) -> + case observer_lib:to_str(get_cell_data(Callback, id, RowInfo)) of + Id -> throw(RowInfo); + _Str -> not_found + end. + change_sort(Col, S0=#holder{parent=Parent, info=Info0, sort=Sort0}) -> NRows = array:size(Info0), InfoList0 = array:to_list(Info0), diff --git a/lib/observer/src/cdv_wx.erl b/lib/observer/src/cdv_wx.erl index 26df60b0a6..ec0c652a27 100644 --- a/lib/observer/src/cdv_wx.erl +++ b/lib/observer/src/cdv_wx.erl @@ -44,6 +44,7 @@ -define(PORT_STR, "Ports"). -define(ETS_STR, "ETS Tables"). -define(TIMER_STR, "Timers"). +-define(SCHEDULER_STR, "Schedulers"). -define(FUN_STR, "Funs"). -define(ATOM_STR, "Atoms"). -define(DIST_STR, "Nodes"). @@ -66,6 +67,7 @@ port_panel, ets_panel, timer_panel, + sched_panel, fun_panel, atom_panel, dist_panel, @@ -171,6 +173,9 @@ setup(#state{frame=Frame, notebook=Notebook}=State) -> %% Timer Panel TimerPanel = add_page(Notebook, ?TIMER_STR, cdv_virtual_list_wx,cdv_timer_cb), + %% Scheduler Panel + SchedPanel = add_page(Notebook, ?SCHEDULER_STR, cdv_virtual_list_wx, cdv_sched_cb), + %% Fun Panel FunPanel = add_page(Notebook, ?FUN_STR, cdv_virtual_list_wx, cdv_fun_cb), @@ -202,6 +207,7 @@ setup(#state{frame=Frame, notebook=Notebook}=State) -> port_panel = PortPanel, ets_panel = EtsPanel, timer_panel = TimerPanel, + sched_panel = SchedPanel, fun_panel = FunPanel, atom_panel = AtomPanel, dist_panel = DistPanel, @@ -335,7 +341,8 @@ check_page_title(Notebook) -> get_active_pid(#state{notebook=Notebook, gen_panel=Gen, pro_panel=Pro, port_panel=Ports, ets_panel=Ets, timer_panel=Timers, fun_panel=Funs, atom_panel=Atoms, dist_panel=Dist, - mod_panel=Mods, mem_panel=Mem, int_panel=Int + mod_panel=Mods, mem_panel=Mem, int_panel=Int, + sched_panel=Sched }) -> Panel = case check_page_title(Notebook) of ?GEN_STR -> Gen; @@ -343,6 +350,7 @@ get_active_pid(#state{notebook=Notebook, gen_panel=Gen, pro_panel=Pro, ?PORT_STR -> Ports; ?ETS_STR -> Ets; ?TIMER_STR -> Timers; + ?SCHEDULER_STR -> Sched; ?FUN_STR -> Funs; ?ATOM_STR -> Atoms; ?DIST_STR -> Dist; diff --git a/lib/observer/src/crashdump_viewer.erl b/lib/observer/src/crashdump_viewer.erl index ef14ba46e2..007fc74279 100644 --- a/lib/observer/src/crashdump_viewer.erl +++ b/lib/observer/src/crashdump_viewer.erl @@ -63,6 +63,7 @@ allocator_info/0, hash_tables/0, index_tables/0, + schedulers/0, expand_binary/1]). %% Library function @@ -114,6 +115,7 @@ -define(proc_heap,proc_heap). -define(proc_messages,proc_messages). -define(proc_stack,proc_stack). +-define(scheduler,scheduler). -define(timer,timer). -define(visible_node,visible_node). @@ -267,6 +269,8 @@ hash_tables() -> call(hash_tables). index_tables() -> call(index_tables). +schedulers() -> + call(schedulers). %%%----------------------------------------------------------------- %%% Called when a link to a process (Pid) is clicked. @@ -431,7 +435,11 @@ handle_call(hash_tables,_From,State=#state{file=File}) -> handle_call(index_tables,_From,State=#state{file=File}) -> IndexTables=index_tables(File), TW = truncated_warning([?hash_table,?index_table]), - {reply,{ok,IndexTables,TW},State}. + {reply,{ok,IndexTables,TW},State}; +handle_call(schedulers,_From,State=#state{file=File}) -> + Schedulers=schedulers(File), + TW = truncated_warning([?scheduler]), + {reply,{ok,Schedulers,TW},State}. @@ -677,9 +685,11 @@ skip(Fd,<<>>) -> val(Fd) -> + val(Fd, "-1"). +val(Fd, NoExist) -> case get_rest_of_line(Fd) of - {eof,[]} -> "-1"; - [] -> "-1"; + {eof,[]} -> NoExist; + [] -> NoExist; {eof,Val} -> Val; Val -> Val end. @@ -967,6 +977,8 @@ get_general_info(Fd,GenInfo) -> get_general_info(Fd,GenInfo#general_info{taints=Val}); "Atoms" -> get_general_info(Fd,GenInfo#general_info{num_atoms=val(Fd)}); + "Calling Thread" -> + get_general_info(Fd,GenInfo#general_info{thread=val(Fd)}); "=" ++ _next_tag -> GenInfo; Other -> @@ -1135,6 +1147,8 @@ all_procinfo(Fd,Fun,Proc,WS,LineHead) -> get_procinfo(Fd,Fun,Proc#proc{arity=Arity--"\r\n"},WS); "Run queue" -> get_procinfo(Fd,Fun,Proc#proc{run_queue=val(Fd)},WS); + "Internal State" -> + get_procinfo(Fd,Fun,Proc#proc{int_state=val(Fd)},WS); "=" ++ _next_tag -> Proc; Other -> @@ -1238,17 +1252,23 @@ maybe_other_node(Id) -> {"<" ++ N, _Rest} -> N; {"#Port<" ++ N, _Rest} -> - N + N; + {_, []} -> + not_found end, + maybe_other_node2(Channel). + +maybe_other_node2(not_found) -> not_found; +maybe_other_node2(Channel) -> Ms = ets:fun2ms( - fun({{Tag,Start},Ch}) when Tag=:=?visible_node, Ch=:=Channel -> + fun({{Tag,Start},Ch}) when Tag=:=?visible_node, Ch=:=Channel -> {"Visible Node",Start}; ({{Tag,Start},Ch}) when Tag=:=?hidden_node, Ch=:=Channel -> {"Hidden Node",Start}; - ({{Tag,Start},Ch}) when Tag=:=?not_connected, Ch=:=Channel -> + ({{Tag,Start},Ch}) when Tag=:=?not_connected, Ch=:=Channel -> {"Not Connected Node",Start} end), - + case ets:select(cdv_dump_index_table,Ms) of [] -> not_found; @@ -1503,7 +1523,7 @@ get_ets_tables(File,Pid,WS) -> end, lookup_and_parse_index(File,{?ets,Pid},ParseFun,"ets"). -get_etsinfo(Fd,EtsTable,WS) -> +get_etsinfo(Fd,EtsTable = #ets_table{details=Ds},WS) -> case line_head(Fd) of "Slot" -> get_etsinfo(Fd,EtsTable#ets_table{slot=list_to_integer(val(Fd))},WS); @@ -1513,7 +1533,7 @@ get_etsinfo(Fd,EtsTable,WS) -> get_etsinfo(Fd,EtsTable#ets_table{name=val(Fd)},WS); "Ordered set (AVL tree), Elements" -> skip_rest_of_line(Fd), - get_etsinfo(Fd,EtsTable#ets_table{type="tree",buckets="-"},WS); + get_etsinfo(Fd,EtsTable#ets_table{data_type="tree"},WS); "Buckets" -> %% A bug in erl_db_hash.c prints a space after the buckets %% - need to strip the string to make list_to_integer/1 happy. @@ -1528,9 +1548,42 @@ get_etsinfo(Fd,EtsTable,WS) -> -1 -> -1; % probably truncated _ -> Words * WS end, - get_etsinfo(Fd,EtsTable#ets_table{memory=Bytes},WS); + get_etsinfo(Fd,EtsTable#ets_table{memory={bytes,Bytes}},WS); "=" ++ _next_tag -> EtsTable; + "Chain Length Min" -> + Val = val(Fd), + get_etsinfo(Fd,EtsTable#ets_table{details=Ds#{chain_min=>Val}},WS); + "Chain Length Avg" -> + Val = try list_to_float(string:strip(val(Fd))) catch _:_ -> "-" end, + get_etsinfo(Fd,EtsTable#ets_table{details=Ds#{chain_avg=>Val}},WS); + "Chain Length Max" -> + Val = val(Fd), + get_etsinfo(Fd,EtsTable#ets_table{details=Ds#{chain_max=>Val}},WS); + "Chain Length Std Dev" -> + Val = val(Fd), + get_etsinfo(Fd,EtsTable#ets_table{details=Ds#{chain_stddev=>Val}},WS); + "Chain Length Expected Std Dev" -> + Val = val(Fd), + get_etsinfo(Fd,EtsTable#ets_table{details=Ds#{chain_exp_stddev=>Val}},WS); + "Fixed" -> + Val = val(Fd), + get_etsinfo(Fd,EtsTable#ets_table{details=Ds#{fixed=>Val}},WS); + "Type" -> + Val = val(Fd), + get_etsinfo(Fd,EtsTable#ets_table{details=Ds#{data_type=>Val}},WS); + "Protection" -> + Val = val(Fd), + get_etsinfo(Fd,EtsTable#ets_table{details=Ds#{protection=>Val}},WS); + "Compressed" -> + Val = val(Fd), + get_etsinfo(Fd,EtsTable#ets_table{details=Ds#{compressed=>Val}},WS); + "Write Concurrency" -> + Val = val(Fd), + get_etsinfo(Fd,EtsTable#ets_table{details=Ds#{write_c=>Val}},WS); + "Read Concurrency" -> + Val = val(Fd), + get_etsinfo(Fd,EtsTable#ets_table{details=Ds#{read_c=>Val}},WS); Other -> unexpected(Fd,Other,"ETS info"), EtsTable @@ -2270,6 +2323,89 @@ get_indextableinfo1(Fd,IndexTable) -> IndexTable end. + +%%----------------------------------------------------------------- +%% Page with scheduler table information +schedulers(File) -> + case lookup_index(?scheduler) of + [] -> + []; + Schedulers -> + Fd = open(File), + R = lists:map(fun({Name,Start}) -> + get_schedulerinfo(Fd,Name,Start) + end, + Schedulers), + close(Fd), + R + end. + +get_schedulerinfo(Fd,Name,Start) -> + pos_bof(Fd,Start), + get_schedulerinfo1(Fd,#sched{name=Name}). + +get_schedulerinfo1(Fd,Sched=#sched{details=Ds}) -> + case line_head(Fd) of + "Current Process" -> + get_schedulerinfo1(Fd,Sched#sched{process=val(Fd, "None")}); + "Current Port" -> + get_schedulerinfo1(Fd,Sched#sched{port=val(Fd, "None")}); + "Run Queue Max Length" -> + RQMax = list_to_integer(val(Fd)), + RQ = RQMax + Sched#sched.run_q, + get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_max=>RQMax}}); + "Run Queue High Length" -> + RQHigh = list_to_integer(val(Fd)), + RQ = RQHigh + Sched#sched.run_q, + get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_high=>RQHigh}}); + "Run Queue Normal Length" -> + RQNorm = list_to_integer(val(Fd)), + RQ = RQNorm + Sched#sched.run_q, + get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_norm=>RQNorm}}); + "Run Queue Low Length" -> + RQLow = list_to_integer(val(Fd)), + RQ = RQLow + Sched#sched.run_q, + get_schedulerinfo1(Fd,Sched#sched{run_q=RQ, details=Ds#{runq_low=>RQLow}}); + "Run Queue Port Length" -> + RQ = list_to_integer(val(Fd)), + get_schedulerinfo1(Fd,Sched#sched{port_q=RQ}); + + "Scheduler Sleep Info Flags" -> + get_schedulerinfo1(Fd,Sched#sched{details=Ds#{sleep_info=>val(Fd, "None")}}); + "Scheduler Sleep Info Aux Work" -> + get_schedulerinfo1(Fd,Sched#sched{details=Ds#{sleep_aux=>val(Fd, "None")}}); + + "Run Queue Flags" -> + get_schedulerinfo1(Fd,Sched#sched{details=Ds#{runq_flags=>val(Fd, "None")}}); + + "Current Process State" -> + get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_state=>val(Fd)}}); + "Current Process Internal State" -> + get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_int_state=>val(Fd)}}); + "Current Process Program counter" -> + get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_prg_cnt=>val(Fd)}}); + "Current Process CP" -> + get_schedulerinfo1(Fd,Sched#sched{details=Ds#{currp_cp=>val(Fd)}}); + "Current Process Limited Stack Trace" -> + %% If there shall be last in scheduler information block + Sched#sched{details=get_limited_stack(Fd, 0, Ds)}; + "=" ++ _next_tag -> + Sched; + Other -> + unexpected(Fd,Other,"scheduler information"), + Sched + end. + +get_limited_stack(Fd, N, Ds) -> + case val(Fd) of + Addr = "0x" ++ _ -> + get_limited_stack(Fd, N+1, Ds#{{currp_stack, N} => Addr}); + "=" ++ _next_tag -> + Ds; + Line -> + get_limited_stack(Fd, N+1, Ds#{{currp_stack, N} => Line}) + end. + %%%----------------------------------------------------------------- %%% Parse memory in crashdump version 0.1 and newer %%% @@ -2572,6 +2708,7 @@ tag_to_atom("proc_dictionary") -> ?proc_dictionary; tag_to_atom("proc_heap") -> ?proc_heap; tag_to_atom("proc_messages") -> ?proc_messages; tag_to_atom("proc_stack") -> ?proc_stack; +tag_to_atom("scheduler") -> ?scheduler; tag_to_atom("timer") -> ?timer; tag_to_atom("visible_node") -> ?visible_node; tag_to_atom(UnknownTag) -> diff --git a/lib/observer/src/crashdump_viewer.hrl b/lib/observer/src/crashdump_viewer.hrl index 47705d0da7..9515e74114 100644 --- a/lib/observer/src/crashdump_viewer.hrl +++ b/lib/observer/src/crashdump_viewer.hrl @@ -36,7 +36,9 @@ num_fun, mem_tot, mem_max, - instr_info}). + instr_info, + thread + }). -record(proc, %% Initial data according to the follwoing: @@ -86,7 +88,8 @@ old_heap_end, memory, stack_dump, - run_queue=?unknown + run_queue=?unknown, + int_state }). -record(port, @@ -98,15 +101,28 @@ monitors, controls}). +-record(sched, + {name, + process, + port, + run_q=0, + port_q=0, + details=#{} + }). + + + -record(ets_table, {pid, slot, id, name, - type="hash", - buckets, + data_type="hash", + buckets="-", size, - memory}). + memory, + details= #{} + }). -record(timer, {pid, diff --git a/lib/observer/src/observer.app.src b/lib/observer/src/observer.app.src index e293990d64..c12353f9e1 100644 --- a/lib/observer/src/observer.app.src +++ b/lib/observer/src/observer.app.src @@ -37,6 +37,7 @@ cdv_proc_cb, cdv_table_wx, cdv_term_cb, + cdv_sched_cb, cdv_timer_cb, cdv_virtual_list_wx, cdv_wx, diff --git a/lib/observer/src/observer_lib.erl b/lib/observer/src/observer_lib.erl index 9592ab5977..40a3eb8831 100644 --- a/lib/observer/src/observer_lib.erl +++ b/lib/observer/src/observer_lib.erl @@ -173,12 +173,17 @@ fill_info([{Str,SubStructure}|Rest], Data) when is_list(SubStructure) -> [{Str, fill_info(SubStructure, Data)}|fill_info(Rest,Data)]; fill_info([{Str,Attrib,SubStructure}|Rest], Data) -> [{Str, Attrib, fill_info(SubStructure, Data)}|fill_info(Rest,Data)]; +fill_info([{Str, Key = {K,N}}|Rest], Data) when is_atom(K), is_integer(N) -> + case get_value(Key, Data) of + undefined -> [undefined | fill_info(Rest, Data)]; + Value -> [{Str, Value} | fill_info(Rest, Data)] + end; fill_info([], _) -> []. -get_value(Key, Data) when is_atom(Key) -> - proplists:get_value(Key,Data); get_value(Fun, Data) when is_function(Fun) -> - Fun(Data). + Fun(Data); +get_value(Key, Data) -> + proplists:get_value(Key,Data). update_info([Fields|Fs], [{_Header, SubStructure}| Rest]) -> update_info2(Fields, SubStructure), @@ -269,6 +274,8 @@ to_str(Pid) when is_pid(Pid) -> pid_to_list(Pid); to_str(No) when is_integer(No) -> integer_to_list(No); +to_str(Float) when is_float(Float) -> + io_lib:format("~.3f", [Float]); to_str(Term) -> io_lib:format("~w", [Term]). diff --git a/lib/observer/src/observer_procinfo.erl b/lib/observer/src/observer_procinfo.erl index 2a840dc49e..d724cd9e96 100644 --- a/lib/observer/src/observer_procinfo.erl +++ b/lib/observer/src/observer_procinfo.erl @@ -150,7 +150,7 @@ handle_event(#wx{event=#wxHtmlLink{linkInfo=#wxHtmlLinkInfo{href=Href}}}, Opened = case lists:keyfind(Id,1,Opened0) of false -> - Win = cdv_detail_wx:start_link(Id,Frame,Callback), + Win = cdv_detail_wx:start_link(Id,[],Frame,Callback), [{Id,Win}|Opened0]; {_,Win} -> wxFrame:raise(Win), diff --git a/lib/observer/src/observer_tv_wx.erl b/lib/observer/src/observer_tv_wx.erl index da4cb8e041..acfa6a2a99 100644 --- a/lib/observer/src/observer_tv_wx.erl +++ b/lib/observer/src/observer_tv_wx.erl @@ -176,10 +176,16 @@ handle_call(Event, From, _State) -> handle_cast(Event, _State) -> error({unhandled_cast, Event}). -handle_info(refresh_interval, State = #state{node=Node, grid=Grid, opt=Opt}) -> - Tables = get_tables(Node, Opt), - Tabs = update_grid(Grid, Opt, Tables), - {noreply, State#state{tabs=Tabs}}; +handle_info(refresh_interval, State = #state{node=Node, grid=Grid, opt=Opt, + tabs=OldTabs}) -> + case get_tables(Node, Opt) of + OldTabs -> + %% no change + {noreply, State}; + Tables -> + Tabs = update_grid(Grid, Opt, Tables), + {noreply, State#state{tabs=Tabs}} + end; handle_info({active, Node}, State = #state{parent=Parent, grid=Grid, opt=Opt, timer=Timer0}) -> diff --git a/lib/orber/doc/src/notes.xml b/lib/orber/doc/src/notes.xml index 1ffd86a1fb..2167a43eee 100644 --- a/lib/orber/doc/src/notes.xml +++ b/lib/orber/doc/src/notes.xml @@ -33,23 +33,7 @@ </header> - <section><title>Orber 3.8</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> Remove the usage of erlang:now() from all Corba - applications and use the new rand module instead of - random. </p> - <p> - Own Id: OTP-12687</p> - </item> - </list> - </section> - -</section> - -<section><title>Orber 3.7.1</title> + <section><title>Orber 3.7.1</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/orber/src/orber.app.src b/lib/orber/src/orber.app.src index 5dda63982f..217c1b247f 100644 --- a/lib/orber/src/orber.app.src +++ b/lib/orber/src/orber.app.src @@ -104,7 +104,7 @@ {applications, [stdlib, kernel, mnesia]}, {env, []}, {mod, {orber, []}}, - {runtime_dependencies, ["stdlib-2.0","ssl-5.3.4","mnesia-4.12","kernel-3.0", + {runtime_dependencies, ["stdlib-2.5","ssl-5.3.4","mnesia-4.12","kernel-3.0", "inets-5.10","erts-7.0"]} ]}. diff --git a/lib/os_mon/c_src/cpu_sup.c b/lib/os_mon/c_src/cpu_sup.c index 20bb9ce391..9e217db105 100644 --- a/lib/os_mon/c_src/cpu_sup.c +++ b/lib/os_mon/c_src/cpu_sup.c @@ -53,7 +53,6 @@ #endif #if defined(__linux__) -#include <string.h> /* strlen */ #define PROCSTAT "/proc/stat" #define BUFFERSIZE (256) @@ -73,6 +72,13 @@ typedef struct { #endif +#if defined(__FreeBSD__) +#include <sys/resource.h> +#include <sys/sysctl.h> +#define CU_BSD_VALUES (6) +#endif + + #define FD_IN (0) #define FD_OUT (1) #define FD_ERR (2) @@ -157,12 +163,16 @@ static int processors_online() { } #endif +#if defined(__FreeBSD__) +void getsysctl(const char *, void *, size_t); +#endif + int main(int argc, char** argv) { char cmd; int rc; int sz; unsigned int *rv; -#if defined(__linux__) +#if defined(__linux__) || defined(__FreeBSD__) unsigned int no_of_cpus = 0; #endif @@ -175,7 +185,14 @@ int main(int argc, char** argv) { #if defined(__linux__) no_of_cpus = processors_online(); if ( (rv = (unsigned int*)malloc(sizeof(unsigned int)*(2 + 2*no_of_cpus*CU_VALUES))) == NULL) { - error("cpu_cup: malloc error"); + error("cpu_sup: malloc error"); + } +#endif + +#if defined(__FreeBSD__) + getsysctl("hw.ncpu", &no_of_cpus, sizeof(int)); + if ( (rv = (unsigned int*)malloc(sizeof(unsigned int)*(2 + 2*no_of_cpus*CU_BSD_VALUES))) == NULL) { + error("cpu_sup: malloc error"); } #endif @@ -204,14 +221,14 @@ int main(int argc, char** argv) { case AVG5: bsd_loadavg(1); break; case AVG15: bsd_loadavg(2); break; #endif -#if defined(__sun__) || defined(__linux__) +#if defined(__sun__) || defined(__linux__) || defined(__FreeBSD__) case UTIL: util_measure(&rv,&sz); sendv(rv, sz); break; #endif case QUIT: free((void*)rv); return 0; default: error("Bad command"); break; } } - return 0; /* supress warnings */ + return 0; /* suppress warnings */ } /* ---------------------------- * @@ -520,6 +537,71 @@ static void util_measure(unsigned int **result_vec, int *result_sz) { #endif /* ---------------------------- * + * FreeBSD stat functions * + * ---------------------------- */ + +#if defined(__FreeBSD__) + +#define EXIT_WITH(msg) (rich_error(msg, __FILE__, __LINE__)) +#define RICH_BUFLEN (213) /* left in error(char*) */ + +void rich_error(const char *reason, const char *file, const int line) { + char buf[RICH_BUFLEN]; + snprintf(buf, RICH_BUFLEN, "%s (%s:%i)", reason, file, line); + error(buf); +} +#undef RICH_BUFLEN + +static void util_measure(unsigned int **result_vec, int *result_sz) { + int no_of_cpus; + size_t size_cpu_times; + unsigned long *cpu_times; + unsigned int *rv = NULL; + int i; + + getsysctl("hw.ncpu", &no_of_cpus, sizeof(int)); + /* Header constant CPUSTATES = #long values per cpu. */ + size_cpu_times = sizeof(long) * CPUSTATES * no_of_cpus; + cpu_times = malloc(size_cpu_times); + if (!cpu_times) { + EXIT_WITH("badalloc"); + } + getsysctl("kern.cp_times", cpu_times, size_cpu_times); + + rv = *result_vec; + rv[0] = no_of_cpus; + rv[1] = CU_BSD_VALUES; + ++rv; /* first value is number of cpus */ + ++rv; /* second value is number of entries */ + + for (i = 0; i < no_of_cpus; ++i) { + int offset = i * CPUSTATES; + rv[ 0] = CU_CPU_ID; rv[ 1] = i; + rv[ 2] = CU_USER; rv[ 3] = cpu_times[CP_USER + offset]; + rv[ 4] = CU_NICE_USER; rv[ 5] = cpu_times[CP_NICE + offset]; + rv[ 6] = CU_KERNEL; rv[ 7] = cpu_times[CP_SYS + offset]; + rv[ 8] = CU_IDLE; rv[ 9] = cpu_times[CP_IDLE + offset]; + rv[10] = CU_HARD_IRQ; rv[11] = cpu_times[CP_INTR + offset]; + rv += CU_BSD_VALUES*2; + } + + *result_sz = 2 + 2*CU_BSD_VALUES * no_of_cpus; +} + +void getsysctl(const char *name, void *ptr, size_t len) +{ + size_t gotlen = len; + if (sysctlbyname(name, ptr, &gotlen, NULL, 0) != 0) { + EXIT_WITH("sysctlbyname failed"); + } + if (gotlen != len) { + EXIT_WITH("sysctlbyname: unexpected length"); + } +} +#endif + + +/* ---------------------------- * * Generic functions * * ---------------------------- */ @@ -581,5 +663,3 @@ static void error(char* err_msg) { ; exit(-1); } - - diff --git a/lib/os_mon/doc/src/cpu_sup.xml b/lib/os_mon/doc/src/cpu_sup.xml index 59da876208..4a8f5bffa0 100644 --- a/lib/os_mon/doc/src/cpu_sup.xml +++ b/lib/os_mon/doc/src/cpu_sup.xml @@ -34,7 +34,7 @@ and CPU utilization. It is part of the OS_Mon application, see <seealso marker="os_mon_app">os_mon(6)</seealso>. Available for Unix, although CPU utilization values (<c>util/0,1</c>) are only - available for Solaris and Linux.</p> + available for Solaris, Linux and FreeBSD.</p> <p>The load values are proportional to how long time a runnable Unix process has to spend in the run queue before it is scheduled. Accordingly, higher values mean more system load. The returned diff --git a/lib/os_mon/doc/src/notes.xml b/lib/os_mon/doc/src/notes.xml index bd4206d748..d3acc1effc 100644 --- a/lib/os_mon/doc/src/notes.xml +++ b/lib/os_mon/doc/src/notes.xml @@ -30,25 +30,6 @@ </header> <p>This document describes the changes made to the OS_Mon application.</p> -<section><title>Os_Mon 2.4</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - cpu_sup should use native sysctl/libkvm calls on BSD</p> - <p> - This avoids forking off with os:cmd every time we just - want to collect the load averages. riak does this every - second, which results in a lot of unnecessary load.</p> - <p> - Own Id: OTP-12730</p> - </item> - </list> - </section> - -</section> - <section><title>Os_Mon 2.3.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/os_mon/src/cpu_sup.erl b/lib/os_mon/src/cpu_sup.erl index 0c26956c57..d8cfd845bc 100644 --- a/lib/os_mon/src/cpu_sup.erl +++ b/lib/os_mon/src/cpu_sup.erl @@ -121,7 +121,7 @@ util(Args) when is_list (Args) -> util(_) -> erlang:error(badarg). --spec util() -> float(). +-spec util() -> float() | {'error', any()}. util() -> case util([]) of @@ -160,7 +160,8 @@ handle_call(?quit, _From, State) -> handle_call({?util, D, PC}, {Client, _Tag}, #state{os_type = {unix, Flavor}} = State) when Flavor == sunos; - Flavor == linux -> + Flavor == linux; + Flavor == freebsd -> case measurement_server_call(State#state.server, {?util, D, PC, Client}) of {error, Reason} -> { reply, diff --git a/lib/os_mon/test/cpu_sup_SUITE.erl b/lib/os_mon/test/cpu_sup_SUITE.erl index 9f58e043db..7da819379c 100644 --- a/lib/os_mon/test/cpu_sup_SUITE.erl +++ b/lib/os_mon/test/cpu_sup_SUITE.erl @@ -64,6 +64,8 @@ all() -> [load_api, util_api, util_values, port, unavailable]; {unix, linux} -> [load_api, util_api, util_values, port, unavailable]; + {unix, freebsd} -> + [load_api, util_api, util_values, port, unavailable]; {unix, _OSname} -> [load_api]; _OS -> [unavailable] end. diff --git a/lib/parsetools/doc/src/notes.xml b/lib/parsetools/doc/src/notes.xml index 1751238d20..c8cb70b6d2 100644 --- a/lib/parsetools/doc/src/notes.xml +++ b/lib/parsetools/doc/src/notes.xml @@ -30,21 +30,6 @@ </header> <p>This document describes the changes made to the Parsetools application.</p> -<section><title>Parsetools 2.1</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> The new <c>-dialyzer()</c> attribute is used for - suppressing Dialyzer warnings in generated code. </p> - <p> - Own Id: OTP-12271</p> - </item> - </list> - </section> - -</section> - <section><title>Parsetools 2.0.12</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/parsetools/src/parsetools.app.src b/lib/parsetools/src/parsetools.app.src index 9eeb8fcc05..a7b258820a 100644 --- a/lib/parsetools/src/parsetools.app.src +++ b/lib/parsetools/src/parsetools.app.src @@ -12,7 +12,7 @@ {env, [{file_util_search_methods,[{"", ""}, {"ebin", "esrc"}, {"ebin", "src"}]} ] }, - {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]} + {runtime_dependencies, ["stdlib-2.5","kernel-3.0","erts-6.0"]} ] }. diff --git a/lib/percept/doc/src/notes.xml b/lib/percept/doc/src/notes.xml index 19d6cd6d01..b51c8fcb4d 100644 --- a/lib/percept/doc/src/notes.xml +++ b/lib/percept/doc/src/notes.xml @@ -32,21 +32,6 @@ </header> <p>This document describes the changes made to the Percept application.</p> -<section><title>Percept 0.8.11</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Fix http server configuration</p> - <p> - Own Id: OTP-12662</p> - </item> - </list> - </section> - -</section> - <section><title>Percept 0.8.10</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/percept/vsn.mk b/lib/percept/vsn.mk index 833ab35aa5..4451354e21 100644 --- a/lib/percept/vsn.mk +++ b/lib/percept/vsn.mk @@ -1 +1 @@ -PERCEPT_VSN = 0.8.11 +PERCEPT_VSN = 0.8.10 diff --git a/lib/public_key/doc/src/notes.xml b/lib/public_key/doc/src/notes.xml index df65ad7004..f241a91eb0 100644 --- a/lib/public_key/doc/src/notes.xml +++ b/lib/public_key/doc/src/notes.xml @@ -34,37 +34,6 @@ <file>notes.xml</file> </header> -<section><title>Public_Key 1.0</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - public_key: Remove legacy switch compact_bit_string</p> - <p> - E.i bitstrings will not be decode as {Unused, Binary}, - they are now Erlang bitstrings.</p> - <p> - Also the compact_bit_string implies the - legacy_erlang_types switch So removing the switch will - also make OCTET STRING values be represented as binaries.</p> - <p> - Undecoded open type will now be wrapped in a - asn1_OPENTYPE tuple.</p> - <p> - This will change some values in records returned by the - public_key API making this change a potentiall - incompatibility.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12110</p> - </item> - </list> - </section> - -</section> - <section><title>Public_Key 0.23</title> <section><title>Improvements and New Features</title> diff --git a/lib/sasl/doc/src/notes.xml b/lib/sasl/doc/src/notes.xml index c0ac0cf79c..95d7c6fa50 100644 --- a/lib/sasl/doc/src/notes.xml +++ b/lib/sasl/doc/src/notes.xml @@ -30,30 +30,6 @@ </header> <p>This document describes the changes made to the SASL application.</p> -<section><title>SASL 2.4.2</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - The undocumented upgrade instruction - <c>{remove_module,PrePurge,PostPurge,DepMods}</c> is - removed. This instruction was added for symmetry reasons - in OTP R7B, but was never documented or tested.</p> - <p> - The existing instruction <c>{add_module,Mod,DepMods}</c> - is now documented, and the complementing instruction - <c>{delete_module,Mod,DepMods}</c> is added.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-11540</p> - </item> - </list> - </section> - -</section> - <section><title>SASL 2.4.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/sasl/doc/src/sasl_app.xml b/lib/sasl/doc/src/sasl_app.xml index 9c3c80bd13..572e550061 100644 --- a/lib/sasl/doc/src/sasl_app.xml +++ b/lib/sasl/doc/src/sasl_app.xml @@ -92,6 +92,13 @@ <item>Installs <c>sasl_report_file_h</c> in the error logger. This makes all reports go to the file <c>FileName</c>. <c>FileName</c> is a string.</item> + <tag><c>{file,FileName,Modes}</c></tag> + <item>Same as <c>{file,FileName}</c> except that the <c>Modes</c> + allows to specify the modes used for opening the <c>FileName</c> + given to the <seealso marker="kernel:file#open/2">file:open/2</seealso> + call. When not specified, the <c>Modes</c> defaults to <c>[write]</c>. + Use <c>[append]</c> for having the <c>FileName</c> open in append mode. + <c>FileName</c> is a string.</item> <tag><c>false</c></tag> <item> <p>No SASL error logger handler is installed.</p> diff --git a/lib/sasl/src/sasl.erl b/lib/sasl/src/sasl.erl index fdea6da13e..4a220f0511 100644 --- a/lib/sasl/src/sasl.erl +++ b/lib/sasl/src/sasl.erl @@ -55,7 +55,9 @@ get_sasl_error_logger() -> case application:get_env(sasl, sasl_error_logger) of {ok, false} -> undefined; {ok, tty} -> tty; - {ok, {file, File}} when is_list(File) -> {file, File}; + {ok, {file, File}} when is_list(File) -> {file, File, [write]}; + {ok, {file, File, Modes}} when is_list(File), is_list(Modes) -> + {file, File, Modes}; {ok, Bad} -> exit({bad_config, {sasl, {sasl_error_logger, Bad}}}); _ -> undefined end. @@ -125,9 +127,9 @@ delete_sasl_error_logger(Type) -> error_logger:delete_report_handler(mod(Type)). mod(tty) -> sasl_report_tty_h; -mod({file, _File}) -> sasl_report_file_h. +mod({file, _File, _Modes}) -> sasl_report_file_h. -args({file, File}, Type) -> {File, type(Type)}; +args({file, File, Modes}, Type) -> {File, Modes, type(Type)}; args(_, Type) -> type(Type). type(error) -> error; diff --git a/lib/sasl/src/sasl_report_file_h.erl b/lib/sasl/src/sasl_report_file_h.erl index f42b4b5ff2..a5bd0ac055 100644 --- a/lib/sasl/src/sasl_report_file_h.erl +++ b/lib/sasl/src/sasl_report_file_h.erl @@ -28,9 +28,9 @@ handle_event/2, handle_call/2, handle_info/2, terminate/2]). -init({File, Type}) -> +init({File, Modes, Type}) when is_list(Modes) -> process_flag(trap_exit, true), - case file:open(File, [write]) of + case file:open(File, Modes) of {ok,Fd} -> {ok, {Fd, File, Type}}; What -> diff --git a/lib/sasl/test/sasl_SUITE.erl b/lib/sasl/test/sasl_SUITE.erl index d7b99d506e..d9ab9e551c 100644 --- a/lib/sasl/test/sasl_SUITE.erl +++ b/lib/sasl/test/sasl_SUITE.erl @@ -26,10 +26,11 @@ %% Test cases must be exported. -export([app_test/1, appup_test/1, - log_mf_h_env/1]). + log_mf_h_env/1, + log_file/1]). all() -> - [log_mf_h_env, app_test, appup_test]. + [log_mf_h_env, log_file, app_test, appup_test]. groups() -> []. @@ -151,10 +152,9 @@ check_appup([],_,_) -> log_mf_h_env(Config) -> PrivDir = ?config(priv_dir,Config), LogDir = filename:join(PrivDir,sasl_SUITE_log_dir), - ok = file:make_dir(LogDir), + ok = filelib:ensure_dir(LogDir), application:stop(sasl), - SaslEnv = application:get_all_env(sasl), - lists:foreach(fun({E,_V}) -> application:unset_env(sasl,E) end, SaslEnv), + clear_env(sasl), ok = application:set_env(sasl,error_logger_mf_dir,LogDir), match_error(missing_config,application:start(sasl)), @@ -178,6 +178,23 @@ log_mf_h_env(Config) -> ok = application:set_env(sasl,error_logger_mf_dir,LogDir), ok = application:start(sasl). +log_file(Config) -> + PrivDir = ?config(priv_dir,Config), + LogDir = filename:join(PrivDir,sasl_SUITE_log_dir), + ok = filelib:ensure_dir(LogDir), + File = filename:join(LogDir, "file.log"), + application:stop(sasl), + clear_env(sasl), + + ok = application:set_env(sasl,sasl_error_logger,{file, File}, [{persistent, true}]), + ok = application:start(sasl), + application:stop(sasl), + ok = application:set_env(sasl,sasl_error_logger,{file, File, [append]}, [{persistent, true}]), + ok = application:start(sasl), + application:stop(sasl), + ok = application:set_env(sasl,sasl_error_logger, tty, [{persistent, false}]), + ok = application:start(sasl). + %%----------------------------------------------------------------- %% Internal @@ -185,3 +202,7 @@ match_error(Expected,{error,{bad_return,{_,{'EXIT',{Expected,{sasl,_}}}}}}) -> ok; match_error(Expected,Actual) -> ?t:fail({unexpected_return,Expected,Actual}). + +clear_env(App) -> + [application:unset_env(App,Opt) || {Opt,_} <- application:get_all_env(App)], + ok. diff --git a/lib/snmp/doc/src/snmp_app.xml b/lib/snmp/doc/src/snmp_app.xml index 86f0981988..e36908a5b9 100644 --- a/lib/snmp/doc/src/snmp_app.xml +++ b/lib/snmp/doc/src/snmp_app.xml @@ -587,7 +587,7 @@ <marker id="manager_server_timeout"></marker> <tag><c><![CDATA[server_timeout() = integer() <optional>]]></c></tag> <item> - <p>Asynchroneous request cleanup time. For every requests, + <p>Asynchronous request cleanup time. For every requests, some info is stored internally, in order to be able to deliver the reply (when it arrives) to the proper destination. If the reply arrives, this info will be deleted. But if diff --git a/lib/snmp/doc/src/snmp_config.xml b/lib/snmp/doc/src/snmp_config.xml index 0ec8bb91cf..d1ee6545dd 100644 --- a/lib/snmp/doc/src/snmp_config.xml +++ b/lib/snmp/doc/src/snmp_config.xml @@ -616,7 +616,7 @@ in so far as it will be converted to the new format if found. <marker id="manager_server_timeout"></marker> <tag><c><![CDATA[server_timeout() = integer() <optional>]]></c></tag> <item> - <p>Asynchroneous request cleanup time. For every requests, + <p>Asynchronous request cleanup time. For every requests, some info is stored internally, in order to be able to deliver the reply (when it arrives) to the proper destination. If the reply arrives, this info will be deleted. But if diff --git a/lib/snmp/src/app/snmp.app.src b/lib/snmp/src/app/snmp.app.src index cbd292e4c3..a55bb389ba 100644 --- a/lib/snmp/src/app/snmp.app.src +++ b/lib/snmp/src/app/snmp.app.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -137,5 +137,5 @@ %% before snmp. {applications, [kernel, stdlib]}, {mod, {snmp_app, []}}, - {runtime_dependencies, ["stdlib-2.0","runtime_tools-1.8.14","mnesia-4.12", + {runtime_dependencies, ["stdlib-2.5","runtime_tools-1.8.14","mnesia-4.12", "kernel-3.0","erts-6.0","crypto-3.3"]}]}. diff --git a/lib/snmp/src/app/snmp.appup.src b/lib/snmp/src/app/snmp.appup.src index 081163b368..a21ff863be 100644 --- a/lib/snmp/src/app/snmp.appup.src +++ b/lib/snmp/src/app/snmp.appup.src @@ -28,6 +28,8 @@ %% {update, snmpa_local_db, soft, soft_purge, soft_purge, []} %% {add_module, snmpm_net_if_mt} [ + {"5.1.2", [ % Only runtime dependencies change + ]}, {"5.1.1", [{restart_application, snmp}]}, {"5.1", [ % Only compiler changes ]}, @@ -47,6 +49,8 @@ %% {remove, {snmpm_net_if_mt, soft_purge, soft_purge}} [ + {"5.1.2", [ % Only runtime dependencies change + ]}, {"5.1.1", [{restart_application, snmp}]}, {"5.1", [ % Only compiler changes ]}, diff --git a/lib/snmp/src/manager/snmpm.erl b/lib/snmp/src/manager/snmpm.erl index 8976322c4e..96e3d55b46 100644 --- a/lib/snmp/src/manager/snmpm.erl +++ b/lib/snmp/src/manager/snmpm.erl @@ -520,7 +520,7 @@ sync_get(UserId, TargetName, Context, Oids, Timeout, ExtraInfo) -> -%% --- asynchroneous get-request --- +%% --- asynchronous get-request --- %% %% The reply will be delivered to the user %% through a call to handle_pdu/5 @@ -588,7 +588,7 @@ sync_get_next(UserId, TargetName, Context, Oids, Timeout, ExtraInfo) -> %% </BACKWARD-COMPAT> -%% --- asynchroneous get_next-request --- +%% --- asynchronous get_next-request --- %% async_get_next2(UserId, TargetName, Oids) -> @@ -654,7 +654,7 @@ sync_set(UserId, TargetName, Context, VarsAndVals, Timeout, ExtraInfo) -> %% </BACKWARD-COMPAT> -%% --- asynchroneous set-request --- +%% --- asynchronous set-request --- %% async_set2(UserId, TargetName, VarsAndVals) -> @@ -746,7 +746,7 @@ sync_get_bulk(UserId, TargetName, NonRep, MaxRep, Context, Oids, Timeout, %% </BACKWARD-COMPAT> -%% --- asynchroneous get-bulk --- +%% --- asynchronous get-bulk --- %% async_get_bulk2(UserId, TargetName, NonRep, MaxRep, Oids) -> diff --git a/lib/snmp/vsn.mk b/lib/snmp/vsn.mk index 67adf0a34f..14da37a225 100644 --- a/lib/snmp/vsn.mk +++ b/lib/snmp/vsn.mk @@ -18,6 +18,6 @@ # %CopyrightEnd% APPLICATION = snmp -SNMP_VSN = 5.1.2 +SNMP_VSN = 5.2 PRE_VSN = APP_VSN = "$(APPLICATION)-$(SNMP_VSN)$(PRE_VSN)" diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index af5b78bff2..c77ee1e77a 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -29,66 +29,46 @@ <file>notes.xml</file> </header> -<section><title>Ssh 4.0</title> +<section><title>Ssh 3.2.4</title> <section><title>Fixed Bugs and Malfunctions</title> <list> <item> <p> - Included test of the 'e' and 'f' parameters in dh key - exchange as specified in rfc 4253 section 8.</p> + Gracefully terminate if sockets is unexpectedly closed.</p> <p> - Own Id: OTP-12649</p> + Own Id: OTP-12782</p> </item> <item> <p> - Fixes the bug that once the rekey_limit bytes (by - default, 1GB) had been transmitted the connection was - rekeyed every minute, not after the next 'rekey_limit'.</p> + Made Codenomicon Defensics test suite pass: <list> + <item>limit number of algorithms in kexinit + message</item> <item>check 'e' and 'f' parameters in + kexdh</item> <item>implement 'keyboard-interactive' user + authentication on server side</item> <item> return plain + text message to bad version exchange message</item> + </list></p> <p> - Thanks to Simon Cornish for the report and the fix!</p> - <p> - Own Id: OTP-12692</p> - </item> - <item> - <p> - Fixes a bug that causes an SFTP connection to always fail - when {timeout, Timeout} option is used with - ssh_sftp:start_channel.</p> - <p> - Thanks to Simon Cornish</p> - <p> - Own Id: OTP-12708</p> + Own Id: OTP-12784</p> </item> </list> </section> +</section> + +<section><title>Ssh 3.2.3</title> - <section><title>Improvements and New Features</title> + <section><title>Fixed Bugs and Malfunctions</title> <list> <item> <p> - The internal group to user_drv protocol has been changed - to be synchronous in order to guarantee that output sent - to a process implementing the user_drv protocol is - printed before replying. This protocol is used by the - standard_output device and the ssh application when - acting as a client. </p> - <p> - This change changes the previous unlimited buffer when - printing to standard_io and other devices that end up in - user_drv to 1KB.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12240</p> - </item> - <item> - <p> - If ssh_connection:subsystem/4 fails we do not want to - crash but rather terminate gracefully.</p> + A new option for handling the SSH_MSG_DEBUG message's + printouts. A fun could be given in the options that will + be called whenever the SSH_MSG_DEBUG message arrives. + This enables the user to format the printout or just + discard it.</p> <p> - Own Id: OTP-12648 Aux Id: seq12834 </p> + Own Id: OTP-12738 Aux Id: seq12860 </p> </item> </list> </section> diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index d49d3ac2a7..c1235715cc 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -35,13 +35,15 @@ <section> <title>SSH</title> - + <marker id="supported"/> <list type="bulleted"> <item>For application dependencies see <seealso marker="SSH_app"> ssh(6)</seealso> </item> <item>Supported SSH version is 2.0.</item> + <item>Supported public key algorithms: ssh-rsa and ssh-dss.</item> <item>Supported MAC algorithms: hmac-sha2-256 and hmac-sha1.</item> <item>Supported encryption algorithms: aes128-ctr, aes128-cb and 3des-cbc.</item> <item>Supported key exchange algorithms: diffie-hellman-group1-sha1.</item> + <item>Supported compression algorithms: none, zlib, [email protected],</item> <item>Supports unicode filenames if the emulator and the underlaying OS support it. See section DESCRIPTION in the <seealso marker="kernel:file">file</seealso> manual page in <c>kernel</c> @@ -57,29 +59,40 @@ this module, or abstractions to indicate the intended use of the data type, or both:</p> <taglist> - <tag><c>boolean()</c></tag> - <item><p>= <c>true | false</c></p></item> - <tag><c>string()</c></tag> - <item><p>= <c>[byte()]</c></p></item> - <tag><c>ssh_daemon_ref()</c></tag> - <item><p>Opaque to the user, - returned by <c>ssh:daemon/[1,2,3]</c></p></item> - <tag><c>ssh_connection_ref()</c></tag> - <item><p>Opaque to the user, - returned by <c>ssh:connect/3</c></p></item> - <tag><c>ip_address()</c></tag> + <tag><c>boolean() =</c></tag> + <item><p><c>true | false</c></p></item> + <tag><c>string() =</c></tag> + <item><p><c>[byte()]</c></p></item> + <tag><c>ssh_daemon_ref() =</c></tag> + <item><p>opaque() - + as returned by <c>ssh:daemon/[1,2,3]</c></p></item> + <tag><c>ssh_connection_ref() =</c></tag> + <item><p>opaque() - as returned by <c>ssh:connect/3</c></p></item> + <tag><c>ip_address() =</c></tag> <item><p><c>inet::ip_address</c></p></item> - <tag><c>subsystem_spec()</c></tag> - <item><p>= <c>{subsystem_name(), - {channel_callback(), channel_init_args()}}</c></p></item> - <tag><c>subsystem_name()</c></tag> - <item><p>= <c>string()</c></p></item> - <tag><c>channel_callback()</c></tag> - <item><p>= <c>atom()</c> - Name of the Erlang module - implementing the subsystem using the <c>ssh_channel</c> behavior, see - <seealso marker="ssh_channel">ssh_channel(3)</seealso></p></item> - <tag><c>channel_init_args()</c></tag> - <item><p>= <c>list()</c></p></item> + <tag><c>subsystem_spec() =</c></tag> + <item><p><c>{subsystem_name(), + {channel_callback(), channel_init_args()}}</c></p></item> + <tag><c>subsystem_name() =</c></tag> + <item><p><c>string()</c></p></item> + <tag><c>channel_callback() =</c></tag> + <item><p><c>atom()</c> - Name of the Erlang module + implementing the subsystem using the <c>ssh_channel</c> behavior, see + <seealso marker="ssh_channel">ssh_channel(3)</seealso></p></item> + <tag><c>channel_init_args() =</c></tag> + <item><p><c>list()</c></p></item> + + <tag><c>algs_list() =</c></tag> + <item><p><c>list( alg_entry() )</c></p></item> + + <tag><c>alg_entry() =</c></tag> + <item><p><c>{kex, simple_algs()} | {public_key, simple_algs()} | {cipher, double_algs()} | {mac, double_algs()} | {compression, double_algs()}</c></p></item> + + <tag><c>simple_algs() =</c></tag> + <item><p><c>list( atom() )</c></p></item> + + <tag><c>double_algs() =</c></tag> + <item><p><c>[{client2serverlist,simple_algs()},{server2client,simple_algs()}] | simple_algs()</c></p></item> </taglist> </section> @@ -161,19 +174,71 @@ and <c>password</c>. However, those optins are not always desirable to use from a security point of view.</p> </item> + + <tag><c><![CDATA[{disconnectfun, fun(Reason:term()) -> _}]]></c></tag> + <item> + <p>Provides a fun to implement your own logging when a server disconnects the client.</p> + </item> + + <tag><c><![CDATA[{unexpectedfun, fun(Message:term(), Peer) -> report | skip }]]></c></tag> + <item> + <p>Provides a fun to implement your own logging or other action when an unexpected message arrives. + If the fun returns <c>report</c> the usual info report is issued but if <c>skip</c> is returned no + report is generated.</p> + <p><c>Peer</c> is in the format of <c>{Host,Port}</c>.</p> + </item> + <tag><c><![CDATA[{public_key_alg, 'ssh-rsa' | 'ssh-dss'}]]></c></tag> <item> + <note> + <p>This option is kept for compatibility. It is ignored if the <c>preferred_algorithms</c> + option is used. The equivalence of <c>{public_key_alg,'ssh-dss'}</c> is + <c>{preferred_algorithms, [{public_key,['ssh-dss','ssh-rsa']}]}</c>.</p> + </note> <p>Sets the preferred public key algorithm to use for user authentication. If the preferred algorithm fails, the other algorithm is tried. The default is to try <c><![CDATA['ssh-rsa']]></c> first.</p> </item> + <tag><c><![CDATA[{pref_public_key_algs, list()}]]></c></tag> <item> + <note> + <p>This option is kept for compatibility. It is ignored if the <c>preferred_algorithms</c> + option is used. The equivalence of <c>{pref_public_key_algs,['ssh-dss']}</c> is + <c>{preferred_algorithms, [{public_key,['ssh-dss']}]}</c>.</p> + </note> <p>List of public key algorithms to try to use. <c>'ssh-rsa'</c> and <c>'ssh-dss'</c> are available. Overrides <c><![CDATA[{public_key_alg, 'ssh-rsa' | 'ssh-dss'}]]></c></p> </item> + + <tag><c><![CDATA[{preferred_algorithms, algs_list()}]]></c></tag> + <item> + <p>List of algorithms to use in the algorithm negotiation. The default <c>algs_list()</c> can + be obtained from <seealso marker="#default_algorithms/0">default_algorithms/0</seealso>. + </p> + <p>Here is an example of this option:</p> + <code> +{preferred_algorithms, + [{public_key,['ssh-rsa','ssh-dss']}, + {cipher,[{client2server,['aes128-ctr']}, + {server2client,['aes128-cbc','3des-cbc']}]}, + {mac,['hmac-sha2-256','hmac-sha1']}, + {compression,[none,zlib]} +} +</code> + <p>The example specifies different algorithms in the two directions (client2server and server2client), for cipher but specifies the same +algorithms for mac and compression in both directions. The kex (key exchange) and public key algorithms are set to their default values, +kex is implicit but public_key is set explicitly.</p> + + <warning> + <p>Changing the values can make a connection less secure. Do not change unless you + know exactly what you are doing. If you do not understand the values then you + are not supposed to change them.</p> + </warning> + </item> + <tag><c><![CDATA[{connect_timeout, timeout()}]]></c></tag> <item> <p>Sets a time-out on the transport layer @@ -227,6 +292,13 @@ <item> <p>Sets a time-out on a connection when no channels are active. Defaults to <c>infinity</c>.</p></item> + <tag><c><![CDATA[{ssh_msg_debug_fun, fun(ConnectionRef::ssh_connection_ref(), AlwaysDisplay::boolean(), Msg::binary(), LanguageTag::binary()) -> _}]]></c></tag> + <item> + <p>Provide a fun to implement your own logging of the SSH message SSH_MSG_DEBUG. The last three parameters are from the message, see RFC4253, section 11.3. The <c>ConnectionRef</c> is the reference to the connection on which the message arrived. The return value from the fun is not checked.</p> + <p>The default behaviour is ignore the message. + To get a printout for each message with <c>AlwaysDisplay = true</c>, use for example <c>{ssh_msg_debug_fun, fun(_,true,M,_)-> io:format("DEBUG: ~p~n", [M]) end}</c></p> + </item> + </taglist> </desc> </func> @@ -296,7 +368,7 @@ an own CLI channel. If set to <c>no_cli</c>, the CLI channels are disabled and only subsystem channels are allowed.</p> </item> - <tag><c><![CDATA[{user_dir, String}]]></c></tag> + <tag><c><![CDATA[{user_dir, string()}]]></c></tag> <item> <p>Sets the user directory. That is, the directory containing <c>ssh</c> configuration files for the user, such as @@ -313,6 +385,7 @@ <c><![CDATA[/etc/ssh]]></c>. For security reasons, this directory is normally accessible only to the root user.</p> </item> + <tag><c><![CDATA[{auth_methods, string()}]]></c></tag> <item> <p>Comma-separated string that determines which @@ -320,6 +393,19 @@ in what order they are tried. Defaults to <c><![CDATA["publickey,keyboard-interactive,password"]]></c></p> </item> + + <tag><c><![CDATA[{auth_method_kb_interactive_data, PromptTexts}]]> + <br/>where: + <br/>PromptTexts = kb_int_tuple() | fun(Peer::{IP::tuple(),Port::integer()}, User::string(), Service::string()) -> kb_int_tuple() + <br/>kb_int_tuple() = {Name::string(), Instruction::string(), Prompt::string(), Echo::boolean()}</c> + </tag> + <item> + <p>Sets the text strings that the daemon sends to the client for presentation to the user when using <c>keyboar-interactive</c> authentication. If the fun/3 is used, it is called when the actual authentication occurs and may therefore return dynamic data like time, remote ip etc.</p> + <p>The parameter <c>Echo</c> guides the client about need to hide the password.</p> + <p>The default value is: + <c>{auth_method_kb_interactive_data, {"SSH server", "Enter password for \""++User++"\"", "password: ", false}></c></p> + </item> + <tag><c><![CDATA[{user_passwords, [{string() = User, string() = Password}]}]]></c></tag> <item> @@ -335,6 +421,33 @@ user. From a security perspective this option makes the server very vulnerable.</p> </item> + + <tag><c><![CDATA[{preferred_algorithms, algs_list()}]]></c></tag> + <item> + <p>List of algorithms to use in the algorithm negotiation. The default <c>algs_list()</c> can + be obtained from <seealso marker="#default_algorithms/0">default_algorithms/0</seealso>. + </p> + <p>Here is an example of this option:</p> + <code> +{preferred_algorithms, + [{public_key,['ssh-rsa','ssh-dss']}, + {cipher,[{client2server,['aes128-ctr']}, + {server2client,['aes128-cbc','3des-cbc']}]}, + {mac,['hmac-sha2-256','hmac-sha1']}, + {compression,[none,zlib]} +} +</code> + <p>The example specifies different algorithms in the two directions (client2server and server2client), for cipher but specifies the same +algorithms for mac and compression in both directions. The kex (key exchange) and public key algorithms are set to their default values, +kex is implicit but public_key is set explicitly.</p> + + <warning> + <p>Changing the values can make a connection less secure. Do not change unless you + know exactly what you are doing. If you do not understand the values then you + are not supposed to change them.</p> + </warning> + </item> + <tag><c><![CDATA[{pwdfun, fun(User::string(), password::string()) -> boolean()}]]></c></tag> <item> <p>Provides a function for password validation. This function is called @@ -409,6 +522,19 @@ Can be used to customize the handling of public keys. </p> </item> + + <tag><c>{profile, atom()}</c></tag> + <item> + <p>Used together with <c>ip-address</c> and <c>port</c> to + uniquely identify a ssh daemon. This can be useful in a + virtualized environment, where there can be more that one + server that has the same <c>ip-address</c> and + <c>port</c>. If this property is not explicitly set, it is + assumed that the the <c>ip-address</c> and <c>port</c> + uniquely identifies the SSH daemon. + </p> + </item> + <tag><c><![CDATA[{fd, file_descriptor()}]]></c></tag> <item> <p>Allows an existing file-descriptor to be used @@ -427,10 +553,46 @@ <item> <p>Provides a fun to implement your own logging when a user disconnects from the server.</p> </item> - </taglist> - </desc> + + <tag><c><![CDATA[{unexpectedfun, fun(Message:term(), Peer) -> report | skip }]]></c></tag> + <item> + <p>Provides a fun to implement your own logging or other action when an unexpected message arrives. + If the fun returns <c>report</c> the usual info report is issued but if <c>skip</c> is returned no + report is generated.</p> + <p><c>Peer</c> is in the format of <c>{Host,Port}</c>.</p> + </item> + + <tag><c><![CDATA[{ssh_msg_debug_fun, fun(ConnectionRef::ssh_connection_ref(), AlwaysDisplay::boolean(), Msg::binary(), LanguageTag::binary()) -> _}]]></c></tag> + <item> + <p>Provide a fun to implement your own logging of the SSH message SSH_MSG_DEBUG. The last three parameters are from the message, see RFC4253, section 11.3. The <c>ConnectionRef</c> is the reference to the connection on which the message arrived. The return value from the fun is not checked.</p> + <p>The default behaviour is ignore the message. + To get a printout for each message with <c>AlwaysDisplay = true</c>, use for example <c>{ssh_msg_debug_fun, fun(_,true,M,_)-> io:format("DEBUG: ~p~n", [M]) end}</c></p> + </item> + + </taglist> + </desc> </func> + <func> + <name>default_algorithms() -> algs_list()</name> + <fsummary>Get a list declaring the supported algorithms</fsummary> + <desc> + <p>Returns a key-value list, where the keys are the different types of algorithms and the values are the + algorithms themselves. An example:</p> + <code> +20> ssh:default_algorithms(). +[{kex,['diffie-hellman-group1-sha1']}, + {public_key,['ssh-rsa','ssh-dss']}, + {cipher,[{client2server,['aes128-ctr','aes128-cbc','3des-cbc']}, + {server2client,['aes128-ctr','aes128-cbc','3des-cbc']}]}, + {mac,[{client2server,['hmac-sha2-256','hmac-sha1']}, + {server2client,['hmac-sha2-256','hmac-sha1']}]}, + {compression,[{client2server,[none,zlib]}, + {server2client,[none,zlib]}]}] +21> +</code> + </desc> + </func> <func> <name>shell(Host) -> </name> diff --git a/lib/ssh/doc/src/ssh_channel.xml b/lib/ssh/doc/src/ssh_channel.xml index b8a03c350a..2fdecf9072 100644 --- a/lib/ssh/doc/src/ssh_channel.xml +++ b/lib/ssh/doc/src/ssh_channel.xml @@ -62,22 +62,22 @@ type, or both:</p> <taglist> - <tag><c>boolean()</c></tag> - <item><p>= <c>true | false</c></p></item> - <tag><c>string()</c></tag> - <item><p>= list of ASCII characters</p></item> - <tag><c>timeout()</c></tag> - <item><p>= <c>infinity | integer()</c> in milliseconds</p></item> - <tag><c>ssh_connection_ref()</c></tag> - <item><p>Opaque to the user, returned by - <c>ssh:connect/3</c> or sent to an SSH channel process</p></item> - <tag><c>ssh_channel_id()</c></tag> - <item><p>= <c>integer()</c></p></item> - <tag><c>ssh_data_type_code()</c></tag> - <item><p>= <c>1</c> ("stderr") | <c>0</c> ("normal") are - the valid values, - see <url href="http://www.ietf.org/rfc/rfc4254.txt">RFC 4254</url> - Section 5.2</p></item> + <tag><c>boolean() =</c></tag> + <item><p><c>true | false</c></p></item> + <tag><c>string() =</c></tag> + <item><p>list of ASCII characters</p></item> + <tag><c>timeout() =</c></tag> + <item><p><c>infinity | integer()</c> in milliseconds</p></item> + <tag><c>ssh_connection_ref() =</c></tag> + <item><p>opaque() -as returned by + <c>ssh:connect/3</c> or sent to an SSH channel process</p></item> + <tag><c>ssh_channel_id() =</c></tag> + <item><p><c>integer()</c></p></item> + <tag><c>ssh_data_type_code() =</c></tag> + <item><p><c>1</c> ("stderr") | <c>0</c> ("normal") are + the valid values, + see <url href="http://www.ietf.org/rfc/rfc4254.txt">RFC 4254</url> + Section 5.2</p></item> </taglist> </section> diff --git a/lib/ssh/doc/src/ssh_client_key_api.xml b/lib/ssh/doc/src/ssh_client_key_api.xml index a8dda042c9..9a892d71fd 100644 --- a/lib/ssh/doc/src/ssh_client_key_api.xml +++ b/lib/ssh/doc/src/ssh_client_key_api.xml @@ -50,16 +50,16 @@ <seealso marker="public_key:public_key_records"> public_key user's guide:</seealso> </p> <taglist> - <tag><c>boolean()</c></tag> - <item><p>= <c>true | false</c></p></item> - <tag><c>string()</c></tag> - <item><p>= <c>[byte()]</c></p></item> - <tag><c>public_key()</c></tag> - <item><p>= <c>#'RSAPublicKey'{}| {integer(), #'Dss-Parms'{}}| term()</c></p></item> - <tag><c>private_key()</c></tag> - <item><p>= <c>#'RSAPrivateKey'{} | #'DSAPrivateKey'{} | term()</c></p></item> - <tag><c>public_key_algorithm()</c></tag> - <item><p>= <c>'ssh-rsa'| 'ssh-dss' | atom()</c></p></item> + <tag><c>boolean() =</c></tag> + <item><p><c>true | false</c></p></item> + <tag><c>string() =</c></tag> + <item><p><c>[byte()]</c></p></item> + <tag><c>public_key() =</c></tag> + <item><p><c>#'RSAPublicKey'{}| {integer(), #'Dss-Parms'{}}| term()</c></p></item> + <tag><c>private_key() =</c></tag> + <item><p><c>#'RSAPrivateKey'{} | #'DSAPrivateKey'{} | term()</c></p></item> + <tag><c>public_key_algorithm() =</c></tag> + <item><p><c>'ssh-rsa'| 'ssh-dss' | atom()</c></p></item> </taglist> </section> diff --git a/lib/ssh/doc/src/ssh_connection.xml b/lib/ssh/doc/src/ssh_connection.xml index 669a361db9..5422633dc3 100644 --- a/lib/ssh/doc/src/ssh_connection.xml +++ b/lib/ssh/doc/src/ssh_connection.xml @@ -56,29 +56,29 @@ type, or both:</p> <taglist> - <tag><c>boolean()</c></tag> - <item><p>= <c>true | false </c></p></item> - <tag><c>string()</c></tag> - <item><p>= list of ASCII characters</p></item> - <tag><c>timeout()</c></tag> - <item><p>= <c>infinity | integer()</c> in milliseconds</p></item> - <tag><c>ssh_connection_ref()</c></tag> - <item><p>Opaque to the user, returned by - <c>ssh:connect/3</c> or sent to an SSH channel processes</p></item> - <tag><c>ssh_channel_id()</c></tag> - <item><p>= <c>integer()</c></p></item> - <tag><c>ssh_data_type_code()</c></tag> - <item><p>= <c>1</c> ("stderr") | <c>0</c> ("normal") are + <tag><c>boolean() =</c></tag> + <item><p><c>true | false </c></p></item> + <tag><c>string() =</c></tag> + <item><p>list of ASCII characters</p></item> + <tag><c>timeout() =</c></tag> + <item><p><c>infinity | integer()</c> in milliseconds</p></item> + <tag><c>ssh_connection_ref() =</c></tag> + <item><p>opaque() -as returned by + <c>ssh:connect/3</c> or sent to an SSH channel processes</p></item> + <tag><c>ssh_channel_id() =</c></tag> + <item><p><c>integer()</c></p></item> + <tag><c>ssh_data_type_code() =</c></tag> + <item><p><c>1</c> ("stderr") | <c>0</c> ("normal") are valid values, see <url href="http://www.ietf.org/rfc/rfc4254.txt">RFC 4254</url> Section 5.2.</p></item> - <tag><c>ssh_request_status() ssh_request_status()</c></tag> - <item><p>= <c>success | failure</c></p></item> - <tag><c>event()</c></tag> - <item><p>= <c>{ssh_cm, ssh_connection_ref(), ssh_event_msg()}</c></p></item> - <tag><c>ssh_event_msg()</c></tag> - <item><p>= <c>data_events() | status_events() | terminal_events()</c></p></item> - <tag><c>reason()</c></tag> - <item><p>= <c>timeout | closed</c></p></item> + <tag><c>ssh_request_status() =</c></tag> + <item><p> <c>success | failure</c></p></item> + <tag><c>event() =</c></tag> + <item><p><c>{ssh_cm, ssh_connection_ref(), ssh_event_msg()}</c></p></item> + <tag><c>ssh_event_msg() =</c></tag> + <item><p><c>data_events() | status_events() | terminal_events()</c></p></item> + <tag><c>reason() =</c></tag> + <item><p><c>timeout | closed</c></p></item> </taglist> <taglist> diff --git a/lib/ssh/doc/src/ssh_server_key_api.xml b/lib/ssh/doc/src/ssh_server_key_api.xml index 34ce7f7660..73dd90c962 100644 --- a/lib/ssh/doc/src/ssh_server_key_api.xml +++ b/lib/ssh/doc/src/ssh_server_key_api.xml @@ -50,20 +50,20 @@ <seealso marker="public_key:public_key_records"> public_key user's guide</seealso>. </p> -<taglist> - <tag><c>boolean()</c></tag> - <item><p>= <c>true | false</c></p></item> - <tag><c>string()</c></tag> - <item><p>= <c>[byte()]</c></p></item> - <tag><c>public_key()</c></tag> - <item><p>= <c>#'RSAPublicKey'{}| {integer(), #'Dss-Parms'{}}| term()</c></p></item> - <tag><c>private_key()</c></tag> - <item><p>= <c>#'RSAPrivateKey'{} | #'DSAPrivateKey'{} | term()</c></p></item> - <tag><c>public_key_algorithm()</c></tag> - <item><p>= <c>'ssh-rsa'| 'ssh-dss' | atom()</c></p></item> + <taglist> + <tag><c>boolean() =</c></tag> + <item><p><c>true | false</c></p></item> + <tag><c>string() =</c></tag> + <item><p><c>[byte()]</c></p></item> + <tag><c>public_key() =</c></tag> + <item><p><c>#'RSAPublicKey'{}| {integer(), #'Dss-Parms'{}}| term()</c></p></item> + <tag><c>private_key() =</c></tag> + <item><p><c>#'RSAPrivateKey'{} | #'DSAPrivateKey'{} | term()</c></p></item> + <tag><c>public_key_algorithm() =</c></tag> + <item><p><c>'ssh-rsa'| 'ssh-dss' | atom()</c></p></item> </taglist> </section> - + <funcs> <func> <name>Module:host_key(Algorithm, DaemonOptions) -> diff --git a/lib/ssh/doc/src/ssh_sftp.xml b/lib/ssh/doc/src/ssh_sftp.xml index 643130fe6b..fc418bc934 100644 --- a/lib/ssh/doc/src/ssh_sftp.xml +++ b/lib/ssh/doc/src/ssh_sftp.xml @@ -43,8 +43,8 @@ </p> <taglist> - <tag><c>ssh_connection_ref()</c></tag> - <item><p>Opaque to the user, returned by <c>ssh:connect/3</c></p></item> + <tag><c>ssh_connection_ref() =</c></tag> + <item><p>opaque() - as returned by <c>ssh:connect/3</c></p></item> <tag><c>timeout()</c></tag> <item><p>= <c>infinity | integer() in milliseconds. Default infinity.</c></p></item> </taglist> diff --git a/lib/ssh/doc/src/ssh_sftpd.xml b/lib/ssh/doc/src/ssh_sftpd.xml index bc2660f595..8b2497e6a3 100644 --- a/lib/ssh/doc/src/ssh_sftpd.xml +++ b/lib/ssh/doc/src/ssh_sftpd.xml @@ -37,16 +37,16 @@ <section> <title>DATA TYPES</title> <taglist> - <tag><c>subsystem_spec()</c></tag> - <item><p>= <c>{subsystem_name(), {channel_callback(), channel_init_args()}}</c></p></item> - <tag><c>subsystem_name()</c></tag> - <item><p>= <c>"sftp"</c></p></item> - <tag><c>channel_callback()</c></tag> - <item><p>= <c>atom()</c> - Name of the Erlang module implementing the subsystem using the + <tag><c>subsystem_spec() =</c></tag> + <item><p><c>{subsystem_name(), {channel_callback(), channel_init_args()}}</c></p></item> + <tag><c>subsystem_name() =</c></tag> + <item><p><c>"sftp"</c></p></item> + <tag><c>channel_callback() =</c></tag> + <item><p><c>atom()</c> - Name of the Erlang module implementing the subsystem using the <c>ssh_channel</c> behavior, see the <seealso marker="ssh_channel">ssh_channel(3)</seealso> manual page.</p></item> - <tag><c>channel_init_args()</c></tag> - <item><p>= <c>list()</c> - The one given as argument to function <c>subsystem_spec/1</c>.</p></item> + <tag><c>channel_init_args() =</c></tag> + <item><p><c>list()</c> - The one given as argument to function <c>subsystem_spec/1</c>.</p></item> </taglist> </section> <funcs> diff --git a/lib/ssh/src/Makefile b/lib/ssh/src/Makefile index 90d71107ad..a06d8acfd4 100644 --- a/lib/ssh/src/Makefile +++ b/lib/ssh/src/Makefile @@ -75,7 +75,7 @@ MODULES= \ ssh_transport \ ssh_xfer -PUBLIC_HRL_FILES= ssh.hrl ssh_userauth.hrl ssh_xfer.hrl +HRL_FILES = ERL_FILES= \ $(MODULES:%=%.erl) \ @@ -95,7 +95,7 @@ APP_TARGET= $(EBIN)/$(APP_FILE) APPUP_SRC= $(APPUP_FILE).src APPUP_TARGET= $(EBIN)/$(APPUP_FILE) -INTERNAL_HRL_FILES = ssh_auth.hrl ssh_connect.hrl ssh_transport.hrl +INTERNAL_HRL_FILES = ssh_auth.hrl ssh_connect.hrl ssh_transport.hrl ssh.hrl ssh_userauth.hrl ssh_xfer.hrl # ---------------------------------------------------- # FLAGS @@ -140,7 +140,7 @@ release_spec: opt $(INSTALL_DATA) $(BEHAVIOUR_TARGET_FILES) $(TARGET_FILES) $(APP_TARGET) \ $(APPUP_TARGET) "$(RELSYSDIR)/ebin" $(INSTALL_DIR) "$(RELSYSDIR)/include" - $(INSTALL_DATA) $(PUBLIC_HRL_FILES) "$(RELSYSDIR)/include" + release_docs_spec: diff --git a/lib/ssh/src/ssh.erl b/lib/ssh/src/ssh.erl index d4b02a024e..86c042781c 100644 --- a/lib/ssh/src/ssh.erl +++ b/lib/ssh/src/ssh.erl @@ -24,11 +24,14 @@ -include("ssh.hrl"). -include("ssh_connect.hrl"). -include_lib("public_key/include/public_key.hrl"). +-include_lib("kernel/include/file.hrl"). -export([start/0, start/1, stop/0, connect/3, connect/4, close/1, connection_info/2, channel_info/3, daemon/1, daemon/2, daemon/3, - stop_listener/1, stop_listener/2, stop_daemon/1, stop_daemon/2, + default_algorithms/0, + stop_listener/1, stop_listener/2, stop_listener/3, + stop_daemon/1, stop_daemon/2, stop_daemon/3, shell/1, shell/2, shell/3]). %%-------------------------------------------------------------------- @@ -157,7 +160,9 @@ daemon(HostAddr, Port, Options0) -> stop_listener(SysSup) -> ssh_system_sup:stop_listener(SysSup). stop_listener(Address, Port) -> - ssh_system_sup:stop_listener(Address, Port). + stop_listener(Address, Port, ?DEFAULT_PROFILE). +stop_listener(Address, Port, Profile) -> + ssh_system_sup:stop_listener(Address, Port, Profile). %%-------------------------------------------------------------------- -spec stop_daemon(pid()) -> ok. @@ -169,8 +174,9 @@ stop_listener(Address, Port) -> stop_daemon(SysSup) -> ssh_system_sup:stop_system(SysSup). stop_daemon(Address, Port) -> - ssh_system_sup:stop_system(Address, Port). - + ssh_system_sup:stop_system(Address, Port, ?DEFAULT_PROFILE). +stop_daemon(Address, Port, Profile) -> + ssh_system_sup:stop_system(Address, Port, Profile). %%-------------------------------------------------------------------- -spec shell(string()) -> _. -spec shell(string(), proplists:proplist()) -> _. @@ -208,6 +214,11 @@ shell(Host, Port, Options) -> end. %%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +default_algorithms() -> + ssh_transport:default_algorithms(). + +%%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- fix_idle_time(SshOptions) -> @@ -226,7 +237,8 @@ start_daemon(Host, Port, Options, Inet) -> end. do_start_daemon(Host, Port, Options, SocketOptions) -> - case ssh_system_sup:system_supervisor(Host, Port) of + Profile = proplists:get_value(profile, Options, ?DEFAULT_PROFILE), + case ssh_system_sup:system_supervisor(Host, Port, Profile) of undefined -> %% It would proably make more sense to call the %% address option host but that is a too big change at the @@ -259,7 +271,7 @@ do_start_daemon(Host, Port, Options, SocketOptions) -> end. handle_options(Opts) -> - try handle_option(proplists:unfold(Opts), [], []) of + try handle_option(algs_compatibility(proplists:unfold(Opts)), [], []) of {Inet, Ssh} -> {handle_ip(Inet), Ssh} catch @@ -267,6 +279,35 @@ handle_options(Opts) -> Error end. + +algs_compatibility(Os) -> + %% Take care of old options 'public_key_alg' and 'pref_public_key_algs' + comp_pk(proplists:get_value(preferred_algorithms,Os), + proplists:get_value(pref_public_key_algs,Os), + proplists:get_value(public_key_alg, Os), + [{K,V} || {K,V} <- Os, + K =/= public_key_alg, + K =/= pref_public_key_algs] + ). + +comp_pk(undefined, undefined, undefined, Os) -> Os; +comp_pk( PrefAlgs, _, _, Os) when PrefAlgs =/= undefined -> Os; + +comp_pk(undefined, undefined, ssh_dsa, Os) -> comp_pk(undefined, undefined, 'ssh-dss', Os); +comp_pk(undefined, undefined, ssh_rsa, Os) -> comp_pk(undefined, undefined, 'ssh-rsa', Os); +comp_pk(undefined, undefined, PK, Os) -> + PKs = [PK | ssh_transport:supported_algorithms(public_key)--[PK]], + [{preferred_algorithms, [{public_key,PKs}] } | Os]; + +comp_pk(undefined, PrefPKs, _, Os) when PrefPKs =/= undefined -> + PKs = [case PK of + ssh_dsa -> 'ssh-dss'; + ssh_rsa -> 'ssh-rsa'; + _ -> PK + end || PK <- PrefPKs], + [{preferred_algorithms, [{public_key,PKs}]} | Os]. + + handle_option([], SocketOptions, SshOptions) -> {SocketOptions, SshOptions}; handle_option([{system_dir, _} = Opt | Rest], SocketOptions, SshOptions) -> @@ -279,8 +320,6 @@ handle_option([{silently_accept_hosts, _} = Opt | Rest], SocketOptions, SshOptio handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{user_interaction, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{public_key_alg, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{connect_timeout, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{user, _} = Opt | Rest], SocketOptions, SshOptions) -> @@ -297,10 +336,6 @@ handle_option([{pwdfun, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{key_cb, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{role, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{compression, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); %%Backwards compatibility handle_option([{allow_user_interaction, Value} | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option({user_interaction, Value}) | SshOptions]); @@ -310,8 +345,12 @@ handle_option([{connectfun, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{disconnectfun, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); +handle_option([{unexpectedfun, _} = Opt | Rest], SocketOptions, SshOptions) -> + handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{failfun, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); +handle_option([{ssh_msg_debug_fun, _} = Opt | Rest], SocketOptions, SshOptions) -> + handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); %%Backwards compatibility should not be underscore between ip and v6 in API handle_option([{ip_v6_disabled, Value} | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option({ipv6_disabled, Value}) | SshOptions]); @@ -329,7 +368,9 @@ handle_option([{exec, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{auth_methods, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{pref_public_key_algs, _} = Opt | Rest], SocketOptions, SshOptions) -> +handle_option([{auth_method_kb_interactive_data, _} = Opt | Rest], SocketOptions, SshOptions) -> + handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); +handle_option([{preferred_algorithms,_} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{quiet_mode, _} = Opt|Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); @@ -349,6 +390,8 @@ handle_option([{minimal_remote_max_packet_size, _} = Opt|Rest], SocketOptions, S handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{id_string, _ID} = Opt|Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); +handle_option([{profile, _ID} = Opt|Rest], SocketOptions, SshOptions) -> + handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, [handle_inet_option(Opt) | SocketOptions], SshOptions). @@ -356,28 +399,17 @@ handle_option([Opt | Rest], SocketOptions, SshOptions) -> handle_ssh_option({minimal_remote_max_packet_size, Value} = Opt) when is_integer(Value), Value >=0 -> Opt; handle_ssh_option({system_dir, Value} = Opt) when is_list(Value) -> - Opt; + check_dir(Opt); handle_ssh_option({user_dir, Value} = Opt) when is_list(Value) -> - Opt; + check_dir(Opt); handle_ssh_option({user_dir_fun, Value} = Opt) when is_function(Value) -> Opt; handle_ssh_option({silently_accept_hosts, Value} = Opt) when is_boolean(Value) -> Opt; handle_ssh_option({user_interaction, Value} = Opt) when is_boolean(Value) -> Opt; -handle_ssh_option({public_key_alg, ssh_dsa}) -> - {public_key_alg, 'ssh-dss'}; -handle_ssh_option({public_key_alg, ssh_rsa}) -> - {public_key_alg, 'ssh-rsa'}; -handle_ssh_option({public_key_alg, Value} = Opt) when Value == 'ssh-rsa'; Value == 'ssh-dss' -> - Opt; -handle_ssh_option({pref_public_key_algs, Value} = Opt) when is_list(Value), length(Value) >= 1 -> - case handle_pref_algs(Value, []) of - {true, NewOpts} -> - NewOpts; - _ -> - throw({error, {eoptions, Opt}}) - end; +handle_ssh_option({preferred_algorithms,[_|_]} = Opt) -> + handle_pref_algs(Opt); handle_ssh_option({connect_timeout, Value} = Opt) when is_integer(Value); Value == infinity -> Opt; handle_ssh_option({max_sessions, Value} = Opt) when is_integer(Value), Value>0 -> @@ -409,14 +441,25 @@ handle_ssh_option({exec, Function} = Opt) when is_function(Function) -> Opt; handle_ssh_option({auth_methods, Value} = Opt) when is_list(Value) -> Opt; +handle_ssh_option({auth_method_kb_interactive_data, {Name,Instruction,Prompt,Echo}} = Opt) when is_list(Name), + is_list(Instruction), + is_list(Prompt), + is_boolean(Echo) -> + Opt; +handle_ssh_option({auth_method_kb_interactive_data, F} = Opt) when is_function(F,3) -> + Opt; handle_ssh_option({infofun, Value} = Opt) when is_function(Value) -> Opt; handle_ssh_option({connectfun, Value} = Opt) when is_function(Value) -> Opt; -handle_ssh_option({disconnectfun , Value} = Opt) when is_function(Value) -> +handle_ssh_option({disconnectfun, Value} = Opt) when is_function(Value) -> + Opt; +handle_ssh_option({unexpectedfun, Value} = Opt) when is_function(Value,2) -> Opt; handle_ssh_option({failfun, Value} = Opt) when is_function(Value) -> Opt; +handle_ssh_option({ssh_msg_debug_fun, Value} = Opt) when is_function(Value,4) -> + Opt; handle_ssh_option({ipv6_disabled, Value} = Opt) when is_boolean(Value) -> throw({error, {{ipv6_disabled, Opt}, option_no_longer_valid_use_inet_option_instead}}); @@ -445,6 +488,8 @@ handle_ssh_option({id_string, random}) -> {id_string, {random,2,5}}; %% 2 - 5 random characters handle_ssh_option({id_string, ID} = Opt) when is_list(ID) -> Opt; +handle_ssh_option({profile, Value} = Opt) when is_atom(Value) -> + Opt; handle_ssh_option(Opt) -> throw({error, {eoptions, Opt}}). @@ -461,23 +506,83 @@ handle_inet_option({reuseaddr, _} = Opt) -> %% Option verified by inet handle_inet_option(Opt) -> Opt. + + %% Check preferred algs -handle_pref_algs([], Acc) -> - {true, lists:reverse(Acc)}; -handle_pref_algs([H|T], Acc) -> - case H of - ssh_dsa -> - handle_pref_algs(T, ['ssh-dss'| Acc]); - ssh_rsa -> - handle_pref_algs(T, ['ssh-rsa'| Acc]); - 'ssh-dss' -> - handle_pref_algs(T, ['ssh-dss'| Acc]); - 'ssh-rsa' -> - handle_pref_algs(T, ['ssh-rsa'| Acc]); - _ -> - false + +handle_pref_algs({preferred_algorithms,Algs}) -> + try alg_duplicates(Algs, [], []) of + [] -> + {preferred_algorithms, + [try ssh_transport:supported_algorithms(Key) + of + DefAlgs -> handle_pref_alg(Key,Vals,DefAlgs) + catch + _:_ -> throw({error, {{eoptions, {preferred_algorithms,Key}}, + "Bad preferred_algorithms key"}}) + end || {Key,Vals} <- Algs] + }; + + Dups -> + throw({error, {{eoptions, {preferred_algorithms,Dups}}, "Duplicates found"}}) + catch + _:_ -> + throw({error, {{eoptions, preferred_algorithms}, "Malformed"}}) end. +alg_duplicates([{K,V}|KVs], Ks, Dups0) -> + Dups = + case lists:member(K,Ks) of + true -> + [K|Dups0]; + false -> + Dups0 + end, + case V--lists:usort(V) of + [] -> + alg_duplicates(KVs, [K|Ks], Dups); + Ds -> + alg_duplicates(KVs, [K|Ks], Dups++Ds) + end; +alg_duplicates([], _Ks, Dups) -> + Dups. + +handle_pref_alg(Key, + Vs=[{client2server,C2Ss=[_|_]},{server2client,S2Cs=[_|_]}], + [{client2server,Sup_C2Ss},{server2client,Sup_S2Cs}] + ) -> + chk_alg_vs(Key, C2Ss, Sup_C2Ss), + chk_alg_vs(Key, S2Cs, Sup_S2Cs), + {Key, Vs}; + +handle_pref_alg(Key, + Vs=[{server2client,[_|_]},{client2server,[_|_]}], + Sup=[{client2server,_},{server2client,_}] + ) -> + handle_pref_alg(Key, lists:reverse(Vs), Sup); + +handle_pref_alg(Key, + Vs=[V|_], + Sup=[{client2server,_},{server2client,_}] + ) when is_atom(V) -> + handle_pref_alg(Key, [{client2server,Vs},{server2client,Vs}], Sup); + +handle_pref_alg(Key, + Vs=[V|_], + Sup=[S|_] + ) when is_atom(V), is_atom(S) -> + chk_alg_vs(Key, Vs, Sup), + {Key, Vs}; + +handle_pref_alg(Key, Vs, _) -> + throw({error, {{eoptions, {preferred_algorithms,[{Key,Vs}]}}, "Badly formed list"}}). + +chk_alg_vs(OptKey, Values, SupportedValues) -> + case (Values -- SupportedValues) of + [] -> Values; + Bad -> throw({error, {{eoptions, {OptKey,Bad}}, "Unsupported value(s) found"}}) + end. + handle_ip(Inet) -> %% Default to ipv4 case lists:member(inet, Inet) of true -> @@ -490,4 +595,31 @@ handle_ip(Inet) -> %% Default to ipv4 [inet | Inet] end end. - + +check_dir({_,Dir} = Opt) -> + case directory_exist_readable(Dir) of + ok -> + Opt; + {error,Error} -> + throw({error, {eoptions,{Opt,Error}}}) + end. + +directory_exist_readable(Dir) -> + case file:read_file_info(Dir) of + {ok, #file_info{type = directory, + access = Access}} -> + case Access of + read -> ok; + read_write -> ok; + _ -> {error, eacces} + end; + + {ok, #file_info{}}-> + {error, enotdir}; + + {error, Error} -> + {error, Error} + end. + + + diff --git a/lib/ssh/src/ssh.hrl b/lib/ssh/src/ssh.hrl index 0c4d34f89c..a02c87505d 100644 --- a/lib/ssh/src/ssh.hrl +++ b/lib/ssh/src/ssh.hrl @@ -31,6 +31,7 @@ -define(SSH_LENGHT_INDICATOR_SIZE, 4). -define(REKEY_TIMOUT, 3600000). -define(REKEY_DATA_TIMOUT, 60000). +-define(DEFAULT_PROFILE, default). -define(FALSE, 0). -define(TRUE, 1). @@ -127,8 +128,10 @@ user, service, userauth_quiet_mode, % boolean() - userauth_supported_methods , % - userauth_methods, + userauth_supported_methods, % string() eg "keyboard-interactive,password" + userauth_methods, % list( string() ) eg ["keyboard-interactive", "password"] + kb_tries_left = 0, % integer(), num tries left for "keyboard-interactive" + kb_data, userauth_preference, available_host_keys, authenticated = false diff --git a/lib/ssh/src/ssh_acceptor.erl b/lib/ssh/src/ssh_acceptor.erl index 34988f17b6..6c431af270 100644 --- a/lib/ssh/src/ssh_acceptor.erl +++ b/lib/ssh/src/ssh_acceptor.erl @@ -21,6 +21,8 @@ -module(ssh_acceptor). +-include("ssh.hrl"). + %% Internal application API -export([start_link/5, number_of_connections/1]). @@ -82,8 +84,10 @@ acceptor_loop(Callback, Port, Address, Opts, ListenSocket, AcceptTimeout) -> end. handle_connection(Callback, Address, Port, Options, Socket) -> - SystemSup = ssh_system_sup:system_supervisor(Address, Port), SSHopts = proplists:get_value(ssh_opts, Options, []), + Profile = proplists:get_value(profile, SSHopts, ?DEFAULT_PROFILE), + SystemSup = ssh_system_sup:system_supervisor(Address, Port, Profile), + MaxSessions = proplists:get_value(max_sessions,SSHopts,infinity), case number_of_connections(SystemSup) < MaxSessions of true -> diff --git a/lib/ssh/src/ssh_acceptor_sup.erl b/lib/ssh/src/ssh_acceptor_sup.erl index 46fdef07d0..e101ce8b39 100644 --- a/lib/ssh/src/ssh_acceptor_sup.erl +++ b/lib/ssh/src/ssh_acceptor_sup.erl @@ -26,7 +26,9 @@ -module(ssh_acceptor_sup). -behaviour(supervisor). --export([start_link/1, start_child/2, stop_child/3]). +-include("ssh.hrl"). + +-export([start_link/1, start_child/2, stop_child/4]). %% Supervisor callback -export([init/1]). @@ -45,14 +47,16 @@ start_child(AccSup, ServerOpts) -> {error, already_present} -> Address = proplists:get_value(address, ServerOpts), Port = proplists:get_value(port, ServerOpts), - stop_child(AccSup, Address, Port), + Profile = proplists:get_value(profile, + proplists:get_value(ssh_opts, ServerOpts), ?DEFAULT_PROFILE), + stop_child(AccSup, Address, Port, Profile), supervisor:start_child(AccSup, Spec); Reply -> Reply end. -stop_child(AccSup, Address, Port) -> - Name = id(Address, Port), +stop_child(AccSup, Address, Port, Profile) -> + Name = id(Address, Port, Profile), case supervisor:terminate_child(AccSup, Name) of ok -> supervisor:delete_child(AccSup, Name); @@ -77,7 +81,8 @@ child_spec(ServerOpts) -> Address = proplists:get_value(address, ServerOpts), Port = proplists:get_value(port, ServerOpts), Timeout = proplists:get_value(timeout, ServerOpts, ?DEFAULT_TIMEOUT), - Name = id(Address, Port), + Profile = proplists:get_value(profile, proplists:get_value(ssh_opts, ServerOpts), ?DEFAULT_PROFILE), + Name = id(Address, Port, Profile), SocketOpts = proplists:get_value(socket_opts, ServerOpts), StartFunc = {ssh_acceptor, start_link, [Port, Address, [{active, false}, @@ -89,6 +94,11 @@ child_spec(ServerOpts) -> Type = worker, {Name, StartFunc, Restart, Shutdown, Type, Modules}. -id(Address, Port) -> - {ssh_acceptor_sup, Address, Port}. +id(Address, Port, Profile) -> + case is_list(Address) of + true -> + {ssh_acceptor_sup, any, Port, Profile}; + false -> + {ssh_acceptor_sup, Address, Port, Profile} + end. diff --git a/lib/ssh/src/ssh_auth.erl b/lib/ssh/src/ssh_auth.erl index 45c4d52d7e..020fb06530 100644 --- a/lib/ssh/src/ssh_auth.erl +++ b/lib/ssh/src/ssh_auth.erl @@ -30,7 +30,8 @@ -export([publickey_msg/1, password_msg/1, keyboard_interactive_msg/1, service_request_msg/1, init_userauth_request_msg/1, userauth_request_msg/1, handle_userauth_request/3, - handle_userauth_info_request/3, handle_userauth_info_response/2 + handle_userauth_info_request/3, handle_userauth_info_response/2, + default_public_key_algorithms/0 ]). %%-------------------------------------------------------------------- @@ -115,33 +116,16 @@ init_userauth_request_msg(#ssh{opts = Opts} = Ssh) -> service = "ssh-connection", method = "none", data = <<>>}, - case proplists:get_value(pref_public_key_algs, Opts, false) of - false -> - FirstAlg = proplists:get_value(public_key_alg, Opts, ?PREFERRED_PK_ALG), - SecondAlg = other_alg(FirstAlg), - Prefs = method_preference(FirstAlg, SecondAlg), - ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, - userauth_preference = Prefs, - userauth_methods = none, - service = "ssh-connection"}); - Algs -> - FirstAlg = lists:nth(1, Algs), - case length(Algs) =:= 2 of - true -> - SecondAlg = other_alg(FirstAlg), - Prefs = method_preference(FirstAlg, SecondAlg), - ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, - userauth_preference = Prefs, - userauth_methods = none, - service = "ssh-connection"}); - _ -> - Prefs = method_preference(FirstAlg), - ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, - userauth_preference = Prefs, - userauth_methods = none, - service = "ssh-connection"}) - end - end; + + + Algs = proplists:get_value(public_key, + proplists:get_value(preferred_algorithms, Opts, []), + default_public_key_algorithms()), + Prefs = method_preference(Algs), + ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, + userauth_preference = Prefs, + userauth_methods = none, + service = "ssh-connection"}); {error, no_user} -> ErrStr = "Could not determine the users name", throw(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_ILLEGAL_USER_NAME, @@ -185,7 +169,8 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, service = "ssh-connection", method = "password", data = <<?FALSE, ?UINT32(Sz), BinPwd:Sz/binary>>}, _, - #ssh{opts = Opts} = Ssh) -> + #ssh{opts = Opts, + userauth_supported_methods = Methods} = Ssh) -> Password = unicode:characters_to_list(BinPwd), case check_password(User, Password, Opts) of true -> @@ -194,7 +179,7 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, false -> {not_authorized, {User, {error,"Bad user or password"}}, ssh_transport:ssh_packet(#ssh_msg_userauth_failure{ - authentications = "", + authentications = Methods, partial_success = false}, Ssh)} end; @@ -207,7 +192,7 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, %% ?UINT32(Sz2), NewBinPwd:Sz2/binary >> }, _, - Ssh) -> + #ssh{userauth_supported_methods = Methods} = Ssh) -> %% Password change without us having sent SSH_MSG_USERAUTH_PASSWD_CHANGEREQ (because we never do) %% RFC 4252 says: %% SSH_MSG_USERAUTH_FAILURE without partial success - The password @@ -216,7 +201,7 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, {not_authorized, {User, {error,"Password change not supported"}}, ssh_transport:ssh_packet(#ssh_msg_userauth_failure{ - authentications = "", + authentications = Methods, partial_success = false}, Ssh)}; handle_userauth_request(#ssh_msg_userauth_request{user = User, @@ -232,7 +217,9 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, service = "ssh-connection", method = "publickey", data = Data}, - SessionId, #ssh{opts = Opts} = Ssh) -> + SessionId, + #ssh{opts = Opts, + userauth_supported_methods = Methods} = Ssh) -> <<?BYTE(HaveSig), ?UINT32(ALen), BAlg:ALen/binary, ?UINT32(KLen), KeyBlob:KLen/binary, SigWLen/binary>> = Data, Alg = binary_to_list(BAlg), @@ -247,7 +234,7 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, false -> {not_authorized, {User, undefined}, ssh_transport:ssh_packet(#ssh_msg_userauth_failure{ - authentications="publickey,password", + authentications = Methods, partial_success = false}, Ssh)} end; ?FALSE -> @@ -259,6 +246,65 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, handle_userauth_request(#ssh_msg_userauth_request{user = User, service = "ssh-connection", + method = "keyboard-interactive", + data = _}, + _, #ssh{opts = Opts, + kb_tries_left = KbTriesLeft, + userauth_supported_methods = Methods} = Ssh) -> + case KbTriesLeft of + N when N<1 -> + {not_authorized, {User, {authmethod, "keyboard-interactive"}}, + ssh_transport:ssh_packet( + #ssh_msg_userauth_failure{authentications = Methods, + partial_success = false}, Ssh)}; + + _ -> + %% RFC4256 + %% The data field contains: + %% - language tag (deprecated). If =/=[] SHOULD use it however. We skip + %% it for simplicity. + %% - submethods. "... the user can give a hint of which actual methods + %% he wants to use. ...". It's a "MAY use" so we skip + %% it. It also needs an understanding between the client + %% and the server. + %% + %% "The server MUST reply with an SSH_MSG_USERAUTH_SUCCESS, + %% SSH_MSG_USERAUTH_FAILURE, or SSH_MSG_USERAUTH_INFO_REQUEST message." + Default = {"SSH server", + "Enter password for \""++User++"\"", + "password: ", + false}, + + {Name, Instruction, Prompt, Echo} = + case proplists:get_value(auth_method_kb_interactive_data, Opts) of + undefined -> + Default; + {_,_,_,_}=V -> + V; + F when is_function(F) -> + {_,PeerName} = Ssh#ssh.peer, + F(PeerName, User, "ssh-connection") + end, + EchoEnc = case Echo of + true -> <<?TRUE>>; + false -> <<?FALSE>> + end, + Msg = #ssh_msg_userauth_info_request{name = unicode:characters_to_list(Name), + instruction = unicode:characters_to_list(Instruction), + language_tag = "", + num_prompts = 1, + data = <<?STRING(unicode:characters_to_binary(Prompt)), + EchoEnc/binary + >> + }, + {not_authorized, {User, undefined}, + ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, + kb_data = Msg + })} + end; + +handle_userauth_request(#ssh_msg_userauth_request{user = User, + service = "ssh-connection", method = Other}, _, #ssh{userauth_supported_methods = Methods} = Ssh) -> {not_authorized, {User, {authmethod, Other}}, @@ -280,6 +326,42 @@ handle_userauth_info_request( #ssh_msg_userauth_info_response{num_responses = NumPrompts, data = Responses}, Ssh)}. +handle_userauth_info_response(#ssh_msg_userauth_info_response{num_responses = 1, + data = <<?UINT32(Sz), Password:Sz/binary>>}, + #ssh{opts = Opts, + kb_tries_left = KbTriesLeft0, + kb_data = InfoMsg, + user = User, + userauth_supported_methods = Methods} = Ssh) -> + KbTriesLeft = KbTriesLeft0 - 1, + case check_password(User, unicode:characters_to_list(Password), Opts) of + true -> + {authorized, User, + ssh_transport:ssh_packet(#ssh_msg_userauth_success{}, Ssh)}; + false when KbTriesLeft > 0 -> + UserAuthInfoMsg = + InfoMsg#ssh_msg_userauth_info_request{ + name = "", + instruction = + lists:concat( + ["Bad user or password, try again. ", + integer_to_list(KbTriesLeft), + " tries left."]) + }, + {not_authorized, {User, undefined}, + ssh_transport:ssh_packet(UserAuthInfoMsg, + Ssh#ssh{kb_tries_left = KbTriesLeft})}; + + false -> + {not_authorized, {User, {error,"Bad user or password"}}, + ssh_transport:ssh_packet(#ssh_msg_userauth_failure{ + authentications = Methods, + partial_success = false}, + Ssh#ssh{kb_data = undefined, + kb_tries_left = 0} + )} + end; + handle_userauth_info_response(#ssh_msg_userauth_info_response{}, _Auth) -> throw(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, @@ -287,20 +369,20 @@ handle_userauth_info_response(#ssh_msg_userauth_info_response{}, "keyboard-interactive", language = "en"}). + +default_public_key_algorithms() -> ?PREFERRED_PK_ALGS. + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- -method_preference(Alg1, Alg2) -> - [{"publickey", ?MODULE, publickey_msg, [Alg1]}, - {"publickey", ?MODULE, publickey_msg,[Alg2]}, - {"password", ?MODULE, password_msg, []}, - {"keyboard-interactive", ?MODULE, keyboard_interactive_msg, []} - ]. -method_preference(Alg1) -> - [{"publickey", ?MODULE, publickey_msg, [Alg1]}, - {"password", ?MODULE, password_msg, []}, - {"keyboard-interactive", ?MODULE, keyboard_interactive_msg, []} - ]. +method_preference(Algs) -> + lists:foldr(fun(A, Acc) -> + [{"publickey", ?MODULE, publickey_msg, [A]} | Acc] + end, + [{"password", ?MODULE, password_msg, []}, + {"keyboard-interactive", ?MODULE, keyboard_interactive_msg, []} + ], + Algs). user_name(Opts) -> Env = case os:type() of @@ -418,27 +500,17 @@ keyboard_interact_fun(KbdInteractFun, Name, Instr, PromptInfos, NumPrompts) -> language = "en"}}) end. -other_alg('ssh-rsa') -> - 'ssh-dss'; -other_alg('ssh-dss') -> - 'ssh-rsa'. decode_public_key_v2(<<?UINT32(Len0), _:Len0/binary, - ?UINT32(Len1), BinE:Len1/binary, - ?UINT32(Len2), BinN:Len2/binary>> + ?UINT32(Len1), E:Len1/big-signed-integer-unit:8, + ?UINT32(Len2), N:Len2/big-signed-integer-unit:8>> ,"ssh-rsa") -> - E = ssh_bits:erlint(Len1, BinE), - N = ssh_bits:erlint(Len2, BinN), {ok, #'RSAPublicKey'{publicExponent = E, modulus = N}}; decode_public_key_v2(<<?UINT32(Len0), _:Len0/binary, - ?UINT32(Len1), BinP:Len1/binary, - ?UINT32(Len2), BinQ:Len2/binary, - ?UINT32(Len3), BinG:Len3/binary, - ?UINT32(Len4), BinY:Len4/binary>> + ?UINT32(Len1), P:Len1/big-signed-integer-unit:8, + ?UINT32(Len2), Q:Len2/big-signed-integer-unit:8, + ?UINT32(Len3), G:Len3/big-signed-integer-unit:8, + ?UINT32(Len4), Y:Len4/big-signed-integer-unit:8>> , "ssh-dss") -> - P = ssh_bits:erlint(Len1, BinP), - Q = ssh_bits:erlint(Len2, BinQ), - G = ssh_bits:erlint(Len3, BinG), - Y = ssh_bits:erlint(Len4, BinY), {ok, {Y, #'Dss-Parms'{p = P, q = Q, g = G}}}; decode_public_key_v2(_, _) -> diff --git a/lib/ssh/src/ssh_auth.hrl b/lib/ssh/src/ssh_auth.hrl index 6cd8e6bf14..764c9f4246 100644 --- a/lib/ssh/src/ssh_auth.hrl +++ b/lib/ssh/src/ssh_auth.hrl @@ -23,7 +23,7 @@ -define(SUPPORTED_AUTH_METHODS, "publickey,keyboard-interactive,password"). --define(PREFERRED_PK_ALG, 'ssh-rsa'). +-define(PREFERRED_PK_ALGS, ['ssh-rsa','ssh-dss']). -define(SSH_MSG_USERAUTH_REQUEST, 50). -define(SSH_MSG_USERAUTH_FAILURE, 51). diff --git a/lib/ssh/src/ssh_bits.erl b/lib/ssh/src/ssh_bits.erl index 8aaff93b9f..d5f8df6fe4 100644 --- a/lib/ssh/src/ssh_bits.erl +++ b/lib/ssh/src/ssh_bits.erl @@ -26,7 +26,7 @@ -include("ssh.hrl"). -export([encode/2]). --export([mpint/1, erlint/2, string/1, name_list/1]). +-export([mpint/1, string/1, name_list/1]). -export([random/1]). -define(name_list(X), @@ -145,11 +145,7 @@ enc(Xs, ['...'| []], _Offset) -> enc([], [],_) -> []. -erlint(Len, BinInt) -> - Sz = Len*8, - <<Int:Sz/big-signed-integer>> = BinInt, - Int. - + %% %% Create a binary with constant bytes %% diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index 4dea284071..e6e5749e07 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -33,7 +33,7 @@ -include("ssh_transport.hrl"). -include("ssh_auth.hrl"). -include("ssh_connect.hrl"). - +-compile(export_all). -export([start_link/3]). %% Internal application API @@ -71,6 +71,7 @@ key_exchange_init_msg, % #ssh_msg_kexinit{} renegotiate = false, % boolean() last_size_rekey = 0, + event_queue = [], connection_queue, address, port, @@ -83,6 +84,11 @@ {next_state, state_name(), term(), timeout()} | {stop, term(), term()}. +-type gen_fsm_sync_return() :: {next_state, state_name(), term()} | + {next_state, state_name(), term(), timeout()} | + {reply, term(), state_name(), term()} | + {stop, term(), term(), term()}. + %%==================================================================== %% Internal application API %%==================================================================== @@ -327,22 +333,25 @@ info(ConnectionHandler, ChannelProcess) -> hello(socket_control, #state{socket = Socket, ssh_params = Ssh} = State) -> VsnMsg = ssh_transport:hello_version_msg(string_version(Ssh)), send_msg(VsnMsg, State), - {ok, [{recbuf, Size}]} = inet:getopts(Socket, [recbuf]), - inet:setopts(Socket, [{packet, line}, {active, once}, {recbuf, ?MAX_PROTO_VERSION}]), - {next_state, hello, State#state{recbuf = Size}}; + case getopt(recbuf, Socket) of + {ok, Size} -> + inet:setopts(Socket, [{packet, line}, {active, once}, {recbuf, ?MAX_PROTO_VERSION}]), + {next_state, hello, State#state{recbuf = Size}}; + {error, Reason} -> + {stop, {shutdown, Reason}, State} + end; hello({info_line, _Line},#state{role = client, socket = Socket} = State) -> %% The server may send info lines before the version_exchange inet:setopts(Socket, [{active, once}]), {next_state, hello, State}; -hello({info_line, _Line},#state{role = server} = State) -> - DisconnectMsg = - #ssh_msg_disconnect{code = - ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "Did not receive expected protocol version exchange", - language = "en"}, - handle_disconnect(DisconnectMsg, State); +hello({info_line, _Line},#state{role = server, + socket = Socket, + transport_cb = Transport } = State) -> + %% as openssh + Transport:send(Socket, "Protocol mismatch."), + {stop, {shutdown,"Protocol mismatch in version exchange."}, State}; hello({version_exchange, Version}, #state{ssh_params = Ssh0, socket = Socket, @@ -433,9 +442,7 @@ key_exchange(#ssh_msg_kex_dh_gex_reply{} = Msg, new_keys(#ssh_msg_newkeys{} = Msg, #state{ssh_params = Ssh0} = State0) -> {ok, Ssh} = ssh_transport:handle_new_keys(Msg, Ssh0), - {NextStateName, State} = - after_new_keys(State0#state{ssh_params = Ssh}), - {next_state, NextStateName, next_packet(State)}. + after_new_keys(next_packet(State0#state{ssh_params = Ssh})). %%-------------------------------------------------------------------- -spec userauth(#ssh_msg_service_request{} | #ssh_msg_service_accept{} | @@ -476,17 +483,22 @@ userauth(#ssh_msg_userauth_request{service = "ssh-connection", service = "ssh-connection", peer = {_, Address}} = Ssh0, opts = Opts, starter = Pid} = State) -> - case ssh_auth:handle_userauth_request(Msg, SessionId, Ssh0) of - {authorized, User, {Reply, Ssh}} -> - send_msg(Reply, State), - Pid ! ssh_connected, - connected_fun(User, Address, Method, Opts), - {next_state, connected, - next_packet(State#state{auth_user = User, ssh_params = Ssh})}; - {not_authorized, {User, Reason}, {Reply, Ssh}} -> - retry_fun(User, Address, Reason, Opts), - send_msg(Reply, State), - {next_state, userauth, next_packet(State#state{ssh_params = Ssh})} + case lists:member(Method, Ssh0#ssh.userauth_methods) of + true -> + case ssh_auth:handle_userauth_request(Msg, SessionId, Ssh0) of + {authorized, User, {Reply, Ssh}} -> + send_msg(Reply, State), + Pid ! ssh_connected, + connected_fun(User, Address, Method, Opts), + {next_state, connected, + next_packet(State#state{auth_user = User, ssh_params = Ssh})}; + {not_authorized, {User, Reason}, {Reply, Ssh}} -> + retry_fun(User, Address, Reason, Opts), + send_msg(Reply, State), + {next_state, userauth, next_packet(State#state{ssh_params = Ssh})} + end; + false -> + userauth(Msg#ssh_msg_userauth_request{method="none"}, State) end; userauth(#ssh_msg_userauth_info_request{} = Msg, @@ -497,10 +509,21 @@ userauth(#ssh_msg_userauth_info_request{} = Msg, {next_state, userauth, next_packet(State#state{ssh_params = Ssh})}; userauth(#ssh_msg_userauth_info_response{} = Msg, - #state{ssh_params = #ssh{role = server} = Ssh0} = State) -> - {ok, {Reply, Ssh}} = ssh_auth:handle_userauth_info_response(Msg, Ssh0), - send_msg(Reply, State), - {next_state, userauth, next_packet(State#state{ssh_params = Ssh})}; + #state{ssh_params = #ssh{role = server, + peer = {_, Address}} = Ssh0, + opts = Opts, starter = Pid} = State) -> + case ssh_auth:handle_userauth_info_response(Msg, Ssh0) of + {authorized, User, {Reply, Ssh}} -> + send_msg(Reply, State), + Pid ! ssh_connected, + connected_fun(User, Address, "keyboard-interactive", Opts), + {next_state, connected, + next_packet(State#state{auth_user = User, ssh_params = Ssh})}; + {not_authorized, {User, Reason}, {Reply, Ssh}} -> + retry_fun(User, Address, Reason, Opts), + send_msg(Reply, State), + {next_state, userauth, next_packet(State#state{ssh_params = Ssh})} + end; userauth(#ssh_msg_userauth_success{}, #state{ssh_params = #ssh{role = client} = Ssh, starter = Pid} = State) -> @@ -559,11 +582,13 @@ userauth(#ssh_msg_userauth_banner{message = Msg}, -spec connected({#ssh_msg_kexinit{}, binary()}, %%| %% #ssh_msg_kexdh_init{}, #state{}) -> gen_fsm_state_return(). %%-------------------------------------------------------------------- -connected({#ssh_msg_kexinit{}, _Payload} = Event, State) -> - kexinit(Event, State#state{renegotiate = true}). -%% ; -%% connected(#ssh_msg_kexdh_init{} = Event, State) -> -%% key_exchange(Event, State#state{renegotiate = true}). +connected({#ssh_msg_kexinit{}, _Payload} = Event, #state{ssh_params = Ssh0} = State0) -> + {KeyInitMsg, SshPacket, Ssh} = ssh_transport:key_exchange_init_msg(Ssh0), + State = State0#state{ssh_params = Ssh, + key_exchange_init_msg = KeyInitMsg, + renegotiate = true}, + send_msg(SshPacket, State), + kexinit(Event, State). %%-------------------------------------------------------------------- -spec handle_event(#ssh_msg_disconnect{} | #ssh_msg_ignore{} | #ssh_msg_debug{} | @@ -581,44 +606,17 @@ handle_event(#ssh_msg_disconnect{description = Desc} = DisconnectMsg, _StateName handle_event(#ssh_msg_ignore{}, StateName, State) -> {next_state, StateName, next_packet(State)}; -handle_event(#ssh_msg_debug{always_display = true, message = DbgMsg}, - StateName, State) -> - io:format("DEBUG: ~p\n", [DbgMsg]), - {next_state, StateName, next_packet(State)}; - -handle_event(#ssh_msg_debug{}, StateName, State) -> +handle_event(#ssh_msg_debug{always_display = Display, message = DbgMsg, language=Lang}, + StateName, #state{opts = Opts} = State) -> + F = proplists:get_value(ssh_msg_debug_fun, Opts, + fun(_ConnRef, _AlwaysDisplay, _Msg, _Language) -> ok end + ), + catch F(self(), Display, DbgMsg, Lang), {next_state, StateName, next_packet(State)}; handle_event(#ssh_msg_unimplemented{}, StateName, State) -> {next_state, StateName, next_packet(State)}; -handle_event({adjust_window, ChannelId, Bytes}, StateName, - #state{connection_state = - #connection{channel_cache = Cache}} = State0) -> - State = - case ssh_channel:cache_lookup(Cache, ChannelId) of - #channel{recv_window_size = WinSize, remote_id = Id} = Channel -> - ssh_channel:cache_update(Cache, Channel#channel{recv_window_size = - WinSize + Bytes}), - Msg = ssh_connection:channel_adjust_window_msg(Id, Bytes), - send_replies([{connection_reply, Msg}], State0); - undefined -> - State0 - end, - {next_state, StateName, next_packet(State)}; - -handle_event({reply_request, success, ChannelId}, StateName, - #state{connection_state = - #connection{channel_cache = Cache}} = State0) -> - State = case ssh_channel:cache_lookup(Cache, ChannelId) of - #channel{remote_id = RemoteId} -> - Msg = ssh_connection:channel_success_msg(RemoteId), - send_replies([{connection_reply, Msg}], State0); - undefined -> - State0 - end, - {next_state, StateName, State}; - handle_event(renegotiate, connected, #state{ssh_params = Ssh0} = State) -> {KeyInitMsg, SshPacket, Ssh} = ssh_transport:key_exchange_init_msg(Ssh0), @@ -630,8 +628,7 @@ handle_event(renegotiate, connected, #state{ssh_params = Ssh0} renegotiate = true})}; handle_event(renegotiate, StateName, State) -> - timer:apply_after(?REKEY_TIMOUT, gen_fsm, send_all_state_event, [self(), renegotiate]), - %% Allready in keyexcahange so ignore + %% Already in key-exchange so safe to ignore {next_state, StateName, State}; %% Rekey due to sent data limit reached? @@ -653,6 +650,38 @@ handle_event(data_size, connected, #state{ssh_params = Ssh0} = State) -> {next_state, connected, next_packet(State)} end; handle_event(data_size, StateName, State) -> + %% Already in key-exchange so safe to ignore + {next_state, StateName, State}; + +handle_event(Event, StateName, State) when StateName /= connected -> + Events = [{event, Event} | State#state.event_queue], + {next_state, StateName, State#state{event_queue = Events}}; + +handle_event({adjust_window, ChannelId, Bytes}, StateName, + #state{connection_state = + #connection{channel_cache = Cache}} = State0) -> + State = + case ssh_channel:cache_lookup(Cache, ChannelId) of + #channel{recv_window_size = WinSize, remote_id = Id} = Channel -> + ssh_channel:cache_update(Cache, Channel#channel{recv_window_size = + WinSize + Bytes}), + Msg = ssh_connection:channel_adjust_window_msg(Id, Bytes), + send_replies([{connection_reply, Msg}], State0); + undefined -> + State0 + end, + {next_state, StateName, next_packet(State)}; + +handle_event({reply_request, success, ChannelId}, StateName, + #state{connection_state = + #connection{channel_cache = Cache}} = State0) -> + State = case ssh_channel:cache_lookup(Cache, ChannelId) of + #channel{remote_id = RemoteId} -> + Msg = ssh_connection:channel_success_msg(RemoteId), + send_replies([{connection_reply, Msg}], State0); + undefined -> + State0 + end, {next_state, StateName, State}; handle_event({request, ChannelPid, ChannelId, Type, Data}, StateName, State0) -> @@ -683,8 +712,62 @@ handle_event({unknown, Data}, StateName, State) -> sockname]} | {channel_info, channel_id(), [recv_window | send_window]} | {close, channel_id()} | stop, term(), state_name(), #state{}) - -> gen_fsm_state_return(). + -> gen_fsm_sync_return(). %%-------------------------------------------------------------------- +handle_sync_event(get_print_info, _From, StateName, State) -> + Reply = + try + {inet:sockname(State#state.socket), + inet:peername(State#state.socket) + } + of + {{ok,Local}, {ok,Remote}} -> {{Local,Remote},io_lib:format("statename=~p",[StateName])}; + _ -> {{"-",0},"-"} + catch + _:_ -> {{"?",0},"?"} + end, + {reply, Reply, StateName, State}; + +handle_sync_event({connection_info, Options}, _From, StateName, State) -> + Info = ssh_info(Options, State, []), + {reply, Info, StateName, State}; + +handle_sync_event({channel_info, ChannelId, Options}, _From, StateName, + #state{connection_state = #connection{channel_cache = Cache}} = State) -> + case ssh_channel:cache_lookup(Cache, ChannelId) of + #channel{} = Channel -> + Info = ssh_channel_info(Options, Channel, []), + {reply, Info, StateName, State}; + undefined -> + {reply, [], StateName, State} + end; + +handle_sync_event({info, ChannelPid}, _From, StateName, + #state{connection_state = + #connection{channel_cache = Cache}} = State) -> + Result = ssh_channel:cache_foldl( + fun(Channel, Acc) when ChannelPid == all; + Channel#channel.user == ChannelPid -> + [Channel | Acc]; + (_, Acc) -> + Acc + end, [], Cache), + {reply, {ok, Result}, StateName, State}; + +handle_sync_event(stop, _, _StateName, #state{connection_state = Connection0, + role = Role} = State0) -> + {disconnect, _Reason, {{replies, Replies}, Connection}} = + ssh_connection:handle_msg(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_BY_APPLICATION, + description = "User closed down connection", + language = "en"}, Connection0, Role), + State = send_replies(Replies, State0), + {stop, normal, ok, State#state{connection_state = Connection}}; + + +handle_sync_event(Event, From, StateName, State) when StateName /= connected -> + Events = [{sync, Event, From} | State#state.event_queue], + {next_state, StateName, State#state{event_queue = Events}}; + handle_sync_event({request, ChannelPid, ChannelId, Type, Data, Timeout}, From, StateName, State0) -> {{replies, Replies}, State1} = handle_request(ChannelPid, ChannelId, Type, Data, @@ -787,46 +870,6 @@ handle_sync_event({recv_window, ChannelId}, _From, StateName, end, {reply, Reply, StateName, next_packet(State)}; -handle_sync_event(get_print_info, _From, StateName, State) -> - Reply = - try - {inet:sockname(State#state.socket), - inet:peername(State#state.socket) - } - of - {{ok,Local}, {ok,Remote}} -> {{Local,Remote},io_lib:format("statename=~p",[StateName])}; - _ -> {{"-",0},"-"} - catch - _:_ -> {{"?",0},"?"} - end, - {reply, Reply, StateName, State}; - -handle_sync_event({connection_info, Options}, _From, StateName, State) -> - Info = ssh_info(Options, State, []), - {reply, Info, StateName, State}; - -handle_sync_event({channel_info, ChannelId, Options}, _From, StateName, - #state{connection_state = #connection{channel_cache = Cache}} = State) -> - case ssh_channel:cache_lookup(Cache, ChannelId) of - #channel{} = Channel -> - Info = ssh_channel_info(Options, Channel, []), - {reply, Info, StateName, State}; - undefined -> - {reply, [], StateName, State} - end; - -handle_sync_event({info, ChannelPid}, _From, StateName, - #state{connection_state = - #connection{channel_cache = Cache}} = State) -> - Result = ssh_channel:cache_foldl( - fun(Channel, Acc) when ChannelPid == all; - Channel#channel.user == ChannelPid -> - [Channel | Acc]; - (_, Acc) -> - Acc - end, [], Cache), - {reply, {ok, Result}, StateName, State}; - handle_sync_event({close, ChannelId}, _, StateName, #state{connection_state = #connection{channel_cache = Cache}} = State0) -> @@ -841,19 +884,7 @@ handle_sync_event({close, ChannelId}, _, StateName, undefined -> State0 end, - {reply, ok, StateName, next_packet(State)}; - -handle_sync_event(stop, _, _StateName, #state{connection_state = Connection0, - role = Role, - opts = Opts} = State0) -> - {disconnect, Reason, {{replies, Replies}, Connection}} = - ssh_connection:handle_msg(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_BY_APPLICATION, - description = "User closed down connection", - language = "en"}, Connection0, Role), - State = send_replies(Replies, State0), - SSHOpts = proplists:get_value(ssh_opts, Opts), - disconnect_fun(Reason, SSHOpts), - {stop, normal, ok, State#state{connection_state = Connection}}. + {reply, ok, StateName, next_packet(State)}. %%-------------------------------------------------------------------- -spec handle_info({atom(), port(), binary()} | {atom(), port()} | @@ -958,15 +989,38 @@ handle_info({check_cache, _ , _}, #connection{channel_cache = Cache}} = State) -> {next_state, StateName, check_cache(State, Cache)}; -handle_info(UnexpectedMessage, StateName, #state{ssh_params = SshParams} = State) -> - Msg = lists:flatten(io_lib:format( - "Unexpected message '~p' received in state '~p'\n" - "Role: ~p\n" - "Peer: ~p\n" - "Local Address: ~p\n", [UnexpectedMessage, StateName, - SshParams#ssh.role, SshParams#ssh.peer, - proplists:get_value(address, SshParams#ssh.opts)])), - error_logger:info_report(Msg), +handle_info(UnexpectedMessage, StateName, #state{opts = Opts, + ssh_params = SshParams} = State) -> + case unexpected_fun(UnexpectedMessage, Opts, SshParams) of + report -> + Msg = lists:flatten( + io_lib:format( + "Unexpected message '~p' received in state '~p'\n" + "Role: ~p\n" + "Peer: ~p\n" + "Local Address: ~p\n", [UnexpectedMessage, StateName, + SshParams#ssh.role, SshParams#ssh.peer, + proplists:get_value(address, SshParams#ssh.opts)])), + error_logger:info_report(Msg); + + skip -> + ok; + + Other -> + Msg = lists:flatten( + io_lib:format("Call to fun in 'unexpectedfun' failed:~n" + "Return: ~p\n" + "Message: ~p\n" + "Role: ~p\n" + "Peer: ~p\n" + "Local Address: ~p\n", [Other, UnexpectedMessage, + SshParams#ssh.role, + element(2,SshParams#ssh.peer), + proplists:get_value(address, SshParams#ssh.opts)] + )), + + error_logger:error_report(Msg) + end, {next_state, StateName, State}. %%-------------------------------------------------------------------- @@ -1122,9 +1176,9 @@ init_ssh(client = Role, Vsn, Version, Options, Socket) -> }; init_ssh(server = Role, Vsn, Version, Options, Socket) -> - AuthMethods = proplists:get_value(auth_methods, Options, ?SUPPORTED_AUTH_METHODS), + AuthMethodsAsList = string:tokens(AuthMethods, ","), {ok, PeerAddr} = inet:peername(Socket), KeyCb = proplists:get_value(key_cb, Options, ssh_file), @@ -1135,60 +1189,46 @@ init_ssh(server = Role, Vsn, Version, Options, Socket) -> io_cb = proplists:get_value(io_cb, Options, ssh_io), opts = Options, userauth_supported_methods = AuthMethods, + userauth_methods = AuthMethodsAsList, + kb_tries_left = 3, peer = {undefined, PeerAddr}, available_host_keys = supported_host_keys(Role, KeyCb, Options) }. supported_host_keys(client, _, Options) -> try - case extract_algs(proplists:get_value(pref_public_key_algs, Options, false), []) of - false -> - ["ssh-rsa", "ssh-dss"]; - Algs -> - Algs + case proplists:get_value(public_key, + proplists:get_value(preferred_algorithms,Options,[]) + ) of + undefined -> + ssh_auth:default_public_key_algorithms(); + L -> + L -- (L--ssh_auth:default_public_key_algorithms()) end + of + [] -> + {stop, {shutdown, "No public key algs"}}; + Algs -> + [atom_to_list(A) || A<-Algs] catch exit:Reason -> {stop, {shutdown, Reason}} end; supported_host_keys(server, KeyCb, Options) -> - lists:foldl(fun(Type, Acc) -> - case available_host_key(KeyCb, Type, Options) of - {error, _} -> - Acc; - Alg -> - [Alg | Acc] - end - end, [], - %% Prefered alg last so no need to reverse - ["ssh-dss", "ssh-rsa"]). -extract_algs(false, _) -> - false; -extract_algs([],[]) -> - false; -extract_algs([], NewList) -> - lists:reverse(NewList); -extract_algs([H|T], NewList) -> - case H of - 'ssh-dss' -> - extract_algs(T, ["ssh-dss"|NewList]); - 'ssh-rsa' -> - extract_algs(T, ["ssh-rsa"|NewList]) - end. -available_host_key(KeyCb, "ssh-dss"= Alg, Opts) -> - case KeyCb:host_key('ssh-dss', Opts) of - {ok, _} -> - Alg; - Other -> - Other - end; -available_host_key(KeyCb, "ssh-rsa" = Alg, Opts) -> - case KeyCb:host_key('ssh-rsa', Opts) of - {ok, _} -> - Alg; - Other -> - Other - end. + Algs= + [atom_to_list(A) || A <- proplists:get_value(public_key, + proplists:get_value(preferred_algorithms,Options,[]), + ssh_auth:default_public_key_algorithms() + ), + available_host_key(KeyCb, A, Options) + ], + Algs. + + +%% Alg :: atom() +available_host_key(KeyCb, Alg, Opts) -> + element(1, catch KeyCb:host_key(Alg, Opts)) == ok. + send_msg(Msg, #state{socket = Socket, transport_cb = Transport}) -> Transport:send(Socket, Msg). @@ -1262,7 +1302,6 @@ generate_event(<<?BYTE(Byte), _/binary>> = Msg, StateName, #state{ role = Role, starter = User, - opts = Opts, renegotiate = Renegotiation, connection_state = Connection0} = State0, EncData) when Byte == ?SSH_MSG_GLOBAL_REQUEST; @@ -1282,8 +1321,17 @@ generate_event(<<?BYTE(Byte), _/binary>> = Msg, StateName, ConnectionMsg = ssh_message:decode(Msg), State1 = generate_event_new_state(State0, EncData), try ssh_connection:handle_msg(ConnectionMsg, Connection0, Role) of - {{replies, Replies}, Connection} -> - State = send_replies(Replies, State1#state{connection_state = Connection}), + {{replies, Replies0}, Connection} -> + if StateName == connected -> + Replies = Replies0, + State2 = State1; + true -> + {ConnReplies, Replies} = + lists:splitwith(fun not_connected_filter/1, Replies0), + Q = State1#state.event_queue ++ ConnReplies, + State2 = State1#state{ event_queue = Q } + end, + State = send_replies(Replies, State2#state{connection_state = Connection}), {next_state, StateName, next_packet(State)}; {noreply, Connection} -> {next_state, StateName, next_packet(State1#state{connection_state = Connection})}; @@ -1293,21 +1341,17 @@ generate_event(<<?BYTE(Byte), _/binary>> = Msg, StateName, User ! {self(), not_connected, Reason}, {stop, {shutdown, normal}, next_packet(State#state{connection_state = Connection})}; - {disconnect, Reason, {{replies, Replies}, Connection}} -> + {disconnect, _Reason, {{replies, Replies}, Connection}} -> State = send_replies(Replies, State1#state{connection_state = Connection}), - SSHOpts = proplists:get_value(ssh_opts, Opts), - disconnect_fun(Reason, SSHOpts), {stop, {shutdown, normal}, State#state{connection_state = Connection}} catch _:Error -> - {disconnect, Reason, {{replies, Replies}, Connection}} = + {disconnect, _Reason, {{replies, Replies}, Connection}} = ssh_connection:handle_msg( #ssh_msg_disconnect{code = ?SSH_DISCONNECT_BY_APPLICATION, description = "Internal error", language = "en"}, Connection0, Role), State = send_replies(Replies, State1#state{connection_state = Connection}), - SSHOpts = proplists:get_value(ssh_opts, Opts), - disconnect_fun(Reason, SSHOpts), {stop, {shutdown, Error}, State#state{connection_state = Connection}} end; @@ -1456,15 +1500,43 @@ next_packet(#state{socket = Socket} = State) -> State. after_new_keys(#state{renegotiate = true} = State) -> - {connected, State#state{renegotiate = false}}; + State1 = State#state{renegotiate = false, event_queue = []}, + lists:foldr(fun after_new_keys_events/2, {next_state, connected, State1}, State#state.event_queue); after_new_keys(#state{renegotiate = false, ssh_params = #ssh{role = client} = Ssh0} = State) -> {Msg, Ssh} = ssh_auth:service_request_msg(Ssh0), send_msg(Msg, State), - {userauth, State#state{ssh_params = Ssh}}; + {next_state, userauth, State#state{ssh_params = Ssh}}; after_new_keys(#state{renegotiate = false, ssh_params = #ssh{role = server}} = State) -> - {userauth, State}. + {next_state, userauth, State}. + +after_new_keys_events({sync, _Event, From}, {stop, _Reason, _StateData}=Terminator) -> + gen_fsm:reply(From, {error, closed}), + Terminator; +after_new_keys_events(_, {stop, _Reason, _StateData}=Terminator) -> + Terminator; +after_new_keys_events({sync, Event, From}, {next_state, StateName, StateData}) -> + case handle_sync_event(Event, From, StateName, StateData) of + {reply, Reply, NextStateName, NewStateData} -> + gen_fsm:reply(From, Reply), + {next_state, NextStateName, NewStateData}; + {next_state, NextStateName, NewStateData}-> + {next_state, NextStateName, NewStateData}; + {stop, Reason, Reply, NewStateData} -> + gen_fsm:reply(From, Reply), + {stop, Reason, NewStateData} + end; +after_new_keys_events({event, Event}, {next_state, StateName, StateData}) -> + case handle_event(Event, StateName, StateData) of + {next_state, NextStateName, NewStateData}-> + {next_state, NextStateName, NewStateData}; + {stop, Reason, NewStateData} -> + {stop, Reason, NewStateData} + end; +after_new_keys_events({connection_reply, _Data} = Reply, {StateName, State}) -> + NewState = send_replies([Reply], State), + {next_state, StateName, NewState}. handle_ssh_packet_data(RemainingSshPacketLen, DecData, EncData, StateName, State) -> @@ -1526,12 +1598,14 @@ handle_disconnect(#ssh_msg_disconnect{} = DisconnectMsg, State, Error) -> handle_disconnect(Type, #ssh_msg_disconnect{description = Desc} = Msg, #state{connection_state = Connection0, role = Role} = State0) -> {disconnect, _, {{replies, Replies}, Connection}} = ssh_connection:handle_msg(Msg, Connection0, Role), State = send_replies(disconnect_replies(Type, Msg, Replies), State0), + disconnect_fun(Desc, State#state.opts), {stop, {shutdown, Desc}, State#state{connection_state = Connection}}. handle_disconnect(Type, #ssh_msg_disconnect{description = Desc} = Msg, #state{connection_state = Connection0, role = Role} = State0, ErrorMsg) -> {disconnect, _, {{replies, Replies}, Connection}} = ssh_connection:handle_msg(Msg, Connection0, Role), State = send_replies(disconnect_replies(Type, Msg, Replies), State0), + disconnect_fun(Desc, State#state.opts), {stop, {shutdown, {Desc, ErrorMsg}}, State#state{connection_state = Connection}}. disconnect_replies(own, Msg, Replies) -> @@ -1625,6 +1699,11 @@ log_error(Reason) -> error_logger:error_report(Report), "Internal error". +not_connected_filter({connection_reply, _Data}) -> + true; +not_connected_filter(_) -> + false. + send_replies([], State) -> State; send_replies([{connection_reply, Data} | Rest], #state{ssh_params = Ssh0} = State) -> @@ -1645,6 +1724,8 @@ send_reply({flow_control, Cache, Channel, From, Msg}) -> send_reply({flow_control, From, Msg}) -> gen_fsm:reply(From, Msg). +disconnect_fun({disconnect,Msg}, Opts) -> + disconnect_fun(Msg, Opts); disconnect_fun(_, undefined) -> ok; disconnect_fun(Reason, Opts) -> @@ -1655,6 +1736,15 @@ disconnect_fun(Reason, Opts) -> catch Fun(Reason) end. +unexpected_fun(UnexpectedMessage, Opts, #ssh{peer={_,Peer}}) -> + case proplists:get_value(unexpectedfun, Opts) of + undefined -> + report; + Fun -> + catch Fun(UnexpectedMessage, Peer) + end. + + check_cache(#state{opts = Opts} = State, Cache) -> %% Check the number of entries in Cache case proplists:get_value(size, ets:info(Cache)) of @@ -1722,3 +1812,12 @@ start_timeout(_,_, infinity) -> ok; start_timeout(Channel, From, Time) -> erlang:send_after(Time, self(), {timeout, {Channel, From}}). + +getopt(Opt, Socket) -> + case inet:getopts(Socket, [Opt]) of + {ok, [{Opt, Value}]} -> + {ok, Value}; + Other -> + {error, {unexpected_getopts_return, Other}} + end. + diff --git a/lib/ssh/src/ssh_info.erl b/lib/ssh/src/ssh_info.erl index 9c79d773a7..fc8f564bc3 100644 --- a/lib/ssh/src/ssh_info.erl +++ b/lib/ssh/src/ssh_info.erl @@ -79,7 +79,7 @@ print_clients(D) -> print_client(D, {undefined,Pid,supervisor,[ssh_connection_handler]}) -> {{Local,Remote},_Str} = ssh_connection_handler:get_print_info(Pid), - io:format(D, " Local=~s Remote=~s~n",[fmt_host_port(Local),fmt_host_port(Remote)]); + io:format(D, " Local=~s Remote=~s ConnectionRef=~p~n",[fmt_host_port(Local),fmt_host_port(Remote),Pid]); print_client(D, Other) -> io:format(D, " [[Other 1: ~p]]~n",[Other]). @@ -134,10 +134,11 @@ walk_sups(D, StartPid) -> io:format(D, "Start at ~p, ~s.~n",[StartPid,dead_or_alive(StartPid)]), walk_sups(D, children(StartPid), _Indent=?inc(0)). -walk_sups(D, [H={_,Pid,SupOrWorker,_}|T], Indent) -> +walk_sups(D, [H={_,Pid,_,_}|T], Indent) -> indent(D, Indent), io:format(D, '~200p ~p is ~s~n',[H,Pid,dead_or_alive(Pid)]), - case SupOrWorker of - supervisor -> walk_sups(D, children(Pid), ?inc(Indent)); + case H of + {_,_,supervisor,[ssh_connection_handler]} -> ok; + {_,Pid,supervisor,_} -> walk_sups(D, children(Pid), ?inc(Indent)); _ -> ok end, walk_sups(D, T, Indent); diff --git a/lib/ssh/src/ssh_message.erl b/lib/ssh/src/ssh_message.erl index 66e7717095..483c6cb4aa 100644 --- a/lib/ssh/src/ssh_message.erl +++ b/lib/ssh/src/ssh_message.erl @@ -421,8 +421,8 @@ decode(<<?BYTE(?SSH_MSG_USERAUTH_INFO_RESPONSE), ?UINT32(Num), Data/binary>>) -> decode(<<?BYTE(?SSH_MSG_KEXINIT), Cookie:128, Data/binary>>) -> decode_kex_init(Data, [Cookie, ssh_msg_kexinit], 10); -decode(<<?BYTE(?SSH_MSG_KEXDH_INIT), ?UINT32(Len), E:Len/binary>>) -> - #ssh_msg_kexdh_init{e = erlint(Len, E) +decode(<<?BYTE(?SSH_MSG_KEXDH_INIT), ?UINT32(Len), E:Len/big-signed-integer-unit:8>>) -> + #ssh_msg_kexdh_init{e = E }; decode(<<?BYTE(?SSH_MSG_KEX_DH_GEX_REQUEST), ?UINT32(Min), ?UINT32(N), ?UINT32(Max)>>) -> #ssh_msg_kex_dh_gex_request{ @@ -442,11 +442,11 @@ decode(<<?BYTE(?SSH_MSG_KEX_DH_GEX_GROUP), g = Generator }; decode(<<?BYTE(?SSH_MSG_KEXDH_REPLY), ?UINT32(Len0), Key:Len0/binary, - ?UINT32(Len1), F:Len1/binary, + ?UINT32(Len1), F:Len1/big-signed-integer-unit:8, ?UINT32(Len2), Hashsign:Len2/binary>>) -> #ssh_msg_kexdh_reply{ public_host_key = decode_host_key(Key), - f = erlint(Len1, F), + f = F, h_sig = decode_sign(Hashsign) }; @@ -514,10 +514,7 @@ decode_kex_init(<<?UINT32(Len), Data:Len/binary, Rest/binary>>, Acc, N) -> Names = string:tokens(unicode:characters_to_list(Data), ","), decode_kex_init(Rest, [Names | Acc], N -1). -erlint(MPIntSize, MPIntValue) -> - Bits = MPIntSize * 8, - <<Integer:Bits/integer>> = MPIntValue, - Integer. + decode_sign(<<?UINT32(Len), _Alg:Len/binary, ?UINT32(_), Signature/binary>>) -> Signature. @@ -525,18 +522,19 @@ decode_sign(<<?UINT32(Len), _Alg:Len/binary, ?UINT32(_), Signature/binary>>) -> decode_host_key(<<?UINT32(Len), Alg:Len/binary, Rest/binary>>) -> decode_host_key(Alg, Rest). -decode_host_key(<<"ssh-rsa">>, <<?UINT32(Len0), E:Len0/binary, - ?UINT32(Len1), N:Len1/binary>>) -> - #'RSAPublicKey'{publicExponent = erlint(Len0, E), - modulus = erlint(Len1, N)}; +decode_host_key(<<"ssh-rsa">>, <<?UINT32(Len0), E:Len0/big-signed-integer-unit:8, + ?UINT32(Len1), N:Len1/big-signed-integer-unit:8>>) -> + #'RSAPublicKey'{publicExponent = E, + modulus = N}; decode_host_key(<<"ssh-dss">>, - <<?UINT32(Len0), P:Len0/binary, - ?UINT32(Len1), Q:Len1/binary, - ?UINT32(Len2), G:Len2/binary, - ?UINT32(Len3), Y:Len3/binary>>) -> - {erlint(Len3, Y), #'Dss-Parms'{p = erlint(Len0, P), q = erlint(Len1, Q), - g = erlint(Len2, G)}}. + <<?UINT32(Len0), P:Len0/big-signed-integer-unit:8, + ?UINT32(Len1), Q:Len1/big-signed-integer-unit:8, + ?UINT32(Len2), G:Len2/big-signed-integer-unit:8, + ?UINT32(Len3), Y:Len3/big-signed-integer-unit:8>>) -> + {Y, #'Dss-Parms'{p = P, + q = Q, + g = G}}. encode_host_key(#'RSAPublicKey'{modulus = N, publicExponent = E}) -> ssh_bits:encode(["ssh-rsa", E, N], [string, mpint, mpint]); diff --git a/lib/ssh/src/ssh_system_sup.erl b/lib/ssh/src/ssh_system_sup.erl index 660fe8bb65..acf94b4b73 100644 --- a/lib/ssh/src/ssh_system_sup.erl +++ b/lib/ssh/src/ssh_system_sup.erl @@ -28,13 +28,15 @@ -behaviour(supervisor). +-include("ssh.hrl"). + -export([start_link/1, stop_listener/1, - stop_listener/2, stop_system/1, - stop_system/2, system_supervisor/2, + stop_listener/3, stop_system/1, + stop_system/3, system_supervisor/3, subsystem_supervisor/1, channel_supervisor/1, connection_supervisor/1, - acceptor_supervisor/1, start_subsystem/2, restart_subsystem/2, - restart_acceptor/2, stop_subsystem/2]). + acceptor_supervisor/1, start_subsystem/2, restart_subsystem/3, + restart_acceptor/3, stop_subsystem/2]). %% Supervisor callback -export([init/1]). @@ -45,14 +47,15 @@ start_link(ServerOpts) -> Address = proplists:get_value(address, ServerOpts), Port = proplists:get_value(port, ServerOpts), - Name = make_name(Address, Port), + Profile = proplists:get_value(profile, proplists:get_value(ssh_opts, ServerOpts), ?DEFAULT_PROFILE), + Name = make_name(Address, Port, Profile), supervisor:start_link({local, Name}, ?MODULE, [ServerOpts]). stop_listener(SysSup) -> stop_acceptor(SysSup). -stop_listener(Address, Port) -> - Name = make_name(Address, Port), +stop_listener(Address, Port, Profile) -> + Name = make_name(Address, Port, Profile), stop_acceptor(whereis(Name)). stop_system(SysSup) -> @@ -60,12 +63,12 @@ stop_system(SysSup) -> spawn(fun() -> sshd_sup:stop_child(Name) end), ok. -stop_system(Address, Port) -> - spawn(fun() -> sshd_sup:stop_child(Address, Port) end), +stop_system(Address, Port, Profile) -> + spawn(fun() -> sshd_sup:stop_child(Address, Port, Profile) end), ok. -system_supervisor(Address, Port) -> - Name = make_name(Address, Port), +system_supervisor(Address, Port, Profile) -> + Name = make_name(Address, Port, Profile), whereis(Name). subsystem_supervisor(SystemSup) -> @@ -103,9 +106,9 @@ stop_subsystem(SystemSup, SubSys) -> end. -restart_subsystem(Address, Port) -> - SysSupName = make_name(Address, Port), - SubSysName = id(ssh_subsystem_sup, Address, Port), +restart_subsystem(Address, Port, Profile) -> + SysSupName = make_name(Address, Port, Profile), + SubSysName = id(ssh_subsystem_sup, Address, Port, Profile), case supervisor:terminate_child(SysSupName, SubSysName) of ok -> supervisor:restart_child(SysSupName, SubSysName); @@ -113,9 +116,9 @@ restart_subsystem(Address, Port) -> Error end. -restart_acceptor(Address, Port) -> - SysSupName = make_name(Address, Port), - AcceptorName = id(ssh_acceptor_sup, Address, Port), +restart_acceptor(Address, Port, Profile) -> + SysSupName = make_name(Address, Port, Profile), + AcceptorName = id(ssh_acceptor_sup, Address, Port, Profile), supervisor:restart_child(SysSupName, AcceptorName). %%%========================================================================= @@ -137,7 +140,8 @@ child_specs(ServerOpts) -> ssh_acceptor_child_spec(ServerOpts) -> Address = proplists:get_value(address, ServerOpts), Port = proplists:get_value(port, ServerOpts), - Name = id(ssh_acceptor_sup, Address, Port), + Profile = proplists:get_value(profile, proplists:get_value(ssh_opts, ServerOpts), ?DEFAULT_PROFILE), + Name = id(ssh_acceptor_sup, Address, Port, Profile), StartFunc = {ssh_acceptor_sup, start_link, [ServerOpts]}, Restart = transient, Shutdown = infinity, @@ -155,12 +159,23 @@ ssh_subsystem_child_spec(ServerOpts) -> {Name, StartFunc, Restart, Shutdown, Type, Modules}. -id(Sup, Address, Port) -> - {Sup, Address, Port}. - -make_name(Address, Port) -> - list_to_atom(lists:flatten(io_lib:format("ssh_system_~p_~p_sup", - [Address, Port]))). +id(Sup, Address, Port, Profile) -> + case is_list(Address) of + true -> + {Sup, any, Port, Profile}; + false -> + {Sup, Address, Port, Profile} + end. + +make_name(Address, Port, Profile) -> + case is_list(Address) of + true -> + list_to_atom(lists:flatten(io_lib:format("ssh_system_~p_~p_~p_sup", + [any, Port, Profile]))); + false -> + list_to_atom(lists:flatten(io_lib:format("ssh_system_~p_~p_~p_sup", + [Address, Port, Profile]))) + end. ssh_subsystem_sup([{_, Child, _, [ssh_subsystem_sup]} | _]) -> Child; @@ -178,3 +193,4 @@ stop_acceptor(Sup) -> supervisor:which_children(Sup)], supervisor:terminate_child(AcceptorSup, Name). + diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index d6414bab6c..ea9bca2390 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -31,6 +31,8 @@ -export([versions/2, hello_version_msg/1]). -export([next_seqnum/1, decrypt_first_block/2, decrypt_blocks/3, + supported_algorithms/0, supported_algorithms/1, + default_algorithms/0, default_algorithms/1, is_valid_mac/3, handle_hello_version/1, key_exchange_init_msg/1, @@ -42,6 +44,68 @@ unpack/3, decompress/2, ssh_packet/2, pack/2, msg_data/1, sign/3, verify/4]). +%%%---------------------------------------------------------------------------- +%%% +%%% There is a difference between supported and default algorithms. The +%%% SUPPORTED algorithms can be handled (maybe untested...). The DEFAULT ones +%%% are announced in ssh_msg_kexinit and in ssh:default_algorithms/0 to the +%%% user. +%%% +%%% A supported algorithm can be requested in the option 'preferred_algorithms', +%%% but may give unexpected results because of being promoted to default. +%%% +%%% This makes it possible to add experimental algorithms (in supported_algorithms) +%%% and test them without letting the default users know about them. +%%% + +default_algorithms() -> [{K,default_algorithms(K)} || K <- algo_classes()]. + +algo_classes() -> [kex, public_key, cipher, mac, compression]. + +default_algorithms(compression) -> + %% Do not announce '[email protected]' because there seem to be problems + supported_algorithms(compression, same(['[email protected]'])); +default_algorithms(Alg) -> + supported_algorithms(Alg). + + +supported_algorithms() -> [{K,supported_algorithms(K)} || K <- algo_classes()]. + +supported_algorithms(kex) -> + ['diffie-hellman-group1-sha1']; +supported_algorithms(public_key) -> + ssh_auth:default_public_key_algorithms(); +supported_algorithms(cipher) -> + Supports = crypto:supports(), + CipherAlgos = [{aes_ctr, 'aes128-ctr'}, {aes_cbc128, 'aes128-cbc'}, {des3_cbc, '3des-cbc'}], + Algs = [SshAlgo || + {CryptoAlgo, SshAlgo} <- CipherAlgos, + lists:member(CryptoAlgo, proplists:get_value(ciphers, Supports, []))], + same(Algs); +supported_algorithms(mac) -> + Supports = crypto:supports(), + HashAlgos = [{sha256, 'hmac-sha2-256'}, {sha, 'hmac-sha1'}], + Algs = [SshAlgo || + {CryptoAlgo, SshAlgo} <- HashAlgos, + lists:member(CryptoAlgo, proplists:get_value(hashs, Supports, []))], + same(Algs); +supported_algorithms(compression) -> + same(['none','zlib','[email protected]']). + + +supported_algorithms(Key, [{client2server,BL1},{server2client,BL2}]) -> + [{client2server,As1},{server2client,As2}] = supported_algorithms(Key), + [{client2server,As1--BL1},{server2client,As2--BL2}]; +supported_algorithms(Key, BlackList) -> + supported_algorithms(Key) -- BlackList. + + + + +same(Algs) -> [{client2server,Algs}, {server2client,Algs}]. + + +%%%---------------------------------------------------------------------------- versions(client, Options)-> Vsn = proplists:get_value(vsn, Options, ?DEFAULT_CLIENT_VERSION), {Vsn, format_version(Vsn, software_version(Options))}; @@ -128,62 +192,45 @@ key_exchange_init_msg(Ssh0) -> kex_init(#ssh{role = Role, opts = Opts, available_host_keys = HostKeyAlgs}) -> Random = ssh_bits:random(16), - Compression = case proplists:get_value(compression, Opts, none) of - openssh_zlib -> ["[email protected]", "none"]; - zlib -> ["zlib", "none"]; - none -> ["none", "zlib"] - end, - kexinit_messsage(Role, Random, Compression, HostKeyAlgs). + PrefAlgs = + case proplists:get_value(preferred_algorithms,Opts) of + undefined -> + default_algorithms(); + Algs0 -> + Algs0 + end, + kexinit_message(Role, Random, PrefAlgs, HostKeyAlgs). key_init(client, Ssh, Value) -> Ssh#ssh{c_keyinit = Value}; key_init(server, Ssh, Value) -> Ssh#ssh{s_keyinit = Value}. -available_ssh_algos() -> - Supports = crypto:supports(), - CipherAlgos = [{aes_ctr, "aes128-ctr"}, {aes_cbc128, "aes128-cbc"}, {des3_cbc, "3des-cbc"}], - Ciphers = [SshAlgo || - {CryptoAlgo, SshAlgo} <- CipherAlgos, - lists:member(CryptoAlgo, proplists:get_value(ciphers, Supports, []))], - HashAlgos = [{sha256, "hmac-sha2-256"}, {sha, "hmac-sha1"}], - Hashs = [SshAlgo || - {CryptoAlgo, SshAlgo} <- HashAlgos, - lists:member(CryptoAlgo, proplists:get_value(hashs, Supports, []))], - {Ciphers, Hashs}. - -kexinit_messsage(client, Random, Compression, HostKeyAlgs) -> - {CipherAlgs, HashAlgs} = available_ssh_algos(), - #ssh_msg_kexinit{ - cookie = Random, - kex_algorithms = ["diffie-hellman-group1-sha1"], - server_host_key_algorithms = HostKeyAlgs, - encryption_algorithms_client_to_server = CipherAlgs, - encryption_algorithms_server_to_client = CipherAlgs, - mac_algorithms_client_to_server = HashAlgs, - mac_algorithms_server_to_client = HashAlgs, - compression_algorithms_client_to_server = Compression, - compression_algorithms_server_to_client = Compression, - languages_client_to_server = [], - languages_server_to_client = [] - }; -kexinit_messsage(server, Random, Compression, HostKeyAlgs) -> - {CipherAlgs, HashAlgs} = available_ssh_algos(), +kexinit_message(_Role, Random, Algs, HostKeyAlgs) -> #ssh_msg_kexinit{ cookie = Random, - kex_algorithms = ["diffie-hellman-group1-sha1"], + kex_algorithms = to_strings( get_algs(kex,Algs) ), server_host_key_algorithms = HostKeyAlgs, - encryption_algorithms_client_to_server = CipherAlgs, - encryption_algorithms_server_to_client = CipherAlgs, - mac_algorithms_client_to_server = HashAlgs, - mac_algorithms_server_to_client = HashAlgs, - compression_algorithms_client_to_server = Compression, - compression_algorithms_server_to_client = Compression, + encryption_algorithms_client_to_server = c2s(cipher,Algs), + encryption_algorithms_server_to_client = s2c(cipher,Algs), + mac_algorithms_client_to_server = c2s(mac,Algs), + mac_algorithms_server_to_client = s2c(mac,Algs), + compression_algorithms_client_to_server = c2s(compression,Algs), + compression_algorithms_server_to_client = s2c(compression,Algs), languages_client_to_server = [], languages_server_to_client = [] }. +c2s(Key, Algs) -> x2y(client2server, Key, Algs). +s2c(Key, Algs) -> x2y(server2client, Key, Algs). + +x2y(DirectionKey, Key, Algs) -> to_strings(proplists:get_value(DirectionKey, get_algs(Key,Algs))). + +get_algs(Key, Algs) -> proplists:get_value(Key, Algs, default_algorithms(Key)). + +to_strings(L) -> lists:map(fun erlang:atom_to_list/1, L). + new_keys_message(Ssh0) -> {SshPacket, Ssh} = ssh_packet(#ssh_msg_newkeys{}, Ssh0), @@ -448,6 +495,7 @@ select_algorithm(Role, Client, Server) -> decompress = Decompression, c_lng = C_Lng, s_lng = S_Lng}, +%%ct:pal("~p~n Client=~p~n Server=~p~n Alg=~p~n",[Role,Client,Server,Alg]), {ok, Alg}. select_encrypt_decrypt(client, Client, Server) -> @@ -537,10 +585,15 @@ alg_final(SSH0) -> {ok,SSH6} = decompress_final(SSH5), SSH6. -select_all(CL, SL) -> +select_all(CL, SL) when length(CL) + length(SL) < 50 -> A = CL -- SL, %% algortihms only used by client %% algorithms used by client and server (client pref) - lists:map(fun(ALG) -> list_to_atom(ALG) end, (CL -- A)). + lists:map(fun(ALG) -> list_to_atom(ALG) end, (CL -- A)); +select_all(_CL, _SL) -> + throw(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, + description = "Too many algorithms", + language = "en"}). + select([], []) -> none; diff --git a/lib/ssh/src/sshd_sup.erl b/lib/ssh/src/sshd_sup.erl index 60222f5172..e879629ccb 100644 --- a/lib/ssh/src/sshd_sup.erl +++ b/lib/ssh/src/sshd_sup.erl @@ -26,8 +26,10 @@ -behaviour(supervisor). +-include("ssh.hrl"). + -export([start_link/1, start_child/1, stop_child/1, - stop_child/2, system_name/1]). + stop_child/3, system_name/1]). %% Supervisor callback -export([init/1]). @@ -40,13 +42,14 @@ start_link(Servers) -> start_child(ServerOpts) -> Address = proplists:get_value(address, ServerOpts), - Port = proplists:get_value(port, ServerOpts), - case ssh_system_sup:system_supervisor(Address, Port) of + Port = proplists:get_value(port, ServerOpts), + Profile = proplists:get_value(profile, proplists:get_value(ssh_opts, ServerOpts), ?DEFAULT_PROFILE), + case ssh_system_sup:system_supervisor(Address, Port, Profile) of undefined -> Spec = child_spec(Address, Port, ServerOpts), case supervisor:start_child(?MODULE, Spec) of {error, already_present} -> - Name = id(Address, Port), + Name = id(Address, Port, Profile), supervisor:delete_child(?MODULE, Name), supervisor:start_child(?MODULE, Spec); Reply -> @@ -60,8 +63,8 @@ start_child(ServerOpts) -> stop_child(Name) -> supervisor:terminate_child(?MODULE, Name). -stop_child(Address, Port) -> - Name = id(Address, Port), +stop_child(Address, Port, Profile) -> + Name = id(Address, Port, Profile), stop_child(Name). system_name(SysSup) -> @@ -87,7 +90,8 @@ init([Servers]) -> %%% Internal functions %%%========================================================================= child_spec(Address, Port, ServerOpts) -> - Name = id(Address, Port), + Profile = proplists:get_value(profile, proplists:get_value(ssh_opts, ServerOpts), ?DEFAULT_PROFILE), + Name = id(Address, Port,Profile), StartFunc = {ssh_system_sup, start_link, [ServerOpts]}, Restart = temporary, Shutdown = infinity, @@ -95,8 +99,13 @@ child_spec(Address, Port, ServerOpts) -> Type = supervisor, {Name, StartFunc, Restart, Shutdown, Type, Modules}. -id(Address, Port) -> - {server, ssh_system_sup, Address, Port}. +id(Address, Port, Profile) -> + case is_list(Address) of + true -> + {server, ssh_system_sup, any, Port, Profile}; + false -> + {server, ssh_system_sup, Address, Port, Profile} + end. system_name([], _ ) -> undefined; diff --git a/lib/ssh/test/Makefile b/lib/ssh/test/Makefile index 740dbd0235..50efc33f98 100644 --- a/lib/ssh/test/Makefile +++ b/lib/ssh/test/Makefile @@ -32,15 +32,18 @@ VSN=$(GS_VSN) MODULES= \ ssh_test_lib \ + ssh_sup_SUITE \ ssh_basic_SUITE \ ssh_to_openssh_SUITE \ ssh_sftp_SUITE \ ssh_sftpd_SUITE \ ssh_sftpd_erlclient_SUITE \ + ssh_upgrade_SUITE \ ssh_connection_SUITE \ ssh_echo_server \ ssh_peername_sockname_server \ - ssh_test_cli + ssh_test_cli \ + ssh_relay HRL_FILES_NEEDED_IN_TEST= \ $(ERL_TOP)/lib/ssh/src/ssh.hrl \ diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index d55d09f2a2..e62feb6857 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -23,12 +23,14 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("kernel/include/inet.hrl"). +-include_lib("kernel/include/file.hrl"). %% Note: This directive should only be used in test suites. -compile(export_all). -define(NEWLINE, <<"\r\n">>). +-define(REKEY_DATA_TMO, 65000). %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- @@ -44,21 +46,33 @@ all() -> {group, dsa_pass_key}, {group, rsa_pass_key}, {group, internal_error}, + connectfun_disconnectfun_server, + connectfun_disconnectfun_client, + {group, renegotiate}, daemon_already_started, server_password_option, server_userpassword_option, + {group, dir_options}, double_close, ssh_connect_timeout, ssh_connect_arg4_timeout, packet_size_zero, ssh_daemon_minimal_remote_max_packet_size_option, + ssh_msg_debug_fun_option_client, + ssh_msg_debug_fun_option_server, + disconnectfun_option_server, + disconnectfun_option_client, + unexpectedfun_option_server, + unexpectedfun_option_client, + preferred_algorithms, id_string_no_opt_client, id_string_own_string_client, id_string_random_client, id_string_no_opt_server, id_string_own_string_server, id_string_random_server, - {group, hardening_tests} + {group, hardening_tests}, + ssh_info_print ]. groups() -> @@ -67,6 +81,7 @@ groups() -> {dsa_pass_key, [], [pass_phrase]}, {rsa_pass_key, [], [pass_phrase]}, {internal_error, [], [internal_error]}, + {renegotiate, [], [rekey, rekey_limit, renegotiate1, renegotiate2]}, {hardening_tests, [], [ssh_connect_nonegtimeout_connected_parallel, ssh_connect_nonegtimeout_connected_sequential, ssh_connect_negtimeout_parallel, @@ -75,19 +90,21 @@ groups() -> max_sessions_ssh_connect_sequential, max_sessions_sftp_start_channel_parallel, max_sessions_sftp_start_channel_sequential - ]} + ]}, + {dir_options, [], [user_dir_option, + system_dir_option]} ]. basic_tests() -> [send, close, peername_sockname, exec, exec_compressed, shell, cli, known_hosts, - idle_time, rekey, openssh_zlib_basic_test, - misc_ssh_options, inet_option]. + idle_time, openssh_zlib_basic_test, misc_ssh_options, inet_option]. %%-------------------------------------------------------------------- init_per_suite(Config) -> + catch crypto:stop(), case catch crypto:start() of ok -> Config; @@ -126,6 +143,48 @@ init_per_group(internal_error, Config) -> ssh_test_lib:setup_dsa(DataDir, PrivDir), file:delete(filename:join(PrivDir, "system/ssh_host_dsa_key")), Config; +init_per_group(dir_options, Config) -> + PrivDir = ?config(priv_dir, Config), + %% Make unreadable dir: + Dir_unreadable = filename:join(PrivDir, "unread"), + ok = file:make_dir(Dir_unreadable), + {ok,F1} = file:read_file_info(Dir_unreadable), + ok = file:write_file_info(Dir_unreadable, + F1#file_info{mode = F1#file_info.mode band (bnot 8#00444)}), + %% Make readable file: + File_readable = filename:join(PrivDir, "file"), + ok = file:write_file(File_readable, <<>>), + + %% Check: + case {file:read_file_info(Dir_unreadable), + file:read_file_info(File_readable)} of + {{ok, Id=#file_info{type=directory, access=Md}}, + {ok, If=#file_info{type=regular, access=Mf}}} -> + AccessOK = + case {Md, Mf} of + {read, _} -> false; + {read_write, _} -> false; + {_, read} -> true; + {_, read_write} -> true; + _ -> false + end, + + case AccessOK of + true -> + %% Save: + [{unreadable_dir, Dir_unreadable}, + {readable_file, File_readable} + | Config]; + false -> + ct:log("File#file_info : ~p~n" + "Dir#file_info : ~p",[If,Id]), + {skip, "File or dir mode settings failed"} + end; + + NotDirFile -> + ct:log("{Dir,File} -> ~p",[NotDirFile]), + {skip, "File/Dir creation failed"} + end; init_per_group(_, Config) -> Config. @@ -285,7 +344,7 @@ exec_compressed(Config) when is_list(Config) -> UserDir = ?config(priv_dir, Config), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir},{user_dir, UserDir}, - {compression, zlib}, + {preferred_algorithms,[{compression, [zlib]}]}, {failfun, fun ssh_test_lib:failfun/2}]), ConnectionRef = @@ -331,25 +390,175 @@ idle_time(Config) -> rekey() -> [{doc, "Idle timeout test"}]. rekey(Config) -> - SystemDir = filename:join(?config(priv_dir, Config), system), + SystemDir = ?config(data_dir, Config), UserDir = ?config(priv_dir, Config), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, - {user_dir, UserDir}, + {user_dir, UserDir}, {failfun, fun ssh_test_lib:failfun/2}, + {user_passwords, + [{"simon", "says"}]}, {rekey_limit, 0}]), + ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, {user_dir, UserDir}, + {user, "simon"}, + {password, "says"}, {user_interaction, false}, {rekey_limit, 0}]), receive - after 200000 -> + after ?REKEY_DATA_TMO -> %%By this time rekeying would have been done ssh:close(ConnectionRef), ssh:stop_daemon(Pid) end. %%-------------------------------------------------------------------- +rekey_limit() -> + [{doc, "Test rekeying by data volume"}]. +rekey_limit(Config) -> + SystemDir = ?config(data_dir, Config), + UserDir = ?config(priv_dir, Config), + DataFile = filename:join(UserDir, "rekey.data"), + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {user_passwords, + [{"simon", "says"}]}]), + {ok, SftpPid, ConnectionRef} = + ssh_sftp:start_channel(Host, Port, [{system_dir, SystemDir}, + {user_dir, UserDir}, + {user, "simon"}, + {password, "says"}, + {rekey_limit, 2500}, + {user_interaction, false}, + {silently_accept_hosts, true}]), + + Kex1 = get_kex_init(ConnectionRef), + + timer:sleep(?REKEY_DATA_TMO), + Kex1 = get_kex_init(ConnectionRef), + + Data = lists:duplicate(9000,1), + ok = ssh_sftp:write_file(SftpPid, DataFile, Data), + + timer:sleep(?REKEY_DATA_TMO), + Kex2 = get_kex_init(ConnectionRef), + + false = (Kex2 == Kex1), + + timer:sleep(?REKEY_DATA_TMO), + Kex2 = get_kex_init(ConnectionRef), + + ok = ssh_sftp:write_file(SftpPid, DataFile, "hi\n"), + + timer:sleep(?REKEY_DATA_TMO), + Kex2 = get_kex_init(ConnectionRef), + + false = (Kex2 == Kex1), + + timer:sleep(?REKEY_DATA_TMO), + Kex2 = get_kex_init(ConnectionRef), + + + ssh_sftp:stop_channel(SftpPid), + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid). + +%%-------------------------------------------------------------------- +renegotiate1() -> + [{doc, "Test rekeying with simulataneous send request"}]. +renegotiate1(Config) -> + SystemDir = ?config(data_dir, Config), + UserDir = ?config(priv_dir, Config), + DataFile = filename:join(UserDir, "renegotiate1.data"), + + {Pid, Host, DPort} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {user_passwords, + [{"simon", "says"}]}]), + RPort = ssh_test_lib:inet_port(), + + {ok,RelayPid} = ssh_relay:start_link({0,0,0,0}, RPort, Host, DPort), + + {ok, SftpPid, ConnectionRef} = + ssh_sftp:start_channel(Host, RPort, [{system_dir, SystemDir}, + {user_dir, UserDir}, + {user, "simon"}, + {password, "says"}, + {user_interaction, false}, + {silently_accept_hosts, true}]), + + Kex1 = get_kex_init(ConnectionRef), + + {ok, Handle} = ssh_sftp:open(SftpPid, DataFile, [write]), + + ok = ssh_sftp:write(SftpPid, Handle, "hi\n"), + + ssh_relay:hold(RelayPid, rx, 20, 1000), + ssh_connection_handler:renegotiate(ConnectionRef), + spawn(fun() -> ok=ssh_sftp:write(SftpPid, Handle, "another hi\n") end), + + timer:sleep(2000), + + Kex2 = get_kex_init(ConnectionRef), + + false = (Kex2 == Kex1), + + ssh_relay:stop(RelayPid), + ssh_sftp:stop_channel(SftpPid), + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid). + +%%-------------------------------------------------------------------- +renegotiate2() -> + [{doc, "Test rekeying with inflight messages from peer"}]. +renegotiate2(Config) -> + SystemDir = ?config(data_dir, Config), + UserDir = ?config(priv_dir, Config), + DataFile = filename:join(UserDir, "renegotiate1.data"), + + {Pid, Host, DPort} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {user_passwords, + [{"simon", "says"}]}]), + RPort = ssh_test_lib:inet_port(), + + {ok,RelayPid} = ssh_relay:start_link({0,0,0,0}, RPort, Host, DPort), + + {ok, SftpPid, ConnectionRef} = + ssh_sftp:start_channel(Host, RPort, [{system_dir, SystemDir}, + {user_dir, UserDir}, + {user, "simon"}, + {password, "says"}, + {user_interaction, false}, + {silently_accept_hosts, true}]), + + Kex1 = get_kex_init(ConnectionRef), + + {ok, Handle} = ssh_sftp:open(SftpPid, DataFile, [write]), + + ok = ssh_sftp:write(SftpPid, Handle, "hi\n"), + + ssh_relay:hold(RelayPid, rx, 20, infinity), + spawn(fun() -> ok=ssh_sftp:write(SftpPid, Handle, "another hi\n") end), + %% need a small pause here to ensure ssh_sftp:write is executed + ct:sleep(10), + ssh_connection_handler:renegotiate(ConnectionRef), + ssh_relay:release(RelayPid, rx), + + timer:sleep(2000), + + Kex2 = get_kex_init(ConnectionRef), + + false = (Kex2 == Kex1), + + ssh_relay:stop(RelayPid), + ssh_sftp:stop_channel(SftpPid), + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid). + +%%-------------------------------------------------------------------- shell() -> [{doc, "Test that ssh:shell/2 works"}]. shell(Config) when is_list(Config) -> @@ -494,6 +703,355 @@ server_userpassword_option(Config) when is_list(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- +system_dir_option(Config) -> + DirUnread = proplists:get_value(unreadable_dir,Config), + FileRead = proplists:get_value(readable_file,Config), + + case ssh_test_lib:daemon([{system_dir, DirUnread}]) of + {error,{eoptions,{{system_dir,DirUnread},eacces}}} -> + ok; + {Pid1,_Host1,Port1} when is_pid(Pid1),is_integer(Port1) -> + ssh:stop_daemon(Pid1), + ct:fail("Didn't detect that dir is unreadable", []) + end, + + case ssh_test_lib:daemon([{system_dir, FileRead}]) of + {error,{eoptions,{{system_dir,FileRead},enotdir}}} -> + ok; + {Pid2,_Host2,Port2} when is_pid(Pid2),is_integer(Port2) -> + ssh:stop_daemon(Pid2), + ct:fail("Didn't detect that option is a plain file", []) + end. + + +user_dir_option(Config) -> + DirUnread = proplists:get_value(unreadable_dir,Config), + FileRead = proplists:get_value(readable_file,Config), + %% Any port will do (beware, implementation knowledge!): + Port = 65535, + + case ssh:connect("localhost", Port, [{user_dir, DirUnread}]) of + {error,{eoptions,{{user_dir,DirUnread},eacces}}} -> + ok; + {error,econnrefused} -> + ct:fail("Didn't detect that dir is unreadable", []) + end, + + case ssh:connect("localhost", Port, [{user_dir, FileRead}]) of + {error,{eoptions,{{user_dir,FileRead},enotdir}}} -> + ok; + {error,econnrefused} -> + ct:fail("Didn't detect that option is a plain file", []) + end. + +%%-------------------------------------------------------------------- +ssh_msg_debug_fun_option_client() -> + [{doc, "validate client that uses the 'ssh_msg_debug_fun' option"}]. +ssh_msg_debug_fun_option_client(Config) -> + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {failfun, fun ssh_test_lib:failfun/2}]), + Parent = self(), + DbgFun = fun(ConnRef,Displ,Msg,Lang) -> Parent ! {msg_dbg,{ConnRef,Displ,Msg,Lang}} end, + + ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {user_interaction, false}, + {ssh_msg_debug_fun,DbgFun}]), + %% Beware, implementation knowledge: + gen_fsm:send_all_state_event(ConnectionRef,{ssh_msg_debug,false,<<"Hello">>,<<>>}), + receive + {msg_dbg,X={ConnectionRef,false,<<"Hello">>,<<>>}} -> + ct:log("Got expected dbg msg ~p",[X]), + ssh:stop_daemon(Pid); + {msg_dbg,X={_,false,<<"Hello">>,<<>>}} -> + ct:log("Got dbg msg but bad ConnectionRef (~p expected) ~p",[ConnectionRef,X]), + ssh:stop_daemon(Pid), + {fail, "Bad ConnectionRef received"}; + {msg_dbg,X} -> + ct:log("Got bad dbg msg ~p",[X]), + ssh:stop_daemon(Pid), + {fail,"Bad msg received"} + after 1000 -> + ssh:stop_daemon(Pid), + {fail,timeout} + end. + +%%-------------------------------------------------------------------- +connectfun_disconnectfun_server(Config) -> + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + + Parent = self(), + Ref = make_ref(), + ConnFun = fun(_,_,_) -> Parent ! {connect,Ref} end, + DiscFun = fun(R) -> Parent ! {disconnect,Ref,R} end, + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {failfun, fun ssh_test_lib:failfun/2}, + {disconnectfun, DiscFun}, + {connectfun, ConnFun}]), + ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {user_interaction, false}]), + receive + {connect,Ref} -> + ssh:close(ConnectionRef), + receive + {disconnect,Ref,R} -> + ct:log("Disconnect result: ~p",[R]), + ssh:stop_daemon(Pid) + after 2000 -> + {fail, "No disconnectfun action"} + end + after 2000 -> + {fail, "No connectfun action"} + end. + +%%-------------------------------------------------------------------- +connectfun_disconnectfun_client(Config) -> + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + + Parent = self(), + Ref = make_ref(), + DiscFun = fun(R) -> Parent ! {disconnect,Ref,R} end, + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {failfun, fun ssh_test_lib:failfun/2}]), + _ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {disconnectfun, DiscFun}, + {user_interaction, false}]), + ssh:stop_daemon(Pid), + receive + {disconnect,Ref,R} -> + ct:log("Disconnect result: ~p",[R]) + after 2000 -> + {fail, "No disconnectfun action"} + end. + +%%-------------------------------------------------------------------- +ssh_msg_debug_fun_option_server() -> + [{doc, "validate client that uses the 'ssh_msg_debug_fun' option"}]. +ssh_msg_debug_fun_option_server(Config) -> + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + + Parent = self(), + DbgFun = fun(ConnRef,Displ,Msg,Lang) -> Parent ! {msg_dbg,{ConnRef,Displ,Msg,Lang}} end, + ConnFun = fun(_,_,_) -> Parent ! {connection_pid,self()} end, + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {failfun, fun ssh_test_lib:failfun/2}, + {connectfun, ConnFun}, + {ssh_msg_debug_fun, DbgFun}]), + _ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {user_interaction, false}]), + receive + {connection_pid,Server} -> + %% Beware, implementation knowledge: + gen_fsm:send_all_state_event(Server,{ssh_msg_debug,false,<<"Hello">>,<<>>}), + receive + {msg_dbg,X={_,false,<<"Hello">>,<<>>}} -> + ct:log("Got expected dbg msg ~p",[X]), + ssh:stop_daemon(Pid); + {msg_dbg,X} -> + ct:log("Got bad dbg msg ~p",[X]), + ssh:stop_daemon(Pid), + {fail,"Bad msg received"} + after 3000 -> + ssh:stop_daemon(Pid), + {fail,timeout2} + end + after 3000 -> + ssh:stop_daemon(Pid), + {fail,timeout1} + end. + +%%-------------------------------------------------------------------- +disconnectfun_option_server(Config) -> + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + + Parent = self(), + DisConnFun = fun(Reason) -> Parent ! {disconnect,Reason} end, + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {failfun, fun ssh_test_lib:failfun/2}, + {disconnectfun, DisConnFun}]), + ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {user_interaction, false}]), + ssh:close(ConnectionRef), + receive + {disconnect,Reason} -> + ct:log("Server detected disconnect: ~p",[Reason]), + ssh:stop_daemon(Pid), + ok + after 3000 -> + receive + X -> ct:log("received ~p",[X]) + after 0 -> ok + end, + {fail,"Timeout waiting for disconnect"} + end. + +%%-------------------------------------------------------------------- +disconnectfun_option_client(Config) -> + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + + Parent = self(), + DisConnFun = fun(Reason) -> Parent ! {disconnect,Reason} end, + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {failfun, fun ssh_test_lib:failfun/2}]), + _ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {user_interaction, false}, + {disconnectfun, DisConnFun}]), + ssh:stop_daemon(Pid), + receive + {disconnect,Reason} -> + ct:log("Client detected disconnect: ~p",[Reason]), + ok + after 3000 -> + receive + X -> ct:log("received ~p",[X]) + after 0 -> ok + end, + {fail,"Timeout waiting for disconnect"} + end. + +%%-------------------------------------------------------------------- +unexpectedfun_option_server(Config) -> + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + + Parent = self(), + ConnFun = fun(_,_,_) -> Parent ! {connection_pid,self()} end, + UnexpFun = fun(Msg,Peer) -> + Parent ! {unexpected,Msg,Peer,self()}, + skip + end, + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {failfun, fun ssh_test_lib:failfun/2}, + {connectfun, ConnFun}, + {unexpectedfun, UnexpFun}]), + _ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {user_interaction, false}]), + receive + {connection_pid,Server} -> + %% Beware, implementation knowledge: + Server ! unexpected_message, + receive + {unexpected, unexpected_message, {{_,_,_,_},_}, _} -> ok; + {unexpected, unexpected_message, Peer, _} -> ct:fail("Bad peer ~p",[Peer]); + M = {unexpected, _, _, _} -> ct:fail("Bad msg ~p",[M]) + after 3000 -> + ssh:stop_daemon(Pid), + {fail,timeout2} + end + after 3000 -> + ssh:stop_daemon(Pid), + {fail,timeout1} + end. + +%%-------------------------------------------------------------------- +unexpectedfun_option_client(Config) -> + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + + Parent = self(), + UnexpFun = fun(Msg,Peer) -> + Parent ! {unexpected,Msg,Peer,self()}, + skip + end, + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {failfun, fun ssh_test_lib:failfun/2}]), + ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {user_interaction, false}, + {unexpectedfun, UnexpFun}]), + %% Beware, implementation knowledge: + ConnectionRef ! unexpected_message, + + receive + {unexpected, unexpected_message, {{_,_,_,_},_}, ConnectionRef} -> + ok; + {unexpected, unexpected_message, Peer, ConnectionRef} -> + ct:fail("Bad peer ~p",[Peer]); + M = {unexpected, _, _, _} -> + ct:fail("Bad msg ~p",[M]) + after 3000 -> + ssh:stop_daemon(Pid), + {fail,timeout} + end. + +%%-------------------------------------------------------------------- known_hosts() -> [{doc, "check that known_hosts is updated correctly"}]. known_hosts(Config) when is_list(Config) -> @@ -822,6 +1380,57 @@ ssh_daemon_minimal_remote_max_packet_size_option(Config) -> ssh:stop_daemon(Server). %%-------------------------------------------------------------------- +%% This test try every algorithm by connecting to an Erlang server +preferred_algorithms(Config) -> + SystemDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + + {Server, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {user_passwords, [{"vego", "morot"}]}, + {failfun, fun ssh_test_lib:failfun/2}]), + Available = ssh:default_algorithms(), + Tests = [[{Tag,[Alg]}] || {Tag, SubAlgs} <- Available, + is_atom(hd(SubAlgs)), + Alg <- SubAlgs] + ++ [[{Tag,[{T1,[A1]},{T2,[A2]}]}] || {Tag, [{T1,As1},{T2,As2}]} <- Available, + A1 <- As1, + A2 <- As2], + ct:log("TESTS: ~p",[Tests]), + [connect_exec_channel(Host,Port,PrefAlgs) || PrefAlgs <- Tests], + ssh:stop_daemon(Server). + + +connect_exec_channel(_Host, Port, Algs) -> + ct:log("Try ~p",[Algs]), + ConnectionRef = ssh_test_lib:connect(Port, [{silently_accept_hosts, true}, + {user_interaction, false}, + {user, "vego"}, + {password, "morot"}, + {preferred_algorithms,Algs} + ]), + chan_exec(ConnectionRef, "2*21.", <<"42\n">>), + ssh:close(ConnectionRef). + +chan_exec(ConnectionRef, Cmnd, Expected) -> + {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), + success = ssh_connection:exec(ConnectionRef, ChannelId0,Cmnd, infinity), + Data0 = {ssh_cm, ConnectionRef, {data, ChannelId0, 0, Expected}}, + case ssh_test_lib:receive_exec_result(Data0) of + expected -> + ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId0); + {unexpected_msg,{ssh_cm, ConnectionRef, {exit_status, ChannelId0, 0}} + = ExitStatus0} -> + ct:pal("0: Collected data ~p", [ExitStatus0]), + ssh_test_lib:receive_exec_result(Data0, + ConnectionRef, ChannelId0); + Other0 -> + ct:fail(Other0) + end. + +%%-------------------------------------------------------------------- id_string_no_opt_client(Config) -> {Server, _Host, Port} = fake_daemon(Config), {error,_} = ssh:connect("localhost", Port, [], 1000), @@ -904,8 +1513,10 @@ ssh_connect_negtimeout(Config, Parallel) -> {failfun, fun ssh_test_lib:failfun/2}]), {ok,Socket} = gen_tcp:connect(Host, Port, []), - ct:pal("And now sleeping 1.2*NegTimeOut (~p ms)...", [round(1.2 * NegTimeOut)]), - receive after round(1.2 * NegTimeOut) -> ok end, + + Factor = 2, + ct:pal("And now sleeping ~p*NegTimeOut (~p ms)...", [Factor, round(Factor * NegTimeOut)]), + ct:sleep(round(Factor * NegTimeOut)), case inet:sockname(Socket) of {ok,_} -> ct:fail("Socket not closed"); @@ -948,8 +1559,11 @@ ssh_connect_nonegtimeout_connected(Config, Parallel) -> ct:pal("---Erlang shell start: ~p~n", [ErlShellStart]), one_shell_op(IO, NegTimeOut), one_shell_op(IO, NegTimeOut), - ct:pal("And now sleeping 1.2*NegTimeOut (~p ms)...", [round(1.2 * NegTimeOut)]), - receive after round(1.2 * NegTimeOut) -> ok end, + + Factor = 2, + ct:pal("And now sleeping ~p*NegTimeOut (~p ms)...", [Factor, round(Factor * NegTimeOut)]), + ct:sleep(round(Factor * NegTimeOut)), + one_shell_op(IO, NegTimeOut) end, exit(Shell, kill). @@ -991,12 +1605,15 @@ openssh_zlib_basic_test(Config) -> {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, {user_dir, UserDir}, + {preferred_algorithms,[{compression, ['[email protected]']}]}, {failfun, fun ssh_test_lib:failfun/2}]), ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, {user_dir, UserDir}, {user_interaction, false}, - {compression, openssh_zlib}]), + {preferred_algorithms,[{compression, ['[email protected]', + none]}]} + ]), ok = ssh:close(ConnectionRef), ssh:stop_daemon(Pid). @@ -1074,6 +1691,7 @@ max_sessions(Config, ParallelLogin, Connect0) when is_function(Connect0,2) -> %% This is expected %% Now stop one connection and try to open one more ok = ssh:close(hd(Connections)), + receive after 250 -> ok end, % sleep so the supervisor has time to count down. Not nice... try Connect(Host,Port) of _ConnectionRef1 -> @@ -1096,6 +1714,74 @@ max_sessions(Config, ParallelLogin, Connect0) when is_function(Connect0,2) -> end. %%-------------------------------------------------------------------- +ssh_info_print(Config) -> + %% Just check that ssh_print:info() crashes + PrivDir = ?config(priv_dir, Config), + PrintFile = filename:join(PrivDir,info), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + + Parent = self(), + UnexpFun = fun(Msg,_Peer) -> + Parent ! {unexpected,Msg,self()}, + skip + end, + ConnFun = fun(_,_,_) -> Parent ! {connect,self()} end, + + {DaemonRef, Host, Port} = + ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {unexpectedfun, UnexpFun}, + {connectfun, ConnFun}, + {failfun, fun ssh_test_lib:failfun/2}]), + ClientConnRef1 = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {unexpectedfun, UnexpFun}, + {user_interaction, false}]), + ClientConnRef2 = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {unexpectedfun, UnexpFun}, + {user_interaction, false}]), + receive + {connect,DaemonConnRef} -> + ct:log("DaemonRef=~p, DaemonConnRef=~p, ClientConnRefs=~p",[DaemonRef, DaemonConnRef, + [ClientConnRef1,ClientConnRef2] + ]) + after 2000 -> + ok + end, + + {ok,D} = file:open(PrintFile, write), + ssh_info:print(D), + ok = file:close(D), + + {ok,Bin} = file:read_file(PrintFile), + ct:log("~s",[Bin]), + + receive + {unexpected, Msg, Pid} -> + ct:log("~p got unexpected msg ~p",[Pid,Msg]), + ct:log("process_info(~p) = ~n~p",[Pid,process_info(Pid)]), + ok = ssh:close(ClientConnRef1), + ok = ssh:close(ClientConnRef2), + ok = ssh:stop_daemon(DaemonRef), + {fail,"unexpected msg"} + after 1000 -> + ok = ssh:close(ClientConnRef1), + ok = ssh:close(ClientConnRef2), + ok = ssh:stop_daemon(DaemonRef) + end. + + +%%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- @@ -1210,3 +1896,18 @@ fake_daemon(_Config) -> {sockname,Server,ServerHost,ServerPort} -> {Server, ServerHost, ServerPort} end. +%% get_kex_init - helper function to get key_exchange_init_msg +get_kex_init(Conn) -> + %% First, validate the key exchange is complete (StateName == connected) + {connected,S} = sys:get_state(Conn), + %% Next, walk through the elements of the #state record looking + %% for the #ssh_msg_kexinit record. This method is robust against + %% changes to either record. The KEXINIT message contains a cookie + %% unique to each invocation of the key exchange procedure (RFC4253) + SL = tuple_to_list(S), + case lists:keyfind(ssh_msg_kexinit, 1, SL) of + false -> + throw(not_found); + KexInit -> + KexInit + end. diff --git a/lib/ssh/test/ssh_connection_SUITE.erl b/lib/ssh/test/ssh_connection_SUITE.erl index db51f65509..f0c337cf2f 100644 --- a/lib/ssh/test/ssh_connection_SUITE.erl +++ b/lib/ssh/test/ssh_connection_SUITE.erl @@ -65,6 +65,7 @@ ptty() -> %%-------------------------------------------------------------------- init_per_suite(Config) -> + catch crypto:stop(), case catch crypto:start() of ok -> Config; diff --git a/lib/ssh/test/ssh_relay.erl b/lib/ssh/test/ssh_relay.erl new file mode 100644 index 0000000000..a4f2bad2e2 --- /dev/null +++ b/lib/ssh/test/ssh_relay.erl @@ -0,0 +1,407 @@ +%%%------------------------------------------------------------------- +%%% @author Simon Cornish <[email protected]> +%%% @copyright (C) 2015, Simon Cornish +%%% @doc +%%% Provide manipulatable TCP-level relaying for testing SSH +%%% @end +%%% Created : 7 May 2015 by Simon Cornish <[email protected]> +%%%------------------------------------------------------------------- +-module(ssh_relay). + +-behaviour(gen_server). + +%% API +-export([start_link/4]). +-export([stop/1]). +-export([hold/4, release/2, release_next/3]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-record(hold, { + port, + n, + tmo, + tref, + q = [] + }). + +-record(state, { + local_addr, + local_port, + peer_addr, + peer_port, + lpid, + local, + peer, + tx_hold, + rx_hold + }). + +-define(ACCEPT_TMO, 200). +%%%=================================================================== +%%% API +%%%=================================================================== +%%-------------------------------------------------------------------- +%% @doc +%% Hold N (or 'all') messages in given direction. +%% Messages will be released after the N+1th message or +%% Tmo ms or 'infinity' +%% +%% Dir is 'tx' for direction local -> peer +%% and 'rx' for direction peer -> local +%% +%% An Error, ealready, is returned if there is already a hold +%% in the given direction +%% +%% @spec hold(Srv, Dir, N, Tmo) -> ok | {error, Error} +%% @end +%%-------------------------------------------------------------------- +hold(Srv, Dir, N, Tmo) -> + gen_server:call(Srv, {hold, Dir, N, Tmo}). + +%%-------------------------------------------------------------------- +%% @doc +%% Release all held messages in given direction. +%% +%% An Error, enoent, is returned if there is no hold +%% in the given direction +%% +%% @spec release(Srv, Dir) -> ok | {error, Error} +%% @end +%%-------------------------------------------------------------------- +release(Srv, Dir) -> + gen_server:call(Srv, {release, Dir}). + +%%-------------------------------------------------------------------- +%% @doc +%% Release all held messages in given direction after the +%% next message in the trigger direction +%% +%% An Error, enoent, is returned if there is no hold +%% in the given direction +%% +%% @spec release_next(Srv, Dir, TriggerDir) -> ok | {error, Error} +%% @end +%%-------------------------------------------------------------------- +release_next(Srv, Dir, TriggerDir) -> + gen_server:call(Srv, {release_next, Dir, TriggerDir}). + +%%-------------------------------------------------------------------- +%% @doc +%% Starts the server +%% +%% @spec start_link() -> {ok, Pid} | ignore | {error, Error} +%% @end +%%-------------------------------------------------------------------- +start_link(ListenAddr, ListenPort, PeerAddr, PeerPort) -> + gen_server:start_link(?MODULE, [ListenAddr, ListenPort, PeerAddr, PeerPort], []). + +stop(Srv) -> + unlink(Srv), + Srv ! stop. + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Initializes the server +%% +%% @spec init(Args) -> {ok, State} | +%% {ok, State, Timeout} | +%% ignore | +%% {stop, Reason} +%% @end +%%-------------------------------------------------------------------- +init([ListenAddr, ListenPort, PeerAddr, PeerPort | Options]) -> + IfAddr = case ListenAddr of + {0,0,0,0} -> + []; + _ -> + [{ifaddr, ListenAddr}] + end, + case gen_tcp:listen(ListenPort, [{reuseaddr, true}, {backlog, 1}, {active, false}, binary | IfAddr]) of + {ok, LSock} -> + Parent = self(), + {LPid, _LMod} = spawn_monitor(fun() -> listen(Parent, LSock) end), + S = #state{local_addr = ListenAddr, + local_port = ListenPort, + lpid = LPid, + peer_addr = PeerAddr, + peer_port = PeerPort + }, + {ok, S}; + Error -> + {stop, Error} + end. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling call messages +%% +%% @spec handle_call(Request, From, State) -> +%% {reply, Reply, State} | +%% {reply, Reply, State, Timeout} | +%% {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, Reply, State} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_call({hold, Dir, N, Tmo}, _From, State) -> + case Dir of + tx -> + do_hold(#state.tx_hold, State#state.peer, N, Tmo, State); + rx -> + do_hold(#state.rx_hold, State#state.local, N, Tmo, State); + _ -> + {reply, {error, einval}, State} + end; +handle_call({release, Dir}, _From, State) -> + case Dir of + tx -> + do_release(#state.tx_hold, State); + rx -> + do_release(#state.rx_hold, State); + _ -> + {reply, {error, einval}, State} + end; +handle_call({release_next, _Dir, _TriggerDir}, _From, State) -> + {reply, {error, nyi}, State}; + +handle_call(Request, _From, State) -> + Reply = {unhandled, Request}, + {reply, Reply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling cast messages +%% +%% @spec handle_cast(Msg, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_cast(_Msg, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling all non call/cast messages +%% +%% @spec handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_info({tcp, Local, Data}, S) when S#state.local == Local -> + S1 = do_local(Data, S), + {noreply, S1}; + +handle_info({tcp_error, Local, Error}, S) when S#state.local == Local -> + S1 = do_local({error, Error}, S), + {noreply, S1}; + +handle_info({tcp_closed, Local}, S) when S#state.local == Local -> + S1 = do_local(closed, S), + {noreply, S1}; + +handle_info({tcp, Peer, Data}, S) when S#state.peer == Peer -> + S1 = do_peer(Data, S), + {noreply, S1}; + +handle_info({tcp_error, Peer, Error}, S) when S#state.peer == Peer -> + S1 = do_peer({error, Error}, S), + {noreply, S1}; + +handle_info({tcp_closed, Peer}, S) when S#state.peer == Peer -> + S1 = do_peer(closed, S), + {noreply, S1}; + +handle_info({accept, Local}, S) -> + S1 = do_accept(Local, S), + {noreply, S1}; + +handle_info({activate, Local}, State) -> + inet:setopts(Local, [{active, true}]), + {noreply, State}; + +handle_info({release, Pos}, S) -> + {reply, _, S1} = do_release(Pos,S), + {noreply, S1}; + +handle_info(stop, State) -> + {stop, normal, State}; + +handle_info({'DOWN', _Ref, _process, LPid, Reason}, S) when S#state.lpid == LPid -> + io:format("Acceptor has finished: ~p~n", [Reason]), + {noreply, S}; + +handle_info(_Info, State) -> + io:format("Unhandled info: ~p~n", [_Info]), + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_server terminates +%% with Reason. The return value is ignored. +%% +%% @spec terminate(Reason, State) -> void() +%% @end +%%-------------------------------------------------------------------- +terminate(_Reason, _State) -> + ok. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%% +%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} +%% @end +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== +do_hold(Pos, _Port, _N, _Tmo, S) when element(Pos, S) /= undefined -> + {reply, {error, ealready}, S}; +do_hold(Pos, Port, N, Tmo, S) -> + TRef = if is_integer(Tmo) andalso Tmo > 0 -> + erlang:send_after(Tmo, self(), {release, Pos}); + true -> + undefined + end, + Hold = #hold{port = Port, n = N, tmo = Tmo, tref = TRef}, + {reply, ok, setelement(Pos, S, Hold)}. + +do_release(HPos, S) when element(HPos, S) == undefined -> + {reply, {error, enoent}, S}; +do_release(HPos, S) -> + #hold{port = Port, tref = TRef, q = Q} = element(HPos, S), + lists:foreach(fun(M) -> gen_tcp:send(Port, M), erlang:yield() end, Q), + catch erlang:cancel_timer(TRef), + receive + {release, HPos} -> ok + after 0 -> + ok + end, + {reply, ok, setelement(HPos, S, undefined)}. + +listen(Parent, LSock) -> + monitor(process, Parent), + do_listen(Parent, LSock). + +do_listen(Parent, LSock) -> + %% So annoying there is no select-like sematic for this + case gen_tcp:accept(LSock, ?ACCEPT_TMO) of + {ok, Sock} -> + Parent ! {accept, Sock}, + gen_tcp:controlling_process(Sock, Parent), + Parent ! {activate, Sock}, + do_flush(Parent, Sock), + gen_tcp:close(LSock); + {error, timeout} -> + receive + DOWN when element(1, DOWN) == 'DOWN' -> + ok; + stop -> + ok + after 1 -> + do_listen(Parent, LSock) + end; + Error -> + gen_tcp:close(LSock), + exit({accept,Error}) + end. + +do_flush(Parent, Sock) -> + receive + {Tcp, Sock, _} = Msg when Tcp == tcp; Tcp == tcp_error -> + Parent ! Msg, + do_flush(Parent, Sock); + {tcp_closed, Sock} = Msg -> + Parent ! Msg, + do_flush(Parent, Sock) + after 1 -> + ok + end. + +do_accept(Local, S) -> + case gen_tcp:connect(S#state.peer_addr, S#state.peer_port, [{active, true}, binary]) of + {ok, Peer} -> + S#state{local = Local, peer = Peer}; + Error -> + exit({connect, Error}) + end. + +do_local(Data, S) when is_binary(Data) -> + TxH = S#state.tx_hold, + if TxH == undefined -> + gen_tcp:send(S#state.peer, Data), + S; + TxH#hold.n == 0 -> + lists:foreach(fun(M) -> gen_tcp:send(S#state.peer, M) end, TxH#hold.q), + gen_tcp:send(S#state.peer, Data), + catch erlang:cancel_timer(TxH#hold.tref), + TxP = #state.tx_hold, + receive + {release, TxP} -> + ok + after 0 -> + ok + end, + S#state{tx_hold = undefined}; + true -> + Q = TxH#hold.q ++ [Data], + N = if is_integer(TxH#hold.n) -> + TxH#hold.n -1; + true -> + TxH#hold.n + end, + S#state{tx_hold = TxH#hold{q = Q, n = N}} + end; +do_local(Error, _S) -> + exit({local, Error}). + +do_peer(Data, S) when is_binary(Data) -> + RxH = S#state.rx_hold, + if RxH == undefined -> + gen_tcp:send(S#state.local, Data), + S; + RxH#hold.n == 0 -> + lists:foreach(fun(M) -> gen_tcp:send(S#state.local, M) end, RxH#hold.q), + gen_tcp:send(S#state.local, Data), + catch erlang:cancel_timer(RxH#hold.tref), + RxP = #state.rx_hold, + receive + {release, RxP} -> + ok + after 0 -> + ok + end, + S#state{rx_hold = undefined}; + true -> + Q = RxH#hold.q ++ [Data], + N = if is_integer(RxH#hold.n) -> + RxH#hold.n -1; + true -> + RxH#hold.n + end, + S#state{rx_hold = RxH#hold{q = Q, n = N}} + end; +do_peer(Error, _S) -> + exit({peer, Error}). + diff --git a/lib/ssh/test/ssh_sftp_SUITE.erl b/lib/ssh/test/ssh_sftp_SUITE.erl index cb74a27638..850b1cbf6b 100644 --- a/lib/ssh/test/ssh_sftp_SUITE.erl +++ b/lib/ssh/test/ssh_sftp_SUITE.erl @@ -49,6 +49,7 @@ all() -> init_per_suite(Config) -> + catch crypto:stop(), case (catch crypto:start()) of ok -> ssh:start(), diff --git a/lib/ssh/test/ssh_sftpd_SUITE.erl b/lib/ssh/test/ssh_sftpd_SUITE.erl index 0ce8eec906..925b02a437 100644 --- a/lib/ssh/test/ssh_sftpd_SUITE.erl +++ b/lib/ssh/test/ssh_sftpd_SUITE.erl @@ -68,6 +68,7 @@ groups() -> %%-------------------------------------------------------------------- init_per_suite(Config) -> + catch crypto:stop(), case (catch crypto:start()) of ok -> DataDir = ?config(data_dir, Config), diff --git a/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl b/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl index cc34cc0793..eac7575486 100644 --- a/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl +++ b/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl @@ -52,6 +52,7 @@ groups() -> init_per_suite(Config) -> catch ssh:stop(), + catch crypto:stop(), case catch crypto:start() of ok -> DataDir = ?config(data_dir, Config), diff --git a/lib/ssh/test/ssh_sup_SUITE.erl b/lib/ssh/test/ssh_sup_SUITE.erl new file mode 100644 index 0000000000..6e1595f9fa --- /dev/null +++ b/lib/ssh/test/ssh_sup_SUITE.erl @@ -0,0 +1,192 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015-2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% + +-module(ssh_sup_SUITE). +-include_lib("common_test/include/ct.hrl"). +-include_lib("ssh/src/ssh.hrl"). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-define(WAIT_FOR_SHUTDOWN, 500). +-define(USER, "Alladin"). +-define(PASSWD, "Sesame"). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- + +all() -> + [default_tree, sshc_subtree, sshd_subtree, sshd_subtree_profile]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + +init_per_suite(Config) -> + Port = ssh_test_lib:inet_port(node()), + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + [{userdir, UserDir},{port, Port}, {host, "localhost"}, {host_ip, any} | Config]. + +end_per_suite(_) -> + ok. + +init_per_testcase(sshc_subtree, Config) -> + ssh:start(), + SystemDir = ?config(data_dir, Config), + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {failfun, fun ssh_test_lib:failfun/2}, + {user_passwords, + [{?USER, ?PASSWD}]}]), + [{server, {Pid, Host, Port}} | Config]; +init_per_testcase(Case, Config) -> + end_per_testcase(Case, Config), + ssh:start(), + Config. +end_per_testcase(sshc_subtree, Config) -> + {Pid,_,_} = ?config(server, Config), + ssh:stop_daemon(Pid), + ssh:stop(); +end_per_testcase(_, _Config) -> + ssh:stop(). + +%%------------------------------------------------------------------------- +%% Test cases +%%------------------------------------------------------------------------- +default_tree() -> + [{doc, "Makes sure the correct processes are started and linked," + "in the default case."}]. +default_tree(Config) when is_list(Config) -> + TopSupChildren = supervisor:which_children(ssh_sup), + 2 = length(TopSupChildren), + {value, {sshc_sup, _, supervisor,[sshc_sup]}} = + lists:keysearch(sshc_sup, 1, TopSupChildren), + {value, {sshd_sup, _,supervisor,[sshd_sup]}} = + lists:keysearch(sshd_sup, 1, TopSupChildren), + [] = supervisor:which_children(sshc_sup), + [] = supervisor:which_children(sshd_sup). + +sshc_subtree() -> + [{doc, "Make sure the sshc subtree is correct"}]. +sshc_subtree(Config) when is_list(Config) -> + {_Pid, Host, Port} = ?config(server, Config), + UserDir = ?config(userdir, Config), + + [] = supervisor:which_children(sshc_sup), + {ok, Pid1} = ssh:connect(Host, Port, [{silently_accept_hosts, true}, + {user_interaction, false}, + {user, ?USER}, {password, ?PASSWD},{user_dir, UserDir}]), + [{_, _,supervisor,[ssh_connection_handler]}] = + supervisor:which_children(sshc_sup), + {ok, Pid2} = ssh:connect(Host, Port, [{silently_accept_hosts, true}, + {user_interaction, false}, + {user, ?USER}, {password, ?PASSWD}, {user_dir, UserDir}]), + [{_,_,supervisor,[ssh_connection_handler]}, + {_,_,supervisor,[ssh_connection_handler]}] = + supervisor:which_children(sshc_sup), + ssh:close(Pid1), + [{_,_,supervisor,[ssh_connection_handler]}] = + supervisor:which_children(sshc_sup), + ssh:close(Pid2), + ct:sleep(?WAIT_FOR_SHUTDOWN), + [] = supervisor:which_children(sshc_sup). + +sshd_subtree() -> + [{doc, "Make sure the sshd subtree is correct"}]. +sshd_subtree(Config) when is_list(Config) -> + HostIP = ?config(host_ip, Config), + Port = ?config(port, Config), + SystemDir = ?config(data_dir, Config), + ssh:daemon(HostIP, Port, [{system_dir, SystemDir}, + {failfun, fun ssh_test_lib:failfun/2}, + {user_passwords, + [{?USER, ?PASSWD}]}]), + [{{server,ssh_system_sup, HostIP, Port, ?DEFAULT_PROFILE}, + Daemon, supervisor, + [ssh_system_sup]}] = + supervisor:which_children(sshd_sup), + check_sshd_system_tree(Daemon, Config), + ssh:stop_daemon(HostIP, Port), + ct:sleep(?WAIT_FOR_SHUTDOWN), + [] = supervisor:which_children(sshd_sup). + +sshd_subtree_profile() -> + [{doc, "Make sure the sshd subtree using profile option is correct"}]. +sshd_subtree_profile(Config) when is_list(Config) -> + HostIP = ?config(host_ip, Config), + Port = ?config(port, Config), + Profile = ?config(profile, Config), + SystemDir = ?config(data_dir, Config), + + {ok, _} = ssh:daemon(HostIP, Port, [{system_dir, SystemDir}, + {failfun, fun ssh_test_lib:failfun/2}, + {user_passwords, + [{?USER, ?PASSWD}]}, + {profile, Profile}]), + [{{server,ssh_system_sup, HostIP,Port,Profile}, + Daemon, supervisor, + [ssh_system_sup]}] = + supervisor:which_children(sshd_sup), + check_sshd_system_tree(Daemon, Config), + ssh:stop_daemon(HostIP, Port, Profile), + ct:sleep(?WAIT_FOR_SHUTDOWN), + [] = supervisor:which_children(sshd_sup). + + +check_sshd_system_tree(Daemon, Config) -> + Host = ?config(host, Config), + Port = ?config(port, Config), + UserDir = ?config(userdir, Config), + {ok, Client} = ssh:connect(Host, Port, [{silently_accept_hosts, true}, + {user_interaction, false}, + {user, ?USER}, {password, ?PASSWD},{user_dir, UserDir}]), + + [{_,SubSysSup, supervisor,[ssh_subsystem_sup]}, + {{ssh_acceptor_sup,_,_,_}, AccSup, supervisor,[ssh_acceptor_sup]}] + = supervisor:which_children(Daemon), + + [{{server,ssh_connection_sup, _,_}, + ConnectionSup, supervisor, + [ssh_connection_sup]}, + {{server,ssh_channel_sup,_ ,_}, + ChannelSup,supervisor, + [ssh_channel_sup]}] = supervisor:which_children(SubSysSup), + + [{{ssh_acceptor_sup,_,_,_},_,worker,[ssh_acceptor]}] = + supervisor:which_children(AccSup), + + [{_, _, worker,[ssh_connection_handler]}] = + supervisor:which_children(ConnectionSup), + + [] = supervisor:which_children(ChannelSup), + + ssh_sftp:start_channel(Client), + + [{_, _,worker,[ssh_channel]}] = + supervisor:which_children(ChannelSup), + ssh:close(Client). + diff --git a/lib/ssh/test/ssh_sup_SUITE_data/id_dsa b/lib/ssh/test/ssh_sup_SUITE_data/id_dsa new file mode 100644 index 0000000000..d306f8b26e --- /dev/null +++ b/lib/ssh/test/ssh_sup_SUITE_data/id_dsa @@ -0,0 +1,13 @@ +-----BEGIN DSA PRIVATE KEY----- +MIIBvAIBAAKBgQDfi2flSTZZofwT4yQT0NikX/LGNT7UPeB/XEWe/xovEYCElfaQ +APFixXvEgXwoojmZ5kiQRKzLM39wBP0jPERLbnZXfOOD0PDnw0haMh7dD7XKVMod +/EigVgHf/qBdM2M8yz1s/rRF7n1UpLSypziKjkzCm7JoSQ2zbWIPdmBIXwIVAMgP +kpr7Sq3O7sHdb8D601DRjoExAoGAMOQxDfB2Fd8ouz6G96f/UOzRMI/Kdv8kYYKW +JIGY+pRYrLPyYzUeJznwZreOJgrczAX+luHnKFWJ2Dnk5CyeXk67Wsr7pJ/4MBMD +OKeIS0S8qoSBN8+Krp79fgA+yS3IfqbkJLtLu4EBaCX4mKQIX4++k44d4U5lc8pt ++9hlEI8CgYEAznKxx9kyC6bVo7LUYKaGhofRFt0SYFc5PVmT2VUGRs1R6+6DPD+e +uEO6IhFct7JFSRbP9p0JD4Uk+3zlZF+XX6b2PsZkeV8f/02xlNGUSmEzCSiNg1AX +Cy/WusYhul0MncWCHMcOZB5rIvU/aP5EJJtn3xrRaz6u0SThF6AnT34CFQC63czE +ZU8w8Q+H7z0j+a+70x2iAw== +-----END DSA PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_sup_SUITE_data/id_rsa b/lib/ssh/test/ssh_sup_SUITE_data/id_rsa new file mode 100644 index 0000000000..9d7e0dd5fb --- /dev/null +++ b/lib/ssh/test/ssh_sup_SUITE_data/id_rsa @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQD1OET+3O/Bvj/dtjxDTXmj1oiJt4sIph5kGy0RfjoPrZfaS+CU +DhakCmS6t2ivxWFgtpKWaoGMZMJqWj6F6ZsumyFl3FPBtujwY/35cgifrI9Ns4Tl +zR1uuengNBmV+WRQ5cd9F2qS6Z8aDQihzt0r8JUqLcK+VQbrmNzboCCQQwIDAQAB +AoGAPQEyqPTt8JUT7mRXuaacjFXiweAXhp9NEDpyi9eLOjtFe9lElZCrsUOkq47V +TGUeRKEm9qSodfTbKPoqc8YaBJGJPhUaTAcha+7QcDdfHBvIsgxvU7ePVnlpXRp3 +CCUEMPhlnx6xBoTYP+fRU0e3+xJIPVyVCqX1jAdUMkzfRoECQQD6ux7B1QJAIWyK +SGkbDUbBilNmzCFNgIpOP6PA+bwfi5d16diTpra5AX09keQABAo/KaP1PdV8Vg0p +z4P3A7G3AkEA+l+AKG6m0kQTTBMJDqOdVPYwe+5GxunMaqmhokpEbuGsrZBl5Dvd +WpcBjR7jmenrhKZRIuA+Fz5HPo/UQJPl1QJBAKxstDkeED8j/S2XoFhPKAJ+6t39 +sUVICVTIZQeXdmzHJXCcUSkw8+WEhakqw/3SyW0oaK2FSWQJFWJUZ+8eJj8CQEh3 +xeduB5kKnS9CvzdeghZqX6QvVosSdtlUmfUYW/BgH5PpHKTP8wTaeld3XldZTpMJ +dKiMkUw2+XYROVUrubUCQD+Na1LhULlpn4ISEtIEfqpdlUhxDgO15Wg8USmsng+x +ICliVOSQtwaZjm8kwaFt0W7XnpnDxbRs37vIEbIMWak= +-----END RSA PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_sup_SUITE_data/ssh_host_dsa_key b/lib/ssh/test/ssh_sup_SUITE_data/ssh_host_dsa_key new file mode 100644 index 0000000000..51ab6fbd88 --- /dev/null +++ b/lib/ssh/test/ssh_sup_SUITE_data/ssh_host_dsa_key @@ -0,0 +1,13 @@ +-----BEGIN DSA PRIVATE KEY----- +MIIBuwIBAAKBgQCClaHzE2ul0gKSUxah5W0W8UiJLy4hXngKEqpaUq9SSdVdY2LK +wVfKH1gt5iuaf1FfzOhsIC9G/GLnjYttXZc92cv/Gfe3gR+s0ni2++MX+T++mE/Q +diltXv/Hp27PybS67SmiFW7I+RWnT2OKlMPtw2oUuKeztCe5UWjaj/y5FQIVAPLA +l9RpiU30Z87NRAHY3NTRaqtrAoGANMRxw8UfdtNVR0CrQj3AgPaXOGE4d+G4Gp4X +skvnCHycSVAjtYxebUkzUzt5Q6f/IabuLUdge3gXrc8BetvrcKbp+XZgM0/Vj2CF +Ymmy3in6kzGZq7Fw1sZaku6AOU8vLa5woBT2vAcHLLT1bLAzj7viL048T6MfjrOP +ef8nHvACgYBhDWFQJ1mf99sg92LalVq1dHLmVXb3PTJDfCO/Gz5NFmj9EZbAtdah +/XcF3DeRF+eEoz48wQF/ExVxSMIhLdL+o+ElpVhlM7Yii+T7dPhkQfEul6zZXu+U +ykSTXYUbtsfTNRFQGBW2/GfnEc0mnIxfn9v10NEWMzlq5z9wT9P0CgIVAN4wtL5W +Lv62jKcdskxNyz2NQoBx +-----END DSA PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_sup_SUITE_data/ssh_host_dsa_key.pub b/lib/ssh/test/ssh_sup_SUITE_data/ssh_host_dsa_key.pub new file mode 100644 index 0000000000..4dbb1305b0 --- /dev/null +++ b/lib/ssh/test/ssh_sup_SUITE_data/ssh_host_dsa_key.pub @@ -0,0 +1,11 @@ +---- BEGIN SSH2 PUBLIC KEY ---- +AAAAB3NzaC1kc3MAAACBAIKVofMTa6XSApJTFqHlbRbxSIkvLiFeeAoSqlpSr1JJ1V1j +YsrBV8ofWC3mK5p/UV/M6GwgL0b8YueNi21dlz3Zy/8Z97eBH6zSeLb74xf5P76YT9B2 +KW1e/8enbs/JtLrtKaIVbsj5FadPY4qUw+3DahS4p7O0J7lRaNqP/LkVAAAAFQDywJfU +aYlN9GfOzUQB2NzU0WqrawAAAIA0xHHDxR9201VHQKtCPcCA9pc4YTh34bganheyS+cI +fJxJUCO1jF5tSTNTO3lDp/8hpu4tR2B7eBetzwF62+twpun5dmAzT9WPYIViabLeKfqT +MZmrsXDWxlqS7oA5Ty8trnCgFPa8BwcstPVssDOPu+IvTjxPox+Os495/yce8AAAAIBh +DWFQJ1mf99sg92LalVq1dHLmVXb3PTJDfCO/Gz5NFmj9EZbAtdah/XcF3DeRF+eEoz48 +wQF/ExVxSMIhLdL+o+ElpVhlM7Yii+T7dPhkQfEul6zZXu+UykSTXYUbtsfTNRFQGBW2 +/GfnEc0mnIxfn9v10NEWMzlq5z9wT9P0Cg== +---- END SSH2 PUBLIC KEY ---- diff --git a/lib/ssh/test/ssh_sup_SUITE_data/ssh_host_rsa_key b/lib/ssh/test/ssh_sup_SUITE_data/ssh_host_rsa_key new file mode 100644 index 0000000000..79968bdd7d --- /dev/null +++ b/lib/ssh/test/ssh_sup_SUITE_data/ssh_host_rsa_key @@ -0,0 +1,16 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDCZX+4FBDwZIh9y/Uxee1VJnEXlowpz2yDKwj8semM4q843337 +zbNfxHmladB1lpz2NqyxI175xMIJuDxogyZdsOxGnFAzAnthR4dqL/RWRWzjaxSB +6IAO9SPYVVlrpZ+1hsjLW79fwXK/yc8VdhRuWTeQiRgYY2ek8+OKbOqz4QIDAQAB +AoGANmvJzJO5hkLuvyDZHKfAnGTtpifcR1wtSa9DjdKUyn8vhKF0mIimnbnYQEmW +NUUb3gXCZLi9PvkpRSVRrASDOZwcjoU/Kvww163vBUVb2cOZfFhyn6o2Sk88Tt++ +udH3hdjpf9i7jTtUkUe+QYPsia+wgvvrmn4QrahLAH86+kECQQDx5gFeXTME3cnW +WMpFz3PPumduzjqgqMMWEccX4FtQkMX/gyGa5UC7OHFyh0N/gSWvPbRHa8A6YgIt +n8DO+fh5AkEAzbqX4DOn8NY6xJIi42q7l/2jIA0RkB6P7YugW5NblhqBZ0XDnpA5 +sMt+rz+K07u9XZtxgh1xi7mNfwY6lEAMqQJBAJBEauCKmRj35Z6OyeQku59SPsnY ++SJEREVvSNw2lH9SOKQQ4wPsYlTGbvKtNVZgAcen91L5MmYfeckYE/fdIZECQQCt +64zxsTnM1I8iFxj/gP/OYlJBikrKt8udWmjaghzvLMEw+T2DExJyb9ZNeT53+UMB +m6O+B/4xzU/djvp+0hbhAkAemIt+rA5kTmYlFndhpvzkSSM8a2EXsO4XIPgGWCTT +tQKS/tTly0ADMjN/TVy11+9d6zcqadNVuHXHGtR4W0GR +-----END RSA PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_sup_SUITE_data/ssh_host_rsa_key.pub b/lib/ssh/test/ssh_sup_SUITE_data/ssh_host_rsa_key.pub new file mode 100644 index 0000000000..75d2025c71 --- /dev/null +++ b/lib/ssh/test/ssh_sup_SUITE_data/ssh_host_rsa_key.pub @@ -0,0 +1,5 @@ +---- BEGIN SSH2 PUBLIC KEY ---- +AAAAB3NzaC1yc2EAAAADAQABAAAAgQDCZX+4FBDwZIh9y/Uxee1VJnEXlowpz2yDKwj8 +semM4q843337zbNfxHmladB1lpz2NqyxI175xMIJuDxogyZdsOxGnFAzAnthR4dqL/RW +RWzjaxSB6IAO9SPYVVlrpZ+1hsjLW79fwXK/yc8VdhRuWTeQiRgYY2ek8+OKbOqz4Q== +---- END SSH2 PUBLIC KEY ---- diff --git a/lib/ssh/test/ssh_test_lib.erl b/lib/ssh/test/ssh_test_lib.erl index 8ca05746db..d08afdfb90 100644 --- a/lib/ssh/test/ssh_test_lib.erl +++ b/lib/ssh/test/ssh_test_lib.erl @@ -361,7 +361,7 @@ do_inet_port(Node) -> openssh_sanity_check(Config) -> ssh:start(), - case ssh:connect("localhost", 22, []) of + case ssh:connect("localhost", 22, [{password,""}]) of {ok, Pid} -> ssh:close(Pid), ssh:stop(), diff --git a/lib/ssh/test/ssh_to_openssh_SUITE.erl b/lib/ssh/test/ssh_to_openssh_SUITE.erl index a61fd2dd41..277e3a1b08 100644 --- a/lib/ssh/test/ssh_to_openssh_SUITE.erl +++ b/lib/ssh/test/ssh_to_openssh_SUITE.erl @@ -60,6 +60,7 @@ groups() -> ]. init_per_suite(Config) -> + catch crypto:stop(), case catch crypto:start() of ok -> case gen_tcp:connect("localhost", 22, []) of @@ -166,9 +167,11 @@ erlang_client_openssh_server_exec_compressed() -> [{doc, "Test that compression option works"}]. erlang_client_openssh_server_exec_compressed(Config) when is_list(Config) -> + CompressAlgs = [zlib, '[email protected]',none], ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, {user_interaction, false}, - {compression, zlib}]), + {preferred_algorithms, + [{compression,CompressAlgs}]}]), {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), success = ssh_connection:exec(ConnectionRef, ChannelId, "echo testing", infinity), @@ -326,8 +329,11 @@ erlang_server_openssh_client_exec_compressed(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), KnownHosts = filename:join(PrivDir, "known_hosts"), +%% CompressAlgs = [zlib, '[email protected]'], % Does not work + CompressAlgs = [zlib], {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, - {compression, zlib}, + {preferred_algorithms, + [{compression, CompressAlgs}]}, {failfun, fun ssh_test_lib:failfun/2}]), ct:sleep(500), diff --git a/lib/ssh/test/ssh_unicode_SUITE.erl b/lib/ssh/test/ssh_unicode_SUITE.erl index cc916673b3..07d51335c6 100644 --- a/lib/ssh/test/ssh_unicode_SUITE.erl +++ b/lib/ssh/test/ssh_unicode_SUITE.erl @@ -55,6 +55,7 @@ all() -> init_per_suite(Config) -> + catch crypto:stop(), case {file:native_name_encoding(), (catch crypto:start())} of {utf8, ok} -> ssh:start(), diff --git a/lib/ssh/test/ssh_upgrade_SUITE.erl b/lib/ssh/test/ssh_upgrade_SUITE.erl new file mode 100644 index 0000000000..861c7ab3dd --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE.erl @@ -0,0 +1,206 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2014-2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/.2 +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(ssh_upgrade_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-record(state, { + config, + root_dir, + server, + client, + connection, + soft + }). + + +%%%================================================================ +%%% +%%% CommonTest callbacks +%%% +all() -> + [ + minor_upgrade, + major_upgrade + ]. + +init_per_suite(Config0) -> + catch crypto:stop(), + try {crypto:start(), erlang:system_info({wordsize, internal}) == + erlang:system_info({wordsize, external})} of + {ok, true} -> + case ct_release_test:init(Config0) of + {skip, Reason} -> + {skip, Reason}; + Config -> + ssh:start(), + Config + end; + {ok, false} -> + {skip, "Test server will not handle halfwordemulator correctly. Skip as halfwordemulator is deprecated"} + catch _:_ -> + {skip, "Crypto did not start"} + end. + +end_per_suite(Config) -> + ct_release_test:cleanup(Config), + ssh:stop(), + crypto:stop(), + UserDir = ?config(priv_dir, Config), + ssh_test_lib:clean_rsa(UserDir). + +init_per_testcase(_TestCase, Config) -> + Config. +end_per_testcase(_TestCase, Config) -> + Config. + +%%%================================================================ +%%% +%%% Test cases +%%% +major_upgrade(Config) when is_list(Config) -> + ct_release_test:upgrade(ssh, major,{?MODULE, #state{config = Config}}, Config). + +minor_upgrade(Config) when is_list(Config) -> + ct_release_test:upgrade(ssh, minor,{?MODULE, #state{config = Config}}, Config). + +%%%================================================================ +%%% +%%% ct_release_test callbacks +%%% + +%%%---------------------------------------------------------------- +%%% Initialyze system before upgrade test starts. +%%% Called by ct_release_test:upgrade/4 +upgrade_init(CTData, State) -> + {ok, AppUp={_, _, Up, _Down}} = ct_release_test:get_appup(CTData, ssh), + ct:pal("AppUp: ~p", [AppUp]), + ct:pal("Up: ~p", [Up]), + case Soft = is_soft(Up) of + %% It is symmetrical, if upgrade is soft so is downgrade + true -> + setup_server_client(State#state{soft = Soft}); + false -> + State#state{soft = Soft} + end. + +%%%---------------------------------------------------------------- +%%% Check that upgrade was successful +%%% Called by ct_release_test:upgrade/4 +upgrade_upgraded(_, #state{soft=false} = State) -> + test_hard(State, "upgrade"); + +upgrade_upgraded(_, State) -> + test_soft(State, "upgrade1"). + +%%%---------------------------------------------------------------- +%%% Check that downgrade was successful. +%%% Called by ct_release_test:upgrade/4 +upgrade_downgraded(_, #state{soft=false} = State) -> + test_hard(State, "downgrade"); + +upgrade_downgraded(_, #state{soft=true} = State) -> + test_soft(State, "downgrade1"). + +%%%================================================================ +%%% +%%% Private functions +%%% + +is_soft([{restart_application, ssh}]) -> + false; +is_soft(_) -> + true. + + +test_hard(State0, FileName) -> + ct:pal("test_hard State0=~p, FileName=~p",[State0, FileName]), + State = setup_server_client(State0), + test_connection(FileName, random_contents(), State). + +test_soft(State0, FileName) -> + ct:pal("test_soft State0=~p, FileName=~p",[State0, FileName]), + State = test_connection(FileName, random_contents(), State0), + setup_server_client( close(State) ). + + +setup_server_client(#state{config=Config} = State) -> + DataDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + + FtpRootDir = filename:join(PrivDir, "ftp_root"), + catch file:make_dir(FtpRootDir), + + SFTP = ssh_sftpd:subsystem_spec([{root,FtpRootDir},{cwd,FtpRootDir}]), + + {Server,Host,Port} = ssh_test_lib:daemon([{system_dir,DataDir}, + {user_passwords,[{"hej","hopp"}]}, + {subsystems,[SFTP]}]), + + {ok, ChannelPid, Connection} = + ssh_sftp:start_channel(Host, Port, [{user_interaction,false}, + {silently_accept_hosts,true}, + {user_dir,DataDir}, + {user,"hej"}, + {password,"hopp"}]), + State#state{server = Server, + client = ChannelPid, + connection = Connection}. + + +test_connection(FileName, FileContents, + #state{client = ChannelPid, + root_dir = FtpRootDir} = State) -> + ct:pal("test_connection Writing with ssh_sftp:write_file",[]), + case ssh_sftp:write_file(ChannelPid, FileName, FileContents) of + ok -> + case ssh_sftp:read_file(ChannelPid, FileName) of + {ok,FileContents} -> + State; + {ok,Unexpected} -> + ct:fail("Expected ~p but got ~p from sftp:read_file(~p,..) in RootDir ~p", + [FileContents,Unexpected,FileName,FtpRootDir] + ); + Other -> + ct:fail("ssh_sftp:read_file(~p,~p) -> ~p~n" + "ssh_sftp:list_dir(~p,\".\") -> ~p", + [ChannelPid,FileName,Other, + ChannelPid, catch ssh_sftp:list_dir(ChannelPid, ".")]) + end; + + Other -> + ct:fail("ssh_sftp:write_file(~p,~p,~p) -> ~p",[ChannelPid,FileName,FileContents,Other]) + end. + + +close(#state{server = Server, + connection = Connection} = State) -> + ssh:close(Connection), + ssh:stop_daemon(Server), + State#state{server = undefined, + client = undefined, + connection = undefined}. + + +random_contents() -> list_to_binary( random_chars(3) ). + +random_chars(N) -> [crypto:rand_uniform($a,$z) || _<-lists:duplicate(N,x)]. diff --git a/lib/ssh/test/ssh_upgrade_SUITE_data/id_dsa b/lib/ssh/test/ssh_upgrade_SUITE_data/id_dsa new file mode 100644 index 0000000000..d306f8b26e --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE_data/id_dsa @@ -0,0 +1,13 @@ +-----BEGIN DSA PRIVATE KEY----- +MIIBvAIBAAKBgQDfi2flSTZZofwT4yQT0NikX/LGNT7UPeB/XEWe/xovEYCElfaQ +APFixXvEgXwoojmZ5kiQRKzLM39wBP0jPERLbnZXfOOD0PDnw0haMh7dD7XKVMod +/EigVgHf/qBdM2M8yz1s/rRF7n1UpLSypziKjkzCm7JoSQ2zbWIPdmBIXwIVAMgP +kpr7Sq3O7sHdb8D601DRjoExAoGAMOQxDfB2Fd8ouz6G96f/UOzRMI/Kdv8kYYKW +JIGY+pRYrLPyYzUeJznwZreOJgrczAX+luHnKFWJ2Dnk5CyeXk67Wsr7pJ/4MBMD +OKeIS0S8qoSBN8+Krp79fgA+yS3IfqbkJLtLu4EBaCX4mKQIX4++k44d4U5lc8pt ++9hlEI8CgYEAznKxx9kyC6bVo7LUYKaGhofRFt0SYFc5PVmT2VUGRs1R6+6DPD+e +uEO6IhFct7JFSRbP9p0JD4Uk+3zlZF+XX6b2PsZkeV8f/02xlNGUSmEzCSiNg1AX +Cy/WusYhul0MncWCHMcOZB5rIvU/aP5EJJtn3xrRaz6u0SThF6AnT34CFQC63czE +ZU8w8Q+H7z0j+a+70x2iAw== +-----END DSA PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_upgrade_SUITE_data/id_rsa b/lib/ssh/test/ssh_upgrade_SUITE_data/id_rsa new file mode 100644 index 0000000000..9d7e0dd5fb --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE_data/id_rsa @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQD1OET+3O/Bvj/dtjxDTXmj1oiJt4sIph5kGy0RfjoPrZfaS+CU +DhakCmS6t2ivxWFgtpKWaoGMZMJqWj6F6ZsumyFl3FPBtujwY/35cgifrI9Ns4Tl +zR1uuengNBmV+WRQ5cd9F2qS6Z8aDQihzt0r8JUqLcK+VQbrmNzboCCQQwIDAQAB +AoGAPQEyqPTt8JUT7mRXuaacjFXiweAXhp9NEDpyi9eLOjtFe9lElZCrsUOkq47V +TGUeRKEm9qSodfTbKPoqc8YaBJGJPhUaTAcha+7QcDdfHBvIsgxvU7ePVnlpXRp3 +CCUEMPhlnx6xBoTYP+fRU0e3+xJIPVyVCqX1jAdUMkzfRoECQQD6ux7B1QJAIWyK +SGkbDUbBilNmzCFNgIpOP6PA+bwfi5d16diTpra5AX09keQABAo/KaP1PdV8Vg0p +z4P3A7G3AkEA+l+AKG6m0kQTTBMJDqOdVPYwe+5GxunMaqmhokpEbuGsrZBl5Dvd +WpcBjR7jmenrhKZRIuA+Fz5HPo/UQJPl1QJBAKxstDkeED8j/S2XoFhPKAJ+6t39 +sUVICVTIZQeXdmzHJXCcUSkw8+WEhakqw/3SyW0oaK2FSWQJFWJUZ+8eJj8CQEh3 +xeduB5kKnS9CvzdeghZqX6QvVosSdtlUmfUYW/BgH5PpHKTP8wTaeld3XldZTpMJ +dKiMkUw2+XYROVUrubUCQD+Na1LhULlpn4ISEtIEfqpdlUhxDgO15Wg8USmsng+x +ICliVOSQtwaZjm8kwaFt0W7XnpnDxbRs37vIEbIMWak= +-----END RSA PRIVATE KEY----- diff --git a/lib/ssh/test/ssh_upgrade_SUITE_data/known_hosts b/lib/ssh/test/ssh_upgrade_SUITE_data/known_hosts new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE_data/known_hosts @@ -0,0 +1 @@ + diff --git a/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_dsa_key b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_dsa_key new file mode 100644 index 0000000000..51ab6fbd88 --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_dsa_key @@ -0,0 +1,13 @@ +-----BEGIN DSA PRIVATE KEY----- +MIIBuwIBAAKBgQCClaHzE2ul0gKSUxah5W0W8UiJLy4hXngKEqpaUq9SSdVdY2LK +wVfKH1gt5iuaf1FfzOhsIC9G/GLnjYttXZc92cv/Gfe3gR+s0ni2++MX+T++mE/Q +diltXv/Hp27PybS67SmiFW7I+RWnT2OKlMPtw2oUuKeztCe5UWjaj/y5FQIVAPLA +l9RpiU30Z87NRAHY3NTRaqtrAoGANMRxw8UfdtNVR0CrQj3AgPaXOGE4d+G4Gp4X +skvnCHycSVAjtYxebUkzUzt5Q6f/IabuLUdge3gXrc8BetvrcKbp+XZgM0/Vj2CF +Ymmy3in6kzGZq7Fw1sZaku6AOU8vLa5woBT2vAcHLLT1bLAzj7viL048T6MfjrOP +ef8nHvACgYBhDWFQJ1mf99sg92LalVq1dHLmVXb3PTJDfCO/Gz5NFmj9EZbAtdah +/XcF3DeRF+eEoz48wQF/ExVxSMIhLdL+o+ElpVhlM7Yii+T7dPhkQfEul6zZXu+U +ykSTXYUbtsfTNRFQGBW2/GfnEc0mnIxfn9v10NEWMzlq5z9wT9P0CgIVAN4wtL5W +Lv62jKcdskxNyz2NQoBx +-----END DSA PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_dsa_key.pub b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_dsa_key.pub new file mode 100644 index 0000000000..4dbb1305b0 --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_dsa_key.pub @@ -0,0 +1,11 @@ +---- BEGIN SSH2 PUBLIC KEY ---- +AAAAB3NzaC1kc3MAAACBAIKVofMTa6XSApJTFqHlbRbxSIkvLiFeeAoSqlpSr1JJ1V1j +YsrBV8ofWC3mK5p/UV/M6GwgL0b8YueNi21dlz3Zy/8Z97eBH6zSeLb74xf5P76YT9B2 +KW1e/8enbs/JtLrtKaIVbsj5FadPY4qUw+3DahS4p7O0J7lRaNqP/LkVAAAAFQDywJfU +aYlN9GfOzUQB2NzU0WqrawAAAIA0xHHDxR9201VHQKtCPcCA9pc4YTh34bganheyS+cI +fJxJUCO1jF5tSTNTO3lDp/8hpu4tR2B7eBetzwF62+twpun5dmAzT9WPYIViabLeKfqT +MZmrsXDWxlqS7oA5Ty8trnCgFPa8BwcstPVssDOPu+IvTjxPox+Os495/yce8AAAAIBh +DWFQJ1mf99sg92LalVq1dHLmVXb3PTJDfCO/Gz5NFmj9EZbAtdah/XcF3DeRF+eEoz48 +wQF/ExVxSMIhLdL+o+ElpVhlM7Yii+T7dPhkQfEul6zZXu+UykSTXYUbtsfTNRFQGBW2 +/GfnEc0mnIxfn9v10NEWMzlq5z9wT9P0Cg== +---- END SSH2 PUBLIC KEY ---- diff --git a/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_rsa_key b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_rsa_key new file mode 100644 index 0000000000..79968bdd7d --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_rsa_key @@ -0,0 +1,16 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDCZX+4FBDwZIh9y/Uxee1VJnEXlowpz2yDKwj8semM4q843337 +zbNfxHmladB1lpz2NqyxI175xMIJuDxogyZdsOxGnFAzAnthR4dqL/RWRWzjaxSB +6IAO9SPYVVlrpZ+1hsjLW79fwXK/yc8VdhRuWTeQiRgYY2ek8+OKbOqz4QIDAQAB +AoGANmvJzJO5hkLuvyDZHKfAnGTtpifcR1wtSa9DjdKUyn8vhKF0mIimnbnYQEmW +NUUb3gXCZLi9PvkpRSVRrASDOZwcjoU/Kvww163vBUVb2cOZfFhyn6o2Sk88Tt++ +udH3hdjpf9i7jTtUkUe+QYPsia+wgvvrmn4QrahLAH86+kECQQDx5gFeXTME3cnW +WMpFz3PPumduzjqgqMMWEccX4FtQkMX/gyGa5UC7OHFyh0N/gSWvPbRHa8A6YgIt +n8DO+fh5AkEAzbqX4DOn8NY6xJIi42q7l/2jIA0RkB6P7YugW5NblhqBZ0XDnpA5 +sMt+rz+K07u9XZtxgh1xi7mNfwY6lEAMqQJBAJBEauCKmRj35Z6OyeQku59SPsnY ++SJEREVvSNw2lH9SOKQQ4wPsYlTGbvKtNVZgAcen91L5MmYfeckYE/fdIZECQQCt +64zxsTnM1I8iFxj/gP/OYlJBikrKt8udWmjaghzvLMEw+T2DExJyb9ZNeT53+UMB +m6O+B/4xzU/djvp+0hbhAkAemIt+rA5kTmYlFndhpvzkSSM8a2EXsO4XIPgGWCTT +tQKS/tTly0ADMjN/TVy11+9d6zcqadNVuHXHGtR4W0GR +-----END RSA PRIVATE KEY----- + diff --git a/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_rsa_key.pub b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_rsa_key.pub new file mode 100644 index 0000000000..75d2025c71 --- /dev/null +++ b/lib/ssh/test/ssh_upgrade_SUITE_data/ssh_host_rsa_key.pub @@ -0,0 +1,5 @@ +---- BEGIN SSH2 PUBLIC KEY ---- +AAAAB3NzaC1yc2EAAAADAQABAAAAgQDCZX+4FBDwZIh9y/Uxee1VJnEXlowpz2yDKwj8 +semM4q843337zbNfxHmladB1lpz2NqyxI175xMIJuDxogyZdsOxGnFAzAnthR4dqL/RW +RWzjaxSB6IAO9SPYVVlrpZ+1hsjLW79fwXK/yc8VdhRuWTeQiRgYY2ek8+OKbOqz4Q== +---- END SSH2 PUBLIC KEY ---- diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index e0992d317c..fe0606b1a3 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -25,61 +25,16 @@ <file>notes.xml</file> </header> <p>This document describes the changes made to the SSL application.</p> - <section><title>SSL 7.0</title> + <section><title>SSL 6.0.1</title> <section><title>Fixed Bugs and Malfunctions</title> <list> <item> <p> - Ignore signature_algorithm (TLS 1.2 extension) sent to - TLS 1.0 or TLS 1.1 server</p> + Terminate gracefully when receving bad input to premaster + secret calculation</p> <p> - Own Id: OTP-12670</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Add new API functions to handle CRL-verification</p> - <p> - Own Id: OTP-10362 Aux Id: kunagi-215 [126] </p> - </item> - <item> - <p> - Remove default support for SSL-3.0, due to Poodle - vunrability in protocol specification.</p> - <p> - Add padding check for TLS-1.0 to remove Poodle - vunrability from TLS 1.0, also add the option - padding_check. This option only affects TLS-1.0 - connections and if set to false it disables the block - cipher padding check to be able to interoperate with - legacy software.</p> - <p> - Remove default support for RC4 cipher suites, as they are - consider too weak.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12390</p> - </item> - <item> - <p> - Add support for TLS ALPN (Application-Layer Protocol - Negotiation) extension.</p> - <p> - Own Id: OTP-12580</p> - </item> - <item> - <p> - Add SNI (Server Name Indication) support for the server - side.</p> - <p> - Own Id: OTP-12736</p> + Own Id: OTP-12783</p> </item> </list> </section> diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index 8a0bf69be4..9122066787 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -123,13 +123,13 @@ The callback <c>gen_tcp</c> is treated specially and calls <c>inet</c> directly.</p> <taglist> - <tag><c>CallbackModule</c></tag> - <item><p><c>= atom()</c></p></item> - <tag><c>DataTag</c></tag> - <item><p><c>= atom()</c></p> + <tag><c>CallbackModule =</c></tag> + <item><p><c>atom()</c></p></item> + <tag><c>DataTag =</c></tag> + <item><p><c>atom()</c></p> <p>Used in socket data message.</p></item> - <tag><c>ClosedTag</c></tag> - <item><p><c>= atom()</c></p> + <tag><c>ClosedTag =</c></tag> + <item><p><c>atom()</c></p> <p>Used in socket close message.</p></item> </taglist> </item> @@ -650,6 +650,27 @@ fun(srp, Username :: string(), UserState :: term()) -> The option <c>sni_fun</c>, and <c>sni_hosts</c> are mutually exclusive.</p></item> + <tag><c>{client_renegotiation, boolean()}</c></tag> + <item>In protocols that support client-initiated renegotiation, the cost + of resources of such an operation is higher for the server than the + client. This can act as a vector for denial of service attacks. The SSL + application already takes measures to counter-act such attempts, + but client-initiated renegotiation can be stricly disabled by setting + this option to <c>false</c>. The default value is <c>true</c>. + Note that disabling renegotiation can result in long-lived connections + becoming unusable due to limits on the number of messages the underlying + cipher suite can encipher. + </item> + + <tag><c>{psk_identity, string()}</c></tag> + <item>Specifies the server identity hint the server presents to the client. + </item> + <tag><c>{log_alert, boolean()}</c></tag> + <item>If false, error reports will not be displayed.</item> + <tag><c>{honor_cipher_order, boolean()}</c></tag> + <item>If true, use the server's preference for cipher selection. If false + (the default), use the client's preference. + </item> </taglist> </section> diff --git a/lib/ssl/doc/src/ssl_crl_cache_api.xml b/lib/ssl/doc/src/ssl_crl_cache_api.xml index 90aa895aff..9230442ae0 100644 --- a/lib/ssl/doc/src/ssl_crl_cache_api.xml +++ b/lib/ssl/doc/src/ssl_crl_cache_api.xml @@ -47,10 +47,10 @@ <taglist> - <tag><c>cache_ref()</c></tag> - <item> = opaque()</item> - <tag><c>dist_point()</c></tag> - <item><p> = #'DistributionPoint'{} see <seealso + <tag><c>cache_ref() =</c></tag> + <item>opaque()</item> + <tag><c>dist_point() =</c></tag> + <item><p>#'DistributionPoint'{} see <seealso marker="public_key:public_key_records"> X509 certificates records</seealso></p></item> </taglist> diff --git a/lib/ssl/doc/src/ssl_session_cache_api.xml b/lib/ssl/doc/src/ssl_session_cache_api.xml index c89d3874a1..28b5f4ce23 100644 --- a/lib/ssl/doc/src/ssl_session_cache_api.xml +++ b/lib/ssl/doc/src/ssl_session_cache_api.xml @@ -40,20 +40,20 @@ <c>ssl_session_cache_api</c>:</p> <taglist> - <tag><c>cache_ref()</c></tag> - <item><p>= <c>opaque()</c></p></item> + <tag><c>cache_ref() =</c></tag> + <item><p><c>opaque()</c></p></item> - <tag><c>key()</c></tag> - <item><p>= <c>{partialkey(), session_id()}</c></p></item> + <tag><c>key() =</c></tag> + <item><p><c>{partialkey(), session_id()}</c></p></item> - <tag><c>partialkey()</c></tag> - <item><p>= <c>opaque()</c></p></item> + <tag><c>partialkey() =</c></tag> + <item><p><c>opaque()</c></p></item> - <tag><c>session_id()</c></tag> - <item><p>= <c>binary()</c></p></item> + <tag><c>session_id() =</c></tag> + <item><p><c>binary()</c></p></item> - <tag><c>session()</c></tag> - <item><p>= <c>opaque()</c></p></item> + <tag><c>session()</c> =</tag> + <item><p><c>opaque()</c></p></item> </taglist> </section> diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl index 610e2c4e41..0c73a49a04 100644 --- a/lib/ssl/src/dtls_connection.erl +++ b/lib/ssl/src/dtls_connection.erl @@ -514,6 +514,7 @@ initial_state(Role, Host, Port, Socket, {SSLOptions, SocketOptions}, User, user_data_buffer = <<>>, session_cache_cb = SessionCacheCb, renegotiation = {false, first}, + allow_renegotiate = SSLOptions#ssl_options.client_renegotiation, start_or_recv_from = undefined, send_queue = queue:new(), protocol_cb = ?MODULE diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 225a9be66f..f8ddfba7e3 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -685,6 +685,7 @@ handle_options(Opts0) -> reuse_session = handle_option(reuse_session, Opts, ReuseSessionFun), reuse_sessions = handle_option(reuse_sessions, Opts, true), secure_renegotiate = handle_option(secure_renegotiate, Opts, false), + client_renegotiation = handle_option(client_renegotiation, Opts, true), renegotiate_at = handle_option(renegotiate_at, Opts, ?DEFAULT_RENEGOTIATE_AT), hibernate_after = handle_option(hibernate_after, Opts, undefined), erl_dist = handle_option(erl_dist, Opts, false), @@ -715,7 +716,7 @@ handle_options(Opts0) -> depth, cert, certfile, key, keyfile, password, cacerts, cacertfile, dh, dhfile, user_lookup_fun, psk_identity, srp_identity, ciphers, - reuse_session, reuse_sessions, ssl_imp, + reuse_session, reuse_sessions, ssl_imp, client_renegotiation, cb_info, renegotiate_at, secure_renegotiate, hibernate_after, erl_dist, alpn_advertised_protocols, sni_hosts, sni_fun, alpn_preferred_protocols, next_protocols_advertised, @@ -857,6 +858,8 @@ validate_option(reuse_sessions, Value) when is_boolean(Value) -> validate_option(secure_renegotiate, Value) when is_boolean(Value) -> Value; +validate_option(client_renegotiation, Value) when is_boolean(Value) -> + Value; validate_option(renegotiate_at, Value) when is_integer(Value) -> erlang:min(Value, ?DEFAULT_RENEGOTIATE_AT); @@ -1226,6 +1229,8 @@ new_ssl_options([{renegotiate_at, Value} | Rest], #ssl_options{} = Opts, RecordC new_ssl_options(Rest, Opts#ssl_options{ renegotiate_at = validate_option(renegotiate_at, Value)}, RecordCB); new_ssl_options([{secure_renegotiate, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> new_ssl_options(Rest, Opts#ssl_options{secure_renegotiate = validate_option(secure_renegotiate, Value)}, RecordCB); +new_ssl_options([{client_renegotiation, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> + new_ssl_options(Rest, Opts#ssl_options{client_renegotiation = validate_option(client_renegotiation, Value)}, RecordCB); new_ssl_options([{hibernate_after, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> new_ssl_options(Rest, Opts#ssl_options{hibernate_after = validate_option(hibernate_after, Value)}, RecordCB); new_ssl_options([{alpn_advertised_protocols, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index 8584e56d6c..47ee4d68fb 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -1573,7 +1573,9 @@ hash_algorithm(?SHA) -> sha; hash_algorithm(?SHA224) -> sha224; hash_algorithm(?SHA256) -> sha256; hash_algorithm(?SHA384) -> sha384; -hash_algorithm(?SHA512) -> sha512. +hash_algorithm(?SHA512) -> sha512; +hash_algorithm(Other) when is_integer(Other) andalso ((Other >= 7) and (Other =< 223)) -> unassigned; +hash_algorithm(Other) when is_integer(Other) andalso ((Other >= 224) and (Other =< 255)) -> Other. sign_algorithm(anon) -> ?ANON; sign_algorithm(rsa) -> ?RSA; @@ -1582,7 +1584,9 @@ sign_algorithm(ecdsa) -> ?ECDSA; sign_algorithm(?ANON) -> anon; sign_algorithm(?RSA) -> rsa; sign_algorithm(?DSA) -> dsa; -sign_algorithm(?ECDSA) -> ecdsa. +sign_algorithm(?ECDSA) -> ecdsa; +sign_algorithm(Other) when is_integer(Other) andalso ((Other >= 4) and (Other =< 223)) -> unassigned; +sign_algorithm(Other) when is_integer(Other) andalso ((Other >= 224) and (Other =< 255)) -> Other. hash_size(null) -> 0; diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index b538fefe53..12a17cb6ac 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -476,19 +476,27 @@ update_handshake_history({Handshake0, _Prev}, Data) -> %% end. premaster_secret(OtherPublicDhKey, MyPrivateKey, #'DHParameter'{} = Params) -> - public_key:compute_key(OtherPublicDhKey, MyPrivateKey, Params); - + try + public_key:compute_key(OtherPublicDhKey, MyPrivateKey, Params) + catch + error:computation_failed -> + throw(?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)) + end; premaster_secret(PublicDhKey, PrivateDhKey, #server_dh_params{dh_p = Prime, dh_g = Base}) -> - crypto:compute_key(dh, PublicDhKey, PrivateDhKey, [Prime, Base]); + try + crypto:compute_key(dh, PublicDhKey, PrivateDhKey, [Prime, Base]) + catch + error:computation_failed -> + throw(?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)) + end; premaster_secret(#client_srp_public{srp_a = ClientPublicKey}, ServerKey, #srp_user{prime = Prime, verifier = Verifier}) -> case crypto:compute_key(srp, ClientPublicKey, ServerKey, {host, [Verifier, Prime, '6a']}) of error -> - ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER); + throw(?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)); PremasterSecret -> PremasterSecret end; - premaster_secret(#server_srp_params{srp_n = Prime, srp_g = Generator, srp_s = Salt, srp_b = Public}, ClientKeys, {Username, Password}) -> case ssl_srp_primes:check_srp_params(Generator, Prime) of @@ -496,21 +504,19 @@ premaster_secret(#server_srp_params{srp_n = Prime, srp_g = Generator, srp_s = Sa DerivedKey = crypto:hash(sha, [Salt, crypto:hash(sha, [Username, <<$:>>, Password])]), case crypto:compute_key(srp, Public, ClientKeys, {user, [DerivedKey, Prime, Generator, '6a']}) of error -> - ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER); + throw(?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)); PremasterSecret -> PremasterSecret end; _ -> - ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER) + throw(?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)) end; - premaster_secret(#client_rsa_psk_identity{ identity = PSKIdentity, exchange_keys = #encrypted_premaster_secret{premaster_secret = EncPMS} }, #'RSAPrivateKey'{} = Key, PSKLookup) -> PremasterSecret = premaster_secret(EncPMS, Key), psk_secret(PSKIdentity, PSKLookup, PremasterSecret); - premaster_secret(#server_dhe_psk_params{ hint = IdentityHint, dh_params = #server_dh_params{dh_y = PublicDhKey} = Params}, @@ -518,7 +524,6 @@ premaster_secret(#server_dhe_psk_params{ LookupFun) -> PremasterSecret = premaster_secret(PublicDhKey, PrivateDhKey, Params), psk_secret(IdentityHint, LookupFun, PremasterSecret); - premaster_secret({rsa_psk, PSKIdentity}, PSKLookup, RSAPremasterSecret) -> psk_secret(PSKIdentity, PSKLookup, RSAPremasterSecret). @@ -527,13 +532,10 @@ premaster_secret(#client_dhe_psk_identity{ dh_public = PublicDhKey}, PrivateKey, #'DHParameter'{} = Params, PSKLookup) -> PremasterSecret = premaster_secret(PublicDhKey, PrivateKey, Params), psk_secret(PSKIdentity, PSKLookup, PremasterSecret). - premaster_secret(#client_psk_identity{identity = PSKIdentity}, PSKLookup) -> psk_secret(PSKIdentity, PSKLookup); - premaster_secret({psk, PSKIdentity}, PSKLookup) -> psk_secret(PSKIdentity, PSKLookup); - premaster_secret(#'ECPoint'{} = ECPoint, #'ECPrivateKey'{} = ECDHKeys) -> public_key:compute_key(ECPoint, ECDHKeys); premaster_secret(EncSecret, #'RSAPrivateKey'{} = RSAPrivateKey) -> @@ -2036,7 +2038,7 @@ psk_secret(PSKIdentity, PSKLookup) -> #alert{} = Alert -> Alert; _ -> - ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER) + throw(?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)) end. psk_secret(PSKIdentity, PSKLookup, PremasterSecret) -> @@ -2048,7 +2050,7 @@ psk_secret(PSKIdentity, PSKLookup, PremasterSecret) -> #alert{} = Alert -> Alert; _ -> - ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER) + throw(?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)) end. handle_psk_identity(_PSKIdentity, LookupFun) diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index baeae68bc4..40eb3d0284 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -110,6 +110,7 @@ reuse_sessions :: boolean(), renegotiate_at, secure_renegotiate, + client_renegotiation, %% undefined if not hibernating, or number of ms of %% inactivity after which ssl_connection will go into %% hibernation diff --git a/lib/ssl/src/ssl_tls_dist_proxy.erl b/lib/ssl/src/ssl_tls_dist_proxy.erl index a22af6b960..d23b42ace5 100644 --- a/lib/ssl/src/ssl_tls_dist_proxy.erl +++ b/lib/ssl/src/ssl_tls_dist_proxy.erl @@ -227,7 +227,10 @@ loop_conn_setup(World, Erts) -> {tcp_closed, Erts} -> ssl:close(World); {ssl_closed, World} -> - gen_tcp:close(Erts) + gen_tcp:close(Erts); + {ssl_error, World, _} -> + + ssl:close(World) end. loop_conn(World, Erts) -> @@ -241,7 +244,9 @@ loop_conn(World, Erts) -> {tcp_closed, Erts} -> ssl:close(World); {ssl_closed, World} -> - gen_tcp:close(Erts) + gen_tcp:close(Erts); + {ssl_error, World, _} -> + ssl:close(World) end. get_ssl_options(Type) -> diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 3304ffcddb..ed7ccb3d70 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -392,6 +392,7 @@ initial_state(Role, Host, Port, Socket, {SSLOptions, SocketOptions, Tracker}, Us user_data_buffer = <<>>, session_cache_cb = SessionCacheCb, renegotiation = {false, first}, + allow_renegotiate = SSLOptions#ssl_options.client_renegotiation, start_or_recv_from = undefined, send_queue = queue:new(), protocol_cb = ?MODULE, diff --git a/lib/ssl/test/ssl_alpn_handshake_SUITE.erl b/lib/ssl/test/ssl_alpn_handshake_SUITE.erl index ccd70fa605..ae76f5849e 100644 --- a/lib/ssl/test/ssl_alpn_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_alpn_handshake_SUITE.erl @@ -384,7 +384,7 @@ ssl_receive_and_assert_alpn(Socket, Protocol, Data) -> ssl_send(Socket, Data) -> ct:log("Connection info: ~p~n", - [ssl:connection_info(Socket)]), + [ssl:connection_information(Socket)]), ssl:send(Socket, Data). ssl_receive(Socket, Data) -> @@ -392,7 +392,7 @@ ssl_receive(Socket, Data) -> ssl_receive(Socket, Data, Buffer) -> ct:log("Connection info: ~p~n", - [ssl:connection_info(Socket)]), + [ssl:connection_information(Socket)]), receive {ssl, Socket, MoreData} -> ct:log("Received ~p~n",[MoreData]), @@ -411,4 +411,4 @@ ssl_receive(Socket, Data, Buffer) -> end. connection_info_result(Socket) -> - ssl:connection_info(Socket). + ssl:connection_information(Socket). diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 50d5fb411f..e131c363d1 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -162,7 +162,8 @@ renegotiate_tests() -> client_no_wrap_sequence_number, server_no_wrap_sequence_number, renegotiate_dos_mitigate_active, - renegotiate_dos_mitigate_passive]. + renegotiate_dos_mitigate_passive, + renegotiate_dos_mitigate_absolute]. cipher_tests() -> [cipher_suites, @@ -384,7 +385,7 @@ new_options_in_accept(Config) when is_list(Config) -> %%-------------------------------------------------------------------- connection_info() -> - [{doc,"Test the API function ssl:connection_info/1"}]. + [{doc,"Test the API function ssl:connection_information/1"}]. connection_info(Config) when is_list(Config) -> ClientOpts = ?config(client_opts, Config), ServerOpts = ?config(server_opts, Config), @@ -2831,7 +2832,7 @@ listen_socket(Config) -> {error, enotconn} = ssl:send(ListenSocket, <<"data">>), {error, enotconn} = ssl:recv(ListenSocket, 0), - {error, enotconn} = ssl:connection_info(ListenSocket), + {error, enotconn} = ssl:connection_information(ListenSocket), {error, enotconn} = ssl:peername(ListenSocket), {error, enotconn} = ssl:peercert(ListenSocket), {error, enotconn} = ssl:session_info(ListenSocket), @@ -2998,8 +2999,36 @@ renegotiate_dos_mitigate_passive(Config) when is_list(Config) -> ssl_test_lib:close(Client). %%-------------------------------------------------------------------- +renegotiate_dos_mitigate_absolute() -> + [{doc, "Mitigate DOS computational attack by not allowing client to initiate renegotiation"}]. +renegotiate_dos_mitigate_absolute(Config) when is_list(Config) -> + ServerOpts = ?config(server_opts, Config), + ClientOpts = ?config(client_opts, Config), + + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + + Server = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {ssl_test_lib, send_recv_result_active, []}}, + {options, [{client_renegotiation, false} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, + renegotiate_rejected, + []}}, + {options, ClientOpts}]), + + ssl_test_lib:check_result(Client, ok, Server, ok), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + +%%-------------------------------------------------------------------- tcp_error_propagation_in_active_mode() -> - [{doc,"Test that process recives {ssl_error, Socket, closed} when tcp error ocurres"}]. + [{doc,"Test that process recives {ssl_error, Socket, closed} when tcp error occurs"}]. tcp_error_propagation_in_active_mode(Config) when is_list(Config) -> ClientOpts = ?config(client_opts, Config), ServerOpts = ?config(server_opts, Config), @@ -3433,23 +3462,43 @@ renegotiate_reuse_session(Socket, Data) -> renegotiate(Socket, Data). renegotiate_immediately(Socket) -> - receive + receive {ssl, Socket, "Hello world"} -> ok; %% Handle 1/n-1 splitting countermeasure Rizzo/Duong-Beast {ssl, Socket, "H"} -> - receive + receive {ssl, Socket, "ello world"} -> ok end end, ok = ssl:renegotiate(Socket), {error, renegotiation_rejected} = ssl:renegotiate(Socket), - ct:sleep(?RENEGOTIATION_DISABLE_TIME +1), + ct:sleep(?RENEGOTIATION_DISABLE_TIME + ?SLEEP), ok = ssl:renegotiate(Socket), ct:log("Renegotiated again"), ssl:send(Socket, "Hello world"), ok. + +renegotiate_rejected(Socket) -> + receive + {ssl, Socket, "Hello world"} -> + ok; + %% Handle 1/n-1 splitting countermeasure Rizzo/Duong-Beast + {ssl, Socket, "H"} -> + receive + {ssl, Socket, "ello world"} -> + ok + end + end, + {error, renegotiation_rejected} = ssl:renegotiate(Socket), + {error, renegotiation_rejected} = ssl:renegotiate(Socket), + ct:sleep(?RENEGOTIATION_DISABLE_TIME +1), + {error, renegotiation_rejected} = ssl:renegotiate(Socket), + ct:log("Failed to renegotiate again"), + ssl:send(Socket, "Hello world"), + ok. + new_config(PrivDir, ServerOpts0) -> CaCertFile = proplists:get_value(cacertfile, ServerOpts0), @@ -3836,10 +3885,10 @@ cipher(CipherSuite, Version, Config, ClientOpts, ServerOpts) -> end. connection_info_result(Socket) -> - ssl:connection_info(Socket). - + {ok, Info} = ssl:connection_information(Socket, [protocol, cipher_suite]), + {ok, {proplists:get_value(protocol, Info), proplists:get_value(cipher_suite, Info)}}. version_info_result(Socket) -> - {ok, {Version, _}} = ssl:connection_info(Socket), + {ok, [{version, Version}]} = ssl:connection_information(Socket, [version]), {ok, Version}. connect_dist_s(S) -> diff --git a/lib/ssl/test/ssl_npn_handshake_SUITE.erl b/lib/ssl/test/ssl_npn_handshake_SUITE.erl index 326f907e66..8e95679306 100644 --- a/lib/ssl/test/ssl_npn_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_npn_handshake_SUITE.erl @@ -332,7 +332,7 @@ ssl_receive_and_assert_npn(Socket, Protocol, Data) -> ssl_send(Socket, Data) -> ct:log("Connection info: ~p~n", - [ssl:connection_info(Socket)]), + [ssl:connection_information(Socket)]), ssl:send(Socket, Data). ssl_receive(Socket, Data) -> @@ -340,7 +340,7 @@ ssl_receive(Socket, Data) -> ssl_receive(Socket, Data, Buffer) -> ct:log("Connection info: ~p~n", - [ssl:connection_info(Socket)]), + [ssl:connection_information(Socket)]), receive {ssl, Socket, MoreData} -> ct:log("Received ~p~n",[MoreData]), @@ -360,4 +360,4 @@ ssl_receive(Socket, Data, Buffer) -> connection_info_result(Socket) -> - ssl:connection_info(Socket). + ssl:connection_information(Socket). diff --git a/lib/ssl/test/ssl_sni_SUITE.erl b/lib/ssl/test/ssl_sni_SUITE.erl index 46cd644e4d..b059ff991b 100644 --- a/lib/ssl/test/ssl_sni_SUITE.erl +++ b/lib/ssl/test/ssl_sni_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2015. All Rights Reserved. +%% Copyright Ericsson AB 2015-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -16,7 +16,6 @@ %% %% %CopyrightEnd% %% - %% -module(ssl_sni_SUITE). @@ -31,7 +30,12 @@ %%-------------------------------------------------------------------- suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> [no_sni_header, sni_match, sni_no_match] ++ [no_sni_header_fun, sni_match_fun, sni_no_match_fun]. +all() -> [no_sni_header, + sni_match, + sni_no_match, + no_sni_header_fun, + sni_match_fun, + sni_no_match_fun]. init_per_suite(Config0) -> catch crypto:stop(), @@ -39,11 +43,11 @@ init_per_suite(Config0) -> ok -> ssl:start(), Result = - (catch make_certs:all(?config(data_dir, Config0), - ?config(priv_dir, Config0))), + (catch make_certs:all(?config(data_dir, Config0), + ?config(priv_dir, Config0))), ct:log("Make certs ~p~n", [Result]), ssl_test_lib:cert_options(Config0) - catch _:_ -> + catch _:_ -> {skip, "Crypto did not start"} end. @@ -76,8 +80,6 @@ sni_no_match_fun(Config) -> %%-------------------------------------------------------------------- %% Internal Functions ------------------------------------------------ %%-------------------------------------------------------------------- - - ssl_recv(SSLSocket, Expect) -> ssl_recv(SSLSocket, "", Expect). @@ -93,20 +95,21 @@ ssl_recv(SSLSocket, CurrentData, ExpectedData) -> end; Other -> ct:fail({unexpected_message, Other}) - after 4000 -> + after 4000 -> ct:fail({timeout, CurrentData, ExpectedData}) end. - - send_and_hostname(SSLSocket) -> ssl:send(SSLSocket, "OK"), {ok, [{sni_hostname, Hostname}]} = ssl:connection_information(SSLSocket, [sni_hostname]), Hostname. -rdnPart([[#'AttributeTypeAndValue'{type=Type, value=Value} | _] | _], Type) -> Value; -rdnPart([_ | Tail], Type) -> rdnPart(Tail, Type); -rdnPart([], _) -> unknown. +rdnPart([[#'AttributeTypeAndValue'{type=Type, value=Value} | _] | _], Type) -> + Value; +rdnPart([_ | Tail], Type) -> + rdnPart(Tail, Type); +rdnPart([], _) -> + unknown. rdn_to_string({utf8String, Binary}) -> erlang:binary_to_list(Binary); @@ -116,12 +119,15 @@ rdn_to_string({printableString, String}) -> recv_and_certificate(SSLSocket) -> ssl_recv(SSLSocket, "OK"), {ok, PeerCert} = ssl:peercert(SSLSocket), - #'OTPCertificate'{tbsCertificate = #'OTPTBSCertificate'{subject = {rdnSequence, Subject}}} = public_key:pkix_decode_cert(PeerCert, otp), + #'OTPCertificate'{tbsCertificate = #'OTPTBSCertificate'{subject = {rdnSequence, Subject}}} + = public_key:pkix_decode_cert(PeerCert, otp), ct:log("Subject of certificate received from server: ~p", [Subject]), rdn_to_string(rdnPart(Subject, ?'id-at-commonName')). run_sni_fun_handshake(Config, SNIHostname, ExpectedSNIHostname, ExpectedCN) -> - ct:log("Start running handshake for sni_fun, Config: ~p, SNIHostname: ~p, ExpectedSNIHostname: ~p, ExpectedCN: ~p", [Config, SNIHostname, ExpectedSNIHostname, ExpectedCN]), + ct:log("Start running handshake for sni_fun, Config: ~p, SNIHostname: ~p, " + "ExpectedSNIHostname: ~p, ExpectedCN: ~p", + [Config, SNIHostname, ExpectedSNIHostname, ExpectedCN]), [{sni_hosts, ServerSNIConf}] = ?config(sni_server_opts, Config), SNIFun = fun(Domain) -> proplists:get_value(Domain, ServerSNIConf, undefined) end, ServerOptions = ?config(server_opts, Config) ++ [{sni_fun, SNIFun}], @@ -142,11 +148,14 @@ run_sni_fun_handshake(Config, SNIHostname, ExpectedSNIHostname, ExpectedCN) -> {host, Hostname}, {from, self()}, {mfa, {?MODULE, recv_and_certificate, []}}, {options, ClientOptions}]), - ssl_test_lib:check_result(Server, ExpectedSNIHostname, Client, ExpectedCN). - + ssl_test_lib:check_result(Server, ExpectedSNIHostname, Client, ExpectedCN), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). run_handshake(Config, SNIHostname, ExpectedSNIHostname, ExpectedCN) -> - ct:log("Start running handshake, Config: ~p, SNIHostname: ~p, ExpectedSNIHostname: ~p, ExpectedCN: ~p", [Config, SNIHostname, ExpectedSNIHostname, ExpectedCN]), + ct:log("Start running handshake, Config: ~p, SNIHostname: ~p, " + "ExpectedSNIHostname: ~p, ExpectedCN: ~p", + [Config, SNIHostname, ExpectedSNIHostname, ExpectedCN]), ServerOptions = ?config(sni_server_opts, Config) ++ ?config(server_opts, Config), ClientOptions = case SNIHostname of @@ -165,4 +174,6 @@ run_handshake(Config, SNIHostname, ExpectedSNIHostname, ExpectedCN) -> {host, Hostname}, {from, self()}, {mfa, {?MODULE, recv_and_certificate, []}}, {options, ClientOptions}]), - ssl_test_lib:check_result(Server, ExpectedSNIHostname, Client, ExpectedCN). + ssl_test_lib:check_result(Server, ExpectedSNIHostname, Client, ExpectedCN), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index 8b98e6f16b..f35c0502ae 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -778,7 +778,12 @@ send_selected_port(_,_,_) -> rsa_suites(CounterPart) -> ECC = is_sane_ecc(CounterPart), - lists:filter(fun({rsa, _, _}) -> + FIPS = is_fips(CounterPart), + lists:filter(fun({rsa, des_cbc, sha}) when FIPS == true -> + false; + ({dhe_rsa, des_cbc, sha}) when FIPS == true -> + false; + ({rsa, _, _}) -> true; ({dhe_rsa, _, _}) -> true; @@ -949,7 +954,8 @@ der_to_pem(File, Entries) -> file:write_file(File, PemBin). cipher_result(Socket, Result) -> - Result = ssl:connection_info(Socket), + {ok, Info} = ssl:connection_information(Socket), + Result = {ok, {proplists:get_value(protocol, Info), proplists:get_value(cipher_suite, Info)}}, ct:log("~p:~p~nSuccessfull connect: ~p~n", [?MODULE,?LINE, Result]), %% Importante to send two packets here %% to properly test "cipher state" handling @@ -1089,6 +1095,25 @@ is_sane_ecc(crypto) -> is_sane_ecc(_) -> true. +is_fips(openssl) -> + VersionStr = os:cmd("openssl version"), + case re:split(VersionStr, "fips") of + [_] -> + false; + _ -> + true + end; +is_fips(crypto) -> + [{_,_, Bin}] = crypto:info_lib(), + case re:split(Bin, <<"fips">>) of + [_] -> + false; + _ -> + true + end; +is_fips(_) -> + false. + cipher_restriction(Config0) -> case is_sane_ecc(openssl) of false -> diff --git a/lib/ssl/test/ssl_to_openssl_SUITE.erl b/lib/ssl/test/ssl_to_openssl_SUITE.erl index 0413415e49..21ce4c4a29 100644 --- a/lib/ssl/test/ssl_to_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_to_openssl_SUITE.erl @@ -1036,7 +1036,7 @@ erlang_client_openssl_server_alpn(Config) when is_list(Config) -> erlang_server_alpn_openssl_client(Config) when is_list(Config) -> Data = "From openssl to erlang", start_erlang_server_and_openssl_client_with_opts(Config, - [{alpn_advertised_protocols, [<<"spdy/2">>]}], + [{alpn_preferred_protocols, [<<"spdy/2">>]}], "", Data, fun(Server, OpensslPort) -> true = port_command(OpensslPort, Data), @@ -1243,15 +1243,16 @@ run_suites(Ciphers, Version, Config, Type) -> ct:fail(cipher_suite_failed_see_test_case_log) end. -client_read_check([], _NewData) -> ok; -client_read_check([Hd | T], NewData) -> - case binary:match(NewData, list_to_binary(Hd)) of +client_read_check([], _Data) -> + ok; +client_read_check([Hd | T], Data) -> + case binary:match(Data, list_to_binary(Hd)) of nomatch -> nomatch; _ -> - client_read_check(T, NewData) + client_read_check(T, Data) end. -client_read_bulk(Port, DataExpected, DataReceived) -> +client_check_result(Port, DataExpected, DataReceived) -> receive {Port, {data, TheData}} -> Data = list_to_binary(TheData), @@ -1261,15 +1262,14 @@ client_read_bulk(Port, DataExpected, DataReceived) -> ok -> ok; _ -> - client_read_bulk(Port, DataExpected, NewData) - end; - _ -> - ct:fail("unexpected_message") - after 4000 -> - ct:fail("timeout") + client_check_result(Port, DataExpected, NewData) + end + after 3000 -> + ct:fail({"Time out on opensssl Client", {expected, DataExpected}, + {got, DataReceived}}) end. -client_read_bulk(Port, DataExpected) -> - client_read_bulk(Port, DataExpected, <<"">>). +client_check_result(Port, DataExpected) -> + client_check_result(Port, DataExpected, <<"">>). send_and_hostname(SSLSocket) -> ssl:send(SSLSocket, "OK"), @@ -1292,9 +1292,12 @@ erlang_server_openssl_client_sni_test(Config, SNIHostname, ExpectedSNIHostname, end, ct:log("Options: ~p", [[ServerOptions, ClientCommand]]), ClientPort = open_port({spawn, ClientCommand}, [stderr_to_stdout]), - ssl_test_lib:check_result(Server, ExpectedSNIHostname), + + %% Client check needs to be done befor server check, + %% or server check might consume client messages ExpectedClientOutput = ["OK", "/CN=" ++ ExpectedCN ++ "/"], - ok = client_read_bulk(ClientPort, ExpectedClientOutput), + client_check_result(ClientPort, ExpectedClientOutput), + ssl_test_lib:check_result(Server, ExpectedSNIHostname), ssl_test_lib:close_port(ClientPort), ssl_test_lib:close(Server), ok. @@ -1318,12 +1321,14 @@ erlang_server_openssl_client_sni_test_sni_fun(Config, SNIHostname, ExpectedSNIHo end, ct:log("Options: ~p", [[ServerOptions, ClientCommand]]), ClientPort = open_port({spawn, ClientCommand}, [stderr_to_stdout]), - ssl_test_lib:check_result(Server, ExpectedSNIHostname), + + %% Client check needs to be done befor server check, + %% or server check might consume client messages ExpectedClientOutput = ["OK", "/CN=" ++ ExpectedCN ++ "/"], - ok = client_read_bulk(ClientPort, ExpectedClientOutput), + client_check_result(ClientPort, ExpectedClientOutput), + ssl_test_lib:check_result(Server, ExpectedSNIHostname), ssl_test_lib:close_port(ClientPort), - ssl_test_lib:close(Server), - ok. + ssl_test_lib:close(Server). cipher(CipherSuite, Version, Config, ClientOpts, ServerOpts) -> @@ -1664,7 +1669,7 @@ erlang_ssl_receive_and_assert_negotiated_protocol(Socket, Protocol, Data) -> erlang_ssl_receive(Socket, Data) -> ct:log("Connection info: ~p~n", - [ssl:connection_info(Socket)]), + [ssl:connection_information(Socket)]), receive {ssl, Socket, Data} -> io:format("Received ~p~n",[Data]), @@ -1683,16 +1688,16 @@ erlang_ssl_receive(Socket, Data) -> end. connection_info(Socket, Version) -> - case ssl:connection_info(Socket) of - {ok, {Version, _} = Info} -> + case ssl:connection_information(Socket, [version]) of + {ok, [{version, Version}] = Info} -> ct:log("Connection info: ~p~n", [Info]), ok; - {ok, {OtherVersion, _}} -> + {ok, [{version, OtherVersion}]} -> {wrong_version, OtherVersion} end. connection_info_result(Socket) -> - ssl:connection_info(Socket). + ssl:connection_information(Socket). delayed_send(Socket, [ErlData, OpenSslData]) -> diff --git a/lib/stdlib/doc/src/Makefile b/lib/stdlib/doc/src/Makefile index a4a2ed9931..d41f91250e 100644 --- a/lib/stdlib/doc/src/Makefile +++ b/lib/stdlib/doc/src/Makefile @@ -102,7 +102,7 @@ XML_REF3_FILES = \ XML_REF6_FILES = stdlib_app.xml XML_PART_FILES = part.xml part_notes.xml part_notes_history.xml -XML_CHAPTER_FILES = io_protocol.xml unicode_usage.xml notes.xml notes_history.xml +XML_CHAPTER_FILES = io_protocol.xml unicode_usage.xml notes.xml notes_history.xml assert_hrl.xml BOOK_FILES = book.xml diff --git a/lib/stdlib/doc/src/array.xml b/lib/stdlib/doc/src/array.xml index b03a2fa0cc..af23cd95d9 100644 --- a/lib/stdlib/doc/src/array.xml +++ b/lib/stdlib/doc/src/array.xml @@ -93,9 +93,6 @@ the default value cannot be confused with the values of set entries.</p> </datatype> <datatype> <name name="array" n_vars="0"/> - <desc> - <p><c>array()</c> is equivalent to <c>array(term())</c>.</p> - </desc> </datatype> <datatype> <name name="array_indx"/> diff --git a/lib/stdlib/doc/src/assert_hrl.xml b/lib/stdlib/doc/src/assert_hrl.xml new file mode 100644 index 0000000000..d812ee16dc --- /dev/null +++ b/lib/stdlib/doc/src/assert_hrl.xml @@ -0,0 +1,160 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE fileref SYSTEM "fileref.dtd"> + +<fileref> + <header> + <copyright> + <year>2012</year><year>2015</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>assert.hrl</title> + <prepared></prepared> + <docno></docno> + <date></date> + <rev></rev> + </header> + <file>assert.hrl</file> + <filesummary>Assert Macros</filesummary> + <description> + <p>The include file <c>assert.hrl</c> provides macros for inserting + assertions in your program code.</p> + <p>These macros are defined in the Stdlib include file + <c>assert.hrl</c>. Include the following directive in the module + from which the function is called:</p> + <code type="none"> +-include_lib("stdlib/include/assert.hrl").</code> + <p>When an assertion succeeds, the assert macro yields the atom + <c>ok</c>. When an assertion fails, an exception of type <c>error</c> is + instead generated. The associated error term will have the form + <c>{Macro, Info}</c>, where <c>Macro</c> is the name of the macro, for + example <c>assertEqual</c>, and <c>Info</c> will be a list of tagged + values such as <c>[{module, M}, {line, L}, ...]</c> giving more + information about the location and cause of the exception. All entries + in the <c>Info</c> list are optional, and you should not rely + programatically on any of them being present.</p> + + <p>If the macro <c>NOASSERT</c> is defined when the <c>assert.hrl</c> + include file is read by the compiler, the macros will be defined as + equivalent to the atom <c>ok</c>. The test will not be performed, and + there will be no cost at runtime.</p> + + <p>For example, using <c>erlc</c> to compile your modules, the following + will disable all assertions:</p> + <code type="none"> +erlc -DNOASSERT=true *.erl</code> + <p>(The value of <c>NOASSERT</c> does not matter, only the fact that it + is defined.)</p> + <p>A few other macros also have effect on the enabling or disabling of + assertions:</p> + <list type="bulleted"> + <item>If <c>NODEBUG</c> is defined, it implies <c>NOASSERT</c>, unless + <c>DEBUG</c> is also defined, which is assumed to take + precedence.</item> + <item>If <c>ASSERT</c> is defined, it overrides <c>NOASSERT</c>, that + is, the assertions will remain enabled.</item> + </list> + <p>If you prefer, you can thus use only <c>DEBUG</c>/<c>NODEBUG</c> as + the main flags to control the behaviour of the assertions (which is + useful if you have other compiler conditionals or debugging macros + controlled by those flags), or you can use <c>ASSERT</c>/<c>NOASSERT</c> + to control only the assert macros.</p> + + </description> + + <section> + </section> + + <section> + <title>Macros</title> + <taglist> + <tag><c>assert(BoolExpr)</c></tag> + <item><p>Tests that <c>BoolExpr</c> completes normally returning + <c>true</c>.</p> + </item> + + <tag><c>assertNot(BoolExpr)</c></tag> + <item><p>Tests that <c>BoolExpr</c> completes normally returning + <c>false</c>.</p> + </item> + + <tag><c>assertMatch(GuardedPattern, Expr)</c></tag> + <item><p>Tests that <c>Expr</c> completes normally yielding a value + that matches <c>GuardedPattern</c>. For example: + <code type="none"> + ?assertMatch({bork, _}, f())</code></p> + <p>Note that a guard <c>when ...</c> can be included: + <code type="none"> + ?assertMatch({bork, X} when X > 0, f())</code></p> + </item> + + <tag><c>assertNotMatch(GuardedPattern, Expr)</c></tag> + <item><p>Tests that <c>Expr</c> completes normally yielding a value + that does not match <c>GuardedPattern</c>.</p> + <p>As in <c>assertMatch</c>, <c>GuardedPattern</c> can have a + <c>when</c> part.</p> + </item> + + <tag><c>assertEqual(ExpectedValue, Expr)</c></tag> + <item><p>Tests that <c>Expr</c> completes normally yielding a value + that is exactly equal to <c>ExpectedValue</c>.</p> + </item> + + <tag><c>assertNotEqual(ExpectedValue, Expr)</c></tag> + <item><p>Tests that <c>Expr</c> completes normally yielding a value + that is not exactly equal to <c>ExpectedValue</c>.</p> + </item> + + <tag><c>assertException(Class, Term, Expr)</c></tag> + <item><p>Tests that <c>Expr</c> completes abnormally with an exception + of type <c>Class</c> and with the associated <c>Term</c>. The + assertion fails if <c>Expr</c> raises a different exception or if it + completes normally returning any value.</p> + <p>Note that both <c>Class</c> and <c>Term</c> can be guarded + patterns, as in <c>assertMatch</c>.</p> + </item> + + <tag><c>assertNotException(Class, Term, Expr)</c></tag> + <item><p>Tests that <c>Expr</c> does not evaluate abnormally with an + exception of type <c>Class</c> and with the associated <c>Term</c>. + The assertion succeeds if <c>Expr</c> raises a different exception or + if it completes normally returning any value.</p> + <p>As in <c>assertException</c>, both <c>Class</c> and <c>Term</c> + can be guarded patterns.</p> + </item> + + <tag><c>assertError(Term, Expr)</c></tag> + <item><p>Equivalent to <c>assertException(error, Term, + Expr)</c></p> + </item> + + <tag><c>assertExit(Term, Expr)</c></tag> + <item><p>Equivalent to <c>assertException(exit, Term, Expr)</c></p> + </item> + + <tag><c>assertThrow(Term, Expr)</c></tag> + <item><p>Equivalent to <c>assertException(throw, Term, Expr)</c></p> + </item> + + </taglist> + </section> + + <section> + <title>SEE ALSO</title> + <p><seealso marker="compiler:compile">compile(3)</seealso></p> + <p><seealso marker="erts:erlc">erlc(3)</seealso></p> + </section> +</fileref> diff --git a/lib/stdlib/doc/src/c.xml b/lib/stdlib/doc/src/c.xml index b49fa6ad67..b43d4786ae 100644 --- a/lib/stdlib/doc/src/c.xml +++ b/lib/stdlib/doc/src/c.xml @@ -232,6 +232,14 @@ compile:file(<anno>File</anno>, <anno>Options</anno> ++ [report_errors, report_w </desc> </func> <func> + <name name="uptime" arity="0"/> + <fsummary>Print node uptime</fsummary> + <desc> + <p>Prints the node uptime (as given by + <c>erlang:statistics(wall_clock)</c>), in human-readable form.</p> + </desc> + </func> + <func> <name>xm(ModSpec) -> void()</name> <fsummary>Cross reference check a module</fsummary> <type> diff --git a/lib/stdlib/doc/src/dict.xml b/lib/stdlib/doc/src/dict.xml index 0771682a25..b456b97578 100644 --- a/lib/stdlib/doc/src/dict.xml +++ b/lib/stdlib/doc/src/dict.xml @@ -46,9 +46,6 @@ </datatype> <datatype> <name name="dict" n_vars="0"/> - <desc> - <p><c>dict()</c> is equivalent to <c>dict(term(), term())</c>.</p> - </desc> </datatype> </datatypes> <funcs> diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index 6b9524ef63..2bfe074c3e 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -1435,7 +1435,9 @@ is_integer(X), is_integer(Y), X + Y < 4711]]></code> <p>Whenever the <c>extended_info</c> option is used, it results in a file not readable by versions of ets prior to that in stdlib-1.15.1</p> - + <p>The <c>sync</c> option, if set to <c>true</c>, ensures that + the content of the file is actually written to the disk before + <c>tab2file</c> returns. Default is <c>{sync, false}</c>.</p> </desc> </func> <func> diff --git a/lib/stdlib/doc/src/gb_sets.xml b/lib/stdlib/doc/src/gb_sets.xml index ea96c14472..99e92d8680 100644 --- a/lib/stdlib/doc/src/gb_sets.xml +++ b/lib/stdlib/doc/src/gb_sets.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2001</year><year>2014</year> + <year>2001</year><year>2015</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -120,9 +120,6 @@ </datatype> <datatype> <name name="set" n_vars="0"/> - <desc> - <p><c>set()</c> is equivalent to <c>set(term())</c>.</p> - </desc> </datatype> <datatype> <name name="iter" n_vars="1"/> @@ -130,9 +127,6 @@ </datatype> <datatype> <name name="iter" n_vars="0"/> - <desc> - <p><c>iter()</c> is equivalent to <c>iter(term())</c>.</p> - </desc> </datatype> </datatypes> <funcs> @@ -306,6 +300,17 @@ </desc> </func> <func> + <name name="iterator_from" arity="2"/> + <fsummary>Return an iterator for a set starting from a specified element</fsummary> + <desc> + <p>Returns an iterator that can be used for traversing the + entries of <c><anno>Set</anno></c>; see <c>next/1</c>. + The difference as compared to the iterator returned by + <c>iterator/1</c> is that the first element greater than + or equal to <c><anno>Element</anno></c> is returned.</p> + </desc> + </func> + <func> <name name="largest" arity="1"/> <fsummary>Return largest element</fsummary> <desc> diff --git a/lib/stdlib/doc/src/gb_trees.xml b/lib/stdlib/doc/src/gb_trees.xml index b2f237e1d7..99ca2d6a9a 100644 --- a/lib/stdlib/doc/src/gb_trees.xml +++ b/lib/stdlib/doc/src/gb_trees.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2001</year><year>2014</year> + <year>2001</year><year>2015</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -64,9 +64,6 @@ </datatype> <datatype> <name name="tree" n_vars="0"/> - <desc> - <p><c>tree()</c> is equivalent to <c>tree(term(), term())</c>.</p> - </desc> </datatype> <datatype> <name name="iter" n_vars="2"/> @@ -74,9 +71,6 @@ </datatype> <datatype> <name name="iter" n_vars="0"/> - <desc> - <p><c>iter()</c> is equivalent to <c>iter(term(), term())</c>.</p> - </desc> </datatype> </datatypes> <funcs> @@ -183,6 +177,17 @@ </desc> </func> <func> + <name name="iterator_from" arity="2"/> + <fsummary>Return an iterator for a tree starting from specified key</fsummary> + <desc> + <p>Returns an iterator that can be used for traversing the + entries of <c><anno>Tree</anno></c>; see <c>next/1</c>. + The difference as compared to the iterator returned by + <c>iterator/1</c> is that the first key greater than + or equal to <c><anno>Key</anno></c> is returned.</p> + </desc> + </func> + <func> <name name="keys" arity="1"/> <fsummary>Return a list of the keys in a tree</fsummary> <desc> diff --git a/lib/stdlib/doc/src/maps.xml b/lib/stdlib/doc/src/maps.xml index e46068230a..7345a9357a 100644 --- a/lib/stdlib/doc/src/maps.xml +++ b/lib/stdlib/doc/src/maps.xml @@ -33,6 +33,28 @@ <funcs> <func> + <name name="filter" arity="2"/> + <fsummary>Choose pairs which satisfy a predicate</fsummary> + <desc> + <p> + Returns a map <c><anno>Map2</anno></c> for which predicate + <c><anno>Pred</anno></c> holds true in <c><anno>Map1</anno></c>. + </p> + <p> + The call will fail with a <c>{badmap,Map}</c> exception if + <c><anno>Map1</anno></c> is not a map or with <c>badarg</c> if + <c><anno>Pred</anno></c> is not a function of arity 2. + </p> + <p>Example:</p> + <code type="none"> +> M = #{a => 2, b => 3, c=> 4, "a" => 1, "b" => 2, "c" => 4}, + Pred = fun(K,V) -> is_atom(K) andalso (V rem 2) =:= 0 end, + maps:filter(Pred,M). +#{a => 2,c => 4} </code> + </desc> + </func> + + <func> <name name="find" arity="2"/> <fsummary></fsummary> <desc> diff --git a/lib/stdlib/doc/src/notes.xml b/lib/stdlib/doc/src/notes.xml index 3914a9bc0e..301a5ee2e8 100644 --- a/lib/stdlib/doc/src/notes.xml +++ b/lib/stdlib/doc/src/notes.xml @@ -30,224 +30,6 @@ </header> <p>This document describes the changes made to the STDLIB application.</p> -<section><title>STDLIB 2.5</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - Fix handling of single dot in filename:join/2</p> - <p> - The reference manual says that filename:join(A,B) is - equivalent to filename:join([A,B]). In some rare cases - this turns out not to be true. For example:</p> - <p> - <c>filename:join("/a/.","b") -> "/a/./b"</c> vs - <c>filename:join(["/a/.","b"]) -> "/a/b"</c>.</p> - <p> - This has been corrected. A single dot is now only kept if - it occurs at the very beginning or the very end of the - resulting path.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12158</p> - </item> - <item> - <p> - The undocumented option <c>generic_debug</c> for - <c>gen_server</c> has been removed.</p> - <p> - Own Id: OTP-12183</p> - </item> - <item> - <p> - erl_lint:icrt_export/4 has been rewritten to make the - code really follow the scoping rules of Erlang, and not - just in most situations by accident.</p> - <p> - Own Id: OTP-12186</p> - </item> - <item> - <p> - Add 'trim_all' option to binary:split/3</p> - <p> - This option can be set to remove _ALL_ empty parts of the - result of a call to binary:split/3.</p> - <p> - Own Id: OTP-12301</p> - </item> - <item> - <p> Correct orddict(3) regarding evaluation order of - <c>fold()</c> and <c>map()</c>. </p> - <p> - Own Id: OTP-12651 Aux Id: seq12832 </p> - </item> - <item> - <p> - Correct <c>maps</c> module error exceptions </p> - <p> - Bad input to maps module function will now yield the - following exceptions: <list> <item>{badmap,NotMap} - or,</item> <item>badarg</item> </list></p> - <p> - Own Id: OTP-12657</p> - </item> - <item> - <p> - It is now possible to paste text in JCL mode (using - Ctrl-Y) that has been copied in the previous shell - session. Also a bug that caused the JCL mode to crash - when pasting text has been fixed.</p> - <p> - Own Id: OTP-12673</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Allow maps for supervisor flags and child specs</p> - <p> - Earlier, supervisor flags and child specs were given as - tuples. While this is kept for backwards compatibility, - it is now also allowed to give these parameters as maps, - see <seealso - marker="stdlib:supervisor#sup_flags">sup_flags</seealso> - and <seealso - marker="stdlib:supervisor#child_spec">child_spec</seealso>.</p> - <p> - Own Id: OTP-11043</p> - </item> - <item> - <p> - A new system message, <c>terminate</c>, is added. This - can be sent with <c>sys:terminate/2,3</c>. If the - receiving process handles system messages properly it - will terminate shortly after receiving this message.</p> - <p> - The new function <c>proc_lib:stop/1,3</c> utilizes this - new system message and monitors the receiving process in - order to facilitate a synchronous stop mechanism for - 'special processes'.</p> - <p> - <c>proc_lib:stop/1,3</c> is used by the following - functions:</p> - <p> - <list> <item><c>gen_server:stop/1,3</c> (new)</item> - <item><c>gen_fsm:stop/1,3</c> (new)</item> - <item><c>gen_event:stop/1,3</c> (modified to be - synchronous)</item> <item><c>wx_object:stop/1,3</c> - (new)</item> </list></p> - <p> - Own Id: OTP-11173 Aux Id: seq12353 </p> - </item> - <item> - <p> - Remove the <c>pg</c> module, which has been deprecated - through OTP-17, is now removed from the STDLIB - application. This module has been marked experimental for - more than 15 years, and has largely been superseded by - the <c>pg2</c> module from the Kernel application.</p> - <p> - Own Id: OTP-11907</p> - </item> - <item> - <p> - New BIF: <c>erlang:get_keys/0</c>, lists all keys - associated with the process dictionary. Note: - <c>erlang:get_keys/0</c> is auto-imported.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12151 Aux Id: seq12521 </p> - </item> - <item> - <p> Add three new functions to <c>io_lib</c>-- - <c>scan_format/2</c>, <c>unscan_format/1</c>, and - <c>build_text/1</c>-- which expose the parsed form of the - format control sequences to make it possible to easily - modify or filter the input to <c>io_lib:format/2</c>. - This can e.g. be used in order to replace unbounded-size - control sequences like <c>~w</c> or <c>~p</c> with - corresponding depth-limited <c>~W</c> and <c>~P</c> - before doing the actual formatting. </p> - <p> - Own Id: OTP-12167</p> - </item> - <item> - <p> Introduce the <c>erl_anno</c> module, an abstraction - of the second element of tokens and tuples in the - abstract format. </p> - <p> - Own Id: OTP-12195</p> - </item> - <item> - <p> - Support variables as Map keys in expressions and patterns</p> - <p>Erlang will accept any expression as keys in Map - expressions and it will accept literals or bound - variables as keys in Map patterns.</p> - <p> - Own Id: OTP-12218</p> - </item> - <item> - <p> The last traces of Mnemosyne Rules have been removed. - </p> - <p> - Own Id: OTP-12257</p> - </item> - <item> - <p> - Properly support maps in match_specs</p> - <p> - Own Id: OTP-12270</p> - </item> - <item> - <p> - New function <c>ets:take/2</c>. Works the same as - <c>ets:delete/2</c> but also returns the deleted - object(s).</p> - <p> - Own Id: OTP-12309</p> - </item> - <item> - <p><c>string:tokens/2</c> is somewhat faster, especially - if the list of separators only contains one separator - character.</p> - <p> - Own Id: OTP-12422 Aux Id: seq12774 </p> - </item> - <item> - <p> - Prevent zip:zip_open/[12] from leaking file descriptors - if parent process dies.</p> - <p> - Own Id: OTP-12566</p> - </item> - <item> - <p> - Add a new random number generator, see <c>rand</c> - module. It have better characteristics and an improved - interface.</p> - <p> - Own Id: OTP-12586 Aux Id: OTP-12501, OTP-12502 </p> - </item> - <item> - <p><c>filename:split/1</c> when given an empty binary - will now return an empty list, to make it consistent with - return value when given an empty list.</p> - <p> - Own Id: OTP-12716</p> - </item> - </list> - </section> - -</section> - <section><title>STDLIB 2.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/stdlib/doc/src/orddict.xml b/lib/stdlib/doc/src/orddict.xml index ec1e43f29c..c853b402d4 100644 --- a/lib/stdlib/doc/src/orddict.xml +++ b/lib/stdlib/doc/src/orddict.xml @@ -48,8 +48,11 @@ <datatypes> <datatype> - <name name="orddict"/> - <desc><p>As returned by new/0.</p></desc> + <name name="orddict" n_vars="2"/> + <desc><p>Dictionary as returned by <c>new/0</c>.</p></desc> + </datatype> + <datatype> + <name name="orddict" n_vars="0"/> </datatype> </datatypes> diff --git a/lib/stdlib/doc/src/queue.xml b/lib/stdlib/doc/src/queue.xml index 9c994154d4..f689412988 100644 --- a/lib/stdlib/doc/src/queue.xml +++ b/lib/stdlib/doc/src/queue.xml @@ -95,9 +95,6 @@ </datatype> <datatype> <name name="queue" n_vars="0"/> - <desc> - <p><c>queue()</c> is equivalent to <c>queue(term())</c>.</p> - </desc> </datatype> </datatypes> diff --git a/lib/stdlib/doc/src/ref_man.xml b/lib/stdlib/doc/src/ref_man.xml index eee4a68ca1..cae62612aa 100644 --- a/lib/stdlib/doc/src/ref_man.xml +++ b/lib/stdlib/doc/src/ref_man.xml @@ -35,6 +35,7 @@ </description> <xi:include href="stdlib_app.xml"/> <xi:include href="array.xml"/> + <xi:include href="assert_hrl.xml"/> <xi:include href="base64.xml"/> <xi:include href="beam_lib.xml"/> <xi:include href="binary.xml"/> diff --git a/lib/stdlib/doc/src/sets.xml b/lib/stdlib/doc/src/sets.xml index 4a31648f8f..eecddb7fd4 100644 --- a/lib/stdlib/doc/src/sets.xml +++ b/lib/stdlib/doc/src/sets.xml @@ -50,9 +50,6 @@ </datatype> <datatype> <name name="set" n_vars="0"/> - <desc> - <p><c>set()</c> is equivalent to <c>set(term())</c>.</p> - </desc> </datatype> </datatypes> <funcs> diff --git a/lib/stdlib/doc/src/supervisor.xml b/lib/stdlib/doc/src/supervisor.xml index ffac1c0bd7..6ff477a42d 100644 --- a/lib/stdlib/doc/src/supervisor.xml +++ b/lib/stdlib/doc/src/supervisor.xml @@ -386,9 +386,15 @@ added to the supervisor and the function returns the same value.</p> <p>If the child process start function returns <c>ignore</c>, - the child specification is added to the supervisor, the pid - is set to <c>undefined</c>, and the function returns - <c>{ok,undefined}</c>.</p> + the child specification is added to the supervisor (unless the + supervisor is a <c>simple_one_for_one</c> supervisor, see below), + the pid is set to <c>undefined</c> and the function returns + <c>{ok,undefined}</c>. + </p> + <p>In the case of a <c>simple_one_for_one</c> supervisor, when a child + process start function returns <c>ignore</c> the functions returns + <c>{ok,undefined}</c> and no child is added to the supervisor. + </p> <p>If the child process start function returns an error tuple or an erroneous value, or if it fails, the child specification is discarded, and the function returns <c>{error,Error}</c> where diff --git a/lib/stdlib/include/assert.hrl b/lib/stdlib/include/assert.hrl new file mode 100644 index 0000000000..239d19a6dc --- /dev/null +++ b/lib/stdlib/include/assert.hrl @@ -0,0 +1,260 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright (C) 2004-2014 Richard Carlsson, Mickaël Rémond +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-ifndef(ASSERT_HRL). +-define(ASSERT_HRL, true). + +%% Asserts are enabled unless NOASSERT is defined, and ASSERT can be used to +%% override it: if both ASSERT and NOASSERT are defined, then ASSERT takes +%% precedence, and NOASSERT will become undefined. +%% +%% Furthermore, if NODEBUG is defined, it implies NOASSERT, unless DEBUG or +%% ASSERT are defined. +%% +%% If asserts are disabled, all assert macros are defined to be the atom +%% 'ok'. If asserts are enabled, all assert macros are defined to yield 'ok' +%% as the result if the test succeeds, and raise an error exception if the +%% test fails. The error term will then have the form {Name, Info} where +%% Name is the name of the macro and Info is a list of tagged tuples. + +%% allow NODEBUG to imply NOASSERT, unless DEBUG +-ifdef(NODEBUG). +-ifndef(DEBUG). +-ifndef(NOASSERT). +-define(NOASSERT, true). +-endif. +-endif. +-endif. + +%% allow ASSERT to override NOASSERT +-ifdef(ASSERT). +-undef(NOASSERT). +-endif. + +%% Assert macros must not depend on any non-kernel or stdlib libraries. +%% +%% We must use fun-call wrappers ((fun () -> ... end)()) to avoid +%% exporting local variables, and furthermore we only use variable names +%% prefixed with "__", that hopefully will not be bound outside the fun. +%% It is not possible to nest assert macros. + +-ifdef(NOASSERT). +-define(assert(BoolExpr),ok). +-else. +%% The assert macro is written the way it is so as not to cause warnings +%% for clauses that cannot match, even if the expression is a constant. +-define(assert(BoolExpr), + begin + ((fun () -> + case (BoolExpr) of + true -> ok; + __V -> erlang:error({assert, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??BoolExpr)}, + {expected, true}, + case __V of false -> {value, __V}; + _ -> {not_boolean,__V} + end]}) + end + end)()) + end). +-endif. + +%% This is the inverse case of assert, for convenience. +-ifdef(NOASSERT). +-define(assertNot(BoolExpr),ok). +-else. +-define(assertNot(BoolExpr), + begin + ((fun () -> + case (BoolExpr) of + false -> ok; + __V -> erlang:error({assert, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??BoolExpr)}, + {expected, false}, + case __V of true -> {value, __V}; + _ -> {not_boolean,__V} + end]}) + end + end)()) + end). +-endif. + +%% This is mostly a convenience which gives more detailed reports. +%% Note: Guard is a guarded pattern, and can not be used for value. +-ifdef(NOASSERT). +-define(assertMatch(Guard, Expr), ok). +-else. +-define(assertMatch(Guard, Expr), + begin + ((fun () -> + case (Expr) of + Guard -> ok; + __V -> erlang:error({assertMatch, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??Expr)}, + {pattern, (??Guard)}, + {value, __V}]}) + end + end)()) + end). +-endif. + +%% This is the inverse case of assertMatch, for convenience. +-ifdef(NOASSERT). +-define(assertNotMatch(Guard, Expr), ok). +-else. +-define(assertNotMatch(Guard, Expr), + begin + ((fun () -> + __V = (Expr), + case __V of + Guard -> erlang:error({assertNotMatch, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??Expr)}, + {pattern, (??Guard)}, + {value, __V}]}); + _ -> ok + end + end)()) + end). +-endif. + +%% This is a convenience macro which gives more detailed reports when +%% the expected LHS value is not a pattern, but a computed value +-ifdef(NOASSERT). +-define(assertEqual(Expect, Expr), ok). +-else. +-define(assertEqual(Expect, Expr), + begin + ((fun (__X) -> + case (Expr) of + __X -> ok; + __V -> erlang:error({assertEqual, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??Expr)}, + {expected, __X}, + {value, __V}]}) + end + end)(Expect)) + end). +-endif. + +%% This is the inverse case of assertEqual, for convenience. +-ifdef(NOASSERT). +-define(assertNotEqual(Unexpected, Expr), ok). +-else. +-define(assertNotEqual(Unexpected, Expr), + begin + ((fun (__X) -> + case (Expr) of + __X -> erlang:error({assertNotEqual, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??Expr)}, + {value, __X}]}); + _ -> ok + end + end)(Unexpected)) + end). +-endif. + +%% Note: Class and Term are patterns, and can not be used for value. +%% Term can be a guarded pattern, but Class cannot. +-ifdef(NOASSERT). +-define(assertException(Class, Term, Expr), ok). +-else. +-define(assertException(Class, Term, Expr), + begin + ((fun () -> + try (Expr) of + __V -> erlang:error({assertException, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??Expr)}, + {pattern, + "{ "++(??Class)++" , "++(??Term) + ++" , [...] }"}, + {unexpected_success, __V}]}) + catch + Class:Term -> ok; + __C:__T -> + erlang:error({assertException, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??Expr)}, + {pattern, + "{ "++(??Class)++" , "++(??Term) + ++" , [...] }"}, + {unexpected_exception, + {__C, __T, + erlang:get_stacktrace()}}]}) + end + end)()) + end). +-endif. + +-define(assertError(Term, Expr), ?assertException(error, Term, Expr)). +-define(assertExit(Term, Expr), ?assertException(exit, Term, Expr)). +-define(assertThrow(Term, Expr), ?assertException(throw, Term, Expr)). + +%% This is the inverse case of assertException, for convenience. +%% Note: Class and Term are patterns, and can not be used for value. +%% Both Class and Term can be guarded patterns. +-ifdef(NOASSERT). +-define(assertNotException(Class, Term, Expr), ok). +-else. +-define(assertNotException(Class, Term, Expr), + begin + ((fun () -> + try (Expr) of + _ -> ok + catch + __C:__T -> + case __C of + Class -> + case __T of + Term -> + erlang:error({assertNotException, + [{module, ?MODULE}, + {line, ?LINE}, + {expression, (??Expr)}, + {pattern, + "{ "++(??Class)++" , " + ++(??Term)++" , [...] }"}, + {unexpected_exception, + {__C, __T, + erlang:get_stacktrace() + }}]}); + _ -> ok + end; + _ -> ok + end + end + end)()) + end). +-endif. + +-endif. % ASSERT_HRL diff --git a/lib/stdlib/src/Makefile b/lib/stdlib/src/Makefile index 55bda60da5..344a5dc099 100644 --- a/lib/stdlib/src/Makefile +++ b/lib/stdlib/src/Makefile @@ -122,6 +122,7 @@ MODULES= \ zip HRL_FILES= \ + ../include/assert.hrl \ ../include/erl_compile.hrl \ ../include/erl_bits.hrl \ ../include/ms_transform.hrl \ diff --git a/lib/stdlib/src/array.erl b/lib/stdlib/src/array.erl index 10d2ccea45..f98a587c55 100644 --- a/lib/stdlib/src/array.erl +++ b/lib/stdlib/src/array.erl @@ -164,7 +164,7 @@ elements :: elements(_) %% the tuple tree }). --opaque array() :: array(term()). +-type array() :: array(term()). -opaque array(Type) :: #array{default :: Type, elements :: elements(Type)}. diff --git a/lib/stdlib/src/c.erl b/lib/stdlib/src/c.erl index 9860adf04d..d5b24d3c32 100644 --- a/lib/stdlib/src/c.erl +++ b/lib/stdlib/src/c.erl @@ -27,7 +27,7 @@ lc_batch/0, lc_batch/1, i/3,pid/3,m/0,m/1, bt/1, q/0, - erlangrc/0,erlangrc/1,bi/1, flush/0, regs/0, + erlangrc/0,erlangrc/1,bi/1, flush/0, regs/0, uptime/0, nregs/0,pwd/0,ls/0,ls/1,cd/1,memory/1,memory/0, xm/1]). -export([display_info/1]). @@ -65,6 +65,7 @@ help() -> "q() -- quit - shorthand for init:stop()\n" "regs() -- information about registered processes\n" "nregs() -- information about all registered processes\n" + "uptime() -- print node uptime\n" "xm(M) -- cross reference check a module\n" "y(File) -- generate a Yecc parser\n">>). @@ -774,6 +775,26 @@ memory() -> erlang:memory(). memory(TypeSpec) -> erlang:memory(TypeSpec). %% +%% uptime/0 +%% + +-spec uptime() -> 'ok'. + +uptime() -> + io:format("~s~n", [uptime(get_uptime())]). + +uptime({D, {H, M, S}}) -> + lists:flatten( + [[ io_lib:format("~p days, ", [D]) || D > 0 ], + [ io_lib:format("~p hours, ", [H]) || D+H > 0 ], + [ io_lib:format("~p minutes and ", [M]) || D+H+M > 0 ], + io_lib:format("~p seconds", [S])]). + +get_uptime() -> + {UpTime, _} = erlang:statistics(wall_clock), + calendar:seconds_to_daystime(UpTime div 1000). + +%% %% Cross Reference Check %% %%-spec xm(module() | file:filename()) -> xref:m/1 return diff --git a/lib/stdlib/src/dict.erl b/lib/stdlib/src/dict.erl index 5a9f63c5e2..d2af9554a1 100644 --- a/lib/stdlib/src/dict.erl +++ b/lib/stdlib/src/dict.erl @@ -70,7 +70,7 @@ }). --opaque dict() :: dict(_, _). +-type dict() :: dict(_, _). -opaque dict(Key, Value) :: #dict{segs :: segs(Key, Value)}. diff --git a/lib/stdlib/src/erl_anno.erl b/lib/stdlib/src/erl_anno.erl index 963b7278a6..fa83375c34 100644 --- a/lib/stdlib/src/erl_anno.erl +++ b/lib/stdlib/src/erl_anno.erl @@ -147,12 +147,10 @@ is_anno2(_, _) -> false. is_filename(T) -> - is_string(T) orelse is_binary(T). + is_list(T) orelse is_binary(T). is_string(T) -> - try lists:all(fun(C) when is_integer(C), C >= 0 -> true end, T) - catch _:_ -> false - end. + is_list(T). -spec column(Anno) -> column() | 'undefined' when Anno :: anno(). diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index 821d81a6b4..b13848c501 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -120,13 +120,13 @@ value_option(Flag, Default, On, OnVal, Off, OffVal, Opts) -> func=[], %Current function warn_format=0, %Warn format calls enabled_warnings=[], %All enabled warnings (ordset). + nowarn_bif_clash=[], %All no warn bif clashes (ordset). errors=[], %Current errors warnings=[], %Current warnings file = "" :: string(), %From last file attribute recdef_top=false :: boolean(), %true in record initialisation %outside any fun or lc xqlc= false :: boolean(), %true if qlc.hrl included - new = false :: boolean(), %Has user-defined 'new/N' called= [] :: [{fa(),line()}], %Called functions usage = #usage{} :: #usage{}, specs = dict:new() %Type specifications @@ -569,6 +569,7 @@ start(File, Opts) -> warn_format = value_option(warn_format, 1, warn_format, 1, nowarn_format, 0, Opts), enabled_warnings = Enabled, + nowarn_bif_clash = nowarn_function(nowarn_bif_clash, Opts), file = File }. @@ -608,22 +609,30 @@ pack_warnings(Ws) -> %% add_warning(ErrorDescriptor, State) -> State' %% add_warning(Line, Error, State) -> State' -add_error(E, St) -> St#lint{errors=[{St#lint.file,E}|St#lint.errors]}. +add_error(E, St) -> add_lint_error(E, St#lint.file, St). add_error(Anno, E, St) -> - {File,Location} = loc(Anno), - add_error({Location,erl_lint,E}, St#lint{file = File}). + {File,Location} = loc(Anno, St), + add_lint_error({Location,erl_lint,E}, File, St). -add_warning(W, St) -> St#lint{warnings=[{St#lint.file,W}|St#lint.warnings]}. +add_lint_error(E, File, St) -> + St#lint{errors=[{File,E}|St#lint.errors]}. + +add_warning(W, St) -> add_lint_warning(W, St#lint.file, St). add_warning(FileLine, W, St) -> - {File,Location} = loc(FileLine), - add_warning({Location,erl_lint,W}, St#lint{file = File}). + {File,Location} = loc(FileLine, St), + add_lint_warning({Location,erl_lint,W}, File, St). + +add_lint_warning(W, File, St) -> + St#lint{warnings=[{File,W}|St#lint.warnings]}. -loc(Anno) -> - File = erl_anno:file(Anno), +loc(Anno, St) -> Location = erl_anno:location(Anno), - {File,Location}. + case erl_anno:file(Anno) of + undefined -> {St#lint.file,Location}; + File -> {File,Location} + end. %% forms([Form], State) -> State' @@ -640,8 +649,6 @@ forms(Forms0, St0) -> St4 = foldl(fun form/2, pre_scan(Forms, St3), Forms), post_traversal_check(Forms, St4). -pre_scan([{function,_L,new,_A,_Cs} | Fs], St) -> - pre_scan(Fs, St#lint{new=true}); pre_scan([{attribute,L,compile,C} | Fs], St) -> case is_warn_enabled(export_all, St) andalso member(export_all, lists:flatten([C])) of @@ -668,11 +675,21 @@ eval_file_attribute(Forms, St) -> eval_file_attr([{attribute,_L,file,{File,_Line}}=Form | Forms], _File) -> [Form | eval_file_attr(Forms, File)]; eval_file_attr([Form0 | Forms], File) -> - Form = set_file(Form0, File), + Form = set_form_file(Form0, File), [Form | eval_file_attr(Forms, File)]; eval_file_attr([], _File) -> []. +%% Sets the file only on the form. This is used on post-traversal. +%% For the remaining of the AST we rely on #lint.file. + +set_form_file({attribute,L,K,V}, File) -> + {attribute,erl_anno:set_file(File, L),K,V}; +set_form_file({function,L,N,A,C}, File) -> + {function,erl_anno:set_file(File, L),N,A,C}; +set_form_file(Form, _File) -> + Form. + set_file(T, File) -> F = fun(Anno) -> erl_anno:set_file(File, Anno) end, erl_parse:map_anno(F, T). @@ -772,8 +789,7 @@ eof(_Line, St0) -> %% bif_clashes(Forms, State0) -> State. -bif_clashes(Forms, St) -> - Nowarn = nowarn_function(nowarn_bif_clash, St#lint.compile), +bif_clashes(Forms, #lint{nowarn_bif_clash=Nowarn} = St) -> Clashes0 = [{Name,Arity} || {function,_L,Name,Arity,_Cs} <- Forms, erl_internal:bif(Name, Arity)], Clashes = ordsets:subtract(ordsets:from_list(Clashes0), Nowarn), @@ -798,10 +814,10 @@ disallowed_compile_flags(Forms, St0) -> %% There are (still) no line numbers in St0#lint.compile. Errors0 = [ {St0#lint.file,{L,erl_lint,disallowed_nowarn_bif_clash}} || {attribute,A,compile,nowarn_bif_clash} <- Forms, - {_,L} <- [loc(A)] ], + {_,L} <- [loc(A, St0)] ], Errors1 = [ {St0#lint.file,{L,erl_lint,disallowed_nowarn_bif_clash}} || {attribute,A,compile,{nowarn_bif_clash, {_,_}}} <- Forms, - {_,L} <- [loc(A)] ], + {_,L} <- [loc(A, St0)] ], Disabled = (not is_warn_enabled(bif_clash, St0)), Errors = if Disabled andalso Errors0 =:= [] -> @@ -926,7 +942,7 @@ behaviour_conflicting(AllBfs, St) -> behaviour_add_conflicts(R, St). behaviour_add_conflicts([{Cb,[{FirstLoc,FirstB}|Cs]}|T], St0) -> - FirstL = element(2, loc(FirstLoc)), + FirstL = element(2, loc(FirstLoc, St0)), St = behaviour_add_conflict(Cs, Cb, FirstL, FirstB, St0), behaviour_add_conflicts(T, St); behaviour_add_conflicts([], St) -> St. @@ -1144,7 +1160,7 @@ check_unused_records(Forms, St0) -> end, St0#lint.records, UsedRecords), Unused = [{Name,FileLine} || {Name,{FileLine,_Fields}} <- dict:to_list(URecs), - element(1, loc(FileLine)) =:= FirstFile], + element(1, loc(FileLine, St0)) =:= FirstFile], foldl(fun ({N,L}, St) -> add_warning(L, {unused_record, N}, St) end, St0, Unused); @@ -1337,14 +1353,15 @@ check_on_load(St) -> St. -spec call_function(line(), atom(), arity(), lint_state()) -> lint_state(). %% Add to both called and calls. -call_function(Line, F, A, #lint{usage=Usage0,called=Cd,func=Func}=St) -> +call_function(Line, F, A, #lint{usage=Usage0,called=Cd,func=Func,file=File}=St) -> #usage{calls = Cs} = Usage0, NA = {F,A}, Usage = case Cs of undefined -> Usage0; _ -> Usage0#usage{calls=dict:append(Func, NA, Cs)} end, - St#lint{called=[{NA,Line}|Cd], usage=Usage}. + Anno = erl_anno:set_file(File, Line), + St#lint{called=[{NA,Anno}|Cd], usage=Usage}. %% function(Line, Name, Arity, Clauses, State) -> State. @@ -2123,7 +2140,7 @@ expr({'receive',Line,Cs,To,ToEs}, Vt, St0) -> {Cvt,St3} = icrt_clauses(Cs, Vt, St2), %% Csvts = [vtnew(Tevt, Vt)|Cvt], %This is just NEW variables! Csvts = [Tevt|Cvt], - Rvt = icrt_export(Csvts, Vt, {'receive',Line}), + Rvt = icrt_export(Csvts, Vt, {'receive',Line}, St3), {vtmerge([Tvt,Tevt,Rvt]),St3}; expr({'fun',Line,Body}, Vt, St) -> %%No one can think funs export! @@ -2826,10 +2843,9 @@ check_record_types([{type, _, field_type, [{atom, AL, FName}, Type]}|Left], check_record_types([], _Name, _DefFields, SeenVars, St, _SeenFields) -> {SeenVars, St}. -used_type(TypePair, L, St) -> - Usage = St#lint.usage, +used_type(TypePair, L, #lint{usage = Usage, file = File} = St) -> OldUsed = Usage#usage.used_types, - UsedTypes = dict:store(TypePair, L, OldUsed), + UsedTypes = dict:store(TypePair, erl_anno:set_file(File, L), OldUsed), St#lint{usage=Usage#usage{used_types=UsedTypes}}. is_default_type({Name, NumberOfTypeVariables}) -> @@ -2984,7 +3000,7 @@ check_unused_types(Forms, #lint{usage=Usage, types=Ts, exp_types=ExpTs}=St) -> UsedTypes = gb_sets:from_list(L), FoldFun = fun(Type, #typeinfo{line = FileLine}, AccSt) -> - case loc(FileLine) of + case loc(FileLine, AccSt) of {FirstFile, _} -> case gb_sets:is_member(Type, UsedTypes) of true -> AccSt; @@ -3022,7 +3038,7 @@ check_local_opaque_types(St) -> icrt_clauses(Cs, In, Vt, St0) -> {Csvt,St1} = icrt_clauses(Cs, Vt, St0), - UpdVt = icrt_export(Csvt, Vt, In), + UpdVt = icrt_export(Csvt, Vt, In, St1), {UpdVt,St1}. %% icrt_clauses(Clauses, ImportVarTable, State) -> @@ -3039,8 +3055,8 @@ icrt_clause({clause,_Line,H,G,B}, Vt0, St0) -> {Bvt,St3} = exprs(B, vtupdate(Vt2, Vt0), St2), {vtupdate(Bvt, Vt2),St3}. -icrt_export(Vts, Vt, {Tag,Attrs}) -> - {_File,Loc} = loc(Attrs), +icrt_export(Vts, Vt, {Tag,Attrs}, St) -> + {_File,Loc} = loc(Attrs, St), icrt_export(lists:merge(Vts), Vt, {Tag,Loc}, length(Vts), []). icrt_export([{V,{{export,_},_,_}}|Vs0], [{V,{{export,_}=S0,_,Ls}}|Vt], @@ -3397,7 +3413,7 @@ vtupdate(Uvt, Vt0) -> %% Return all new variables in UpdVarTable as unsafe. vtunsafe({Tag,FileLine}, Uvt, Vt) -> - {_File,Line} = loc(FileLine), + Line = erl_anno:location(FileLine), [{V,{{unsafe,{Tag,Line}},U,Ls}} || {V,{_,U,Ls}} <- vtnew(Uvt, Vt)]. %% vtmerge(VarTable, VarTable) -> VarTable. @@ -3781,8 +3797,7 @@ is_autoimport_suppressed(NoAutoSet,{Func,Arity}) -> gb_sets:is_element({Func,Arity},NoAutoSet). %% Predicate to find out if a function specific bif-clash suppression (old deprecated) is present bif_clash_specifically_disabled(St,{F,A}) -> - Nowarn = nowarn_function(nowarn_bif_clash, St#lint.compile), - lists:member({F,A},Nowarn). + lists:member({F,A},St#lint.nowarn_bif_clash). %% Predicate to find out if an autoimported guard_bif is not overriden in some way %% Guard Bif without module name is disallowed if diff --git a/lib/stdlib/src/erl_parse.yrl b/lib/stdlib/src/erl_parse.yrl index e328e065e3..274bb2a782 100644 --- a/lib/stdlib/src/erl_parse.yrl +++ b/lib/stdlib/src/erl_parse.yrl @@ -125,22 +125,19 @@ top_type_100 -> type_200 : '$1'. top_type_100 -> type_200 '|' top_type_100 : lift_unions('$1','$3'). type_200 -> type_300 '..' type_300 : {type, ?anno('$1'), range, - [skip_paren('$1'), - skip_paren('$3')]}. + ['$1', '$3']}. type_200 -> type_300 : '$1'. -type_300 -> type_300 add_op type_400 : ?mkop2(skip_paren('$1'), - '$2', skip_paren('$3')). +type_300 -> type_300 add_op type_400 : ?mkop2('$1', '$2', '$3'). type_300 -> type_400 : '$1'. -type_400 -> type_400 mult_op type_500 : ?mkop2(skip_paren('$1'), - '$2', skip_paren('$3')). +type_400 -> type_400 mult_op type_500 : ?mkop2('$1', '$2', '$3'). type_400 -> type_500 : '$1'. -type_500 -> prefix_op type : ?mkop1('$1', skip_paren('$2')). +type_500 -> prefix_op type : ?mkop1('$1', '$2'). type_500 -> type : '$1'. -type -> '(' top_type ')' : {paren_type, ?anno('$2'), ['$2']}. +type -> '(' top_type ')' : '$2'. type -> var : '$1'. type -> atom : '$1'. type -> atom '(' ')' : build_gen_type('$1'). @@ -524,6 +521,7 @@ Erlang code. -export([normalise/1,abstract/1,tokens/1,tokens/2]). -export([abstract/2]). -export([inop_prec/1,preop_prec/1,func_prec/0,max_prec/0]). +-export([type_inop_prec/1,type_preop_prec/1]). -export([map_anno/2, fold_anno/3, mapfold_anno/3, new_anno/1, anno_to_term/1, anno_from_term/1]). -export([set_line/2,get_attribute/2,get_attributes/1]). @@ -671,11 +669,6 @@ lift_unions(T1, {type, _Aa, union, List}) -> lift_unions(T1, T2) -> {type, ?anno(T1), union, [T1, T2]}. -skip_paren({paren_type,_A,[Type]}) -> - skip_paren(Type); -skip_paren(Type) -> - Type. - build_gen_type({atom, Aa, tuple}) -> {type, Aa, tuple, any}; build_gen_type({atom, Aa, map}) -> @@ -687,7 +680,7 @@ build_gen_type({atom, Aa, Name}) -> build_bin_type([{var, _, '_'}|Left], Int) -> build_bin_type(Left, Int); build_bin_type([], Int) -> - skip_paren(Int); + Int; build_bin_type([{var, Aa, _}|_], _) -> ret_err(Aa, "Bad binary type"). @@ -807,8 +800,7 @@ record_fields([{typed,Expr,TypeInfo}|Fields]) -> {atom, Aa, _} -> case has_undefined(TypeInfo) of false -> - TypeInfo2 = maybe_add_paren(TypeInfo), - lift_unions(abstract2(undefined, Aa), TypeInfo2); + lift_unions(abstract2(undefined, Aa), TypeInfo); true -> TypeInfo end @@ -822,18 +814,11 @@ has_undefined({atom,_,undefined}) -> true; has_undefined({ann_type,_,[_,T]}) -> has_undefined(T); -has_undefined({paren_type,_,[T]}) -> - has_undefined(T); has_undefined({type,_,union,Ts}) -> lists:any(fun has_undefined/1, Ts); has_undefined(_) -> false. -maybe_add_paren({ann_type,A,T}) -> - {paren_type,A,[{ann_type,A,T}]}; -maybe_add_paren(T) -> - T. - term(Expr) -> try normalise(Expr) catch _:_R -> ret_err(?anno(Expr), "bad attribute") @@ -1099,6 +1084,39 @@ func_prec() -> {800,700}. max_prec() -> 900. +-type prec() :: non_neg_integer(). + +-type type_inop() :: '::' | '|' | '..' | '+' | '-' | 'bor' | 'bxor' + | 'bsl' | 'bsr' | '*' | '/' | 'div' | 'rem' | 'band'. + +-type type_preop() :: '+' | '-' | 'bnot' | '#'. + +-spec type_inop_prec(type_inop()) -> {prec(), prec(), prec()}. + +type_inop_prec('=') -> {150,100,100}; +type_inop_prec('::') -> {160,150,150}; +type_inop_prec('|') -> {180,170,170}; +type_inop_prec('..') -> {300,200,300}; +type_inop_prec('+') -> {400,400,500}; +type_inop_prec('-') -> {400,400,500}; +type_inop_prec('bor') -> {400,400,500}; +type_inop_prec('bxor') -> {400,400,500}; +type_inop_prec('bsl') -> {400,400,500}; +type_inop_prec('bsr') -> {400,400,500}; +type_inop_prec('*') -> {500,500,600}; +type_inop_prec('/') -> {500,500,600}; +type_inop_prec('div') -> {500,500,600}; +type_inop_prec('rem') -> {500,500,600}; +type_inop_prec('band') -> {500,500,600}; +type_inop_prec('#') -> {800,700,800}. + +-spec type_preop_prec(type_preop()) -> {prec(), prec()}. + +type_preop_prec('+') -> {600,700}; +type_preop_prec('-') -> {600,700}; +type_preop_prec('bnot') -> {600,700}; +type_preop_prec('#') -> {700,800}. + %%% [Experimental]. The parser just copies the attributes of the %%% scanner tokens to the abstract format. This design decision has %%% been hidden to some extent: use set_line() and get_attribute() to diff --git a/lib/stdlib/src/erl_pp.erl b/lib/stdlib/src/erl_pp.erl index 623a29f923..6da585b72e 100644 --- a/lib/stdlib/src/erl_pp.erl +++ b/lib/stdlib/src/erl_pp.erl @@ -27,7 +27,8 @@ -import(lists, [append/1,foldr/3,mapfoldl/3,reverse/1,reverse/2]). -import(io_lib, [write/1,format/2]). --import(erl_parse, [inop_prec/1,preop_prec/1,func_prec/0,max_prec/0]). +-import(erl_parse, [inop_prec/1,preop_prec/1,func_prec/0,max_prec/0, + type_inop_prec/1, type_preop_prec/1]). -define(MAXLINE, 72). @@ -271,49 +272,64 @@ typeattr(Tag, {TypeName,Type,Args}, _Opts) -> {first,leaf("-"++atom_to_list(Tag)++" "), typed(call({atom,a0(),TypeName}, Args, 0, options(none)), Type)}. -ltype({ann_type,_Line,[V,T]}) -> - typed(lexpr(V, options(none)), T); -ltype({paren_type,_Line,[T]}) -> - [$(,ltype(T),$)]; -ltype({type,_Line,union,Ts}) -> - {seq,[],[],[' |'],ltypes(Ts)}; -ltype({type,_Line,list,[T]}) -> +ltype(T) -> + ltype(T, 0). + +ltype({ann_type,_Line,[V,T]}, Prec) -> + {_L,P,_R} = type_inop_prec('::'), + E = typed(lexpr(V, options(none)), T), + maybe_paren(P, Prec, E); +ltype({paren_type,_Line,[T]}, P) -> + %% Generated before Erlang/OTP 18. + ltype(T, P); +ltype({type,_Line,union,Ts}, Prec) -> + {_L,P,R} = type_inop_prec('|'), + E = {seq,[],[],[' |'],ltypes(Ts, R)}, + maybe_paren(P, Prec, E); +ltype({type,_Line,list,[T]}, _) -> {seq,$[,$],$,,[ltype(T)]}; -ltype({type,_Line,nonempty_list,[T]}) -> +ltype({type,_Line,nonempty_list,[T]}, _) -> {seq,$[,$],[$,],[ltype(T),leaf("...")]}; -ltype({type,Line,nil,[]}) -> - lexpr({nil,Line}, 0, options(none)); -ltype({type,Line,map,any}) -> +ltype({type,Line,nil,[]}, _) -> + lexpr({nil,Line}, options(none)); +ltype({type,Line,map,any}, _) -> simple_type({atom,Line,map}, []); -ltype({type,_Line,map,Pairs}) -> - map_type(Pairs); -ltype({type,Line,tuple,any}) -> +ltype({type,_Line,map,Pairs}, Prec) -> + {P,_R} = type_preop_prec('#'), + E = map_type(Pairs), + maybe_paren(P, Prec, E); +ltype({type,Line,tuple,any}, _) -> simple_type({atom,Line,tuple}, []); -ltype({type,_Line,tuple,Ts}) -> - tuple_type(Ts, fun ltype/1); -ltype({type,_Line,record,[{atom,_,N}|Fs]}) -> - record_type(N, Fs); -ltype({type,_Line,range,[_I1,_I2]=Es}) -> - expr_list(Es, '..', fun lexpr/2, options(none)); -ltype({type,_Line,binary,[I1,I2]}) -> +ltype({type,_Line,tuple,Ts}, _) -> + tuple_type(Ts, fun ltype/2); +ltype({type,_Line,record,[{atom,_,N}|Fs]}, Prec) -> + {P,_R} = type_preop_prec('#'), + E = record_type(N, Fs), + maybe_paren(P, Prec, E); +ltype({type,_Line,range,[_I1,_I2]=Es}, Prec) -> + {_L,P,R} = type_inop_prec('..'), + F = fun(E, Opts) -> lexpr(E, R, Opts) end, + E = expr_list(Es, '..', F, options(none)), + maybe_paren(P, Prec, E); +ltype({type,_Line,binary,[I1,I2]}, _) -> binary_type(I1, I2); % except binary() -ltype({type,_Line,'fun',[]}) -> +ltype({type,_Line,'fun',[]}, _) -> leaf("fun()"); -ltype({type,_,'fun',[{type,_,any},_]}=FunType) -> +ltype({type,_,'fun',[{type,_,any},_]}=FunType, _) -> [fun_type(['fun',$(], FunType),$)]; -ltype({type,_Line,'fun',[{type,_,product,_},_]}=FunType) -> +ltype({type,_Line,'fun',[{type,_,product,_},_]}=FunType, _) -> [fun_type(['fun',$(], FunType),$)]; -ltype({type,Line,T,Ts}) -> +ltype({type,Line,T,Ts}, _) -> %% Compatibility. Before 18.0. simple_type({atom,Line,T}, Ts); -ltype({user_type,Line,T,Ts}) -> +ltype({user_type,Line,T,Ts}, _) -> simple_type({atom,Line,T}, Ts); -ltype({remote_type,Line,[M,F,Ts]}) -> +ltype({remote_type,Line,[M,F,Ts]}, _) -> simple_type({remote,Line,M,F}, Ts); -ltype({atom,_,T}) -> +ltype({atom,_,T}, _) -> leaf(write(T)); -ltype(E) -> - lexpr(E, 0, options(none)). +ltype(E, P) -> + lexpr(E, P, options(none)). binary_type(I1, I2) -> B = [[] || {integer,_,0} <- [I1]] =:= [], @@ -327,42 +343,37 @@ map_type(Fs) -> {first,[$#],map_pair_types(Fs)}. map_pair_types(Fs) -> - tuple_type(Fs, fun map_pair_type/1). + tuple_type(Fs, fun map_pair_type/2). -map_pair_type({type,_Line,map_field_assoc,[Ktype,Vtype]}) -> - map_assoc_typed(ltype(Ktype), Vtype). +map_pair_type({type,_Line,map_field_assoc,[Ktype,Vtype]}, Prec) -> + map_assoc_typed(ltype(Ktype), Vtype, Prec). -map_assoc_typed(B, {type,_,union,Ts}) -> - {first,[B,$\s],{seq,[],[],[],map_assoc_union_type(Ts)}}; -map_assoc_typed(B, Type) -> - {list,[{cstep,[B," =>"],ltype(Type)}]}. +map_assoc_typed(B, {type,_,union,Ts}, Prec) -> + {first,[B,$\s],{seq,[],[],[],map_assoc_union_type(Ts, Prec)}}; +map_assoc_typed(B, Type, Prec) -> + {list,[{cstep,[B," =>"],ltype(Type, Prec)}]}. -map_assoc_union_type([T|Ts]) -> - [[leaf("=> "),ltype(T)] | ltypes(Ts, fun union_elem/1)]. +map_assoc_union_type([T|Ts], Prec) -> + [[leaf("=> "),ltype(T)] | ltypes(Ts, fun union_elem/2, Prec)]. record_type(Name, Fields) -> {first,[record_name(Name)],field_types(Fields)}. field_types(Fs) -> - tuple_type(Fs, fun field_type/1). + tuple_type(Fs, fun field_type/2). -field_type({type,_Line,field_type,[Name,Type]}) -> +field_type({type,_Line,field_type,[Name,Type]}, _Prec) -> typed(lexpr(Name, options(none)), Type). -typed(B, {type,_,union,Ts}) -> - %% Special layout for :: followed by union. - {first,[B,$\s],{seq,[],[],[],union_type(Ts)}}; typed(B, Type) -> - {list,[{cstep,[B,' ::'],ltype(Type)}]}. + {_L,_P,R} = type_inop_prec('::'), + {list,[{cstep,[B,' ::'],ltype(Type, R)}]}. -union_type([T|Ts]) -> - [[leaf(":: "),ltype(T)] | ltypes(Ts, fun union_elem/1)]. - -union_elem(T) -> - [leaf(" | "),ltype(T)]. +union_elem(T, Prec) -> + [leaf(" | "),ltype(T, Prec)]. tuple_type(Ts, F) -> - {seq,${,$},[$,],ltypes(Ts, F)}. + {seq,${,$},[$,],ltypes(Ts, F, 0)}. specattr(SpecKind, {FuncSpec,TypeSpecs}) -> Func = case FuncSpec of @@ -399,16 +410,16 @@ type_args({type,_line,product,Ts}) -> targs(Ts). simple_type(Tag, Types) -> - {first,lexpr(Tag, 0, options(none)),targs(Types)}. + {first,lexpr(Tag, options(none)),targs(Types)}. targs(Ts) -> - {seq,$(,$),[$,],ltypes(Ts)}. + {seq,$(,$),[$,],ltypes(Ts, 0)}. -ltypes(Ts) -> - ltypes(Ts, fun ltype/1). +ltypes(Ts, Prec) -> + ltypes(Ts, fun ltype/2, Prec). -ltypes(Ts, F) -> - [F(T) || T <- Ts]. +ltypes(Ts, F, Prec) -> + [F(T, Prec) || T <- Ts]. attr(Name, Args) -> call({var,a0(),format("-~s", [Name])}, Args, 0, options(none)). diff --git a/lib/stdlib/src/ets.erl b/lib/stdlib/src/ets.erl index 1df069755d..bec64861c3 100644 --- a/lib/stdlib/src/ets.erl +++ b/lib/stdlib/src/ets.erl @@ -739,7 +739,8 @@ do_filter(Tab, Key, F, A, Ack) -> -record(filetab_options, { object_count = false :: boolean(), - md5sum = false :: boolean() + md5sum = false :: boolean(), + sync = false :: boolean() }). -spec tab2file(Tab, Filename) -> 'ok' | {'error', Reason} when @@ -754,7 +755,7 @@ tab2file(Tab, File) -> Tab :: tab(), Filename :: file:name(), Options :: [Option], - Option :: {'extended_info', [ExtInfo]}, + Option :: {'extended_info', [ExtInfo]} | {'sync', boolean()}, ExtInfo :: 'md5sum' | 'object_count', Reason :: term(). @@ -835,6 +836,15 @@ tab2file(Tab, File, Options) -> List -> LogFun(NewState1,[['$end_of_table',List]]) end, + case FtOptions#filetab_options.sync of + true -> + case disk_log:sync(Name) of + ok -> ok; + {error, Reason2} -> throw(Reason2) + end; + false -> + ok + end, disk_log:close(Name) catch throw:TReason -> @@ -887,23 +897,24 @@ md5terms(State, [H|T]) -> {FinState, [B|TL]}. parse_ft_options(Options) when is_list(Options) -> - {Opt,Rest} = case (catch lists:keytake(extended_info,1,Options)) of - false -> - {[],Options}; - {value,{extended_info,L},R} when is_list(L) -> - {L,R} - end, - case Rest of - [] -> - parse_ft_info_options(#filetab_options{}, Opt); - Other -> - throw({unknown_option, Other}) - end; -parse_ft_options(Malformed) -> + {ok, parse_ft_options(Options, #filetab_options{}, false)}. + +parse_ft_options([], FtOpt, _) -> + FtOpt; +parse_ft_options([{sync,true} | Rest], FtOpt, EI) -> + parse_ft_options(Rest, FtOpt#filetab_options{sync = true}, EI); +parse_ft_options([{sync,false} | Rest], FtOpt, EI) -> + parse_ft_options(Rest, FtOpt, EI); +parse_ft_options([{extended_info,L} | Rest], FtOpt0, false) -> + FtOpt1 = parse_ft_info_options(FtOpt0, L), + parse_ft_options(Rest, FtOpt1, true); +parse_ft_options([Other | _], _, _) -> + throw({unknown_option, Other}); +parse_ft_options(Malformed, _, _) -> throw({malformed_option, Malformed}). parse_ft_info_options(FtOpt,[]) -> - {ok,FtOpt}; + FtOpt; parse_ft_info_options(FtOpt,[object_count | T]) -> parse_ft_info_options(FtOpt#filetab_options{object_count = true}, T); parse_ft_info_options(FtOpt,[md5sum | T]) -> @@ -1297,18 +1308,30 @@ create_tab(I, TabArg) -> {name, Name} = lists:keyfind(name, 1, I), {type, Type} = lists:keyfind(type, 1, I), {protection, P} = lists:keyfind(protection, 1, I), - {named_table, Val} = lists:keyfind(named_table, 1, I), {keypos, _Kp} = Keypos = lists:keyfind(keypos, 1, I), {size, Sz} = lists:keyfind(size, 1, I), - Comp = case lists:keyfind(compressed, 1, I) of - {compressed, true} -> [compressed]; - {compressed, false} -> []; - false -> [] - end, + L1 = [Type, P, Keypos], + L2 = case lists:keyfind(named_table, 1, I) of + {named_table, true} -> [named_table | L1]; + {named_table, false} -> L1 + end, + L3 = case lists:keyfind(compressed, 1, I) of + {compressed, true} -> [compressed | L2]; + {compressed, false} -> L2; + false -> L2 + end, + L4 = case lists:keyfind(write_concurrency, 1, I) of + {write_concurrency, _}=Wcc -> [Wcc | L3]; + _ -> L3 + end, + L5 = case lists:keyfind(read_concurrency, 1, I) of + {read_concurrency, _}=Rcc -> [Rcc | L4]; + false -> L4 + end, case TabArg of [] -> try - Tab = ets:new(Name, [Type, P, Keypos] ++ named_table(Val) ++ Comp), + Tab = ets:new(Name, L5), {ok, Tab, Sz} catch _:_ -> throw(cannot_create_table) @@ -1317,8 +1340,6 @@ create_tab(I, TabArg) -> {ok, TabArg, Sz} end. -named_table(true) -> [named_table]; -named_table(false) -> []. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% tabfile_info/1 reads the head information in an ets table dumped to diff --git a/lib/stdlib/src/gb_sets.erl b/lib/stdlib/src/gb_sets.erl index 393fb07229..d099737d8f 100644 --- a/lib/stdlib/src/gb_sets.erl +++ b/lib/stdlib/src/gb_sets.erl @@ -137,6 +137,10 @@ %% approach is that it does not require the complete list of all %% elements to be built in memory at one time. %% +%% - iterator_from(X, S): returns an iterator that can be used for +%% traversing the elements of set S greater than or equal to X; +%% see `next'. +%% %% - next(T): returns {X, T1} where X is the smallest element referred %% to by the iterator T, and T1 is the new iterator to be used for %% traversing the remaining elements, or the atom `none' if no @@ -157,8 +161,8 @@ insert/2, add/2, delete/2, delete_any/2, balance/1, union/2, union/1, intersection/2, intersection/1, is_disjoint/2, difference/2, is_subset/2, to_list/1, from_list/1, from_ordset/1, smallest/1, - largest/1, take_smallest/1, take_largest/1, iterator/1, next/1, - filter/2, fold/3, is_set/1]). + largest/1, take_smallest/1, take_largest/1, iterator/1, + iterator_from/2, next/1, filter/2, fold/3, is_set/1]). %% `sets' compatibility aliases: @@ -199,11 +203,10 @@ -export_type([set/0, set/1, iter/0, iter/1]). -type gb_set_node(Element) :: 'nil' | {Element, _, _}. --type gb_set_node() :: gb_set_node(_). -opaque set(Element) :: {non_neg_integer(), gb_set_node(Element)}. --opaque set() :: set(_). +-type set() :: set(_). -opaque iter(Element) :: [gb_set_node(Element)]. --opaque iter() :: [gb_set_node()]. +-type iter() :: iter(_). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -500,6 +503,22 @@ iterator({_, L, _} = T, As) -> iterator(nil, As) -> As. +-spec iterator_from(Element, Set) -> Iter when + Set :: set(Element), + Iter :: iter(Element). + +iterator_from(S, {_, T}) -> + iterator_from(S, T, []). + +iterator_from(S, {K, _, T}, As) when K < S -> + iterator_from(S, T, As); +iterator_from(_, {_, nil, _} = T, As) -> + [T | As]; +iterator_from(S, {_, L, _} = T, As) -> + iterator_from(S, L, [T | As]); +iterator_from(_, nil, As) -> + As. + -spec next(Iter1) -> {Element, Iter2} | 'none' when Iter1 :: iter(Element), Iter2 :: iter(Element). diff --git a/lib/stdlib/src/gb_trees.erl b/lib/stdlib/src/gb_trees.erl index 7069b61873..2cbfd8fd2a 100644 --- a/lib/stdlib/src/gb_trees.erl +++ b/lib/stdlib/src/gb_trees.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2014. All Rights Reserved. +%% Copyright Ericsson AB 2001-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -102,6 +102,10 @@ %% approach is that it does not require the complete list of all %% elements to be built in memory at one time. %% +%% - iterator_from(K, T): returns an iterator that can be used for +%% traversing the entries of tree T with key greater than or +%% equal to K; see `next'. +%% %% - next(S): returns {X, V, S1} where X is the smallest key referred to %% by the iterator S, and S1 is the new iterator to be used for %% traversing the remaining entries, or the atom `none' if no entries @@ -117,7 +121,7 @@ update/3, enter/3, delete/2, delete_any/2, balance/1, is_defined/2, keys/1, values/1, to_list/1, from_orddict/1, smallest/1, largest/1, take_smallest/1, take_largest/1, - iterator/1, next/1, map/2]). + iterator/1, iterator_from/2, next/1, map/2]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -156,11 +160,10 @@ -type gb_tree_node(K, V) :: 'nil' | {K, V, gb_tree_node(K, V), gb_tree_node(K, V)}. --type gb_tree_node() :: gb_tree_node(_, _). -opaque tree(Key, Value) :: {non_neg_integer(), gb_tree_node(Key, Value)}. --opaque tree() :: tree(_, _). +-type tree() :: tree(_, _). -opaque iter(Key, Value) :: [gb_tree_node(Key, Value)]. --opaque iter() :: [gb_tree_node()]. +-type iter() :: iter(_, _). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -529,6 +532,29 @@ iterator({_, _, L, _} = T, As) -> iterator(nil, As) -> As. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +-spec iterator_from(Key, Tree) -> Iter when + Tree :: tree(Key, Value), + Iter :: iter(Key, Value). + +iterator_from(S, {_, T}) -> + iterator_1_from(S, T). + +iterator_1_from(S, T) -> + iterator_from(S, T, []). + +iterator_from(S, {K, _, _, T}, As) when K < S -> + iterator_from(S, T, As); +iterator_from(_, {_, _, nil, _} = T, As) -> + [T | As]; +iterator_from(S, {_, _, L, _} = T, As) -> + iterator_from(S, L, [T | As]); +iterator_from(_, nil, As) -> + As. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + -spec next(Iter1) -> 'none' | {Key, Value, Iter2} when Iter1 :: iter(Key, Value), Iter2 :: iter(Key, Value). diff --git a/lib/stdlib/src/maps.erl b/lib/stdlib/src/maps.erl index 3877c150ec..533ff08726 100644 --- a/lib/stdlib/src/maps.erl +++ b/lib/stdlib/src/maps.erl @@ -19,7 +19,8 @@ -module(maps). --export([get/3,fold/3, map/2, size/1, +-export([get/3,filter/2,fold/3, map/2, + size/1, without/2, with/2]). @@ -145,6 +146,19 @@ get(Key,Map,Default) -> erlang:error({badmap,Map},[Key,Map,Default]). +-spec filter(Pred,Map1) -> Map2 when + Pred :: fun((Key, Value) -> boolean()), + Key :: term(), + Value :: term(), + Map1 :: map(), + Map2 :: map(). + +filter(Pred,Map) when is_function(Pred,2), is_map(Map) -> + maps:from_list([{K,V}||{K,V}<-maps:to_list(Map),Pred(K,V)]); +filter(Pred,Map) -> + erlang:error(error_type(Map),[Pred,Map]). + + -spec fold(Fun,Init,Map) -> Acc when Fun :: fun((K, V, AccIn) -> AccOut), Init :: term(), @@ -169,10 +183,7 @@ fold(Fun,Init,Map) -> V2 :: term(). map(Fun,Map) when is_function(Fun, 2), is_map(Map) -> - maps:from_list(lists:map(fun - ({K,V}) -> - {K,Fun(K,V)} - end,maps:to_list(Map))); + maps:from_list([{K,Fun(K,V)}||{K,V}<-maps:to_list(Map)]); map(Fun,Map) -> erlang:error(error_type(Map),[Fun,Map]). diff --git a/lib/stdlib/src/orddict.erl b/lib/stdlib/src/orddict.erl index af5d917840..cbdf25d757 100644 --- a/lib/stdlib/src/orddict.erl +++ b/lib/stdlib/src/orddict.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -25,11 +25,13 @@ -export([store/3,append/3,append_list/3,update/3,update/4,update_counter/3]). -export([fold/3,map/2,filter/2,merge/3]). --export_type([orddict/0]). +-export_type([orddict/0, orddict/2]). %%--------------------------------------------------------------------------- --type orddict() :: [{Key :: term(), Value :: term()}]. +-type orddict() :: orddict(_, _). + +-type orddict(Key, Value) :: [{Key, Value}]. %%--------------------------------------------------------------------------- @@ -38,8 +40,7 @@ new() -> []. -spec is_key(Key, Orddict) -> boolean() when - Key :: term(), - Orddict :: orddict(). + Orddict :: orddict(Key, Value :: term()). is_key(Key, [{K,_}|_]) when Key < K -> false; is_key(Key, [{K,_}|Dict]) when Key > K -> is_key(Key, Dict); @@ -47,14 +48,14 @@ is_key(_Key, [{_K,_Val}|_]) -> true; %Key == K is_key(_, []) -> false. -spec to_list(Orddict) -> List when - Orddict :: orddict(), - List :: [{Key :: term(), Value :: term()}]. + Orddict :: orddict(Key, Value), + List :: [{Key, Value}]. to_list(Dict) -> Dict. -spec from_list(List) -> Orddict when - List :: [{Key :: term(), Value :: term()}], - Orddict :: orddict(). + List :: [{Key, Value}], + Orddict :: orddict(Key, Value). from_list([]) -> []; from_list([{_,_}]=Pair) -> Pair; @@ -73,17 +74,13 @@ is_empty([]) -> true; is_empty([_|_]) -> false. -spec fetch(Key, Orddict) -> Value when - Key :: term(), - Value :: term(), - Orddict :: orddict(). + Orddict :: orddict(Key, Value). fetch(Key, [{K,_}|D]) when Key > K -> fetch(Key, D); fetch(Key, [{K,Value}|_]) when Key == K -> Value. -spec find(Key, Orddict) -> {'ok', Value} | 'error' when - Key :: term(), - Orddict :: orddict(), - Value :: term(). + Orddict :: orddict(Key, Value). find(Key, [{K,_}|_]) when Key < K -> error; find(Key, [{K,_}|D]) when Key > K -> find(Key, D); @@ -91,17 +88,16 @@ find(_Key, [{_K,Value}|_]) -> {ok,Value}; %Key == K find(_, []) -> error. -spec fetch_keys(Orddict) -> Keys when - Orddict :: orddict(), - Keys :: [term()]. + Orddict :: orddict(Key, Value :: term()), + Keys :: [Key]. fetch_keys([{Key,_}|Dict]) -> [Key|fetch_keys(Dict)]; fetch_keys([]) -> []. -spec erase(Key, Orddict1) -> Orddict2 when - Key :: term(), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value). erase(Key, [{K,_}=E|Dict]) when Key < K -> [E|Dict]; erase(Key, [{K,_}=E|Dict]) when Key > K -> @@ -110,10 +106,8 @@ erase(_Key, [{_K,_Val}|Dict]) -> Dict; %Key == K erase(_, []) -> []. -spec store(Key, Value, Orddict1) -> Orddict2 when - Key :: term(), - Value :: term(), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value). store(Key, New, [{K,_}|_]=Dict) when Key < K -> [{Key,New}|Dict]; @@ -124,10 +118,8 @@ store(Key, New, [{_K,_Old}|Dict]) -> %Key == K store(Key, New, []) -> [{Key,New}]. -spec append(Key, Value, Orddict1) -> Orddict2 when - Key :: term(), - Value :: term(), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value). append(Key, New, [{K,_}|_]=Dict) when Key < K -> [{Key,[New]}|Dict]; @@ -138,10 +130,9 @@ append(Key, New, [{_K,Old}|Dict]) -> %Key == K append(Key, New, []) -> [{Key,[New]}]. -spec append_list(Key, ValList, Orddict1) -> Orddict2 when - Key :: term(), - ValList :: [Value :: term()], - Orddict1 :: orddict(), - Orddict2 :: orddict(). + ValList :: [Value], + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value). append_list(Key, NewList, [{K,_}|_]=Dict) when Key < K -> [{Key,NewList}|Dict]; @@ -153,10 +144,9 @@ append_list(Key, NewList, []) -> [{Key,NewList}]. -spec update(Key, Fun, Orddict1) -> Orddict2 when - Key :: term(), - Fun :: fun((Value1 :: term()) -> Value2 :: term()), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Fun :: fun((Value1 :: Value) -> Value2 :: Value), + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value). update(Key, Fun, [{K,_}=E|Dict]) when Key > K -> [E|update(Key, Fun, Dict)]; @@ -164,11 +154,10 @@ update(Key, Fun, [{K,Val}|Dict]) when Key == K -> [{Key,Fun(Val)}|Dict]. -spec update(Key, Fun, Initial, Orddict1) -> Orddict2 when - Key :: term(), - Initial :: term(), - Fun :: fun((Value1 :: term()) -> Value2 :: term()), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Initial :: Value, + Fun :: fun((Value1 :: Value) -> Value2 :: Value), + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value). update(Key, _, Init, [{K,_}|_]=Dict) when Key < K -> [{Key,Init}|Dict]; @@ -179,10 +168,9 @@ update(Key, Fun, _Init, [{_K,Val}|Dict]) -> %Key == K update(Key, _, Init, []) -> [{Key,Init}]. -spec update_counter(Key, Increment, Orddict1) -> Orddict2 when - Key :: term(), - Increment :: number(), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value), + Increment :: number(). update_counter(Key, Incr, [{K,_}|_]=Dict) when Key < K -> [{Key,Incr}|Dict]; @@ -193,28 +181,30 @@ update_counter(Key, Incr, [{_K,Val}|Dict]) -> %Key == K update_counter(Key, Incr, []) -> [{Key,Incr}]. -spec fold(Fun, Acc0, Orddict) -> Acc1 when - Fun :: fun((Key :: term(), Value :: term(), AccIn :: term()) -> AccOut :: term()), - Acc0 :: term(), - Acc1 :: term(), - Orddict :: orddict(). + Fun :: fun((Key, Value, AccIn) -> AccOut), + Orddict :: orddict(Key, Value), + Acc0 :: Acc, + Acc1 :: Acc, + AccIn :: Acc, + AccOut :: Acc. fold(F, Acc, [{Key,Val}|D]) -> fold(F, F(Key, Val, Acc), D); fold(F, Acc, []) when is_function(F, 3) -> Acc. -spec map(Fun, Orddict1) -> Orddict2 when - Fun :: fun((Key :: term(), Value1 :: term()) -> Value2 :: term()), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Fun :: fun((Key, Value1) -> Value2), + Orddict1 :: orddict(Key, Value1), + Orddict2 :: orddict(Key, Value2). map(F, [{Key,Val}|D]) -> [{Key,F(Key, Val)}|map(F, D)]; map(F, []) when is_function(F, 2) -> []. -spec filter(Pred, Orddict1) -> Orddict2 when - Pred :: fun((Key :: term(), Value :: term()) -> boolean()), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Pred :: fun((Key, Value) -> boolean()), + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value). filter(F, [{Key,Val}=E|D]) -> case F(Key, Val) of @@ -224,10 +214,10 @@ filter(F, [{Key,Val}=E|D]) -> filter(F, []) when is_function(F, 2) -> []. -spec merge(Fun, Orddict1, Orddict2) -> Orddict3 when - Fun :: fun((Key :: term(), Value1 :: term(), Value2 :: term()) -> Value :: term()), - Orddict1 :: orddict(), - Orddict2 :: orddict(), - Orddict3 :: orddict(). + Fun :: fun((Key, Value1, Value2) -> Value), + Orddict1 :: orddict(Key, Value1), + Orddict2 :: orddict(Key, Value2), + Orddict3 :: orddict(Key, Value). merge(F, [{K1,_}=E1|D1], [{K2,_}=E2|D2]) when K1 < K2 -> [E1|merge(F, D1, [E2|D2])]; diff --git a/lib/stdlib/src/queue.erl b/lib/stdlib/src/queue.erl index 472d503b99..d2e6848258 100644 --- a/lib/stdlib/src/queue.erl +++ b/lib/stdlib/src/queue.erl @@ -48,7 +48,7 @@ -opaque queue(Item) :: {list(Item), list(Item)}. --opaque queue() :: queue(_). +-type queue() :: queue(_). %% Creation, inspection and conversion diff --git a/lib/stdlib/src/sets.erl b/lib/stdlib/src/sets.erl index 167a676281..041d281148 100644 --- a/lib/stdlib/src/sets.erl +++ b/lib/stdlib/src/sets.erl @@ -70,7 +70,7 @@ segs :: segs(_) % Segments }). --opaque set() :: set(_). +-type set() :: set(_). -opaque set(Element) :: #set{segs :: segs(Element)}. diff --git a/lib/stdlib/src/shell_default.erl b/lib/stdlib/src/shell_default.erl index 3fe359af0e..0fca7ff8c7 100644 --- a/lib/stdlib/src/shell_default.erl +++ b/lib/stdlib/src/shell_default.erl @@ -23,7 +23,7 @@ -module(shell_default). -export([help/0,lc/1,c/1,c/2,nc/1,nl/1,l/1,i/0,pid/3,i/3,m/0,m/1, - memory/0,memory/1, + memory/0,memory/1,uptime/0, erlangrc/1,bi/1, regs/0, flush/0,pwd/0,ls/0,ls/1,cd/1, y/1, y/2, xm/1, bt/1, q/0, @@ -92,6 +92,7 @@ pid(X,Y,Z) -> c:pid(X,Y,Z). pwd() -> c:pwd(). q() -> c:q(). regs() -> c:regs(). +uptime() -> c:uptime(). xm(Mod) -> c:xm(Mod). y(File) -> c:y(File). y(File, Opts) -> c:y(File, Opts). diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src index a27a35dca2..c33130cf8c 100644 --- a/lib/stdlib/src/stdlib.app.src +++ b/lib/stdlib/src/stdlib.app.src @@ -104,7 +104,7 @@ dets]}, {applications, [kernel]}, {env, []}, - {runtime_dependencies, ["sasl-2.4","kernel-3.0.2","erts-7.0","crypto-3.3", + {runtime_dependencies, ["sasl-2.4","kernel-4.0","erts-7.0","crypto-3.3", "compiler-5.0"]} ]}. diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src index ee87a8ddb2..b3569c2848 100644 --- a/lib/stdlib/src/stdlib.appup.src +++ b/lib/stdlib/src/stdlib.appup.src @@ -1,7 +1,7 @@ %% -*- erlang -*- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2014. All Rights Reserved. +%% Copyright Ericsson AB 1999-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -17,9 +17,7 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"2\\.[1-3](\\.[0-9]+)*">>,[restart_new_emulator]}, %% 17.1-17.3 - {<<"2\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}], %% 17.0 + [{<<"2\\.[0-4](\\.[0-9]+)*">>,[restart_new_emulator]}], %% 17.0-17.5 %% Down to - max one major revision back - [{<<"2\\.[1-3](\\.[0-9]+)*">>,[restart_new_emulator]}, %% 17.1-17.3 - {<<"2\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}] %% 17.0 + [{<<"2\\.[0-4](\\.[0-9]+)*">>,[restart_new_emulator]}] %% 17.0-17.5 }. diff --git a/lib/stdlib/src/supervisor.erl b/lib/stdlib/src/supervisor.erl index 67655b1145..1d7396adee 100644 --- a/lib/stdlib/src/supervisor.erl +++ b/lib/stdlib/src/supervisor.erl @@ -381,7 +381,7 @@ handle_call({start_child, EArgs}, _From, State) when ?is_simple(State) -> #child{mfargs = {M, F, A}} = Child, Args = A ++ EArgs, case do_start_child_i(M, F, Args) of - {ok, undefined} when Child#child.restart_type =:= temporary -> + {ok, undefined} -> {reply, {ok, undefined}, State}; {ok, Pid} -> NState = save_dynamic_child(Child#child.restart_type, Pid, Args, State), diff --git a/lib/stdlib/src/zip.erl b/lib/stdlib/src/zip.erl index 3c67bd67c6..f986c0081d 100644 --- a/lib/stdlib/src/zip.erl +++ b/lib/stdlib/src/zip.erl @@ -24,7 +24,7 @@ list_dir/1, list_dir/2, table/1, table/2, t/1, tt/1]). -%% unzipping peicemeal +%% unzipping piecemeal -export([openzip_open/1, openzip_open/2, openzip_get/1, openzip_get/2, openzip_t/1, openzip_tt/1, diff --git a/lib/stdlib/test/Makefile b/lib/stdlib/test/Makefile index 61eb34d565..d4ab674486 100644 --- a/lib/stdlib/test/Makefile +++ b/lib/stdlib/test/Makefile @@ -107,7 +107,8 @@ RELSYSDIR = $(RELEASE_PATH)/stdlib_test ERL_MAKE_FLAGS += ERL_COMPILE_FLAGS += -I$(ERL_TOP)/lib/test_server/include \ - -I$(ERL_TOP)/lib/kernel/include + -I$(ERL_TOP)/lib/kernel/include \ + -I$(ERL_TOP)/lib/stdlib/include EBIN = . diff --git a/lib/stdlib/test/binary_module_SUITE.erl b/lib/stdlib/test/binary_module_SUITE.erl index 5248870744..8d26c77c9b 100644 --- a/lib/stdlib/test/binary_module_SUITE.erl +++ b/lib/stdlib/test/binary_module_SUITE.erl @@ -993,43 +993,51 @@ random_parts(X,N) -> random_ref_comp(doc) -> ["Test pseudorandomly generated cases against reference imlementation"]; random_ref_comp(Config) when is_list(Config) -> - ?line put(success_counter,0), - ?line random:seed({1271,769940,559934}), - ?line do_random_match_comp(5000,{1,40},{30,1000}), + put(success_counter,0), + random:seed({1271,769940,559934}), + Nr = {1,40}, + Hr = {30,1000}, + I1 = 1500, + I2 = 5, + do_random_match_comp(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_match_comp2(5000,{1,40},{30,1000}), + do_random_match_comp2(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_match_comp3(5000,{1,40},{30,1000}), + do_random_match_comp3(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_match_comp4(5000,{1,40},{30,1000}), + do_random_match_comp4(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_matches_comp(5000,{1,40},{30,1000}), + do_random_matches_comp(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_matches_comp2(5000,{1,40},{30,1000}), + do_random_matches_comp2(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_matches_comp3(5,{1,40},{30,1000}), - ?line erts_debug:set_internal_state(available_internal_state,true), - ?line io:format("oldlimit: ~p~n",[ erts_debug:set_internal_state(binary_loop_limit,100)]), - ?line do_random_match_comp(5000,{1,40},{30,1000}), - ?line do_random_matches_comp3(5,{1,40},{30,1000}), - ?line io:format("limit was: ~p~n",[ erts_debug:set_internal_state(binary_loop_limit,default)]), - ?line erts_debug:set_internal_state(available_internal_state,false), + do_random_matches_comp3(I2,Nr,Hr), + erts_debug:set_internal_state(available_internal_state,true), + io:format("oldlimit: ~p~n",[ erts_debug:set_internal_state(binary_loop_limit,100)]), + do_random_match_comp(I1,Nr,Hr), + do_random_matches_comp3(I2,Nr,Hr), + io:format("limit was: ~p~n",[ erts_debug:set_internal_state(binary_loop_limit,default)]), + erts_debug:set_internal_state(available_internal_state,false), ok. random_ref_sr_comp(doc) -> ["Test pseudorandomly generated cases against reference imlementation of split and replace"]; random_ref_sr_comp(Config) when is_list(Config) -> - ?line put(success_counter,0), - ?line random:seed({1271,769940,559934}), - ?line do_random_split_comp(5000,{1,40},{30,1000}), + put(success_counter,0), + random:seed({1271,769940,559934}), + Nr = {1,40}, + Hr = {30,1000}, + I1 = 1500, + do_random_split_comp(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_replace_comp(5000,{1,40},{30,1000}), + do_random_replace_comp(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_split_comp2(5000,{1,40},{30,1000}), + do_random_split_comp2(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_replace_comp2(5000,{1,40},{30,1000}), + do_random_replace_comp2(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), ok. + random_ref_fla_comp(doc) -> ["Test pseudorandomly generated cases against reference imlementation of split and replace"]; random_ref_fla_comp(Config) when is_list(Config) -> diff --git a/lib/stdlib/test/dict_SUITE.erl b/lib/stdlib/test/dict_SUITE.erl index 69814e12ce..ab624e8dd2 100644 --- a/lib/stdlib/test/dict_SUITE.erl +++ b/lib/stdlib/test/dict_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2013. All Rights Reserved. +%% Copyright Ericsson AB 2008-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -25,16 +25,16 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, - create/1,store/1]). + create/1,store/1,iterate/1]). -include_lib("test_server/include/test_server.hrl"). --import(lists, [foldl/3,reverse/1]). +-import(lists, [foldl/3]). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [create, store]. + [create, store, iterate]. groups() -> []. @@ -93,6 +93,48 @@ store_1(List, M) -> D0. %%% +%%% Test specifics for gb_trees. +%%% + +iterate(Config) when is_list(Config) -> + test_all(fun iterate_1/1). + +iterate_1(M) -> + case M(module, []) of + gb_trees -> iterate_2(M); + _ -> ok + end, + M(empty, []). + +iterate_2(M) -> + random:seed(1, 2, 42), + iter_tree(M, 1000). + +iter_tree(_M, 0) -> + ok; +iter_tree(M, N) -> + L = [{I, I} || I <- lists:seq(1, N)], + T = M(from_list, L), + L = lists:reverse(iterate_tree(M, T)), + R = random:uniform(N), + KV = lists:reverse(iterate_tree_from(M, R, T)), + KV = [P || P={K,_} <- L, K >= R], + iter_tree(M, N-1). + +iterate_tree(M, Tree) -> + I = M(iterator, Tree), + iterate_tree_1(M, M(next, I), []). + +iterate_tree_from(M, Start, Tree) -> + I = M(iterator_from, {Start, Tree}), + iterate_tree_1(M, M(next, I), []). + +iterate_tree_1(_, none, R) -> + R; +iterate_tree_1(M, {K, V, I}, R) -> + iterate_tree_1(M, M(next, I), [{K, V} | R]). + +%%% %%% Helper functions. %%% diff --git a/lib/stdlib/test/dict_test_lib.erl b/lib/stdlib/test/dict_test_lib.erl index 4fdb4fa0bd..81d26ce5f8 100644 --- a/lib/stdlib/test/dict_test_lib.erl +++ b/lib/stdlib/test/dict_test_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2013. All Rights Reserved. +%% Copyright Ericsson AB 2008-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,6 +29,9 @@ new(Mod, Eq) -> (module, []) -> Mod; (size, D) -> Mod:size(D); (is_empty, D) -> Mod:is_empty(D); + (iterator, S) -> Mod:iterator(S); + (iterator_from, {Start, S}) -> Mod:iterator_from(Start, S); + (next, I) -> Mod:next(I); (to_list, D) -> to_list(Mod, D) end. diff --git a/lib/stdlib/test/erl_anno_SUITE.erl b/lib/stdlib/test/erl_anno_SUITE.erl index 7632fbd324..d024f6907d 100644 --- a/lib/stdlib/test/erl_anno_SUITE.erl +++ b/lib/stdlib/test/erl_anno_SUITE.erl @@ -89,7 +89,6 @@ is_anno(_Config) -> false = erl_anno:is_anno([{generated,true}]), false = erl_anno:is_anno([{location,1},{file,nofile}]), false = erl_anno:is_anno([{location,1},{text,notext}]), - false = erl_anno:is_anno([{location,1},{text,[a,b,c]}]), true = erl_anno:is_anno(erl_anno:new(1)), A0 = erl_anno:new({1, 17}), diff --git a/lib/stdlib/test/erl_pp_SUITE.erl b/lib/stdlib/test/erl_pp_SUITE.erl index 1d63c8e17e..afeeb5bfd4 100644 --- a/lib/stdlib/test/erl_pp_SUITE.erl +++ b/lib/stdlib/test/erl_pp_SUITE.erl @@ -1149,7 +1149,7 @@ otp_11100(Config) when is_list(Config) -> {a,{type,A1,range,[{integer,A1,1},{foo,bar}]},[]}}), "-type foo(INVALID-FORM:{foo,bar}:) :: A.\n" = pf({attribute,A1,type,{foo,{var,A1,'A'},[{foo,bar}]}}), - "-type foo() :: (INVALID-FORM:{foo,bar}: :: []).\n" = + "-type foo() :: INVALID-FORM:{foo,bar}: :: [].\n" = pf({attribute,A1,type, {foo,{paren_type,A1, [{ann_type,A1,[{foo,bar},{type,A1,nil,[]}]}]}, diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 5d1d4b7358..f47c2c518d 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -3063,13 +3063,13 @@ time_lookup(Config) when is_list(Config) -> "~p ets lookups/s",[Values]))}. time_lookup_do(Opts) -> - ?line Tab = ets_new(foo,Opts), - ?line fill_tab(Tab,foo), - ?line ets:insert(Tab,{{a,key},foo}), - ?line {Time,_} = ?t:timecall(test_server,do_times, - [10000,ets,lookup,[Tab,{a,key}]]), - ?line true = ets:delete(Tab), - round(10000 / Time). % lookups/s + Tab = ets_new(foo,Opts), + fill_tab(Tab,foo), + ets:insert(Tab,{{a,key},foo}), + {Time,_} = ?t:timecall(test_server,do_times, + [100000,ets,lookup,[Tab,{a,key}]]), + true = ets:delete(Tab), + round(100000 / Time). % lookups/s badlookup(doc) -> ["Check proper return values from bad lookups in existing/non existing " @@ -4080,21 +4080,39 @@ tab2file(doc) -> ["Check the ets:tab2file function on an empty " "ets table."]; tab2file(suite) -> []; tab2file(Config) when is_list(Config) -> - %% Write an empty ets table to a file, read back and check properties. - ?line Tab = ets_new(ets_SUITE_foo_tab, [named_table, set, private, - {keypos, 2}]), ?line FName = filename:join([?config(priv_dir, Config),"tab2file_case"]), - ?line ok = ets:tab2file(Tab, FName), - ?line true = ets:delete(Tab), + tab2file_do(FName, []), + tab2file_do(FName, [{sync,true}]), + tab2file_do(FName, [{sync,false}]), + {'EXIT',{{badmatch,{error,_}},_}} = (catch tab2file_do(FName, [{sync,yes}])), + {'EXIT',{{badmatch,{error,_}},_}} = (catch tab2file_do(FName, [sync])), + ok. + +tab2file_do(FName, Opts) -> + %% Write an empty ets table to a file, read back and check properties. + ?line Tab = ets_new(ets_SUITE_foo_tab, [named_table, set, public, + {keypos, 2}, + compressed, + {write_concurrency,true}, + {read_concurrency,true}]), + catch file:delete(FName), + Res = ets:tab2file(Tab, FName, Opts), + true = ets:delete(Tab), + ok = Res, % ?line EtsMem = etsmem(), ?line {ok, Tab2} = ets:file2tab(FName), - ?line private = ets:info(Tab2, protection), + public = ets:info(Tab2, protection), ?line true = ets:info(Tab2, named_table), ?line 2 = ets:info(Tab2, keypos), ?line set = ets:info(Tab2, type), + true = ets:info(Tab2, compressed), + Smp = erlang:system_info(smp_support), + Smp = ets:info(Tab2, read_concurrency), + Smp = ets:info(Tab2, write_concurrency), ?line true = ets:delete(Tab2), ?line verify_etsmem(EtsMem). + tab2file2(doc) -> ["Check the ets:tab2file function on a ", "filled set/bag type ets table."]; @@ -4310,7 +4328,7 @@ tabfile_ext4(Config) when is_list(Config) -> {error,Y} = ets:file2tab(FName,[{verify,true}]), ets:tab2file(TL,FName,[{extended_info,[md5sum]}]), {X,Y} - end || N <- lists:seq(400,500) ], + end || N <- lists:seq(500,600) ], io:format("~p~n",[Res]), file:delete(FName), ok. diff --git a/lib/stdlib/test/io_proto_SUITE.erl b/lib/stdlib/test/io_proto_SUITE.erl index 858a78b1d2..78432789cd 100644 --- a/lib/stdlib/test/io_proto_SUITE.erl +++ b/lib/stdlib/test/io_proto_SUITE.erl @@ -482,7 +482,7 @@ unicode_options_gen(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), AllModes = [utf8,utf16,{utf16,big},{utf16,little}, utf32,{utf32,big},{utf32,little}], - FSize = 17*1024, + FSize = 9*1024, NumItersRead = 2, NumItersWrite = 2, Dir = filename:join(PrivDir, "GENDATA1"), diff --git a/lib/stdlib/test/maps_SUITE.erl b/lib/stdlib/test/maps_SUITE.erl index 1d9c041a74..f8f241d834 100644 --- a/lib/stdlib/test/maps_SUITE.erl +++ b/lib/stdlib/test/maps_SUITE.erl @@ -34,18 +34,21 @@ -export([init_per_testcase/2]). -export([end_per_testcase/2]). --export([t_get_3/1, +-export([t_get_3/1, t_filter_2/1, t_fold_3/1,t_map_2/1,t_size_1/1, t_with_2/1,t_without_2/1]). --define(badmap(V,F,Args), {'EXIT', {{badmap,V}, [{maps,F,Args,_}|_]}}). --define(badarg(F,Args), {'EXIT', {badarg, [{maps,F,Args,_}|_]}}). +%-define(badmap(V,F,Args), {'EXIT', {{badmap,V}, [{maps,F,Args,_}|_]}}). +%-define(badarg(F,Args), {'EXIT', {badarg, [{maps,F,Args,_}|_]}}). +% silly broken hipe +-define(badmap(V,F,_Args), {'EXIT', {{badmap,V}, [{maps,F,_,_}|_]}}). +-define(badarg(F,_Args), {'EXIT', {badarg, [{maps,F,_,_}|_]}}). suite() -> [{ct_hooks, [ts_install_cth]}]. all() -> - [t_get_3, + [t_get_3,t_filter_2, t_fold_3,t_map_2,t_size_1, t_with_2,t_without_2]. @@ -99,6 +102,16 @@ t_with_2(_Config) -> ?badarg(with,[a,#{}]) = (catch maps:with(a,#{})), ok. +t_filter_2(Config) when is_list(Config) -> + M = #{a => 2, b => 3, c=> 4, "a" => 1, "b" => 2, "c" => 4}, + Pred1 = fun(K,V) -> is_atom(K) andalso (V rem 2) =:= 0 end, + Pred2 = fun(K,V) -> is_list(K) andalso (V rem 2) =:= 0 end, + #{a := 2,c := 4} = maps:filter(Pred1,M), + #{"b" := 2,"c" := 4} = maps:filter(Pred2,M), + %% error case + ?badmap(a,filter,[_,a]) = (catch maps:filter(fun(_,_) -> ok end,id(a))), + ?badarg(filter,[<<>>,#{}]) = (catch maps:filter(id(<<>>),#{})), + ok. t_fold_3(Config) when is_list(Config) -> Vs = lists:seq(1,200), diff --git a/lib/stdlib/test/qlc_SUITE.erl b/lib/stdlib/test/qlc_SUITE.erl index 348c308f5d..56829fac5c 100644 --- a/lib/stdlib/test/qlc_SUITE.erl +++ b/lib/stdlib/test/qlc_SUITE.erl @@ -6120,7 +6120,7 @@ otp_6964(Config) when is_list(Config) -> lists:flatten(qlc:format_error(ErrReply)), qlc_SUITE:install_error_logger(), 20000 = length(F(warning_msg)), - {error, joining} = qlc_SUITE:read_error_logger(), + {warning, joining} = qlc_SUITE:read_error_logger(), 20000 = length(F(info_msg)), {info, joining} = qlc_SUITE:read_error_logger(), 20000 = length(F(error_msg)), @@ -6155,8 +6155,8 @@ otp_6964(Config) when is_list(Config) -> {error, caching} = qlc_SUITE:read_error_logger(), {error, caching} = qlc_SUITE:read_error_logger(), 1 = length(F(warning_msg)), - {error, caching} = qlc_SUITE:read_error_logger(), - {error, caching} = qlc_SUITE:read_error_logger(), + {warning, caching} = qlc_SUITE:read_error_logger(), + {warning, caching} = qlc_SUITE:read_error_logger(), 1 = length(F(info_msg)), {info, caching} = qlc_SUITE:read_error_logger(), {info, caching} = qlc_SUITE:read_error_logger(), @@ -6188,7 +6188,7 @@ otp_6964(Config) when is_list(Config) -> L = F(info_msg), {info, sorting} = qlc_SUITE:read_error_logger(), L = F(warning_msg), - {error, sorting} = qlc_SUITE:read_error_logger(), + {warning, sorting} = qlc_SUITE:read_error_logger(), qlc_SUITE:uninstall_error_logger(), ets:delete(E1), ets:delete(E2)">>], @@ -6215,7 +6215,7 @@ otp_6964(Config) when is_list(Config) -> R = lists:sort(F(error_msg)), {error, caching} = qlc_SUITE:read_error_logger(), R = lists:sort(F(warning_msg)), - {error, caching} = qlc_SUITE:read_error_logger(), + {warning, caching} = qlc_SUITE:read_error_logger(), qlc_SUITE:uninstall_error_logger(), ErrReply = F(not_allowed), {error,qlc,{tmpdir_usage,caching}} = ErrReply, @@ -8178,6 +8178,8 @@ read_error_logger() -> {error, Why}; {info, Why} -> {info, Why}; + {warning, Why} -> + {warning, Why}; {error, Pid, Tuple} -> {error, Pid, Tuple} after 1000 -> @@ -8192,8 +8194,7 @@ read_error_logger() -> init(Tester) -> {ok, Tester}. -handle_event({error, _GL, {_Pid, _Msg, [Why, _]}}, Tester) - when is_atom(Why) -> +handle_event({error, _GL, {_Pid, _Msg, [Why, _]}}, Tester) when is_atom(Why) -> Tester ! {error, Why}, {ok, Tester}; handle_event({error, _GL, {_Pid, _Msg, [P, T]}}, Tester) when is_pid(P) -> @@ -8202,6 +8203,9 @@ handle_event({error, _GL, {_Pid, _Msg, [P, T]}}, Tester) when is_pid(P) -> handle_event({info_msg, _GL, {_Pid, _Msg, [Why, _]}}, Tester) -> Tester ! {info, Why}, {ok, Tester}; +handle_event({warning_msg, _GL, {_Pid, _Msg, [Why, _]}}, Tester) when is_atom(Why) -> + Tester ! {warning, Why}, + {ok, Tester}; handle_event(_Event, State) -> {ok, State}. diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index 9a1f37aa75..39ce1bd89a 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -33,7 +33,7 @@ -include_lib("test_server/include/test_server.hrl"). % Default timetrap timeout (set in init_per_testcase). --define(default_timeout, ?t:minutes(1)). +-define(default_timeout, ?t:minutes(3)). -define(LOOP, 1000000). init_per_testcase(_Case, Config) -> diff --git a/lib/stdlib/test/sets_SUITE.erl b/lib/stdlib/test/sets_SUITE.erl index c0cf1fc7e8..24f5d65f82 100644 --- a/lib/stdlib/test/sets_SUITE.erl +++ b/lib/stdlib/test/sets_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2013. All Rights Reserved. +%% Copyright Ericsson AB 2004-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -28,7 +28,7 @@ create/1,add_element/1,del_element/1, subtract/1,intersection/1,union/1,is_subset/1, is_set/1,fold/1,filter/1, - take_smallest/1,take_largest/1]). + take_smallest/1,take_largest/1, iterate/1]). -include_lib("test_server/include/test_server.hrl"). @@ -48,7 +48,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [create, add_element, del_element, subtract, intersection, union, is_subset, is_set, fold, filter, - take_smallest, take_largest]. + take_smallest, take_largest, iterate]. groups() -> []. @@ -426,6 +426,44 @@ take_largest_3(S0, List0, M) -> take_largest_3(S, List, M) end. +iterate(Config) when is_list(Config) -> + test_all(fun iterate_1/1). + +iterate_1(M) -> + case M(module, []) of + gb_sets -> iterate_2(M); + _ -> ok + end, + M(empty, []). + +iterate_2(M) -> + random:seed(1, 2, 42), + iter_set(M, 1000). + +iter_set(_M, 0) -> + ok; +iter_set(M, N) -> + L = [I || I <- lists:seq(1, N)], + T = M(from_list, L), + L = lists:reverse(iterate_set(M, T)), + R = random:uniform(N), + S = lists:reverse(iterate_set(M, R, T)), + S = [E || E <- L, E >= R], + iter_set(M, N-1). + +iterate_set(M, Set) -> + I = M(iterator, Set), + iterate_set_1(M, M(next, I), []). + +iterate_set(M, Start, Set) -> + I = M(iterator_from, {Start, Set}), + iterate_set_1(M, M(next, I), []). + +iterate_set_1(_, none, R) -> + R; +iterate_set_1(M, {E, I}, R) -> + iterate_set_1(M, M(next, I), [E | R]). + %%% %%% Helper functions. %%% diff --git a/lib/stdlib/test/sets_test_lib.erl b/lib/stdlib/test/sets_test_lib.erl index 86f009a8f9..772139406d 100644 --- a/lib/stdlib/test/sets_test_lib.erl +++ b/lib/stdlib/test/sets_test_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2013. All Rights Reserved. +%% Copyright Ericsson AB 2004-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -34,7 +34,10 @@ new(Mod, Eq) -> (is_empty, S) -> is_empty(Mod, S); (is_set, S) -> Mod:is_set(S); (is_subset, {S,Set}) -> is_subset(Mod, Eq, S, Set); + (iterator, S) -> Mod:iterator(S); + (iterator_from, {Start, S}) -> Mod:iterator_from(Start, S); (module, []) -> Mod; + (next, I) -> Mod:next(I); (singleton, E) -> singleton(Mod, E); (size, S) -> Mod:size(S); (subtract, {S1,S2}) -> subtract(Mod, S1, S2); diff --git a/lib/stdlib/test/stdlib_SUITE.erl b/lib/stdlib/test/stdlib_SUITE.erl index 206eb4fd74..8ab30eb62b 100644 --- a/lib/stdlib/test/stdlib_SUITE.erl +++ b/lib/stdlib/test/stdlib_SUITE.erl @@ -30,7 +30,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [app_test, appup_test, {group,upgrade}]. + [app_test, appup_test, assert_test, {group,upgrade}]. groups() -> [{upgrade,[minor_upgrade,major_upgrade]}]. @@ -185,3 +185,68 @@ upgrade_upgraded(_CtData,State) -> State. upgrade_downgraded(_CtData,State) -> State. + + +-include_lib("stdlib/include/assert.hrl"). +-include_lib("stdlib/include/assert.hrl"). % test repeated inclusion +assert_test(suite) -> + []; +assert_test(doc) -> + ["Assert macros test."]; +assert_test(_Config) -> + ok = ?assert(true), + {'EXIT',{{assert, _},_}} = (catch ?assert(false)), + {'EXIT',{{assert, Info1},_}} = (catch ?assert(0)), + {not_boolean,0} = lists:keyfind(not_boolean,1,Info1), + + ok = ?assertNot(false), + {'EXIT',{{assert, _},_}} = (catch ?assertNot(true)), + {'EXIT',{{assert, Info2},_}} = (catch ?assertNot(0)), + {not_boolean,0} = lists:keyfind(not_boolean,1,Info2), + + ok = ?assertMatch({foo,_}, {foo,bar}), + {'EXIT',{{assertMatch,_},_}} = + (catch ?assertMatch({foo,_}, {foo})), + + ok = ?assertMatch({foo,N} when N > 0, {foo,1}), + {'EXIT',{{assertMatch,_},_}} = + (catch ?assertMatch({foo,N} when N > 0, {foo,0})), + + ok = ?assertNotMatch({foo,_}, {foo,bar,baz}), + {'EXIT',{{assertNotMatch,_},_}} = + (catch ?assertNotMatch({foo,_}, {foo,baz})), + + ok = ?assertNotMatch({foo,N} when N > 0, {foo,0}), + {'EXIT',{{assertNotMatch,_},_}} = + (catch ?assertNotMatch({foo,N} when N > 0, {foo,1})), + + ok = ?assertEqual(1.0, 1.0), + {'EXIT',{{assertEqual,_},_}} = (catch ?assertEqual(1, 1.0)), + + ok = ?assertNotEqual(1, 1.0), + {'EXIT',{{assertNotEqual,_},_}} = (catch ?assertNotEqual(1.0, 1.0)), + + ok = ?assertException(error, badarith, 1/0), + ok = ?assertException(exit, foo, exit(foo)), + ok = ?assertException(throw, foo, throw(foo)), + ok = ?assertException(throw, {foo,_}, throw({foo,bar})), + ok = ?assertException(throw, {foo,N} when N > 0, throw({foo,1})), + {'EXIT',{{assertException,Why1},_}} = + (catch ?assertException(error, badarith, 0/1)), + true = lists:keymember(unexpected_success,1,Why1), + {'EXIT',{{assertException,Why2},_}} = + (catch ?assertException(error, badarith, 1/length(0))), + true = lists:keymember(unexpected_exception,1,Why2), + {'EXIT',{{assertException,Why3},_}} = + (catch ?assertException(throw, {foo,N} when N > 0, throw({foo,0}))), + true = lists:keymember(unexpected_exception,1,Why3), + + ok = ?assertNotException(throw, {foo,baz}, throw({foo,bar})), + {'EXIT',{{assertNotException,Why4},_}} = + (catch ?assertNotException(throw, {foo,bar}, throw({foo,bar}))), + true = lists:keymember(unexpected_exception,1,Why4), + + ok = ?assertError(badarith, 1/0), + ok = ?assertExit(foo, exit(foo)), + ok = ?assertThrow(foo, throw(foo)), + ok. diff --git a/lib/stdlib/test/supervisor_SUITE.erl b/lib/stdlib/test/supervisor_SUITE.erl index 9dcf19707c..015b09f35e 100644 --- a/lib/stdlib/test/supervisor_SUITE.erl +++ b/lib/stdlib/test/supervisor_SUITE.erl @@ -37,6 +37,7 @@ sup_start_ignore_child/1, sup_start_ignore_temporary_child/1, sup_start_ignore_temporary_child_start_child/1, sup_start_ignore_temporary_child_start_child_simple/1, + sup_start_ignore_permanent_child_start_child_simple/1, sup_start_error_return/1, sup_start_fail/1, sup_start_map/1, sup_start_map_faulty_specs/1, sup_stop_infinity/1, sup_stop_timeout/1, sup_stop_brutal_kill/1, @@ -99,6 +100,7 @@ groups() -> sup_start_ignore_child, sup_start_ignore_temporary_child, sup_start_ignore_temporary_child_start_child, sup_start_ignore_temporary_child_start_child_simple, + sup_start_ignore_permanent_child_start_child_simple, sup_start_error_return, sup_start_fail]}, {sup_start_map, [], [sup_start_map, sup_start_map_faulty_specs]}, @@ -250,6 +252,27 @@ sup_start_ignore_temporary_child_start_child_simple(Config) [1,1,0,1] = get_child_counts(sup_test). %%------------------------------------------------------------------------- +%% Tests what happens if child's init-callback returns ignore for a +%% permanent child when child is started with start_child/2, and the +%% supervisor is simple_one_for_one. +%% Child spec shall NOT be saved!!! +sup_start_ignore_permanent_child_start_child_simple(Config) + when is_list(Config) -> + process_flag(trap_exit, true), + Child1 = {child1, {supervisor_1, start_child, [ignore]}, + permanent, 1000, worker, []}, + {ok, Pid} = start_link({ok, {{simple_one_for_one, 2, 3600}, [Child1]}}), + + {ok, undefined} = supervisor:start_child(sup_test, []), + {ok, CPid2} = supervisor:start_child(sup_test, []), + + [{undefined, CPid2, worker, []}] = supervisor:which_children(sup_test), + [1,1,0,1] = get_child_counts(sup_test), + + %% Regression test: check that the supervisor terminates without error. + exit(Pid, shutdown), + check_exit_reason(Pid, shutdown). +%%------------------------------------------------------------------------- %% Tests what happens if init-callback returns a invalid value. sup_start_error_return(Config) when is_list(Config) -> process_flag(trap_exit, true), diff --git a/lib/stdlib/test/unicode_SUITE.erl b/lib/stdlib/test/unicode_SUITE.erl index 613be99ccd..9f5d485df6 100644 --- a/lib/stdlib/test/unicode_SUITE.erl +++ b/lib/stdlib/test/unicode_SUITE.erl @@ -87,8 +87,9 @@ ex_binaries_errors_utf8(Config) when is_list(Config) -> %% Now, try with longer binary (trapping) BrokenPart = list_to_binary(lists:seq(128,255)), BrokenSz = byte_size(BrokenPart), + Seq255 = lists:seq(1,255), [ begin - OKList = lists:flatten(lists:duplicate(N,lists:seq(1,255))), + OKList = lists:flatten(lists:duplicate(N,Seq255)), OKBin = unicode:characters_to_binary(OKList), OKLen = length(OKList), %% Copy to avoid that the binary get's writable diff --git a/lib/syntax_tools/doc/overview.edoc b/lib/syntax_tools/doc/overview.edoc index df02ad0b3a..3111633a99 100644 --- a/lib/syntax_tools/doc/overview.edoc +++ b/lib/syntax_tools/doc/overview.edoc @@ -2,79 +2,34 @@ Syntax Tools overview page - @author Richard Carlsson <[email protected]> -@copyright 1997-2004 Richard Carlsson +@copyright 1997-2014 Richard Carlsson @version {@version} -@title Erlang Syntax Tools +@title Erlang Syntax and Metaprogramming tools -@doc This package contains modules for handling abstract Erlang syntax -trees, in a way that is compatible with the "parse trees" of the -standard library module `erl_parse', together with utilities for reading -source files in unusual ways and pretty-printing syntax trees. Also -included is an amazing module merger and renamer called Igor, as well as -an automatic code-cleaner. +@doc This package contains modules for handling abstract syntax trees (ASTs) +in Erlang, in a way that is compatible with the "abstract format" parse +trees of the stdlib module `erl_parse', together with utilities for reading +source files, {@link erl_prettypr. pretty-printing syntax trees}, {@link +igor. merging and renaming modules}, {@link erl_tidy. cleaning up obsolete +constructs}, and doing {@link merl. metaprogramming} in Erlang. -<p>The abstract layer (defined in {@link erl_syntax}) is nicely +The abstract layer (defined in {@link erl_syntax}) is nicely structured and the node types are context-independent. The layer makes it possible to transparently attach source-code comments and user annotations to nodes of the tree. Using the abstract layer makes applications less sensitive to changes in the {@link //stdlib/erl_parse} -data structures, only requiring the {@link erl_syntax} module to be -up-to-date.</p> +data structures, only requiring the `erl_syntax' module to be up-to-date. -<p>The pretty printer {@link erl_prettypr} is implemented on top of the +The pretty printer {@link erl_prettypr} is implemented on top of the library module {@link prettypr}: this is a powerful and flexible generic -pretty printing library, which is also distributed separately.</p> - -<p>For a short demonstration of parsing and pretty-printing, simply -compile the included module <a -href="../examples/demo.erl"><code>demo.erl</code></a>, and execute -<code>demo:run()</code> from the Erlang shell. It will compile the -remaining modules and give you further instructions.</p> - -<p>Also try the {@link erl_tidy} module, as follows: -<pre> - erl_tidy:dir("any-erlang-source-dir", [test, old_guard_tests]).</pre> -("<code>test</code>" assures that no files are modified).</p> - -<p>News in 1.4: -<ul> - <li>Added support for {@link erl_syntax:cond_expr/1. cond-expressions}, - {@link erl_syntax:try_expr/4. try-expressions} and - {@link erl_syntax:class_qualifier/2. class-qualifier patterns}.</li> - <li>Added support for parameterized modules.</li> - <li>{@link igor. Igor} is officially included.</li> - <li>Quick-parse functionality added to {@link epp_dodger}.</li> -</ul> -</p> - -<p>News in 1.3: -<ul> - <li>Added support for qualified names (as used by "packages").</li> - <li>Various internal changes.</li> -</ul> -</p> +pretty printing library, which is also distributed separately. -<p>News in 1.2: -<ul> - <li>HTML Documentation (generated with EDoc).</li> - <li>A few bug fixes and some minor interface changes (sorry for any - inconvenience).</li> -</ul> -</p> +For a short demonstration of parsing and pretty-printing, simply +compile the included module <a href="../examples/demo.erl">`demo.erl'</a>, +and execute `demo:run()' from the Erlang shell. It will compile the +remaining modules and give you further instructions. -<p>News in 1.1: -<ul> - <li>Module {@link erl_tidy}: check or tidy either a single module, or a - whole directory tree recursively. Rewrites and reformats the code - without losing comments or expanding macros. Safe mode allows - generating reports without modifying files.</li> - <li>Module {@link erl_syntax_lib}: contains support functions for easier - analysis of the source code structure.</li> - <li>Module {@link epp_dodger}: Bypasses the Erlang preprocessor - avoids - macro expansion, file inclusion, conditional compilation, etc. - Allows you to find/modify particular definitions/applications of - macros, and other things previously not possible.</li> -</ul> -</p> +Also try the {@link erl_tidy} module, as follows: +```erl_tidy:dir("any-erlang-source-dir", [test, old_guard_tests]).''' +(the `test' option assures that no files are modified). diff --git a/lib/syntax_tools/doc/src/Makefile b/lib/syntax_tools/doc/src/Makefile index 2502bf877a..b7c599a9b9 100644 --- a/lib/syntax_tools/doc/src/Makefile +++ b/lib/syntax_tools/doc/src/Makefile @@ -50,6 +50,8 @@ XML_REF3_FILES = \ erl_syntax_lib.xml \ erl_tidy.xml \ igor.xml \ + merl.xml \ + merl_transform.xml \ prettypr.xml XML_PART_FILES = part.xml part_notes.xml diff --git a/lib/syntax_tools/doc/src/notes.xml b/lib/syntax_tools/doc/src/notes.xml index 8f245083c4..408f6d5bac 100644 --- a/lib/syntax_tools/doc/src/notes.xml +++ b/lib/syntax_tools/doc/src/notes.xml @@ -31,20 +31,6 @@ <p>This document describes the changes made to the Syntax_Tools application.</p> -<section><title>Syntax_Tools 1.7</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p>Use the new <c>erl_anno</c> module.</p> - <p> - Own Id: OTP-12732</p> - </item> - </list> - </section> - -</section> - <section><title>Syntax_Tools 1.6.18</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/syntax_tools/doc/src/ref_man.xml b/lib/syntax_tools/doc/src/ref_man.xml index 598f656011..2b114c8528 100644 --- a/lib/syntax_tools/doc/src/ref_man.xml +++ b/lib/syntax_tools/doc/src/ref_man.xml @@ -29,12 +29,11 @@ </header> <description> <p><em>Syntax_Tools</em> contains modules for handling abstract - Erlang syntax trees, in a way that is compatible with the "parse - trees" of the STDLIB module <c>erl_parse</c>, together with - utilities for reading source files in unusual ways and - pretty-printing syntax trees. Also included is an amazing module - merger and renamer called Igor, as well as an automatic - code-cleaner.</p> + Erlang syntax trees, in a way that is compatible with the "external + format" parse trees of the STDLIB module <c>erl_parse</c>, together + with utilities for reading source files, pretty-printing syntax trees, + merging and renaming modules, cleaning up obsolete constructs, and + doing metaprogramming in Erlang.</p> </description> <xi:include href="epp_dodger.xml"/> <xi:include href="erl_comment_scan.xml"/> @@ -44,6 +43,8 @@ <xi:include href="erl_syntax_lib.xml"/> <xi:include href="erl_tidy.xml"/> <xi:include href="igor.xml"/> + <xi:include href="merl.xml"/> + <xi:include href="merl_transform.xml"/> <xi:include href="prettypr.xml"/> </application> diff --git a/lib/syntax_tools/examples/merl/Makefile b/lib/syntax_tools/examples/merl/Makefile new file mode 100644 index 0000000000..13a9703733 --- /dev/null +++ b/lib/syntax_tools/examples/merl/Makefile @@ -0,0 +1,22 @@ +EBIN=../../ebin +INCLUDES=../../include +SOURCES=merl_build.erl lisp.erl lispc.erl basic.erl basicc.erl +HEADERS=$(INCLUDES)/merl.hrl +OBJECTS=$(SOURCES:%.erl=%.beam) +ERLC_FLAGS=+debug_info -I$(INCLUDES) -pa $(EBIN) + +all: $(OBJECTS) test + +%.beam: %.erl $(HEADERS) Makefile + erlc $(ERLC_FLAGS) -o ./ $< + +# additional dependencies due to the parse transform +lispc.beam basicc.beam: $(EBIN)/merl_transform.beam $(EBIN)/merl.beam + +clean: + -rm -f $(OBJECTS) + +test: + erl -noshell -pa $(EBIN) \ + -eval 'eunit:test([lisp, lispc, basic, basicc],[])' \ + -s init stop diff --git a/lib/syntax_tools/examples/merl/basic.erl b/lib/syntax_tools/examples/merl/basic.erl new file mode 100644 index 0000000000..9030059d11 --- /dev/null +++ b/lib/syntax_tools/examples/merl/basic.erl @@ -0,0 +1,77 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012 Richard Carlsson +%% @doc Trivial Basic interpreter in Erlang + +-module(basic). + +-export([run/2]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(INTERPRETED, true). +-include("basic_test.erl"). + +run(N, Prog) -> + ets:new(var, [private, named_table]), + ets:new(line, [private, named_table, ordered_set]), + lists:foreach(fun (T) -> ets:insert(line, T) end, Prog), + goto(N). + +stop(N) -> + ets:delete(var), + ets:delete(line), + N. + +goto('$end_of_table') -> stop(0); +goto(L) -> + L1 = ets:next(line, L), + %% user-supplied line numbers might not exist + case ets:lookup(line, L) of + [{_, X}] -> + stmt(X, L1); + _ -> + goto(L1) + end. + +stmt({print, S, As}, L) -> io:format(S, [expr(A) || A <- As]), goto(L); +stmt({set, V, X}, L) -> ets:insert(var, {V, expr(X)}), goto(L); +stmt({goto, X}, _L) -> goto(expr(X)); +stmt({stop, X}, _L) -> stop(expr(X)); +stmt({iff, X, A, B}, _L) -> + case expr(X) of + 0 -> goto(B); + _ -> goto(A) + end. + +expr(X) when is_number(X) ; is_list(X) -> + X; +expr(X) when is_atom(X) -> + case ets:lookup(var, X) of + [] -> 0; + [{_,V}] -> V + end; +expr({plus, X, Y}) -> + expr(X) + expr(Y); +expr({equal, X, Y}) -> + bool(expr(X) == expr(Y)); +expr({gt, X, Y}) -> + bool(expr(X) > expr(Y)); +expr({knot, X}) -> + case expr(X) of + 0 -> 1; + _ -> 0 + end. + +bool(true) -> 1; +bool(false) -> 0. diff --git a/lib/syntax_tools/examples/merl/basic_test.erl b/lib/syntax_tools/examples/merl/basic_test.erl new file mode 100644 index 0000000000..ff35de6325 --- /dev/null +++ b/lib/syntax_tools/examples/merl/basic_test.erl @@ -0,0 +1,77 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012 Richard Carlsson +%% @doc Tests. For including in another module. + +%-module(basic_test). +%-import(basic, run/1) + +-export([basic_fib/1]). + +-include_lib("eunit/include/eunit.hrl"). + +basics_test_() -> + [?_assertEqual(42, run(1,[{1,{stop, 42}}])), + ?_assertEqual("hello", run(1,[{1,{stop,"hello"}}])), + ?_assertEqual(0, run(1,[{1,{print, "hello ~w", [42]}}])), + ?_assertEqual(5, run(1,[{1,{stop, {plus, 2, 3}}}])), + ?_assertEqual(5, run(1,[{1,{stop,{plus, 8, -3}}}])), + ?_assertEqual(0, run(1,[{1,{stop,{equal, 0, 1}}}])), + ?_assertEqual(1, run(1,[{1,{stop,{equal, 1, 1}}}])), + ?_assertEqual(0, run(1,[{1,{stop,{gt, 0, 1}}}])), + ?_assertEqual(0, run(1,[{1,{stop,{gt, 1, 1}}}])), + ?_assertEqual(1, run(1,[{1,{stop,{gt, 2, 1}}}])), + ?_assertEqual(0, run(1,[{1,{stop,{knot, 42}}}])), + ?_assertEqual(1, run(1,[{1,{stop,{knot, 0}}}])), + ?_assertEqual(42, run(1,[{1,{set, x, 42}}, {2,{stop,x}}])), + ?_assertEqual(17, run(1,[{1,{iff, 1, 2, 3}}, + {2,{stop, 17}}, + {3,{stop, 42}}])), + ?_assertEqual(42, run(1,[{1,{iff, 0, 2, 3}}, + {2,{stop, 17}}, + {3,{stop, 42}}])), + ?_assertEqual(17, run(1,[{1,{iff, 1, 2, 3}}, + {2,{stop, 17}}, + {3,{stop, -1}}])), + ?_assertEqual(42, run(1,[{1,{iff, 0, 2, 3}}, + {2,{stop, -1}}, + {3,{stop, 42}}])) + + + ]. + + +fib_test_() -> + [?_assertEqual(fib(N), basic_fib(N)) || N <- lists:seq(1,15) + ]. + + +fib(N) when N > 1 -> + fib(N-1) + fib(N-2); +fib(_) -> + 1. + +basic_fib(N) -> + run(1, + [{1,{set,x,0}}, + {2,{set,a,1}}, + {3,{set,b,0}}, + {10,{iff, {equal, x, N}, 20, 30}}, + {20,{stop,a}}, + {30,{print,"~w, ~w, ~w\n",[x,a,b]}}, + {31,{set,t,a}}, + {32,{set,a,{plus,a,b}}}, + {33,{set,b,t}}, + {34,{set,x,{plus,x,1}}}, + {40,{goto,10}} + ]). diff --git a/lib/syntax_tools/examples/merl/basicc.erl b/lib/syntax_tools/examples/merl/basicc.erl new file mode 100644 index 0000000000..531ac51538 --- /dev/null +++ b/lib/syntax_tools/examples/merl/basicc.erl @@ -0,0 +1,149 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012 Richard Carlsson +%% @doc Basic compiler in Erlang. + +-module(basicc). + +-export([run/2, make_lines/1, bool/1]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(INTERPRETED, true). +-include("basic_test.erl"). + +-include("merl.hrl"). + +run(N, Prog) -> + compile(Prog, tmp), + tmp:run(N, Prog). + +make_lines(Prog) -> + ets:new(line, [private, named_table, ordered_set]), + lists:foreach(fun ({L,_}) -> ets:insert(line, {L,label(L)}) end, Prog). + +compile(Prog, ModName) -> + make_lines(Prog), + Fs0 = lists:map(fun ({L, X}) -> + {true, label(L), + case stmt(X) of + {Stmt, false} -> + [?Q("() -> _@Stmt")]; + {Stmt, true} -> + Next = case ets:next(line, L) of + '$end_of_table' -> + ?Q("stop(0)"); + L1 -> + Label = label(L1), + ?Q("_@Label@()") + end, + [?Q("() -> _@Stmt, _@Next")] + end} + end, Prog), + ets:delete(line), + Run = ?Q(["(N, Prog) ->", + " ets:new(var, [private, named_table]),", + " basicc:make_lines(Prog),", + " goto(N)" + ]), + Stop = ?Q(["(R) ->", + " ets:delete(var),", + " ets:delete(line),", + " R" + ]), + Goto = ?Q(["(L) ->", + " case ets:lookup(line, L) of", + " [{_, X}] -> apply(tmp, X, []);", + " _ ->", + " case ets:next(line, L) of", + " '$end_of_table' -> stop(0);", + " L1 -> goto(L1)", + " end", + " end"]), + Fs = [{true, run, [Run]}, + {false, stop, [Stop]}, + {true, goto, [Goto]} + | Fs0], + Forms = merl_build:module_forms( + lists:foldl(fun ({X, Name, Cs}, S) -> + merl_build:add_function(X, Name, Cs, S) + end, + merl_build:init_module(ModName), + Fs)), + %% %% Write source to file for debugging + %% file:write_file(lists:concat([ModName, "_gen.erl"]), + %% erl_prettypr:format(erl_syntax:form_list(Forms), + %% [{paper,160},{ribbon,80}])), + merl:compile_and_load(Forms, [verbose]). + +label(L) -> + list_to_atom("label_" ++ integer_to_list(L)). + +stmt({print, S, As}) -> + Exprs = [expr(A) || A <- As], + {[?Q(["io:format(_@S@, [_@Exprs])"])], true}; +stmt({set, V, X}) -> + Expr = expr(X), + {[?Q(["ets:insert(var, {_@V@, _@Expr})"])], true}; +stmt({goto, X}) -> + {[jump(X)], false}; +stmt({stop, X}) -> + Expr = expr(X), + {[?Q(["stop(_@Expr)"])], false}; +stmt({iff, X, A, B}) -> + Cond = expr(X), + True = jump(A), + False = jump(B), + {?Q(["case _@Cond of", + " 0 -> _@False;", + " _ -> _@True", + "end"]), + false}. + +jump(X) -> + case ets:lookup(line, X) of + [{_, F}] -> + ?Q(["_@F@()"]); + true -> + Expr = expr(X), + [?Q(["goto(_@Expr)"])] + end. + +expr(X) when is_number(X) ; is_list(X) -> + ?Q("_@X@"); +expr(X) when is_atom(X) -> + ?Q(["case ets:lookup(var, _@X@) of", + " [] -> 0;", + " [{_,V}] -> V", + "end"]); +expr({plus, X, Y}) -> + ExprX = expr(X), + ExprY = expr(Y), + ?Q("_@ExprX + _@ExprY"); +expr({equal, X, Y}) -> + ExprX = expr(X), + ExprY = expr(Y), + ?Q("basicc:bool(_@ExprX == _@ExprY)"); +expr({gt, X, Y}) -> + ExprX = expr(X), + ExprY = expr(Y), + ?Q("basicc:bool(_@ExprX > _@ExprY)"); +expr({knot, X}) -> + Expr = expr(X), + ?Q(["case _@Expr of", + " 0 -> 1;", + " _ -> 0", + "end"]). + +bool(true) -> 1; +bool(false) -> 0. diff --git a/lib/syntax_tools/examples/merl/lisp.erl b/lib/syntax_tools/examples/merl/lisp.erl new file mode 100644 index 0000000000..371dc6b261 --- /dev/null +++ b/lib/syntax_tools/examples/merl/lisp.erl @@ -0,0 +1,160 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012 Richard Carlsson +%% @doc Trivial Lisp interpreter in Erlang. + +-module(lisp). + +-export([eval/1]). + +-export([init/0, equal/2, gt/2, knot/1]). + +-record(st, {env}). + +-define(INTERPRETED, true). +-include("lisp_test.erl"). + +eval(P) -> + {X, _} = eval(P, init()), + X. + +init() -> + Env = [{print, {builtin, fun do_print/2}} + ,{list, {builtin, fun do_list/2}} + ,{apply, {builtin, fun do_apply/2}} + ,{plus, {builtin, fun do_plus/2}} + ,{equal, {builtin, fun do_equal/2}} + ,{gt, {builtin, fun do_gt/2}} + ,{knot, {builtin, fun do_knot/2}} + ,{y, y()} + ], + #st{env=dict:from_list(Env)}. + +eval([lambda, Ps, B], #st{env=E}=St) when is_list(Ps) -> + case lists:all(fun is_atom/1, Ps) andalso + (length(Ps) =:= length(lists:usort(Ps))) of + true -> {{lambda, Ps, B, E}, St}; + false -> throw(bad_lambda) + end; +eval([lambda | _], _) -> + throw(bad_lambda); +eval([def, A, V, B], #st{env=E0}=St) when is_atom(A) -> + {V1, St1} = eval(V, St), + E1 = bind(A, V1, E0), + {X, St2} = eval(B, St1#st{env=E1}), + {X, St2#st{env=E0}}; +eval([def | _], _) -> + throw(bad_def); +eval([quote, A], St) -> + {A, St}; +eval([quote | _], _) -> + throw(bad_quote); +eval([iff, X, A, B], St) -> + case eval(X, St) of + {[], St1} -> eval(B, St1); + {_, St1} -> eval(A, St1) + end; +eval([do], _St0) -> + throw(bad_do); +eval([do | As], St0) -> + lists:foldl(fun (X, {_,St}) -> eval(X, St) end, {[],St0}, As); +eval([_|_]=L, St) -> + {[F | As], St1} = lists:mapfoldl(fun eval/2, St, L), + call(F, As, St1); +eval(A, St) when is_atom(A) -> + {deref(A, St), St}; +eval(C, St) -> + {C, St}. + +%% UTILITY FUNCTIONS + +deref(A, #st{env=E}) -> + case dict:find(A, E) of + {ok, V} -> V; + error -> throw({undefined, A}) + end. + +bind(A, V, E) -> + dict:store(A, V, E). + +bind_args([P | Ps], [A | As], E) -> + bind_args(Ps, As, dict:store(P, A, E)); +bind_args([], [], E) -> + E; +bind_args(_, _, _) -> + throw(bad_arity). + +call({lambda, Ps, B, E}, As, #st{env=E0}=St) -> + {X, St1} = eval(B, St#st{env=bind_args(Ps, As, E)}), + {X, St1#st{env=E0}}; +call({builtin, F}, As, St) -> + F(As, St); +call(X, _, _) -> + throw({bad_fun, X}). + +bool(true) -> 1; +bool(false) -> []. + +%% BUILTINS + +y() -> + {Y, _} = eval([lambda, [f], + [[lambda, [x], [f, [lambda, [y], [[x, x], y]]]], + [lambda, [x], [f, [lambda, [y], [[x, x], y]]]]]], + #st{env=dict:new()}), + Y. + +do_print([S | Xs], St) -> + io:format(S, Xs), + {[], St}; +do_print(_, _) -> + throw(bad_print). + +do_list(As, St) -> + {As, St}. + +do_apply([F, As], St) -> + call(F, As, St); +do_apply(_, _) -> + throw(bad_apply). + +do_plus([X, Y], St) when is_number(X), is_number(Y) -> + {X + Y, St}; +do_plus(As, _) -> + throw({bad_plus, As}). + +do_equal([X, Y], St) -> + {equal(X, Y), St}; +do_equal(As, _) -> + throw({bad_equal, As}). + +equal(X, Y) -> + bool(X =:= Y). + +do_gt([X, Y], St) -> + {gt(X, Y), St}; +do_gt(As, _) -> + throw({bad_gt, As}). + +gt(X, Y) -> + bool(X > Y). + +do_knot([X], St) -> + {knot(X), St}; +do_knot(As, _) -> + throw({bad_gt, As}). + +knot([]) -> + 1; +knot(_) -> + []. diff --git a/lib/syntax_tools/examples/merl/lisp_test.erl b/lib/syntax_tools/examples/merl/lisp_test.erl new file mode 100644 index 0000000000..cab8134b8f --- /dev/null +++ b/lib/syntax_tools/examples/merl/lisp_test.erl @@ -0,0 +1,98 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012 Richard Carlsson +%% @doc Tests. For including in another module. + +%-module(lisp_test). +%-import(lisp, eval/1) + +-export([fib/1, lisp_fib/1]). + +-include_lib("eunit/include/eunit.hrl"). + +basics_test_() -> + [?_assertEqual(42, eval(42)), + ?_assertEqual("hello", eval([quote, "hello"])), + ?_assertEqual(print, eval([quote, print])), + ?_assertMatch([17,[1,2],42], eval([list,17,[list,1,2],42])), + ?_assertEqual([], eval([print, [quote, "hello ~w"], [list, 42]])), + ?_assertEqual(5, eval([plus, 2, 3])), + ?_assertEqual(5, eval([plus, 8, -3])), + ?_assertEqual([], eval([equal, 0, 1])), + ?_assertEqual(1, eval([equal, 1, 1])), + ?_assertEqual([], eval([gt, 0, 1])), + ?_assertEqual([], eval([gt, 1, 1])), + ?_assertEqual(1, eval([gt, 2, 1])), + ?_assertEqual([], eval([knot, 42])), + ?_assertEqual(1, eval([knot, []])), + ?_assertEqual(42, eval([do, 17, 42])), + ?_assertEqual([], eval([apply, print, [quote, ["~p", [42]]]])), + ?_assertEqual(42, eval([iff, [], 17, 42])), + ?_assertEqual(17, eval([iff, 1, 17, 42])), + ?_assertEqual(42, eval([iff, [], [apply], 42])), + ?_assertEqual(17, eval([iff, 1, 17, [apply]])), + ?_assertEqual(17, eval([def, foo, 17, foo])), + ?_assertEqual(17, eval([def, bar, 42, [def, foo, 17, foo]])), + ?_assertEqual(42, eval([def, bar, 42, [def, foo, 17, bar]])), + ?_assertEqual(17, eval([def, foo, 42, [def, foo, 17, foo]])) + ]. + +-ifdef(INTERPRETED). +interpreter_basics_test_() -> + [?_assertThrow({undefined, foo}, eval(foo)), + ?_assertMatch({builtin,_}, eval(print)), + ?_assertThrow(bad_do, eval([do])), + ?_assertThrow(bad_apply, eval([apply])), + ?_assertThrow({undefined, foo}, eval([def, bar, 17, foo])) + ]. + +interpreter_lambda_test_() -> + [?_assertMatch({lambda,_,_,_}, eval([lambda, [], 42])), + ?_assertMatch({lambda,_,_,_}, eval([lambda, [x], x])), + ?_assertMatch({lambda,_,_,_}, eval([lambda, [x,y], 42])) + ]. +-endif. + +lambda_test_() -> + [?_assertThrow(bad_lambda, eval([lambda])), + ?_assertThrow(bad_lambda, eval([lambda, []])), + ?_assertThrow(bad_lambda, eval([lambda, [], 17, 42])), + ?_assertThrow(bad_lambda, eval([lambda, 17, 42])), + ?_assertThrow(bad_lambda, eval([lambda, [17], 42])), + ?_assertThrow(bad_lambda, eval([lambda, [foo, foo], 42])), + ?_assertEqual(42, eval([[lambda, [x], x], 42])), + ?_assertEqual([42, 17], eval([[lambda, [x], [list, x, 17]], 42])), + ?_assertEqual([42, 17], eval([def, f, [def, y, 42, + [lambda, [x], [list, y, x]]], + [f, 17]])) + ]. + +fib_test_() -> + [?_assertEqual(fib(N), lisp_fib(N)) || N <- lists:seq(1,15) + ]. + + +fib(N) when N > 1 -> + fib(N-1) + fib(N-2); +fib(_) -> + 1. + +lisp_fib(N) -> + eval([def, fib, + [y, [lambda, [f], [lambda, [x], + [iff, [gt, x, 1], + [plus, [f, [plus,x,-1]], [f, [plus,x,-2]]], + 1] + ]]], + [fib, N] + ]). diff --git a/lib/syntax_tools/examples/merl/lispc.erl b/lib/syntax_tools/examples/merl/lispc.erl new file mode 100644 index 0000000000..97072cdab7 --- /dev/null +++ b/lib/syntax_tools/examples/merl/lispc.erl @@ -0,0 +1,102 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012 Richard Carlsson +%% @doc Lisp compiler in Erlang. + +-module(lispc). + +-export([eval/1]). + +-record(st, {}). + +-include("lisp_test.erl"). + +-include("merl.hrl"). + +eval(Lisp) -> + compile(Lisp, tmp), + tmp:eval(). + +compile(Lisp, ModName) -> + {Code, _} = gen(Lisp, #st{}), + Main = ?Q(["() ->", + " __print = fun (S, Xs) -> io:format(S,Xs), [] end,", + " __apply = fun erlang:apply/2,", + " __plus = fun erlang:'+'/2,", + " __equal = fun lisp:equal/2,", + " __gt = fun lisp:gt/2,", + " __knot = fun lisp:knot/1,", + " __y = fun (F) ->", + " (fun (X) -> F(fun (Y) -> (X(X))(Y) end) end)", + " (fun (X) -> F(fun (Y) -> (X(X))(Y) end) end)", + " end,", + " _@Code"]), + Forms = merl_build:module_forms( + merl_build:add_function(true, eval, [Main], + merl_build:init_module(ModName))), + %% %% Write source to file for debugging + %% file:write_file(lists:concat([ModName, "_gen.erl"]), + %% erl_prettypr:format(erl_syntax:form_list(Forms), + %% [{paper,160},{ribbon,80}])), + merl:compile_and_load(Forms, [verbose]). + +var(Atom) -> + merl:var(list_to_atom("__" ++ atom_to_list(Atom))). + +gen([lambda, Ps, B], St) when is_list(Ps) -> + case lists:all(fun is_atom/1, Ps) andalso + (length(Ps) =:= length(lists:usort(Ps))) of + true -> + Vars = [var(P) || P <- Ps], + {Body, St1} = gen(B, St), + {?Q("fun (_@Vars) -> _@Body end"), St1}; + false -> + throw(bad_lambda) + end; +gen([lambda | _], _) -> + throw(bad_lambda); +gen([def, A, V, B], St) when is_atom(A) -> + Var = var(A), + {Val, St1} = gen(V, St), + {Body, St2} = gen(B, St1), + {?Q("(fun (_@Var) -> _@Body end)(_@Val)"), St2}; +gen([def | _], _) -> + throw(bad_def); +gen([quote, A], St) -> + {merl:term(A), St}; +gen([quote | _], _) -> + throw(bad_quote); +gen([iff, X, A, B], St) -> + {Cond, St1} = gen(X, St), + {True, St2} = gen(A, St1), + {False, St3} = gen(B, St2), + {?Q(["case _@Cond of", + " [] -> _@False;", + " _ -> _@True", + "end"]), + St3}; +gen([do], _) -> + throw(bad_do); +gen([do | As], St0) -> + {Body, St1} = lists:mapfoldl(fun gen/2, St0, As), + {?Q("begin _@Body end"), St1}; +gen([list | As], St0) -> + {Elem, St1} = lists:mapfoldl(fun gen/2, St0, As), + {?Q("[ _@Elem ]"), St1}; +gen([_|_]=L, St) -> + {[F | As], St1} = lists:mapfoldl(fun gen/2, St, L), + {?Q("((_@F)(_@As))"), St1}; +gen(A, St) when is_atom(A) -> + {var(A), St}; +gen(C, St) -> + {merl:term(C), St}. diff --git a/lib/syntax_tools/examples/merl/merl_build.erl b/lib/syntax_tools/examples/merl/merl_build.erl new file mode 100644 index 0000000000..c539f8e2af --- /dev/null +++ b/lib/syntax_tools/examples/merl/merl_build.erl @@ -0,0 +1,104 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012 Richard Carlsson +%% @doc Making it simple to build a module with merl + +-module(merl_build). + +-export([init_module/1, module_forms/1, add_function/4, add_record/3, + add_import/3, add_attribute/3, set_file/2]). + +-import(merl, [term/1]). + +-include("merl.hrl"). + +-type filename() :: string(). + +-record(module, { name :: atom() + , file :: filename() + , exports=[] :: [{atom(), integer()}] + , imports=[] :: [{atom(), [{atom(), integer()}]}] + , attributes=[] :: [{filename(), atom(), [term()]}] + , records=[] :: [{filename(), atom(), + [{atom(), merl:tree()}]}] + , functions=[] :: [{filename(), atom(), [merl:tree()]}] + }). + +%% TODO: init module from a list of forms (from various sources) + +%% @doc Create a new module representation, using the given module name. +init_module(Name) when is_atom(Name) -> + %% use the module name as the default file name - better than nothing + #module{name=Name, file=atom_to_list(Name)}. + +%% @doc Get the list of syntax tree forms for a module representation. This can +%% be passed to compile/2. +module_forms(#module{name=Name, + exports=Xs, + imports=Is, + records=Rs, + attributes=As, + functions=Fs}) + when is_atom(Name), Name =/= undefined -> + Module = ?Q("-module('@Name@')."), + Exported = [erl_syntax:arity_qualifier(term(N), term(A)) + || {N,A} <- ordsets:from_list(Xs)], + Export = ?Q("-export(['@_Exported'/1])."), + Imports = [?Q("-import('@M@', ['@_NAs'/1]).") + || {M, Ns} <- Is, + NAs <- [[erl_syntax:arity_qualifier(term(N), term(A)) + || {N,A} <- ordsets:from_list(Ns)]] + ], + Attrs = [?Q("-file(\"'@File@\",1). -'@N@'('@T@').") + || {File, N, T} <- lists:reverse(As)], + Records = [?Q("-file(\"'@File@\",1). -record('@N@',{'@_RFs'=[]}).") + || {File, N, Es} <- lists:reverse(Rs), + RFs <- [[erl_syntax:record_field(term(F), V) + || {F,V} <- Es]] + ], + Functions = [?Q("-file(\"'@File@\",1). '@_F'() -> [].") + || {File, N, Cs} <- lists:reverse(Fs), + F <- [erl_syntax:function(term(N), Cs)]], + lists:flatten([Module, Export, Imports, Attrs, Records, Functions]). + +%% @doc Set the source file name for all subsequently added functions, +%% records, and attributes. +set_file(Filename, #module{}=M) -> + M#module{file=filename:flatten(Filename)}. + +%% @doc Add a function to a module representation. +add_function(Exported, Name, Clauses, + #module{file=File, exports=Xs, functions=Fs}=M) + when is_boolean(Exported), is_atom(Name), Clauses =/= [] -> + Arity = length(erl_syntax:clause_patterns(hd(Clauses))), + Xs1 = case Exported of + true -> [{Name,Arity} | Xs]; + false -> Xs + end, + M#module{exports=Xs1, functions=[{File, Name, Clauses} | Fs]}. + +%% @doc Add a record declaration to a module representation. +add_record(Name, Fields, #module{file=File, records=Rs}=M) + when is_atom(Name) -> + M#module{records=[{File, Name, Fields} | Rs]}. + +%% @doc Add a "wild" attribute, such as `-compile(Opts)' to a module +%% representation. Note that such attributes can only have a single argument. +add_attribute(Name, Term, #module{file=File, attributes=As}=M) + when is_atom(Name) -> + M#module{attributes=[{File, Name, Term} | As]}. + +%% @doc Add an import declaration to a module representation. +add_import(From, Names, #module{imports=Is}=M) + when is_atom(From), is_list(Names) -> + M#module{imports=[{From, Names} | Is]}. diff --git a/lib/syntax_tools/include/merl.hrl b/lib/syntax_tools/include/merl.hrl new file mode 100644 index 0000000000..e44a78dece --- /dev/null +++ b/lib/syntax_tools/include/merl.hrl @@ -0,0 +1,29 @@ +%% --------------------------------------------------------------------- +%% Header file for merl +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-ifndef(MERL_HRL). + + +%% Quoting a piece of code +-define(Q(Text), merl:quote(?LINE, Text)). + +%% Quasi-quoting code, substituting metavariables listed in Env +-define(Q(Text, Env), merl:qquote(?LINE, Text, Env)). + + +-ifndef(MERL_NO_TRANSFORM). +-compile({parse_transform, merl_transform}). +-endif. + + +-endif. diff --git a/lib/syntax_tools/src/Makefile b/lib/syntax_tools/src/Makefile index c9fbad8f9a..2e91adf8af 100644 --- a/lib/syntax_tools/src/Makefile +++ b/lib/syntax_tools/src/Makefile @@ -22,6 +22,9 @@ RELSYSDIR = $(RELEASE_PATH)/lib/syntax_tools-$(VSN) # EBIN = ../ebin +INCLUDE=../include + +ERL_COMPILE_FLAGS += -pa $(EBIN) -pa ./ -I$(INCLUDE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native @@ -30,10 +33,15 @@ ERL_COMPILE_FLAGS += +nowarn_shadow_vars +warn_unused_import -Werror # +warn_mis SOURCES=erl_syntax.erl erl_prettypr.erl erl_syntax_lib.erl \ erl_comment_scan.erl erl_recomment.erl erl_tidy.erl \ - epp_dodger.erl prettypr.erl igor.erl + epp_dodger.erl prettypr.erl igor.erl \ + merl.erl merl_transform.erl + +INCLUDE_FILES = merl.hrl OBJECTS=$(SOURCES:%.erl=$(EBIN)/%.$(EMULATOR)) $(APP_TARGET) $(APPUP_TARGET) +INCLUDE_DELIVERABLES = $(INCLUDE_FILES:%=$(INCLUDE)/%) + APP_FILE= syntax_tools.app APP_SRC= $(APP_FILE).src APP_TARGET= $(EBIN)/$(APP_FILE) @@ -52,6 +60,7 @@ all: $(OBJECTS) clean: + rm -f ./merl_transform.beam rm -f $(OBJECTS) rm -f core *~ @@ -64,6 +73,15 @@ realclean: clean $(EBIN)/%.$(EMULATOR):%.erl $(erlc_verbose)erlc -W $(ERL_COMPILE_FLAGS) -o$(EBIN) $< +# special rules and dependencies to apply the transform to itself +$(EBIN)/merl_transform.beam: $(EBIN)/merl.beam ./merl_transform.beam \ + ../include/merl.hrl \ + $(EBIN)/erl_syntax.beam $(EBIN)/erl_syntax_lib.beam +./merl_transform.beam: ./merl_transform.erl $(EBIN)/merl.beam \ + ../include/merl.hrl + $(V_ERLC) -DMERL_NO_TRANSFORM $(ERL_COMPILE_FLAGS) -o ./ $< + + # ---------------------------------------------------- # Special Build Targets # ---------------------------------------------------- @@ -84,6 +102,8 @@ release_spec: opt $(INSTALL_DATA) $(OBJECTS) "$(RELSYSDIR)/ebin" $(INSTALL_DIR) "$(RELSYSDIR)/src" $(INSTALL_DATA) $(SOURCES) "$(RELSYSDIR)/src" + $(INSTALL_DIR) "$(RELSYSDIR)/include" + $(INSTALL_DATA) $(INCLUDE_DELIVERABLES) "$(RELSYSDIR)/include" release_docs_spec: diff --git a/lib/syntax_tools/src/merl.erl b/lib/syntax_tools/src/merl.erl new file mode 100644 index 0000000000..690306c17b --- /dev/null +++ b/lib/syntax_tools/src/merl.erl @@ -0,0 +1,1230 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% Note: EDoc uses @@ and @} as escape sequences, so in the doc text below, +%% `@@' must be written `@@@@' and `@}' must be written `@@}'. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2010-2015 Richard Carlsson +%% +%% @doc Metaprogramming in Erlang. +%% Merl is a more user friendly interface to the `erl_syntax' module, making +%% it easy both to build new ASTs from scratch and to +%% match and decompose existing ASTs. For details that are outside the scope +%% of Merl itself, please see the documentation of {@link erl_syntax}. +%% +%% == Quick start == +%% +%% To enable the full power of Merl, your module needs to include the Merl +%% header file: +%% ```-include_lib("syntax_tools/include/merl.hrl").''' +%% +%% Then, you can use the `?Q(Text)' macros in your code to create ASTs or match +%% on existing ASTs. For example: +%% ```Tuple = ?Q("{foo, 42}"), +%% ?Q("{foo, _@Number}") = Tuple, +%% Call = ?Q("foo:bar(_@Number)")''' +%% +%% Calling `merl:print(Call)' will then print the following code: +%% ```foo:bar(42)''' +%% +%% The `?Q' macros turn the quoted code fragments into ASTs, and lifts +%% metavariables such as `_@Tuple' and `_@Number' to the level of your Erlang +%% code, so you can use the corresponding Erlang variables `Tuple' and `Number' +%% directly. This is the most straightforward way to use Merl, and in many +%% cases it's all you need. +%% +%% You can even write case switches using `?Q' macros as patterns. For example: +%% ```case AST of +%% ?Q("{foo, _@Foo}") -> handle(Foo); +%% ?Q("{bar, _@Bar}") when erl_syntax:is_integer(Bar) -> handle(Bar); +%% _ -> handle_default() +%% end''' +%% +%% These case switches only allow `?Q(...)' or `_' as clause patterns, and the +%% guards may contain any expressions, not just Erlang guard expressions. +%% +%% If the macro `MERL_NO_TRANSFORM' is defined before the `merl.hrl' header +%% file is included, the parse transform used by Merl will be disabled, and in +%% that case, the match expressions `?Q(...) = ...', case switches using +%% `?Q(...)' patterns, and automatic metavariables like `_@Tuple' cannot be +%% used in your code, but the Merl macros and functions still work. To do +%% metavariable substitution, you need to use the `?Q(Text, Map)' macro, e.g.: +%% ```Tuple = ?Q("{foo, _@bar, _@baz}", [{bar, Bar}, {baz,Baz}])''' +%% +%% The text given to a `?Q(Text)' macro can be either a single string, or a +%% list of strings. The latter is useful when you need to split a long +%% expression over multiple lines, e.g.: +%% ```?Q(["case _@Expr of", +%% " {foo, X} -> f(X);", +%% " {bar, X} -> g(X)", +%% " _ -> h(X)" +%% "end"])''' +%% If there is a syntax error somewhere in the text (like the missing semicolon +%% in the second clause above) this allows Merl to generate an error message +%% pointing to the exact line in your source code. (Just remember to +%% comma-separate the strings in the list, otherwise Erlang will concatenate +%% the string fragments as if they were a single string.) +%% +%% == Metavariable syntax == +%% +%% There are several ways to write a metavariable in your quoted code: +%% <ul> +%% <li>Atoms starting with `@', for example `` '@foo' '' or `` '@Foo' ''</li> +%% <li>Variables starting with `_@', for example `_@bar' or `_@Bar'</li> +%% <li>Strings starting with ``"'@'', for example ``"'@File"''</li> +%% <li>Integers starting with 909, for example `9091' or `909123'</li> +%% </ul> +%% Following the prefix, one or more `_' or `0' characters may be used to +%% indicate "lifting" of the variable one or more levels, and after that, a `@' +%% or `9' character indicates a glob metavariable (matching zero or more +%% elements in a sequence) rather than a normal metavariable. For example: +%% <ul> +%% <li>`` '@_foo' '' is lifted one level, and `_@__foo' is lifted two +%% levels</li> +%% <li>`_@@@@bar' is a glob variable, and `_@_@bar' is a lifted glob +%% variable</li> +%% <li>`90901' is a lifted variable,`90991' is a glob variable, and `9090091' +%% is a glob variable lifted two levels</li> +%% </ul> +%% (Note that the last character in the name is never considered to be a lift +%% or glob marker, hence, `_@__' and `90900' are only lifted one level, not +%% two. Also note that globs only matter for matching; when doing +%% substitutions, a non-glob variable can be used to inject a sequence of +%% elements, and vice versa.) +%% +%% If the name after the prefix and any lift and glob markers is `_' or `0', +%% the variable is treated as an anonymous catch-all pattern in matches. For +%% example, `_@_', `_@@@@_', `_@__', or even `_@__@_'. +%% +%% Finally, if the name without any prefixes or lift/glob markers begins with +%% an uppercase character, as in `_@Foo' or `_@_@Foo', it will become a +%% variable on the Erlang level, and can be used to easily deconstruct and +%% construct syntax trees: +%% ```case Input of +%% ?Q("{foo, _@Number}") -> ?Q("foo:bar(_@Number)"); +%% ...''' +%% We refer to these as "automatic metavariables". If in addition the name ends +%% with `@', as in `_@Foo@', the value of the variable as an Erlang term will +%% be automatically converted to the corresponding abstract syntax tree when +%% used to construct a larger tree. For example, in: +%% ```Bar = {bar, 42}, +%% Foo = ?Q("{foo, _@Bar@@}")''' +%% (where Bar is just some term, not a syntax tree) the result `Foo' will be a +%% syntax tree representing `{foo, {bar, 42}}'. This avoids the need for +%% temporary variables in order to inject data, as in +%% ```TmpBar = erl_syntax:abstract(Bar), +%% Foo = ?Q("{foo, _@TmpBar}")''' +%% +%% If the context requires an integer rather than a variable, an atom, or a +%% string, you cannot use the uppercase convention to mark an automatic +%% metavariable. Instead, if the integer (without the `909'-prefix and +%% lift/glob markers) ends in a `9', the integer will become an Erlang-level +%% variable prefixed with `Q', and if it ends with `99' it will also be +%% automatically abstracted. For example, the following will increment the +%% arity of the exported function f: +%% ```case Form of +%% ?Q("-export([f/90919]).") -> +%% Q2 = erl_syntax:concrete(Q1) + 1, +%% ?Q("-export([f/909299])."); +%% ...''' +%% +%% == When to use the various forms of metavariables == +%% +%% Merl can only parse a fragment of text if it follows the basic syntactical +%% rules of Erlang. In most places, a normal Erlang variable can be used as +%% metavariable, for example: +%% ```?Q("f(_@Arg)") = Expr''' +%% but if you want to match on something like the name of a function, you have +%% to use an atom as metavariable: +%% ```?Q("'@Name'() -> _@@@@_." = Function''' +%% (note the anonymous glob variable `_@@@@_' to ignore the function body). +%% +%% In some contexts, only a string or an integer is allowed. For example, the +%% directive `-file(Name, Line)' requires that `Name' is a string literal and +%% `Line' an integer literal: +%% +%% ```?Q("-file(\"'@File\", 9090).") = ?Q("-file(\"foo.erl\", 42).")).''' +%% This will extract the string literal `"foo.erl"' into the variable `Foo'. +%% Note the use of the anonymous variable `9090' to ignore the line number. To +%% match and also bind a metavariable that must be an integer literal, we can +%% use the convention of ending the integer with a 9, turning it into a +%% Q-prefixed variable on the Erlang level (see the previous section). +%% +%% === Globs === +%% +%% Whenever you want to match out a number of elements in a sequence (zero or +%% more) rather than a fixed set of elements, you need to use a glob. For +%% example: +%% ```?Q("{_@@@@Elements}") = ?Q({a, b, c})''' +%% will bind Elements to the list of individual syntax trees representing the +%% atoms `a', `b', and `c'. This can also be used with static prefix and suffix +%% elements in the sequence. For example: +%% ```?Q("{a, b, _@@@@Elements}") = ?Q({a, b, c, d})''' +%% will bind Elements to the list of the `c' and `d' subtrees, and +%% ```?Q("{_@@@@Elements, c, d}") = ?Q({a, b, c, d})''' +%% will bind Elements to the list of the `a' and `b' subtrees. You can even use +%% plain metavariables in the prefix or suffix: +%% ```?Q("{_@First, _@@@@Rest}") = ?Q({a, b, c})''' +%% or +%% ```?Q("{_@@@@_, _@Last}") = ?Q({a, b, c})''' +%% (ignoring all but the last element). You cannot however have two globs as +%% part of the same sequence. +%% +%% === Lifted metavariables === +%% +%% In some cases, the Erlang syntax rules make it impossible to place a +%% metavariable directly where you would like it. For example, you cannot +%% write: +%% ```?Q("-export([_@@@@Name]).")''' +%% to match out all name/arity pairs in the export list, or to insert a list of +%% exports in a declaration, because the Erlang parser only allows elements on +%% the form `A/I' (where `A' is an atom and `I' an integer) in the export list. +%% A variable like the above is not allowed, but neither is a single atom or +%% integer, so `` '@@@@Name' '' or `909919' wouldn't work either. +%% +%% What you have to do in such cases is to write your metavariable in a +%% syntactically valid position, and use lifting markers to denote where it +%% should really apply, as in: +%% ```?Q("-export(['@@_@@Name'/0]).")''' +%% This causes the variable to be lifted (after parsing) to the next higher +%% level in the syntax tree, replacing that entire subtree. In this case, the +%% `` '@@_@@Name'/0 '' will be replaced with `` '@@@@Name' '', and the ``/0'' +%% part was just used as dummy notation and will be discarded. +%% +%% You may even need to apply lifting more than once. To match the entire +%% export list as a single syntax tree, you can write: +%% ```?Q("-export(['@@__Name'/0]).")''' +%% using two underscores, but with no glob marker this time. This will make the +%% entire ``['@@__Name'/0]'' part be replaced with `` '@@Name' ''. +%% +%% Sometimes, the tree structure of a code fragment isn't very obvious, and +%% parts of the structure may be invisible when printed as source code. For +%% instance, a simple function definition like the following: +%% ```zero() -> 0.''' +%% consists of the name (the atom `zero'), and a list of clauses containing the +%% single clause `() -> 0'. The clause consists of an argument list (empty), a +%% guard (empty), and a body (which is always a list of expressions) containing +%% the single expression `0'. This means that to match out the name and the +%% list of clauses of any function, you'll need to use a pattern like +%% ``?Q("'@Name'() -> _@_@Body.")'', using a dummy clause whose body is a glob +%% lifted one level. +%% +%% To visualize the structure of a syntax tree, you can use the function +%% `merl:show(T)', which prints a summary. For example, entering +%% ```merl:show(merl:quote("inc(X, Y) when Y > 0 -> X + Y."))''' +%% in the Erlang shell will print the following (where the `+' signs separate +%% groups of subtrees on the same level): +%% ```function: inc(X, Y) when ... -> X + Y. +%% atom: inc +%% + +%% clause: (X, Y) when ... -> X + Y +%% variable: X +%% variable: Y +%% + +%% disjunction: Y > 0 +%% conjunction: Y > 0 +%% infix_expr: Y > 0 +%% variable: Y +%% + +%% operator: > +%% + +%% integer: 0 +%% + +%% infix_expr: X + Y +%% variable: X +%% + +%% operator: + +%% + +%% variable: Y''' +%% +%% This shows another important non-obvious case: a clause guard, even if it's +%% as simple as `Y > 0', always consists of a single disjunction of one or more +%% conjunctions of tests, much like a tuple of tuples. Thus: +%% <ul> +%% <li>``"when _@Guard ->"'' will only match a guard with exactly one +%% test</li> +%% <li>``"when _@@@@Guard ->"'' will match a guard with one or more +%% comma-separated tests (but no semicolons), binding `Guard' to the list +%% of tests</li> +%% <li>``"when _@_Guard ->"'' will match just like the previous pattern, but +%% binds `Guard' to the conjunction subtree</li> +%% <li>``"when _@_@Guard ->"'' will match an arbitrary nonempty guard, +%% binding `Guard' to the list of conjunction subtrees</li> +%% <li>``"when _@__Guard ->"'' will match like the previous pattern, but +%% binds `Guard' to the whole disjunction subtree</li> +%% <li>and finally, ``"when _@__@Guard ->"'' will match any clause, +%% binding `Guard' to `[]' if the guard is empty and to `[Disjunction]' +%% otherwise</li> +%% </ul> +%% +%% Thus, the following pattern matches all possible clauses: +%% ```"(_@@Args) when _@__@Guard -> _@@Body"''' +%% @end + +-module(merl). + +-export([term/1, var/1, print/1, show/1]). + +-export([quote/1, quote/2, qquote/2, qquote/3]). + +-export([template/1, tree/1, subst/2, tsubst/2, alpha/2, match/2, switch/2]). + +-export([template_vars/1, meta_template/1]). + +-export([compile/1, compile/2, compile_and_load/1, compile_and_load/2]). + +%% NOTE: this module may not include merl.hrl! + +-type tree() :: erl_syntax:syntaxTree(). + +-type tree_or_trees() :: tree() | [tree()]. + +-type pattern() :: tree() | template(). + +-type pattern_or_patterns() :: pattern() | [pattern()]. + +-type env() :: [{Key::id(), pattern_or_patterns()}]. + +-type id() :: atom() | integer(). + +%% A list of strings or binaries is assumed to represent individual lines, +%% while a flat string or binary represents source code containing newlines. +-type text() :: string() | binary() | [string()] | [binary()]. + +-type location() :: erl_anno:location(). + + +%% ------------------------------------------------------------------------ +%% Compiling and loading code directly to memory + +%% @equiv compile(Code, []) +compile(Code) -> + compile(Code, []). + +%% @doc Compile a syntax tree or list of syntax trees representing a module +%% into a binary BEAM object. +%% @see compile_and_load/2 +%% @see compile/1 +compile(Code, Options) when not is_list(Code)-> + case type(Code) of + form_list -> compile(erl_syntax:form_list_elements(Code)); + _ -> compile([Code], Options) + end; +compile(Code, Options0) when is_list(Options0) -> + Forms = [erl_syntax:revert(F) || F <- Code], + Options = [verbose, report_errors, report_warnings, binary | Options0], + compile:noenv_forms(Forms, Options). + + +%% @equiv compile_and_load(Code, []) +compile_and_load(Code) -> + compile_and_load(Code, []). + +%% @doc Compile a syntax tree or list of syntax trees representing a module +%% and load the resulting module into memory. +%% @see compile/2 +%% @see compile_and_load/1 +compile_and_load(Code, Options) -> + case compile(Code, Options) of + {ok, ModuleName, Binary} -> + _ = code:load_binary(ModuleName, "", Binary), + {ok, Binary}; + Other -> Other + end. + + +%% ------------------------------------------------------------------------ +%% Utility functions + + +-spec var(atom()) -> tree(). + +%% @doc Create a variable. + +var(Name) -> + erl_syntax:variable(Name). + + +-spec term(term()) -> tree(). + +%% @doc Create a syntax tree for a constant term. + +term(Term) -> + erl_syntax:abstract(Term). + + +%% @doc Pretty-print a syntax tree or template to the standard output. This +%% is a utility function for development and debugging. + +print(Ts) when is_list(Ts) -> + lists:foreach(fun print/1, Ts); +print(T) -> + io:put_chars(erl_prettypr:format(tree(T))), + io:nl(). + +%% @doc Print the structure of a syntax tree or template to the standard +%% output. This is a utility function for development and debugging. + +show(Ts) when is_list(Ts) -> + lists:foreach(fun show/1, Ts); +show(T) -> + io:put_chars(pp(tree(T), 0)), + io:nl(). + +pp(T, I) -> + [lists:duplicate(I, $\s), + limit(lists:flatten([atom_to_list(type(T)), ": ", + erl_prettypr:format(erl_syntax_lib:limit(T,3))]), + 79-I), + $\n, + pp_1(lists:filter(fun (X) -> X =/= [] end, subtrees(T)), I+2) + ]. + +pp_1([G], I) -> + pp_2(G, I); +pp_1([G | Gs], I) -> + [pp_2(G, I), lists:duplicate(I, $\s), "+\n" | pp_1(Gs, I)]; +pp_1([], _I) -> + []. + +pp_2(G, I) -> + [pp(E, I) || E <- G]. + +%% limit string to N characters, stay on a single line and compact whitespace +limit([$\n | Cs], N) -> limit([$\s | Cs], N); +limit([$\r | Cs], N) -> limit([$\s | Cs], N); +limit([$\v | Cs], N) -> limit([$\s | Cs], N); +limit([$\t | Cs], N) -> limit([$\s | Cs], N); +limit([$\s, $\s | Cs], N) -> limit([$\s | Cs], N); +limit([C | Cs], N) when C < 32 -> limit(Cs, N); +limit([C | Cs], N) when N > 3 -> [C | limit(Cs, N-1)]; +limit([_C1, _C2, _C3, _C4 | _Cs], 3) -> "..."; +limit(Cs, 3) -> Cs; +limit([_C1, _C2, _C3 | _], 2) -> ".."; +limit(Cs, 2) -> Cs; +limit([_C1, _C2 | _], 1) -> "."; +limit(Cs, 1) -> Cs; +limit(_, _) -> []. + +%% ------------------------------------------------------------------------ +%% Parsing and instantiating code fragments + + +-spec qquote(Text::text(), Env::env()) -> tree_or_trees(). + +%% @doc Parse text and substitute meta-variables. +%% +%% @equiv qquote(1, Text, Env) + +qquote(Text, Env) -> + qquote(1, Text, Env). + + +-spec qquote(StartPos::location(), Text::text(), Env::env()) -> tree_or_trees(). + +%% @doc Parse text and substitute meta-variables. Takes an initial scanner +%% starting position as first argument. +%% +%% The macro `?Q(Text, Env)' expands to `merl:qquote(?LINE, Text, Env)'. +%% +%% @see quote/2 + +qquote(StartPos, Text, Env) -> + subst(quote(StartPos, Text), Env). + + +-spec quote(Text::text()) -> tree_or_trees(). + +%% @doc Parse text. +%% +%% @equiv quote(1, Text) + +quote(Text) -> + quote(1, Text). + + +-spec quote(StartPos::location(), Text::text()) -> tree_or_trees(). + +%% @doc Parse text. Takes an initial scanner starting position as first +%% argument. +%% +%% The macro `?Q(Text)' expands to `merl:quote(?LINE, Text, Env)'. +%% +%% @see quote/1 + +quote({Line, Col}, Text) + when is_integer(Line), is_integer(Col) -> + quote_1(Line, Col, Text); +quote(StartPos, Text) when is_integer(StartPos) -> + quote_1(StartPos, undefined, Text). + +quote_1(StartLine, StartCol, Text) -> + %% be backwards compatible as far as R12, ignoring any starting column + StartPos = case erlang:system_info(version) of + "5.6" ++ _ -> StartLine; + "5.7" ++ _ -> StartLine; + "5.8" ++ _ -> StartLine; + _ when StartCol =:= undefined -> StartLine; + _ -> {StartLine, StartCol} + end, + FlatText = flatten_text(Text), + {ok, Ts, _} = erl_scan:string(FlatText, StartPos), + merge_comments(StartLine, erl_comment_scan:string(FlatText), parse_1(Ts)). + +parse_1(Ts) -> + %% if dot tokens are present, it is assumed that the text represents + %% complete forms, not dot-terminated expressions or similar + case split_forms(Ts) of + {ok, Fs} -> parse_forms(Fs); + error -> + parse_2(Ts) + end. + +split_forms(Ts) -> + split_forms(Ts, [], []). + +split_forms([{dot,_}=T|Ts], Fs, As) -> + split_forms(Ts, [lists:reverse(As, [T]) | Fs], []); +split_forms([T|Ts], Fs, As) -> + split_forms(Ts, Fs, [T|As]); +split_forms([], Fs, []) -> + {ok, lists:reverse(Fs)}; +split_forms([], [], _) -> + error; % no dot tokens found - not representing form(s) +split_forms([], _, [T|_]) -> + fail("incomplete form after ~p", [T]). + +parse_forms([Ts | Tss]) -> + case erl_parse:parse_form(Ts) of + {ok, Form} -> [Form | parse_forms(Tss)]; + {error, R} -> parse_error(R) + end; +parse_forms([]) -> + []. + +parse_2(Ts) -> + %% one or more comma-separated expressions? + %% (recall that Ts has no dot tokens if we get to this stage) + case erl_parse:parse_exprs(Ts ++ [{dot,0}]) of + {ok, Exprs} -> Exprs; + {error, E} -> + parse_3(Ts ++ [{'end',0}, {dot,0}], [E]) + end. + +parse_3(Ts, Es) -> + %% try-clause or clauses? + case erl_parse:parse_exprs([{'try',0}, {atom,0,true}, {'catch',0} | Ts]) of + {ok, [{'try',_,_,_,_,_}=X]} -> + %% get the right kind of qualifiers in the clause patterns + erl_syntax:try_expr_handlers(X); + {error, E} -> + parse_4(Ts, [E|Es]) + end. + +parse_4(Ts, Es) -> + %% fun-clause or clauses? (`(a)' is also a pattern, but `(a,b)' isn't, + %% so fun-clauses must be tried before normal case-clauses + case erl_parse:parse_exprs([{'fun',0} | Ts]) of + {ok, [{'fun',_,{clauses,Cs}}]} -> Cs; + {error, E} -> + parse_5(Ts, [E|Es]) + end. + +parse_5(Ts, Es) -> + %% case-clause or clauses? + case erl_parse:parse_exprs([{'case',0}, {atom,0,true}, {'of',0} | Ts]) of + {ok, [{'case',_,_,Cs}]} -> Cs; + {error, E} -> + %% select the best error to report + parse_error(lists:last(lists:sort([E|Es]))) + end. + +-dialyzer({nowarn_function, parse_error/1}). % no local return + +parse_error({L, M, R}) when is_atom(M), is_integer(L) -> + fail("~w: ~s", [L, M:format_error(R)]); +parse_error({{L,C}, M, R}) when is_atom(M), is_integer(L), is_integer(C) -> + fail("~w:~w: ~s", [L,C,M:format_error(R)]); +parse_error({_, M, R}) when is_atom(M) -> + fail(M:format_error(R)); +parse_error(R) -> + fail("unknown parse error: ~p", [R]). + +%% ------------------------------------------------------------------------ +%% Templates, substitution and matching + +%% Leaves are normal syntax trees, and inner nodes are tuples +%% {template,Type,Attrs,Groups} where Groups are lists of lists of nodes. +%% Metavariables are 1-tuples {VarName}, where VarName is an atom or an +%% integer. {'_'} and {0} work as anonymous variables in matching. Glob +%% metavariables are tuples {'*',VarName}, and {'*','_'} and {'*',0} are +%% anonymous globs. + +%% Note that although template() :: tree() | ..., it is implied that these +%% syntax trees are free from metavariables, so pattern() :: tree() | +%% template() is in fact a wider type than template(). + +-type template() :: tree() + | {id()} + | {'*',id()} + | {template, atom(), term(), [[template()]]}. + +-type template_or_templates() :: template() | [template()]. + +-spec template(pattern_or_patterns()) -> template_or_templates(). + +%% @doc Turn a syntax tree or list of trees into a template or templates. +%% Templates can be instantiated or matched against, and reverted back to +%% normal syntax trees using {@link tree/1}. If the input is already a +%% template, it is not modified further. +%% +%% @see subst/2 +%% @see match/2 +%% @see tree/1 + +template(Trees) when is_list(Trees) -> + [template_0(T) || T <- Trees]; +template(Tree) -> + template_0(Tree). + +template_0({template, _, _, _}=Template) -> Template; +template_0({'*',_}=Template) -> Template; +template_0({_}=Template) -> Template; +template_0(Tree) -> + case template_1(Tree) of + false -> Tree; + {Name} when is_list(Name) -> + fail("bad metavariable: '~s'", [tl(Name)]); % drop v/n from name + Template -> Template + end. + +%% returns either a template or a lifted metavariable {String}, or 'false' +%% if Tree contained no metavariables +template_1(Tree) -> + case subtrees(Tree) of + [] -> + case metavar(Tree) of + {"v_"++Cs}=V when Cs =/= [] -> V; % to be lifted + {"n0"++Cs}=V when Cs =/= [] -> V; % to be lifted + {"v@"++Cs} when Cs =/= [] -> {'*',list_to_atom(Cs)}; + {"n9"++Cs} when Cs =/= [] -> {'*',list_to_integer(Cs)}; + {"v"++Cs} -> {list_to_atom(Cs)}; + {"n"++Cs} -> {list_to_integer(Cs)}; + false -> false + end; + Gs -> + case template_2(Gs, [], false) of + Gs1 when is_list(Gs1) -> + {template, type(Tree), erl_syntax:get_attrs(Tree), Gs1}; + Other -> + Other + end + end. + +template_2([G | Gs], As, Bool) -> + case template_3(G, [], false) of + {"v_"++Cs}=V when Cs =/= [] -> V; % lift further + {"n0"++Cs}=V when Cs =/= [] -> V; % lift further + {"v@"++Cs} when Cs =/= [] -> {'*',list_to_atom(Cs)}; % stop + {"n9"++Cs} when Cs =/= [] -> {'*',list_to_integer(Cs)}; % stop + {"v"++Cs} when is_list(Cs) -> {list_to_atom(Cs)}; % stop + {"n"++Cs} when is_list(Cs) -> {list_to_integer(Cs)}; % stop + false -> template_2(Gs, [G | As], Bool); + G1 -> template_2(Gs, [G1 | As], true) + end; +template_2([], _As, false) -> false; +template_2([], As, true) -> lists:reverse(As). + +template_3([T | Ts], As, Bool) -> + case template_1(T) of + {"v_"++Cs} when Cs =/= [] -> {"v"++Cs}; % lift + {"n0"++Cs} when Cs =/= [] -> {"n"++Cs}; % lift + false -> template_3(Ts, [T | As], Bool); + T1 -> template_3(Ts, [T1 | As], true) + end; +template_3([], _As, false) -> false; +template_3([], As, true) -> lists:reverse(As). + + +%% @doc Turn a template into a syntax tree representing the template. +%% Meta-variables in the template are turned into normal Erlang variables if +%% their names (after the metavariable prefix characters) begin with an +%% uppercase character. E.g., `_@Foo' in the template becomes the variable +%% `Foo' in the meta-template. Furthermore, variables ending with `@' are +%% automatically wrapped in a call to merl:term/1, so e.g. `_@Foo@ in the +%% template becomes `merl:term(Foo)' in the meta-template. + +-spec meta_template(template_or_templates()) -> tree_or_trees(). + +meta_template(Templates) when is_list(Templates) -> + [meta_template_1(T) || T <- Templates]; +meta_template(Template) -> + meta_template_1(Template). + +meta_template_1({template, Type, Attrs, Groups}) -> + erl_syntax:tuple( + [erl_syntax:atom(template), + erl_syntax:atom(Type), + erl_syntax:abstract(Attrs), + erl_syntax:list([erl_syntax:list([meta_template_1(T) || T <- G]) + || G <- Groups])]); +meta_template_1({Var}=V) -> + meta_template_2(Var, V); +meta_template_1({'*',Var}=V) -> + meta_template_2(Var, V); +meta_template_1(Leaf) -> + erl_syntax:abstract(Leaf). + +meta_template_2(Var, V) when is_atom(Var) -> + case atom_to_list(Var) of + [C|_]=Name when C >= $A, C =< $Z ; C >= $À, C =< $Þ, C /= $× -> + case lists:reverse(Name) of + "@"++([_|_]=RevRealName) -> % don't allow empty RealName + RealName = lists:reverse(RevRealName), + erl_syntax:application(erl_syntax:atom(merl), + erl_syntax:atom(term), + [erl_syntax:variable(RealName)]); + _ -> + %% plain automatic metavariable + erl_syntax:variable(Name) + end; + _ -> + erl_syntax:abstract(V) + end; +meta_template_2(Var, V) when is_integer(Var) -> + if Var > 9, (Var rem 10) =:= 9 -> + %% at least 2 digits, ends in 9: make it a Q-variable + if Var > 99, (Var rem 100) =:= 99 -> + %% at least 3 digits, ends in 99: wrap in merl:term/1 + Name = "Q" ++ integer_to_list(Var div 100), + erl_syntax:application(erl_syntax:atom(merl), + erl_syntax:atom(term), + [erl_syntax:variable(Name)]); + true -> + %% plain automatic Q-variable + Name = integer_to_list(Var div 10), + erl_syntax:variable("Q" ++ Name) + end; + true -> + erl_syntax:abstract(V) + end. + + + +-spec template_vars(template_or_templates()) -> [id()]. + +%% @doc Return an ordered list of the metavariables in the template. + +template_vars(Template) -> + template_vars(Template, []). + +template_vars(Templates, Vars) when is_list(Templates) -> + lists:foldl(fun template_vars_1/2, Vars, Templates); +template_vars(Template, Vars) -> + template_vars_1(Template, Vars). + +template_vars_1({template, _, _, Groups}, Vars) -> + lists:foldl(fun (G, V) -> lists:foldl(fun template_vars_1/2, V, G) end, + Vars, Groups); +template_vars_1({Var}, Vars) -> + ordsets:add_element(Var, Vars); +template_vars_1({'*',Var}, Vars) -> + ordsets:add_element(Var, Vars); +template_vars_1(_, Vars) -> + Vars. + + +-spec tree(template_or_templates()) -> tree_or_trees(). + +%% @doc Revert a template to a normal syntax tree. Any remaining +%% metavariables are turned into `@'-prefixed atoms or `909'-prefixed +%% integers. +%% @see template/1 + +tree(Templates) when is_list(Templates) -> + [tree_1(T) || T <- Templates]; +tree(Template) -> + tree_1(Template). + +tree_1({template, Type, Attrs, Groups}) -> + %% flattening here is needed for templates created via source transforms + Gs = [lists:flatten([tree_1(T) || T <- G]) || G <- Groups], + erl_syntax:set_attrs(make_tree(Type, Gs), Attrs); +tree_1({Var}) when is_atom(Var) -> + erl_syntax:atom(list_to_atom("@"++atom_to_list(Var))); +tree_1({Var}) when is_integer(Var) -> + erl_syntax:integer(list_to_integer("909"++integer_to_list(Var))); +tree_1({'*',Var}) when is_atom(Var) -> + erl_syntax:atom(list_to_atom("@@"++atom_to_list(Var))); +tree_1({'*',Var}) when is_integer(Var) -> + erl_syntax:integer(list_to_integer("9099"++integer_to_list(Var))); +tree_1(Leaf) -> + Leaf. % any syntax tree, not necessarily atomic (due to substitutions) + + +-spec subst(pattern_or_patterns(), env()) -> tree_or_trees(). + +%% @doc Substitute metavariables in a pattern or list of patterns, yielding +%% a syntax tree or list of trees as result. Both for normal metavariables +%% and glob metavariables, the substituted value may be a single element or +%% a list of elements. For example, if a list representing `1, 2, 3' is +%% substituted for `var' in either of `[foo, _@var, bar]' or `[foo, _@@var, +%% bar]', the result represents `[foo, 1, 2, 3, bar]'. + +subst(Trees, Env) when is_list(Trees) -> + [subst_0(T, Env) || T <- Trees]; +subst(Tree, Env) -> + subst_0(Tree, Env). + +subst_0(Tree, Env) -> + tree_1(subst_1(template(Tree), Env)). + + +-spec tsubst(pattern_or_patterns(), env()) -> template_or_templates(). + +%% @doc Like subst/2, but does not convert the result from a template back +%% to a tree. Useful if you want to do multiple separate substitutions. +%% @see subst/2 +%% @see tree/1 + +tsubst(Trees, Env) when is_list(Trees) -> + [subst_1(template(T), Env) || T <- Trees]; +tsubst(Tree, Env) -> + subst_1(template(Tree), Env). + +subst_1({template, Type, Attrs, Groups}, Env) -> + Gs1 = [lists:flatten([subst_1(T, Env) || T <- G]) || G <- Groups], + {template, Type, Attrs, Gs1}; +subst_1({Var}=V, Env) -> + case lists:keyfind(Var, 1, Env) of + {Var, TreeOrTrees} -> TreeOrTrees; + false -> V + end; +subst_1({'*',Var}=V, Env) -> + case lists:keyfind(Var, 1, Env) of + {Var, TreeOrTrees} -> TreeOrTrees; + false -> V + end; +subst_1(Leaf, _Env) -> + Leaf. + + +-spec alpha(pattern_or_patterns(), [{id(), id()}]) -> template_or_templates(). + +%% @doc Alpha converts a pattern (renames variables). Similar to tsubst/1, +%% but only renames variables (including globs). +%% @see tsubst/2 + +alpha(Trees, Env) when is_list(Trees) -> + [alpha_1(template(T), Env) || T <- Trees]; +alpha(Tree, Env) -> + alpha_1(template(Tree), Env). + +alpha_1({template, Type, Attrs, Groups}, Env) -> + Gs1 = [lists:flatten([alpha_1(T, Env) || T <- G]) || G <- Groups], + {template, Type, Attrs, Gs1}; +alpha_1({Var}=V, Env) -> + case lists:keyfind(Var, 1, Env) of + {Var, NewVar} -> {NewVar}; + false -> V + end; +alpha_1({'*',Var}=V, Env) -> + case lists:keyfind(Var, 1, Env) of + {Var, NewVar} -> {'*',NewVar}; + false -> V + end; +alpha_1(Leaf, _Env) -> + Leaf. + + +-spec match(pattern_or_patterns(), tree_or_trees()) -> + {ok, env()} | error. + +%% @doc Match a pattern against a syntax tree (or patterns against syntax +%% trees) returning an environment mapping variable names to subtrees; the +%% environment is always sorted on keys. Note that multiple occurrences of +%% metavariables in the pattern is not allowed, but is not checked. +%% +%% @see template/1 +%% @see switch/2 + +match(Patterns, Trees) when is_list(Patterns), is_list(Trees) -> + try {ok, match_1(Patterns, Trees, [])} + catch + error -> error + end; +match(Patterns, Tree) when is_list(Patterns) -> match(Patterns, [Tree]); +match(Pattern, Trees) when is_list(Trees) -> match([Pattern], Trees); +match(Pattern, Tree) -> + try {ok, match_template(template(Pattern), Tree, [])} + catch + error -> error + end. + +match_1([P|Ps], [T | Ts], Dict) -> + match_1(Ps, Ts, match_template(template(P), T, Dict)); +match_1([], [], Dict) -> + Dict; +match_1(_, _, _Dict) -> + erlang:error(merl_match_arity). + +%% match a template against a syntax tree +match_template({template, Type, _, Gs}, Tree, Dict) -> + case type(Tree) of + Type -> match_template_1(Gs, subtrees(Tree), Dict); + _ -> throw(error) % type mismatch + end; +match_template({Var}, _Tree, Dict) + when Var =:= '_' ; Var =:= 0 -> + Dict; % anonymous variable +match_template({Var}, Tree, Dict) -> + orddict:store(Var, Tree, Dict); +match_template(Tree1, Tree2, Dict) -> + %% if Tree1 is not a template, Tree1 and Tree2 are both syntax trees + case compare_trees(Tree1, Tree2) of + true -> Dict; + false -> throw(error) % different trees + end. + +match_template_1([G1 | Gs1], [G2 | Gs2], Dict) -> + match_template_2(G1, G2, match_template_1(Gs1, Gs2, Dict)); +match_template_1([], [], Dict) -> + Dict; +match_template_1(_, _, _Dict) -> + throw(error). % shape mismatch + +match_template_2([{Var} | Ts1], [_ | Ts2], Dict) + when Var =:= '_' ; Var =:= 0 -> + match_template_2(Ts1, Ts2, Dict); % anonymous variable +match_template_2([{Var} | Ts1], [Tree | Ts2], Dict) -> + match_template_2(Ts1, Ts2, orddict:store(Var, Tree, Dict)); +match_template_2([{'*',Var} | Ts1], Ts2, Dict) -> + match_glob(lists:reverse(Ts1), lists:reverse(Ts2), Var, Dict); +match_template_2([T1 | Ts1], [T2 | Ts2], Dict) -> + match_template_2(Ts1, Ts2, match_template(T1, T2, Dict)); +match_template_2([], [], Dict) -> + Dict; +match_template_2(_, _, _Dict) -> + throw(error). % shape mismatch + +%% match the tails in reverse order; no further globs allowed +match_glob([{'*',Var} | _], _, _, _) -> + fail("multiple glob variables in same match group: ~w", [Var]); +match_glob([T1 | Ts1], [T2 | Ts2], Var, Dict) -> + match_glob(Ts1, Ts2, Var, match_template(T1, T2, Dict)); +match_glob([], _Group, Var, Dict) when Var =:= '_' ; Var =:= 0 -> + Dict; % anonymous glob variable +match_glob([], Group, Var, Dict) -> + orddict:store(Var, lists:reverse(Group), Dict); +match_glob(_, _, _, _Dict) -> + throw(error). % shape mismatch + + +%% compare two syntax trees for equivalence +compare_trees(T1, T2) -> + Type1 = type(T1), + case type(T2) of + Type1 -> + case subtrees(T1) of + [] -> + case subtrees(T2) of + [] -> compare_leaves(Type1, T1, T2); + _Gs2 -> false % shape mismatch + end; + Gs1 -> + case subtrees(T2) of + [] -> false; % shape mismatch + Gs2 -> compare_trees_1(Gs1, Gs2) + end + end; + _Type2 -> + false % different tree types + end. + +compare_trees_1([G1 | Gs1], [G2 | Gs2]) -> + compare_trees_2(G1, G2) andalso compare_trees_1(Gs1, Gs2); +compare_trees_1([], []) -> + true; +compare_trees_1(_, _) -> + false. % shape mismatch + +compare_trees_2([T1 | Ts1], [T2 | Ts2]) -> + compare_trees(T1, T2) andalso compare_trees_2(Ts1, Ts2); +compare_trees_2([], []) -> + true; +compare_trees_2(_, _) -> + false. % shape mismatch + +compare_leaves(Type, T1, T2) -> + case Type of + atom -> + erl_syntax:atom_value(T1) + =:= erl_syntax:atom_value(T2); + char -> + erl_syntax:char_value(T1) + =:= erl_syntax:char_value(T2); + float -> + erl_syntax:float_value(T1) + =:= erl_syntax:float_value(T2); + integer -> + erl_syntax:integer_value(T1) + =:= erl_syntax:integer_value(T2); + string -> + erl_syntax:string_value(T1) + =:= erl_syntax:string_value(T2); + operator -> + erl_syntax:operator_name(T1) + =:= erl_syntax:operator_name(T2); + text -> + erl_syntax:text_string(T1) + =:= erl_syntax:text_string(T2); + variable -> + erl_syntax:variable_name(T1) + =:= erl_syntax:variable_name(T2); + _ -> + true % trivially equal nodes + end. + + +%% @doc Match against one or more clauses with patterns and optional guards. +%% +%% Note that clauses following a default action will be ignored. +%% +%% @see match/2 + +-type switch_clause() :: + {pattern_or_patterns(), guarded_actions()} + | {pattern_or_patterns(), guard_test(), switch_action()} + | default_action(). + +-type guarded_actions() :: guarded_action() | [guarded_action()]. + +-type guarded_action() :: switch_action() | {guard_test(), switch_action()}. + +-type switch_action() :: fun( (env()) -> any() ). + +-type guard_test() :: fun( (env()) -> boolean() ). + +-type default_action() :: fun( () -> any() ). + + +-spec switch(tree_or_trees(), [switch_clause()]) -> any(). + +switch(Trees, [{Patterns, GuardedActions} | Cs]) when is_list(GuardedActions) -> + switch_1(Trees, Patterns, GuardedActions, Cs); +switch(Trees, [{Patterns, GuardedAction} | Cs]) -> + switch_1(Trees, Patterns, [GuardedAction], Cs); +switch(Trees, [{Patterns, Guard, Action} | Cs]) -> + switch_1(Trees, Patterns, [{Guard, Action}], Cs); +switch(_Trees, [Default | _Cs]) when is_function(Default, 0) -> + Default(); +switch(_Trees, []) -> + erlang:error(merl_switch_clause); +switch(_Tree, _) -> + erlang:error(merl_switch_badarg). + +switch_1(Trees, Patterns, GuardedActions, Cs) -> + case match(Patterns, Trees) of + {ok, Env} -> + switch_2(Env, GuardedActions, Trees, Cs); + error -> + switch(Trees, Cs) + end. + +switch_2(Env, [{Guard, Action} | Bs], Trees, Cs) + when is_function(Guard, 1), is_function(Action, 1) -> + case Guard(Env) of + true -> Action(Env); + false -> switch_2(Env, Bs, Trees, Cs) + end; +switch_2(Env, [Action | _Bs], _Trees, _Cs) when is_function(Action, 1) -> + Action(Env); +switch_2(_Env, [], Trees, Cs) -> + switch(Trees, Cs); +switch_2(_Env, _, _Trees, _Cs) -> + erlang:error(merl_switch_badarg). + + +%% ------------------------------------------------------------------------ +%% Internal utility functions + +-dialyzer({nowarn_function, fail/1}). % no local return + +fail(Text) -> + fail(Text, []). + +fail(Fs, As) -> + throw({error, lists:flatten(io_lib:format(Fs, As))}). + +flatten_text([L | _]=Lines) when is_list(L) -> + lists:foldr(fun(S, T) -> S ++ [$\n | T] end, "", Lines); +flatten_text([B | _]=Lines) when is_binary(B) -> + lists:foldr(fun(S, T) -> binary_to_list(S) ++ [$\n | T] end, "", Lines); +flatten_text(Text) when is_binary(Text) -> + binary_to_list(Text); +flatten_text(Text) -> + Text. + +-spec metavar(tree()) -> {string()} | false. + +%% Check if a syntax tree represents a metavariable. If not, 'false' is +%% returned; otherwise, this returns a 1-tuple with a string containing the +%% variable name including lift/glob prefixes but without any leading +%% metavariable prefix, and instead prefixed with "v" for a variable or "i" +%% for an integer. +%% +%% Metavariables are atoms starting with @, variables starting with _@, +%% strings starting with "'@, or integers starting with 909. Following the +%% prefix, one or more _ or 0 characters (unless it's the last character in +%% the name) may be used to indicate "lifting" of the variable one or more +%% levels , and after that, a @ or 9 character indicates a glob metavariable +%% rather than a normal metavariable. If the name after the prefix is _ or +%% 0, the variable is treated as an anonymous catch-all pattern in matches. + +metavar(Tree) -> + case type(Tree) of + atom -> + case erl_syntax:atom_name(Tree) of + "@" ++ Cs when Cs =/= [] -> {"v"++Cs}; + _ -> false + end; + variable -> + case erl_syntax:variable_literal(Tree) of + "_@" ++ Cs when Cs =/= [] -> {"v"++Cs}; + _ -> false + end; + integer -> + case erl_syntax:integer_value(Tree) of + N when N >= 9090 -> + case integer_to_list(N) of + "909" ++ Cs -> {"n"++Cs}; + _ -> false + end; + _ -> false + end; + string -> + case erl_syntax:string_value(Tree) of + "'@" ++ Cs -> {"v"++Cs}; + _ -> false + end; + _ -> + false + end. + +%% wrappers around erl_syntax functions to provide more uniform shape of +%% generic subtrees (maybe this can be fixed in syntax_tools one day) + +type(T) -> + case erl_syntax:type(T) of + nil -> list; + Type -> Type + end. + +subtrees(T) -> + case erl_syntax:type(T) of + tuple -> + [erl_syntax:tuple_elements(T)]; %% don't treat {} as a leaf + nil -> + [[], []]; %% don't treat [] as a leaf, but as a list + list -> + case erl_syntax:list_suffix(T) of + none -> + [erl_syntax:list_prefix(T), []]; + S -> + [erl_syntax:list_prefix(T), [S]] + end; + binary_field -> + [[erl_syntax:binary_field_body(T)], + erl_syntax:binary_field_types(T)]; + clause -> + case erl_syntax:clause_guard(T) of + none -> + [erl_syntax:clause_patterns(T), [], + erl_syntax:clause_body(T)]; + G -> + [erl_syntax:clause_patterns(T), [G], + erl_syntax:clause_body(T)] + end; + receive_expr -> + case erl_syntax:receive_expr_timeout(T) of + none -> + [erl_syntax:receive_expr_clauses(T), [], []]; + E -> + [erl_syntax:receive_expr_clauses(T), [E], + erl_syntax:receive_expr_action(T)] + end; + record_expr -> + case erl_syntax:record_expr_argument(T) of + none -> + [[], [erl_syntax:record_expr_type(T)], + erl_syntax:record_expr_fields(T)]; + V -> + [[V], [erl_syntax:record_expr_type(T)], + erl_syntax:record_expr_fields(T)] + end; + record_field -> + case erl_syntax:record_field_value(T) of + none -> + [[erl_syntax:record_field_name(T)], []]; + V -> + [[erl_syntax:record_field_name(T)], [V]] + end; + _ -> + erl_syntax:subtrees(T) + end. + +make_tree(list, [P, []]) -> erl_syntax:list(P); +make_tree(list, [P, [S]]) -> erl_syntax:list(P, S); +make_tree(tuple, [E]) -> erl_syntax:tuple(E); +make_tree(binary_field, [[B], Ts]) -> erl_syntax:binary_field(B, Ts); +make_tree(clause, [P, [], B]) -> erl_syntax:clause(P, none, B); +make_tree(clause, [P, [G], B]) -> erl_syntax:clause(P, G, B); +make_tree(receive_expr, [C, [], _A]) -> erl_syntax:receive_expr(C); +make_tree(receive_expr, [C, [E], A]) -> erl_syntax:receive_expr(C, E, A); +make_tree(record_expr, [[], [T], F]) -> erl_syntax:record_expr(T, F); +make_tree(record_expr, [[E], [T], F]) -> erl_syntax:record_expr(E, T, F); +make_tree(record_field, [[N], []]) -> erl_syntax:record_field(N); +make_tree(record_field, [[N], [E]]) -> erl_syntax:record_field(N, E); +make_tree(Type, Groups) -> + erl_syntax:make_tree(Type, Groups). + +merge_comments(_StartLine, [], [T]) -> T; +merge_comments(_StartLine, [], Ts) -> Ts; +merge_comments(StartLine, Comments, Ts) -> + merge_comments(StartLine, Comments, Ts, []). + +merge_comments(_StartLine, [], [], [T]) -> T; +merge_comments(_StartLine, [], [T], []) -> T; +merge_comments(_StartLine, [], Ts, Acc) -> + lists:reverse(Acc, Ts); +merge_comments(StartLine, Cs, [], Acc) -> + merge_comments(StartLine, [], [], + [erl_syntax:set_pos( + erl_syntax:comment(Indent, Text), + StartLine + Line - 1) + || {Line, _, Indent, Text} <- Cs] ++ Acc); +merge_comments(StartLine, [C|Cs], [T|Ts], Acc) -> + {Line, _Col, Indent, Text} = C, + CommentLine = StartLine + Line - 1, + case erl_syntax:get_pos(T) of + Pos when Pos < CommentLine -> + %% TODO: traverse sub-tree rather than only the top level nodes + merge_comments(StartLine, [C|Cs], Ts, [T|Acc]); + CommentLine -> + Tc = erl_syntax:add_postcomments( + [erl_syntax:comment(Indent, Text)], T), + merge_comments(StartLine, Cs, [Tc|Ts], Acc); + _ -> + Tc = erl_syntax:add_precomments( + [erl_syntax:comment(Indent, Text)], T), + merge_comments(StartLine, Cs, [Tc|Ts], Acc) + end. diff --git a/lib/syntax_tools/src/merl_tests.erl b/lib/syntax_tools/src/merl_tests.erl new file mode 100644 index 0000000000..c1aae3100e --- /dev/null +++ b/lib/syntax_tools/src/merl_tests.erl @@ -0,0 +1,539 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012-2015 Richard Carlsson +%% @doc Unit tests for merl. +%% @private + +-module(merl_tests). + +%-define(MERL_NO_TRANSFORM, true). +-include("merl.hrl"). + +-include_lib("eunit/include/eunit.hrl"). + + +%% utilities + +f(Ts) when is_list(Ts) -> + lists:flatmap(fun erl_prettypr:format/1, Ts); +f(T) -> + erl_prettypr:format(T). + +fe(Env) -> [{Key, f(T)} || {Key, T} <- Env]. + +g_exported_() -> + %% for testing the parse transform, autoexported to avoid complaints + {ok, merl:quote(?LINE, "42")}. + + +ok({ok, X}) -> X. + + +%% +%% tests +%% + +parse_error_test_() -> + [?_assertThrow({error, "1: syntax error before: '{'" ++ _}, + f(merl:quote("{"))) + ]. + +term_test_() -> + [?_assertEqual(tuple, erl_syntax:type(merl:term({}))), + ?_assertEqual("{foo, 42}", f(merl:term({foo, 42}))) + ]. + +quote_form_test_() -> + [?_assertEqual("f(X) -> {ok, X}.", + f(?Q("f(X) -> {ok, X}."))), + ?_assertEqual("-module(foo).", + f(?Q("-module(foo)."))), + ?_assertEqual("-import(bar, [f/1, g/2]).", + f(?Q("-import(bar, [f/1, g/2])."))), + ?_assertEqual(("-module(foo)." + "-export([f/1])." + "f(X) -> {ok, X}."), + f(?Q(["-module(foo).", + "-export([f/1]).", + "f(X) -> {ok, X}."]))) + ]. + +quote_term_test_() -> + [?_assertEqual("foo", + f(?Q("foo"))), + ?_assertEqual("42", + f(?Q("42"))), + ?_assertEqual("{foo, 42}", + f(?Q("{foo, 42}"))), + ?_assertEqual(("1" ++ "2" ++ "3"), + f(?Q("1, 2, 3"))), + ?_assertEqual(("foo" "42" "{}" "true"), + f(?Q("foo, 42, {}, (true)"))) + ]. + +quote_expr_test_() -> + [?_assertEqual("2 + 2", + f(?Q("2 + 2"))), + ?_assertEqual("f(foo, 42)", + f(?Q("f(foo, 42)"))), + ?_assertEqual("case X of\n a -> 1;\n b -> 2\nend", + f(?Q("case X of a -> 1; b -> 2 end"))), + ?_assertEqual(("2 + 2" ++ "f(42)" ++ "catch 22"), + f(?Q("2 + 2, f(42), catch 22"))) + ]. + +quote_try_clause_test_() -> + [?_assertEqual("(error:R) when R =/= foo -> ok", + f(?Q("error:R when R =/= foo -> ok"))), + %% note that without any context, clauses are printed as fun-clauses + ?_assertEqual(("(error:badarg) -> badarg" + "(exit:normal) -> normal" + "(_) -> other"), + f(?Q(["error:badarg -> badarg;", + "exit:normal -> normal;" + "_ -> other"]))) + ]. + +quote_fun_clause_test_() -> + [?_assertEqual("(X, Y) when X < Y -> {ok, X}", + f(?Q("(X, Y) when X < Y -> {ok, X}"))), + ?_assertEqual(("(X, Y) when X < Y -> less" + "(X, Y) when X > Y -> greater" + "(_, _) -> equal"), + f(?Q(["(X, Y) when X < Y -> less;", + "(X, Y) when X > Y -> greater;" + "(_, _) -> equal"])))]. + +quote_case_clause_test_() -> + [?_assertEqual("({X, Y}) when X < Y -> X", + f(?Q("{X, Y} when X < Y -> X"))), + ?_assertEqual(("({X, Y}) when X < Y -> -1" + "({X, Y}) when X > Y -> 1" + "(_) -> 0"), + f(?Q(["{X, Y} when X < Y -> -1;", + "{X, Y} when X > Y -> 1;" + "_ -> 0"])))]. + +quote_comment_test_() -> + [?_assertEqual("%% comment preserved\n" + "{foo, 42}", + f(?Q(["%% comment preserved", + "{foo, 42}"]))), + ?_assertEqual("{foo, 42}" + "%% comment preserved\n", + f(?Q(["{foo, 42}", + "%% comment preserved"]))), + ?_assertEqual(" % just a comment (with indent)\n", + f(?Q(" % just a comment (with indent)"))) + ]. + +metavar_test_() -> + [?_assertEqual("'@foo'", f(merl:tree(merl:template(?Q("'@foo'"))))), + ?_assertEqual("'@foo'", f(merl:tree(merl:template(?Q("_@foo"))))), + ?_assertEqual("'@foo'", f(merl:tree(merl:template(?Q("\"'@foo\""))))), + ?_assertEqual("{'@foo'}", f(merl:tree(merl:template(?Q("{_@foo}"))))), + ?_assertEqual("'@foo'", f(merl:tree(merl:template(?Q("{_@_foo}"))))), + ?_assertEqual("909123", f(merl:tree(merl:template(?Q("{9090123}"))))), + ?_assertEqual("{'@foo'}", + f(merl:tree(merl:template(?Q("{{{_@__foo}}}"))))), + ?_assertEqual("{909123}", + f(merl:tree(merl:template(?Q("{{{90900123}}}"))))), + ?_assertEqual("{'@@foo'}", + f(merl:tree(merl:template(?Q("{{{_@__@foo}}}"))))), + ?_assertEqual("{9099123}", + f(merl:tree(merl:template(?Q("{{{909009123}}}"))))) + ]. + +subst_test_() -> + [?_assertEqual("42", + f(merl:subst(?Q("_@foo"), [{foo, merl:term(42)}]))), + ?_assertEqual("'@foo'", + f(merl:subst(?Q("_@foo"), []))), + ?_assertEqual("{42}", + f(merl:subst(?Q("{_@foo}"), + [{foo, merl:term(42)}]))), + ?_assertEqual("{'@foo'}", + f(merl:subst(?Q("{_@foo}"), []))), + ?_assertEqual("fun bar/0", + f(merl:subst(merl:template(?Q("fun '@foo'/0")), + [{foo, merl:term(bar)}]))), + ?_assertEqual("fun foo/3", + f(merl:subst(merl:template(?Q("fun foo/9091")), + [{1, merl:term(3)}]))), + ?_assertEqual("[42]", + f(merl:subst(merl:template(?Q("[_@foo]")), + [{foo, merl:term(42)}]))), + ?_assertEqual("[foo, bar]", + f(merl:subst(merl:template(?Q("[_@foo]")), + [{foo, [merl:term(foo),merl:term(bar)]}]))), + ?_assertEqual("{fee, fie, foe, fum}", + f(merl:subst(merl:template(?Q("{fee, _@foo, fum}")), + [{foo, [merl:term(fie),merl:term(foe)]}]))), + ?_assertEqual("[foo, bar]", + f(merl:subst(merl:template(?Q("[_@@foo]")), + [{foo, [merl:term(foo),merl:term(bar)]}]))), + ?_assertEqual("{fee, fie, foe, fum}", + f(merl:subst(merl:template(?Q("{fee, _@@foo, fum}")), + [{foo, [merl:term(fie),merl:term(foe)]}]))), + ?_assertEqual("['@@foo']", + f(merl:subst(merl:template(?Q("[_@@foo]")), []))), + ?_assertEqual("foo", + f(merl:subst(merl:template(?Q("[_@_foo]")), + [{foo, merl:term(foo)}]))), + ?_assertEqual("{'@foo'}", + f(merl:subst(merl:template(?Q("{[_@_foo]}")), []))), + ?_assertEqual("{'@@foo'}", + f(merl:subst(merl:template(?Q("{[_@_@foo]}")), []))), + ?_assertEqual("-export([foo/1, bar/2]).", + f(merl:subst(merl:template(?Q("-export(['@_@foo'/0]).")), + [{foo, [erl_syntax:arity_qualifier( + merl:term(foo), + merl:term(1)), + erl_syntax:arity_qualifier( + merl:term(bar), + merl:term(2)) + ]} + ]))) + ]. + +match_test_() -> + [?_assertEqual({ok, []}, merl:match(?Q("foo"), ?Q("foo"))), + ?_assertEqual(error, merl:match(?Q("foo"), ?Q("bar"))), + ?_assertEqual({ok,[]}, merl:match(?Q("{foo,42}"), ?Q("{foo,42}"))), + ?_assertEqual(error, merl:match(?Q("{foo,42}"), ?Q("{foo,bar}"))), + ?_assertEqual({ok,[]}, merl:match(?Q("[foo,[42]]"), ?Q("[foo,[42]]"))), + ?_assertEqual(error, merl:match(?Q("[foo,[42]]"), ?Q("[foo,{42}]"))), + ?_assertEqual({ok,[]}, merl:match(?Q("[foo,[_@_]]"), + ?Q("[foo,[42]]"))), + ?_assertEqual({ok,[]}, merl:match(?Q("[foo,[9090]]"), + ?Q("[foo,[42]]"))), + ?_assertEqual({ok,[]}, merl:match(?Q("{_@_,[_@_,2]}"), + ?Q("{foo,[1,2]}"))), + ?_assertEqual(error, merl:match(?Q("{_@_,[_@_,2]}"), + ?Q("{foo,[1,3]}"))), + ?_assertEqual({ok,[]}, merl:match(?Q("[foo,[9090,9090]]"), + ?Q("[foo,[1,2]]"))), + ?_assertEqual(error, merl:match(?Q("[foo,[9090,9090]]"), + ?Q("[foo,[1,2,3]]"))), + ?_assertEqual([{foo,"42"}], + fe(ok(merl:match(?Q("_@foo"), ?Q("42"))))), + ?_assertEqual([{foo,"42"}], + fe(ok(merl:match(?Q("{_@foo}"), ?Q("{42}"))))), + ?_assertEqual([{1,"0"},{foo,"bar"}], + fe(ok(merl:match(?Q("fun '@foo'/9091"), + ?Q("fun bar/0"))))), + ?_assertEqual([{line,"17"},{text,"\"hello\""}], + fe(ok(merl:match(?Q("{_@line, _@text}"), + ?Q("{17, \"hello\"}"))))), + ?_assertEqual([{line,"17"},{text,"\"hello\""}], + fe(ok(merl:match(?Q("foo(_@line, _@text)"), + ?Q("foo(17, \"hello\")"))))), + ?_assertEqual([{foo,""}], + fe(ok(merl:match(?Q("f(_@@foo)"), + ?Q("f()"))))), + ?_assertEqual([{foo,"fee"}], + fe(ok(merl:match(?Q("f(_@@foo)"), + ?Q("f(fee)"))))), + ?_assertEqual([{foo,"feefiefum"}], + fe(ok(merl:match(?Q("f(_@@foo)"), + ?Q("f(fee, fie, fum)"))))), + ?_assertEqual([{foo,""}], + fe(ok(merl:match(?Q("[_@@foo]"), + ?Q("[]"))))), + ?_assertEqual([{foo,"fee"}], + fe(ok(merl:match(?Q("[_@@foo]"), + ?Q("[fee]"))))), + ?_assertEqual([{foo,"feefiefoefum"}], + fe(ok(merl:match(?Q("[_@@foo]"), + ?Q("[fee, fie, foe, fum]"))))), + ?_assertEqual([{foo,""}], + fe(ok(merl:match(?Q("{_@@foo}"), + ?Q("{}"))))), + ?_assertEqual([{foo,"fee"}], + fe(ok(merl:match(?Q("{_@@foo}"), + ?Q("{fee}"))))), + ?_assertEqual([{foo,"feefiefoefum"}], + fe(ok(merl:match(?Q("{_@@foo}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertEqual([{foo,"fie"}], + fe(ok(merl:match(?Q("{fee, _@@foo}"), + ?Q("{fee, fie}"))))), + ?_assertEqual([{foo,"fiefoefum"}], + fe(ok(merl:match(?Q("{fee, _@@foo}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertEqual([{foo,"fie"}], + fe(ok(merl:match(?Q("{_@@foo, foe, fum}"), + ?Q("{fie, foe, fum}"))))), + ?_assertEqual([{foo,"feefie"}], + fe(ok(merl:match(?Q("{_@@foo, foe, fum}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertEqual([{foo,"fie"}], + fe(ok(merl:match(?Q("{fee, _@@foo, fum}"), + ?Q("{fee, fie, fum}"))))), + ?_assertEqual([{foo,"fiefoe"}], + fe(ok(merl:match(?Q("{fee, _@@foo, fum}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertEqual([{foo,"fiefoe"},{post,"fum"},{pre,"fee"}], + fe(ok(merl:match(?Q("{_@pre, _@@foo, _@post}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertThrow({error, "multiple glob variables"++_}, + fe(ok(merl:match(?Q("{_@@foo, _@@bar}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertEqual([], + fe(ok(merl:match(?Q("{fee, _@@_}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertEqual([], + fe(ok(merl:match(?Q("{_@@_, foe, fum}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertEqual([{post,"fum"},{pre,"fee"}], + fe(ok(merl:match(?Q("{_@pre, _@@_, _@post}"), + ?Q("{fee, fie, foe, fum}"))))) + ]. + +switch_test_() -> + [?_assertEqual(42, merl:switch(?Q("foo"), [fun () -> 42 end])), + ?_assertEqual(17, merl:switch(?Q("foo"), [fun () -> 17 end, + fun () -> 42 end])), + ?_assertEqual(17, merl:switch(?Q("foo"), [{?Q("foo"), + fun ([]) -> 17 end}, + fun () -> 42 end])), + ?_assertEqual(17, + merl:switch(?Q("foo"), [{?Q("bar"), fun ([]) -> 0 end}, + {?Q("foo"), fun ([]) -> 17 end}, + fun () -> 42 end])), + ?_assertEqual([{foo,"17"}], + merl:switch(?Q("{foo,17}"), + [{?Q("{bar, _@foo}"), fun (_) -> 0 end}, + {?Q("{foo, _@foo}"), fun fe/1}, + fun () -> 42 end])), + ?_assertEqual(17, + merl:switch(?Q("{foo, 17}"), + [{?Q("{foo, _@foo}"), + fun ([{foo, X}]) -> f(X) =:= "17" end, + fun (_) -> 17 end}, + fun () -> 42 end])), + ?_assertEqual([{foo,"17"}], + merl:switch(?Q("{foo, 17}"), + [{?Q("{foo, _@foo}"), + fun ([{foo, X}]) -> f(X) =:= "42" end, + fun (_) -> 0 end}, + {?Q("{foo, _@foo}"), fun fe/1}, + fun () -> 42 end])), + ?_assertEqual(17, + merl:switch(?Q("{foo, 17}"), + [{?Q("{foo, _@foo}"), + [{fun ([{foo, X}]) -> f(X) =:= "17" end, + fun (_) -> 17 end}, + fun (_) -> 0 end]}, + fun () -> 42 end])), + ?_assertEqual([{foo,"17"}], + merl:switch(?Q("{foo, 17}"), + [{?Q("{foo, _@foo}"), + [{fun ([{foo, X}]) -> f(X) =:= "42" end, + fun (_) -> 0 end}, + fun fe/1]}, + fun () -> 42 end])) + ]. + +-ifndef(MERL_NO_TRANSFORM). + +inline_meta_test_() -> + [?_assertEqual("{foo}", + f(begin + Foo = ?Q("foo"), + ?Q("{_@Foo}") + end)), + ?_assertEqual("{foo, '@bar'}", + f(begin + Foo = ?Q("foo"), + ?Q("{_@Foo,_@bar}") + end)), + ?_assertEqual("{foo, '@bar'}", + f(begin + Q1 = ?Q("foo"), + ?Q("{90919,_@bar}") + end)) + ]. + +inline_meta_autoabstract_test_() -> + [?_assertEqual("{foo}", + f(begin + Foo = foo, + ?Q("{_@Foo@}") + end)), + ?_assertEqual("{foo, '@bar@'}", + f(begin + Foo = foo, + ?Q("{_@Foo@,_@bar@}") + end)), + ?_assertEqual("{foo, '@bar@'}", + f(begin + Q1 = foo, + ?Q("{909199,_@bar@}") + end)) + ]. + +meta_match_test_() -> + [?_assertEqual("{[bar], baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + ?Q("{foo, _@Bar, '@Baz'}") = Tree, + ?Q("{_@Bar, _@Baz}") + end)), + ?_assertEqual("{[bar], baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + ?Q("{foo, 90919, 90929}") = Tree, + ?Q("{_@Q1, _@Q2}") + end)), + ?_assertError({badmatch,error}, + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + ?Q("{fie, _@Bar, '@Baz'}") = Tree, + ?Q("{_@Bar, _@Baz}") + end)) + ]. + +meta_case_test_() -> + [?_assertEqual("{[bar], baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + case Tree of + ?Q("{foo, _@Bar, '@Baz'}") -> ?Q("{_@Bar, _@Baz}") + end + end)), + ?_assertEqual("{foo, [bar], baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + case Tree of + ?Q("{fie, _@Bar, '@Baz'}") -> ?Q("{_@Bar, _@Baz}"); + _ -> Tree + end + end)), + ?_assertError(merl_switch_clause, + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + case Tree of + ?Q("{fie, _@Bar, '@Baz'}") -> ?Q("{_@Bar, _@Baz}") + end + end)), + ?_assertEqual("{foo, 4}", + f(begin + Tree = ?Q("{foo, 3}"), + case Tree of + ?Q("{foo, _@N}") -> + N1 = erl_syntax:concrete(N) + 1, + ?Q("{foo, _@N1@}"); + _ -> Tree + end + end)), + ?_assertEqual("-export([f/4]).", + f(begin + Tree = ?Q("-export([f/3])."), + case Tree of + ?Q("-export([f/90919]).") -> + Q2 = erl_syntax:concrete(Q1) + 1, + ?Q("-export([f/909299])."); + _ -> Tree + end + end)), + ?_assertEqual("{1, [bar], baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + case Tree of + ?Q("{foo, _@Bar, '@Baz'}") -> + ?Q("{1, _@Bar, _@Baz}"); + ?Q("{fie, _@Bar, '@Baz'}") -> + ?Q("{2, _@Bar, _@Baz}"); + _ -> Tree + end + end)), + ?_assertEqual("{2, [bar], baz()}", + f(begin + Tree = ?Q("{fie, [bar], baz()}"), + case Tree of + ?Q("{foo, _@Bar, '@Baz'}") -> + ?Q("{1, _@Bar, _@Baz}"); + ?Q("{fie, _@Bar, '@Baz'}") -> + ?Q("{2, _@Bar, _@Baz}"); + _ -> Tree + end + end)), + ?_assertEqual("{2, baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + case Tree of + ?Q("{foo, [_@Bar], '@Baz'}") + when erl_syntax:is_atom(Bar, foo) -> + ?Q("{1, _@Baz}"); + ?Q("{foo, [_@Bar], '@Baz'}") + when erl_syntax:is_atom(Bar, bar) -> + ?Q("{2, _@Baz}"); + ?Q("{foo, [_@Bar], '@Baz'}") -> + ?Q("{3, _@Baz}"); + _ -> Tree + end + end)), + ?_assertEqual("{2, 42}", + f(begin + Tree = ?Q("{foo, [bar], 42}"), + case Tree of + ?Q("{foo, [_@Bar], '@Baz'}") + when erl_syntax:is_atom(Bar, bar), + erl_syntax:is_integer(Baz, 17) -> + ?Q("{1, _@Bar}"); + ?Q("{foo, [_@Bar], '@Baz'}") + when erl_syntax:is_atom(Bar, bar), + erl_syntax:is_integer(Baz, 42) -> + ?Q("{2, _@Baz}"); + ?Q("{foo, [_@Bar], '@Baz'}") -> + ?Q("{3, _@Baz}"); + _ -> Tree + end + end)), + ?_assertEqual("{2, 42}", + f(begin + Tree = ?Q("{foo, [baz], 42}"), + case Tree of + ?Q("{foo, [_@Bar], '@Baz'}") + when erl_syntax:is_atom(Bar, bar), + erl_syntax:is_integer(Baz, 17) + ; erl_syntax:is_atom(Bar, baz), + erl_syntax:is_integer(Baz, 17) -> + ?Q("{1, _@Bar}"); + ?Q("{foo, [_@Bar], '@Baz'}") + when erl_syntax:is_atom(Bar, bar), + erl_syntax:is_integer(Baz, 42) + ; erl_syntax:is_atom(Bar, baz), + erl_syntax:is_integer(Baz, 42) -> + ?Q("{2, _@Baz}"); + ?Q("{foo, [_@Bar], '@Baz'}") -> + ?Q("{3, _@Baz}"); + _ -> Tree + end + end)), + ?_assertEqual("{2, foo, Bar, Baz, Bar(), Baz()}", + f(begin + Tree = ?Q("foo(Bar, Baz) -> Bar(), Baz()."), + case Tree of + ?Q("'@Func'(_@Args) -> _@Body.") -> + ?Q("{1, _@Func, _@Args, _@Body}"); + ?Q("'@Func'(_@@Args) -> _@@Body.") -> + ?Q("{2, _@Func, _@Args, _@Body}"); + ?Q("'@Func'(_@Args, Baz) -> _@Body1, _@Body2.") -> + ?Q("{3, _@Func, _@Args, _@Body1, _@Body2}") + end + end)) + ]. + +-endif. diff --git a/lib/syntax_tools/src/merl_transform.erl b/lib/syntax_tools/src/merl_transform.erl new file mode 100644 index 0000000000..66b06c8137 --- /dev/null +++ b/lib/syntax_tools/src/merl_transform.erl @@ -0,0 +1,262 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012-2015 Richard Carlsson +%% @doc Parse transform for merl. Enables the use of automatic metavariables +%% and using quasi-quotes in matches and case switches. Also optimizes calls +%% to functions in `merl' by partially evaluating them, turning strings to +%% templates, etc., at compile-time. +%% +%% Using `-include_lib("syntax_tools/include/merl.hrl").' enables this +%% transform, unless the macro `MERL_NO_TRANSFORM' is defined first. + +-module(merl_transform). + +-export([parse_transform/2]). + +%% NOTE: We cannot use inline metavariables or any other parse transform +%% features in this module, because it must be possible to compile it with +%% the parse transform disabled! +-include("merl.hrl"). + +%% TODO: unroll calls to switch? it will probably get messy + +%% TODO: use Igor to make resulting code independent of merl at runtime? + +parse_transform(Forms, _Options) -> + erl_syntax:revert_forms(expand(erl_syntax:form_list(Forms))). + +expand(Tree0) -> + Tree = pre(Tree0), + post(case erl_syntax:subtrees(Tree) of + [] -> + Tree; + Gs -> + erl_syntax:update_tree(Tree, + [[expand(T) || T <- G] || G <- Gs]) + end). + +pre(T) -> + merl:switch( + T, + [{?Q("merl:quote(_@line, _@text) = _@expr"), + fun ([{expr,_}, {line,Line}, {text,Text}]) -> + erl_syntax:is_literal(Text) andalso erl_syntax:is_literal(Line) + end, + fun ([{expr,Expr}, {line,Line}, {text,Text}]) -> + pre_expand_match(Expr, erl_syntax:concrete(Line), + erl_syntax:concrete(Text)) + end}, + {?Q(["case _@expr of", + " merl:quote(_@_, _@text) when _@__@_ -> _@@_; _@_@_ -> 0", + "end"]), + fun case_guard/1, + fun (As) -> case_body(As, T) end}, + fun () -> T end + ]). + +case_guard([{expr,_}, {text,Text}]) -> + erl_syntax:is_literal(Text). + +case_body([{expr,Expr}, {text,_Text}], T) -> + pre_expand_case(Expr, erl_syntax:case_expr_clauses(T), + erl_syntax:get_pos(T)). + +post(T) -> + merl:switch( + T, + [{?Q("merl:_@function(_@@args)"), + [{fun ([{args, As}, {function, F}]) -> + lists:all(fun erl_syntax:is_literal/1, [F|As]) + end, + fun ([{args, As}, {function, F}]) -> + Line = erl_syntax:get_pos(F), + [F1|As1] = lists:map(fun erl_syntax:concrete/1, [F|As]), + eval_call(Line, F1, As1, T) + end}, + fun ([{args, As}, {function, F}]) -> + merl:switch( + F, + [{?Q("qquote"), fun ([]) -> expand_qquote(As, T, 1) end}, + {?Q("subst"), fun ([]) -> expand_template(F, As, T) end}, + {?Q("match"), fun ([]) -> expand_template(F, As, T) end}, + fun () -> T end + ]) + end]}, + fun () -> T end]). + +expand_qquote([Line, Text, Env], T, _) -> + case erl_syntax:is_literal(Line) of + true -> + expand_qquote([Text, Env], T, erl_syntax:concrete(Line)); + false -> + T + end; +expand_qquote([Text, Env], T, Line) -> + case erl_syntax:is_literal(Text) of + true -> + As = [Line, erl_syntax:concrete(Text)], + %% expand further if possible + expand(merl:qquote(Line, "merl:subst(_@tree, _@env)", + [{tree, eval_call(Line, quote, As, T)}, + {env, Env}])); + false -> + T + end; +expand_qquote(_As, T, _StartPos) -> + T. + +expand_template(F, [Pattern | Args], T) -> + case erl_syntax:is_literal(Pattern) of + true -> + Line = erl_syntax:get_pos(Pattern), + As = [erl_syntax:concrete(Pattern)], + merl:qquote(Line, "merl:_@function(_@pattern, _@args)", + [{function, F}, + {pattern, eval_call(Line, template, As, T)}, + {args, Args}]); + false -> + T + end; +expand_template(_F, _As, T) -> + T. + +eval_call(Line, F, As, T) -> + try apply(merl, F, As) of + T1 when F =:= quote -> + %% lift metavariables in a template to Erlang variables + Template = merl:template(T1), + Vars = merl:template_vars(Template), + case lists:any(fun is_inline_metavar/1, Vars) of + true when is_list(T1) -> + merl:qquote(Line, "merl:tree([_@template])", + [{template, merl:meta_template(Template)}]); + true -> + merl:qquote(Line, "merl:tree(_@template)", + [{template, merl:meta_template(Template)}]); + false -> + merl:term(T1) + end; + T1 -> + merl:term(T1) + catch + throw:_Reason -> T + end. + +pre_expand_match(Expr, Line, Text) -> + {Template, Out, _Vars} = rewrite_pattern(Line, Text), + merl:qquote(Line, "{ok, _@out} = merl:match(_@template, _@expr)", + [{expr, Expr}, + {out, Out}, + {template, erl_syntax:abstract(Template)}]). + +rewrite_pattern(Line, Text) -> + %% we must rewrite the metavariables in the pattern to use lowercase, + %% and then use real matching to bind the Erlang-level variables + T0 = merl:template(merl:quote(Line, Text)), + Vars = [V || V <- merl:template_vars(T0), is_inline_metavar(V)], + {merl:alpha(T0, [{V, var_to_tag(V)} || V <- Vars]), + erl_syntax:list([erl_syntax:tuple([erl_syntax:abstract(var_to_tag(V)), + erl_syntax:variable(var_name(V))]) + || V <- Vars]), + Vars}. + +var_name(V) when is_integer(V) -> + V1 = if V > 99, (V rem 100) =:= 99 -> + V div 100; + V > 9, (V rem 10) =:= 9 -> + V div 10; + true -> V + end, + list_to_atom("Q" ++ integer_to_list(V1)); +var_name(V) -> V. + +var_to_tag(V) when is_integer(V) -> V; +var_to_tag(V) -> + list_to_atom(string:to_lower(atom_to_list(V))). + +pre_expand_case(Expr, Clauses, Line) -> + merl:qquote(Line, "merl:switch(_@expr, _@clauses)", + [{clauses, erl_syntax:list([pre_expand_case_clause(C) + || C <- Clauses])}, + {expr, Expr}]). + +pre_expand_case_clause(T) -> + %% note that the only allowed non ``?Q(...) -> ...'' clause is ``_ -> ...'' + merl:switch( + T, + [{?Q("(merl:quote(_@line, _@text)) when _@__@guard -> _@@body"), + fun ([{body,_}, {guard,_}, {line,Line}, {text,Text}]) -> + erl_syntax:is_literal(Text) andalso erl_syntax:is_literal(Line) + end, + fun ([{body,Body}, {guard,Guard}, {line,Line}, {text,Text}]) -> + pre_expand_case_clause(Body, Guard, erl_syntax:concrete(Line), + erl_syntax:concrete(Text)) + end}, + {?Q("_ -> _@@body"), + fun (Env) -> merl:qquote("fun () -> _@body end", Env) end} + ]). + +pre_expand_case_clause(Body, Guard, Line, Text) -> + %% this is similar to a meta-match ``?Q("...") = Term'' + %% (note that the guards may in fact be arbitrary expressions) + {Template, Out, Vars} = rewrite_pattern(Line, Text), + GuardExprs = rewrite_guard(Guard), + Param = [{body, Body}, + {guard,GuardExprs}, + {out, Out}, + {template, erl_syntax:abstract(Template)}, + {unused, dummy_uses(Vars)}], + case GuardExprs of + [] -> + merl:qquote(Line, ["{_@template, ", + " fun (_@out) -> _@unused, _@body end}"], + Param); + _ -> + merl:qquote(Line, ["{_@template, ", + " fun (_@out) -> _@unused, _@guard end, ", + " fun (_@out) -> _@unused, _@body end}"], + Param) + end. + +%% We have to insert dummy variable uses at the beginning of the "guard" and +%% "body" function bodies to avoid warnings for unused variables in the +%% generated code. (Expansions at the Erlang level can't be marked up as +%% compiler generated to allow later compiler stages to ignore them.) +dummy_uses(Vars) -> + [?Q("_ = _@var", [{var, erl_syntax:variable(var_name(V))}]) + || V <- Vars]. + +rewrite_guard([]) -> []; +rewrite_guard([D]) -> [make_orelse(erl_syntax:disjunction_body(D))]. + +make_orelse([]) -> []; +make_orelse([C]) -> make_andalso(erl_syntax:conjunction_body(C)); +make_orelse([C | Cs]) -> + ?Q("_@expr orelse _@rest", + [{expr, make_andalso(erl_syntax:conjunction_body(C))}, + {rest, make_orelse(Cs)}]). + +make_andalso([E]) -> E; +make_andalso([E | Es]) -> + ?Q("_@expr andalso _@rest", [{expr, E}, {rest, make_andalso(Es)}]). + +is_inline_metavar(Var) when is_atom(Var) -> + is_erlang_var(atom_to_list(Var)); +is_inline_metavar(Var) when is_integer(Var) -> + Var > 9 andalso (Var rem 10) =:= 9; +is_inline_metavar(_) -> false. + +is_erlang_var([C|_]) when C >= $A, C =< $Z ; C >= $À, C =< $Þ, C /= $× -> + true; +is_erlang_var(_) -> + false. diff --git a/lib/syntax_tools/src/syntax_tools.app.src b/lib/syntax_tools/src/syntax_tools.app.src index 83dcb5fe23..dd4ac46055 100644 --- a/lib/syntax_tools/src/syntax_tools.app.src +++ b/lib/syntax_tools/src/syntax_tools.app.src @@ -11,8 +11,11 @@ erl_syntax_lib, erl_tidy, igor, + merl, + merl_transform, prettypr]}, {registered,[]}, {applications, [stdlib]}, {env, []}, - {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]}]}. + {runtime_dependencies, + ["compiler-6.0","erts-6.0","kernel-3.0","stdlib-2.5"]}]}. diff --git a/lib/syntax_tools/test/Makefile b/lib/syntax_tools/test/Makefile index f67e3f8984..569c044b1a 100644 --- a/lib/syntax_tools/test/Makefile +++ b/lib/syntax_tools/test/Makefile @@ -6,7 +6,8 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk # ---------------------------------------------------- MODULES= \ - syntax_tools_SUITE + syntax_tools_SUITE \ + merl_SUITE ERL_FILES= $(MODULES:%=%.erl) diff --git a/lib/syntax_tools/test/merl_SUITE.erl b/lib/syntax_tools/test/merl_SUITE.erl new file mode 100644 index 0000000000..08b0f7a696 --- /dev/null +++ b/lib/syntax_tools/test/merl_SUITE.erl @@ -0,0 +1,91 @@ +%% ``The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved via the world wide web at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% The Initial Developer of the Original Code is Ericsson Utvecklings AB. +%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings +%% AB. All Rights Reserved.'' +%% +-module(merl_SUITE). + +-include_lib("test_server/include/test_server.hrl"). + +%% include the Merl header file +-include_lib("syntax_tools/include/merl.hrl"). + +%% for assert macros +-include_lib("eunit/include/eunit.hrl"). + +%% Test server specific exports +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2]). + +%% Test cases +-export([merl_smoke_test/1]). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [merl_smoke_test]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + +-define(tokens2str(X), ??X). + +merl_smoke_test(Config) when is_list(Config) -> + ?assertThrow({error, "1: syntax error before: '{'" ++ _}, + f(merl:quote("{"))), + ?assertEqual(tuple, erl_syntax:type(merl:term({}))), + ?assertEqual("{foo, 42}", f(merl:term({foo, 42}))), + ?assertEqual("f(X) -> {ok, X}.", f(?Q("f(X) -> {ok, X}."))), + ?assertEqual("{foo, 42}", f(?Q("{foo, 42}"))), + ?assertEqual("2 + 2", f(?Q("2 + 2"))), + ?assertEqual("%% comment preserved\n{foo, 42}", + f(?Q(["%% comment preserved", "{foo, 42}"]))), + ?assertEqual("'@foo'", f(merl:tree(merl:template(?Q("'@foo'"))))), + ?assertEqual("42", f(merl:subst(?Q("_@foo"), [{foo, merl:term(42)}]))), + ?assertEqual({ok, []}, merl:match(?Q("foo"), ?Q("foo"))), + ?assertEqual(42, merl:switch(?Q("foo"), [fun () -> 42 end])), + ?assertEqual("{foo}", f(begin Foo = ?Q("foo"), ?Q("{_@Foo}") end)), + ?assertEqual("{foo}", f(begin Foo = foo, ?Q("{_@Foo@}") end)), + ?assertEqual("{[bar], baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + ?Q("{foo, _@Bar, '@Baz'}") = Tree, + ?Q("{_@Bar, _@Baz}") + end)), + ?assertEqual("{[bar], baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + case Tree of + ?Q("{foo, _@Bar, '@Baz'}") -> ?Q("{_@Bar, _@Baz}") + end + end)), + ok. + +%% utilities + +f(Ts) when is_list(Ts) -> + lists:flatmap(fun erl_prettypr:format/1, Ts); +f(T) -> + erl_prettypr:format(T). diff --git a/lib/test_server/doc/src/notes.xml b/lib/test_server/doc/src/notes.xml index 6cf7a2b52d..e996d2b4a3 100644 --- a/lib/test_server/doc/src/notes.xml +++ b/lib/test_server/doc/src/notes.xml @@ -32,55 +32,6 @@ <file>notes.xml</file> </header> -<section><title>Test_Server 3.9</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - The status of an aborted test due to test suite - compilation error has changed from 'auto_skipped' to - 'failed'. This affects both the textual log file, event - handling and CT hook callbacks. The logging of - compilation failures has also been improved, especially - in the case of multiple test suites failing compilation.</p> - <p> - Own Id: OTP-10816</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - The Test Server application has been marked as obsolete - and will be removed from OTP in the next major release - (OTP 19.0).</p> - <p> - Own Id: OTP-10923 Aux Id: OTP-12705 </p> - </item> - <item> - <p> - When running OTP tests using the ts interface, it is now - possible to specify so called test categories per OTP - application. A test category is represented by a CT test - specification and defines an arbitrary subset of existing - test suites, groups and cases. Examples of test - categories are 'smoke' (smoke tests) and 'bench' - (benchmarks). (Call ts:help() for more info). Also, - functions for reading terms from the current test - specification during test, ct:get_testspec_terms/0 and - ct:get_testspec_terms/1, have been implemented.</p> - <p> - Own Id: OTP-11962</p> - </item> - </list> - </section> - -</section> - <section><title>Test_Server 3.8.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/test_server/src/erl2html2.erl b/lib/test_server/src/erl2html2.erl index 9101212852..b0b5c40965 100644 --- a/lib/test_server/src/erl2html2.erl +++ b/lib/test_server/src/erl2html2.erl @@ -170,7 +170,11 @@ get_line(Anno) -> %%% Find the line number of the last expression in the function find_clause_lines([{clause,CL,_Params,_Op,Exprs}], CLs) -> % last clause try tuple_to_list(lists:last(Exprs)) of - [_Type,ExprLine | _] -> + [_Type,ExprLine | _] when is_integer(ExprLine) -> + {lists:reverse([{clause,get_line(CL)}|CLs]), get_line(ExprLine)}; + [tree,_ | Exprs1] -> + find_clause_lines([{clause,CL,undefined,undefined,Exprs1}], CLs); + [macro,{_var,ExprLine,_MACRO} | _] when is_integer(ExprLine) -> {lists:reverse([{clause,get_line(CL)}|CLs]), get_line(ExprLine)}; _ -> {lists:reverse([{clause,get_line(CL)}|CLs]), get_line(CL)} @@ -188,18 +192,18 @@ build_html(SFd,DFd,Encoding,FuncsAndCs) -> build_html(SFd,DFd,Encoding,file:read_line(SFd),1,FuncsAndCs, false,undefined). -%% function start line found -build_html(SFd,DFd,Enc,{ok,Str},L0,[{F,A,L0,LastL}|FuncsAndCs], - _IsFuncDef,_FAndLastL) -> - FALink = test_server_ctrl:uri_encode(F++"-"++integer_to_list(A),utf8), - file:write(DFd,["<a name=\"",to_raw_list(FALink,Enc),"\"/>"]), - build_html(SFd,DFd,Enc,{ok,Str},L0,FuncsAndCs,true,{F,LastL}); %% line of last expression in function found build_html(SFd,DFd,Enc,{ok,Str},LastL,FuncsAndCs,_IsFuncDef,{F,LastL}) -> LastLineLink = test_server_ctrl:uri_encode(F++"-last_expr",utf8), file:write(DFd,["<a name=\"", to_raw_list(LastLineLink,Enc),"\"/>"]), build_html(SFd,DFd,Enc,{ok,Str},LastL,FuncsAndCs,true,undefined); +%% function start line found +build_html(SFd,DFd,Enc,{ok,Str},L0,[{F,A,L0,LastL}|FuncsAndCs], + _IsFuncDef,_FAndLastL) -> + FALink = test_server_ctrl:uri_encode(F++"-"++integer_to_list(A),utf8), + file:write(DFd,["<a name=\"",to_raw_list(FALink,Enc),"\"/>"]), + build_html(SFd,DFd,Enc,{ok,Str},L0,FuncsAndCs,true,{F,LastL}); build_html(SFd,DFd,Enc,{ok,Str},L,[{clause,L}|FuncsAndCs], _IsFuncDef,FAndLastL) -> build_html(SFd,DFd,Enc,{ok,Str},L,FuncsAndCs,true,FAndLastL); diff --git a/lib/test_server/test/erl2html2_SUITE.erl b/lib/test_server/test/erl2html2_SUITE.erl index 908985c879..796b84dedd 100644 --- a/lib/test_server/test/erl2html2_SUITE.erl +++ b/lib/test_server/test/erl2html2_SUITE.erl @@ -130,15 +130,7 @@ groups() -> %% @end %%-------------------------------------------------------------------- all() -> - [m1]. - -%%-------------------------------------------------------------------- -%% @spec TestCase() -> Info -%% Info = [tuple()] -%% @end -%%-------------------------------------------------------------------- -m1() -> - []. + [macros_defined, macros_undefined]. %%-------------------------------------------------------------------- %% @spec TestCase(Config0) -> @@ -149,19 +141,29 @@ m1() -> %% Comment = term() %% @end %%-------------------------------------------------------------------- -m1(Config) -> - {Src,Dst} = convert_module("m1",Config), +macros_defined(Config) -> + %% let erl2html2 use epp as parser + DataDir = ?config(data_dir,Config), + InclDir = filename:join(DataDir, "include"), + {Src,Dst} = convert_module("m1",[InclDir],Config), {true,L} = check_line_numbers(Src,Dst), - ok = check_link_targets(Src,Dst,L,[{baz,0}]), + ok = check_link_targets(Src,Dst,L,[{baz,0}],[]), ok. -convert_module(Mod,Config) -> +macros_undefined(Config) -> + %% let erl2html2 use epp_dodger as parser + {Src,Dst} = convert_module("m1",[],Config), + {true,L} = check_line_numbers(Src,Dst), + ok = check_link_targets(Src,Dst,L,[{baz,0}],[{quux,0}]), + ok. + +convert_module(Mod,InclDirs,Config) -> DataDir = ?config(data_dir,Config), PrivDir = ?config(priv_dir,Config), Src = filename:join(DataDir,Mod++".erl"), Dst = filename:join(PrivDir,Mod++".erl.html"), io:format("<a href=\"~s\">~s</a>\n",[Src,filename:basename(Src)]), - ok = erl2html2:convert(Src, Dst, [], "<html><body>"), + ok = erl2html2:convert(Src, Dst, InclDirs, "<html><body>"), io:format("<a href=\"~s\">~s</a>\n",[Dst,filename:basename(Dst)]), {Src,Dst}. @@ -229,36 +231,46 @@ check_line_number(Last,Line,OrigLine) -> %% function. %% The test module has -compile(export_all), so all functions are %% found by listing the exported ones. -check_link_targets(Src,Dst,L,RmFncs) -> +check_link_targets(Src,Dst,L,RmFncs,ShouldRemain) -> Mod = list_to_atom(filename:basename(filename:rootname(Src))), Exports = Mod:module_info(exports)--[{module_info,0},{module_info,1}|RmFncs], - {ok,{[],L},_} = xmerl_sax_parser:file(Dst, - [{event_fun,fun sax_event/3}, - {event_state,{Exports,0}}]), + LastExprFuncs = [Func || {Func,_A} <- Exports], + {ok,{FAs,Fs,L},_} = + xmerl_sax_parser:file(Dst, + [{event_fun,fun sax_event/3}, + {event_state,{Exports,LastExprFuncs,0}}]), + true = (length(FAs) == length(ShouldRemain)), + [] = [FA || FA <- FAs, not lists:member(FA,ShouldRemain)], + [] = [F || F <- Fs, not lists:keymember(F,1,ShouldRemain)], ok. sax_event(Event,_Loc,State) -> sax_event(Event,State). -sax_event({startElement,_Uri,"a",_QN,Attrs},{Exports,PrevLine}) -> +sax_event({startElement,_Uri,"a",_QN,Attrs},{Exports,LastExprFuncs,PrevLine}) -> {_,_,"name",Name} = lists:keyfind("name",3,Attrs), case catch list_to_integer(Name) of Line when is_integer(Line) -> case PrevLine + 1 of Line -> -% erlang:display({found_line,Line}), - {Exports,Line}; + {Exports,LastExprFuncs,Line}; Other -> ct:fail({unexpected_line_number_target,Other}) end; {'EXIT',_} -> - {match,[FStr,AStr]} = - re:run(Name,"^(.*)-([0-9]+)$",[{capture,all_but_first,list}]), + {match,[FStr,EndStr]} = + re:run(Name,"^(.*)-(last_expr|[0-9]+)$", + [{capture,all_but_first,list}]), F = list_to_atom(http_uri:decode(FStr)), - A = list_to_integer(AStr), -% erlang:display({found_fnc,F,A}), - A = proplists:get_value(F,Exports), - {lists:delete({F,A},Exports),PrevLine} + case EndStr of + "last_expr" -> + true = lists:member(F,LastExprFuncs), + {Exports,lists:delete(F,LastExprFuncs),PrevLine}; + _ -> + A = list_to_integer(EndStr), + A = proplists:get_value(F,Exports), + {lists:delete({F,A},Exports),LastExprFuncs,PrevLine} + end end; sax_event(_,State) -> State. diff --git a/lib/test_server/test/erl2html2_SUITE_data/include/header3.hrl b/lib/test_server/test/erl2html2_SUITE_data/include/header3.hrl new file mode 100644 index 0000000000..2a20850a3a --- /dev/null +++ b/lib/test_server/test/erl2html2_SUITE_data/include/header3.hrl @@ -0,0 +1 @@ +-define(EPP_SWITCH, on). diff --git a/lib/test_server/test/erl2html2_SUITE_data/m1.erl b/lib/test_server/test/erl2html2_SUITE_data/m1.erl index 156f1d0a51..1d405963a5 100644 --- a/lib/test_server/test/erl2html2_SUITE_data/m1.erl +++ b/lib/test_server/test/erl2html2_SUITE_data/m1.erl @@ -7,9 +7,15 @@ -include("header1.hrl"). -include("header2.hrl"). +-include("header3.hrl"). -define(MACRO1,value). +%% This macro is used to select parser in erl2html2. +%% If EPP_SWITCH is defined epp is used, else epp_dodger. +epp_switch() -> + ?EPP_SWITCH. + %%% Comment foo(x) -> %% Comment diff --git a/lib/tools/doc/src/cprof.xml b/lib/tools/doc/src/cprof.xml index 553597837e..bfddb9f5a8 100644 --- a/lib/tools/doc/src/cprof.xml +++ b/lib/tools/doc/src/cprof.xml @@ -66,7 +66,7 @@ <func> <name>analyse() -> {AllCallCount, ModAnalysisList}</name> <name>analyse(Limit) -> {AllCallCount, ModAnalysisList}</name> - <name>analyse(Mod) -> ModAnlysis</name> + <name>analyse(Mod) -> ModAnalysis</name> <name>analyse(Mod, Limit) -> ModAnalysis</name> <fsummary>Collect and analyse call counters.</fsummary> <type> diff --git a/lib/tools/doc/src/notes.xml b/lib/tools/doc/src/notes.xml index 85610d623a..38b57b73a9 100644 --- a/lib/tools/doc/src/notes.xml +++ b/lib/tools/doc/src/notes.xml @@ -30,65 +30,6 @@ </header> <p>This document describes the changes made to the Tools application.</p> -<section><title>Tools 2.8</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - In order to improve performance of the cover tool, new - functions are added for cover compilation and analysis on - multiple files. This allows for more parallelisation.</p> - <p> - Some improvements of the data base access is also done in - order to improve the performance when analysing and - resetting cover data.</p> - <p> - Minor incompatibility: An error reason from - analyse_to_file is changed from no_source_code_found to - {no_source_code_found,Module}.</p> - <p> - *** POTENTIAL INCOMPATIBILITY ***</p> - <p> - Own Id: OTP-12330 Aux Id: seq12757 </p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Allow maps for supervisor flags and child specs</p> - <p> - Earlier, supervisor flags and child specs were given as - tuples. While this is kept for backwards compatibility, - it is now also allowed to give these parameters as maps, - see <seealso - marker="stdlib:supervisor#sup_flags">sup_flags</seealso> - and <seealso - marker="stdlib:supervisor#child_spec">child_spec</seealso>.</p> - <p> - Own Id: OTP-11043</p> - </item> - <item> - <p> - Remove Mnemosyne rules support.</p> - <p> - Own Id: OTP-12511</p> - </item> - <item> - <p> - Add printout of total number of calls and time in eprof</p> - <p> - Own Id: OTP-12681</p> - </item> - </list> - </section> - -</section> - <section><title>Tools 2.7.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/tools/emacs/erlang.el b/lib/tools/emacs/erlang.el index 3610356355..ca194703bb 100644 --- a/lib/tools/emacs/erlang.el +++ b/lib/tools/emacs/erlang.el @@ -880,10 +880,10 @@ resulting regexp is surrounded by \\_< and \\_>." "dt_restore_tag" "dt_spread_tag" "dunlink" + "convert_time_unit" "external_size" "finish_after_on_load" "finish_loading" - "flush_monitor_message" "format_cpu_topology" "fun_info" "fun_info_mfa" @@ -913,6 +913,7 @@ resulting regexp is surrounded by \\_< and \\_>." "memory" "module_info" "monitor_node" + "monotonic_time" "nif_error" "phash" "phash2" @@ -946,13 +947,17 @@ resulting regexp is surrounded by \\_< and \\_>." "system_info" "system_monitor" "system_profile" + "system_time" "trace" "trace_delivered" "trace_info" "trace_pattern" + "time_offset" + "timestamp" "universaltime" "universaltime_to_localtime" "universaltime_to_posixtime" + "unique_integer" "yield") "Erlang built-in functions (BIFs) that needs erlang: prefix")) diff --git a/lib/tools/src/cover.erl b/lib/tools/src/cover.erl index 71e17e0ba1..9ec5e809bc 100644 --- a/lib/tools/src/cover.erl +++ b/lib/tools/src/cover.erl @@ -782,7 +782,7 @@ main_process_loop(State) -> {From, {{analyse_to_file, Opts},Module}} -> S = try Loaded = is_loaded(Module, State), - spawn(fun() -> + spawn_link(fun() -> ?SPAWN_DBG(analyse_to_file,{Module,Opts}), do_parallel_analysis_to_file( Module, Opts, Loaded, From, State) @@ -1017,14 +1017,24 @@ load_compiled([{Module,File,Binary,InitialTable}|Compiled],Acc) -> %% Make sure the #bump{} records are available *before* the %% module is loaded. insert_initial_data(InitialTable), - NewAcc = - case code:load_binary(Module, ?TAG, Binary) of - {module,Module} -> - add_compiled(Module, File, Acc); - _ -> - do_clear(Module), - Acc - end, + Sticky = case code:is_sticky(Module) of + true -> + code:unstick_mod(Module), + true; + false -> + false + end, + NewAcc = case code:load_binary(Module, ?TAG, Binary) of + {module,Module} -> + add_compiled(Module, File, Acc); + _ -> + do_clear(Module), + Acc + end, + case Sticky of + true -> code:stick_mod(Module); + false -> ok + end, load_compiled(Compiled,NewAcc); load_compiled([],Acc) -> Acc. @@ -2143,7 +2153,13 @@ find_source(Module, File0) -> throw_file(filename:join([BeamDir, "..", "src", Base])), %% Not in ../src: look for source path in compile info, but %% first look relative the beam directory. - Info = lists:keyfind(source, 1, Module:module_info(compile)), + Info = + try lists:keyfind(source, 1, Module:module_info(compile)) + catch error:undef -> + %% The module might have been imported + %% and the beam not available + throw({beam, File0}) + end, false == Info andalso throw({beam, File0}), %% stripped {source, SrcFile} = Info, throw_file(splice(BeamDir, SrcFile)), %% below ../src diff --git a/lib/tools/src/tools.app.src b/lib/tools/src/tools.app.src index a4e5d85f92..8458941761 100644 --- a/lib/tools/src/tools.app.src +++ b/lib/tools/src/tools.app.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2012. All Rights Reserved. +%% Copyright Ericsson AB 1996-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -40,7 +40,7 @@ {env, [{file_util_search_methods,[{"", ""}, {"ebin", "esrc"}, {"ebin", "src"}]} ] }, - {runtime_dependencies, ["webtool-0.8.10","stdlib-2.0","runtime_tools-1.8.14", + {runtime_dependencies, ["webtool-0.8.10","stdlib-2.5","runtime_tools-1.8.14", "kernel-3.0","inets-5.10","erts-7.0", "compiler-5.0"]} ] diff --git a/lib/tools/test/cover_SUITE.erl b/lib/tools/test/cover_SUITE.erl index 368fa6c3d1..bc85f3c045 100644 --- a/lib/tools/test/cover_SUITE.erl +++ b/lib/tools/test/cover_SUITE.erl @@ -29,7 +29,8 @@ export_import/1, otp_5031/1, eif/1, otp_5305/1, otp_5418/1, otp_6115/1, otp_7095/1, otp_8188/1, otp_8270/1, otp_8273/1, otp_8340/1, - otp_10979_hanging_node/1, compile_beam_opts/1, eep37/1]). + otp_10979_hanging_node/1, compile_beam_opts/1, eep37/1, + analyse_no_beam/1]). -export([do_coverage/1]). @@ -52,7 +53,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> NoStartStop = [eif,otp_5305,otp_5418,otp_7095,otp_8273, - otp_8340,otp_8188,compile_beam_opts,eep37], + otp_8340,otp_8188,compile_beam_opts,eep37, + analyse_no_beam], StartStop = [start, compile, analyse, misc, stop, distribution, reconnect, die_and_reconnect, dont_reconnect_after_stop, stop_node_after_disconnect, @@ -1687,6 +1689,43 @@ compile_beam_opts(Config) when is_list(Config) -> ok = file:set_cwd(Cwd), ok. +analyse_no_beam(doc) -> + ["Don't crash if beam is not available"]; +analyse_no_beam(suite) -> []; +analyse_no_beam(Config) when is_list(Config) -> + {ok, Cwd} = file:get_cwd(), + ok = file:set_cwd(?config(data_dir, Config)), + + code:purge(t), + code:delete(t), + + {ok,_} = file:copy("compile_beam/t.erl", "t.erl"), + {ok,t} = compile:file(t, [debug_info]), + {module,t} = code:load_file(t), + {ok,t} = cover:compile_beam(t), + t:f(), + ok = cover:export("t.coverdata"), + + code:purge(t), + code:delete(t), + + %% this is just so that cover realises (without stopping) + %% that this module is not cover compiled any more + {error, {not_cover_compiled,t}} = cover:analyse(t), + + %% source and beam not available any more + ok = file:delete("t.erl"), + ok = file:delete("t.beam"), + + ok = cover:import("t.coverdata"), + + {error,{no_source_code_found,t}} = cover:analyse_to_file(t), + {result,[],[{no_source_code_found,t}]} = cover:analyse_to_file([t]), + + ok = file:delete("t.coverdata"), + ok = file:set_cwd(Cwd), + ok. + %%--Auxiliary------------------------------------------------------------ analyse_expr(Expr, Config) -> diff --git a/lib/tools/test/cover_SUITE_data/compile_beam/t.erl b/lib/tools/test/cover_SUITE_data/compile_beam/t.erl new file mode 100644 index 0000000000..96dc2f4209 --- /dev/null +++ b/lib/tools/test/cover_SUITE_data/compile_beam/t.erl @@ -0,0 +1,6 @@ +-module(t). + +-export([f/0]). + +f() -> + ok. diff --git a/lib/tools/test/lcnt_SUITE.erl b/lib/tools/test/lcnt_SUITE.erl index 010dffe138..de68486b1b 100644 --- a/lib/tools/test/lcnt_SUITE.erl +++ b/lib/tools/test/lcnt_SUITE.erl @@ -97,12 +97,12 @@ t_conflicts_file([File|Files]) -> {ok, _} = lcnt:start(), ok = lcnt:load(File), ok = lcnt:conflicts(), - THs = [-1, 0, 100, 1000], + THs = [-1, 5], Print = [name , id , type , entry , tries , colls , ratio , time , duration], Opts = [ [{sort, Sort}, {reverse, Rev}, {max_locks, ML}, {combine, Combine}, {thresholds, [TH]}, {print, [Print]}] || - Sort <- [name , id , type , tries , colls , ratio , time , entry], - ML <- [none, 1 , 32, 4096], + Sort <- [name , type , tries , colls , ratio , time], + ML <- [none, 32], Combine <- [true, false], TH <- [{tries, Tries} || Tries <- THs] ++ [{colls, Colls} || Colls <- THs] ++ [{time, Time} || Time <- THs], Rev <- [true, false] @@ -131,12 +131,12 @@ t_locations_file([File|Files]) -> {ok, _} = lcnt:start(), ok = lcnt:load(File), ok = lcnt:locations(), - THs = [-1, 0, 100, 1000], + THs = [-1, 0, 100], Print = [name , id , type , entry , tries , colls , ratio , time , duration], Opts = [ [{full_id, Id}, {sort, Sort}, {max_locks, ML}, {combine, Combine}, {thresholds, [TH]}, {print, Print}] || Sort <- [name , id , type , tries , colls , ratio , time , entry], - ML <- [none, 1 , 64], + ML <- [none, 64], Combine <- [true, false], TH <- [{tries, Tries} || Tries <- THs] ++ [{colls, Colls} || Colls <- THs] ++ [{time, Time} || Time <- THs], Id <- [true, false] diff --git a/lib/typer/doc/src/notes.xml b/lib/typer/doc/src/notes.xml index 045b55ffe4..23e22759d6 100644 --- a/lib/typer/doc/src/notes.xml +++ b/lib/typer/doc/src/notes.xml @@ -30,20 +30,6 @@ </header> <p>This document describes the changes made to TypEr.</p> -<section><title>TypEr 0.9.9</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> Properly extract annotations from core code. </p> - <p> - Own Id: OTP-12727</p> - </item> - </list> - </section> - -</section> - <section><title>TypEr 0.9.8</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/webtool/doc/src/notes.xml b/lib/webtool/doc/src/notes.xml index 4800dd6df4..e571668c91 100644 --- a/lib/webtool/doc/src/notes.xml +++ b/lib/webtool/doc/src/notes.xml @@ -31,23 +31,6 @@ <p>This document describes the changes made to the Webtool application.</p> -<section><title>WebTool 0.9</title> - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - The Webtool application has been marked as obsolete and - will be removed from OTP in the next major release (OTP - 19.0).</p> - <p> - Own Id: OTP-10922 Aux Id: OTP-12705 </p> - </item> - </list> - </section> - -</section> - <section><title>WebTool 0.8.10</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/wx/api_gen/wx_gen_cpp.erl b/lib/wx/api_gen/wx_gen_cpp.erl index 8e32aeddc8..8cbc448563 100644 --- a/lib/wx/api_gen/wx_gen_cpp.erl +++ b/lib/wx/api_gen/wx_gen_cpp.erl @@ -207,7 +207,7 @@ gen_funcs(Defs) -> " }~n"), w(" case WXE_BIN_INCR:~n driver_binary_inc_refc(Ecmd.bin[0]->bin);~n break;~n",[]), w(" case WXE_BIN_DECR:~n driver_binary_dec_refc(Ecmd.bin[0]->bin);~n break;~n",[]), - w(" case WXE_INIT_OPENGL:~n wxe_initOpenGL(rt, bp);~n break;~n",[]), + w(" case WXE_INIT_OPENGL:~n wxe_initOpenGL(&rt, bp);~n break;~n",[]), Res = [gen_class(Class) || Class <- Defs], @@ -910,11 +910,24 @@ is_dc(Class) -> Parents = wx_gen_erl:parents(Class), lists:member("wxDC", Parents) orelse lists:member("wxGraphicsContext", Parents). -build_return_vals(Type,Ps) -> +build_return_vals(Type,Ps0) -> + Ps = [P || P = #param{in=In} <- Ps0, In =/= true], HaveType = case Type of void -> 0; _ -> 1 end, - NoOut = lists:sum([1 || #param{in=In} <- Ps, In =/= true]) + HaveType, + NoOut = length(Ps) + HaveType, OutTupSz = if NoOut > 1 -> NoOut; true -> 0 end, + CountFloats = fun(#param{type=#type{base=Float, single=true}}, Acc) + when Float =:= float; Float =:= double -> + Acc + 1; + (_, Acc) -> + Acc + end, + NofFloats = lists:foldl(CountFloats, 1, Ps), + case NofFloats > 1 of + true -> %%io:format("Floats ~p:~p ~p ~n",[get(current_class),get(current_func), NofFloats]); + w(" rt.ensureFloatCount(~p);~n",[NofFloats]); + false -> ignore + end, build_ret_types(Type,Ps), if OutTupSz > 1 -> w(" rt.addTupleCount(~p);~n",[OutTupSz]); @@ -923,12 +936,11 @@ build_return_vals(Type,Ps) -> Ps. build_ret_types(void,Ps) -> - Calc = fun(#param{name=N,in=False,type=T}, Free) when False =/= true -> - case build_ret(N, {arg, False}, T) of + Calc = fun(#param{name=N,in=In,type=T}, Free) -> + case build_ret(N, {arg, In}, T) of ok -> Free; Other -> [Other|Free] - end; - (_, Free) -> Free + end end, lists:foldl(Calc, [], Ps); build_ret_types(Type,Ps) -> @@ -936,12 +948,11 @@ build_ret_types(Type,Ps) -> ok -> []; FreeStr -> [FreeStr] end, - Calc = fun(#param{name=N,in=False,type=T}, FreeAcc) when False =/= true -> - case build_ret(N, {arg, False}, T) of + Calc = fun(#param{name=N,in=In,type=T}, FreeAcc) -> + case build_ret(N, {arg, In}, T) of ok -> FreeAcc; FreeMe -> [FreeMe|FreeAcc] - end; - (_, FreeAcc) -> FreeAcc + end end, lists:foldl(Calc, Free, Ps). @@ -1016,7 +1027,6 @@ build_ret(Name,_,#type{name="wxArrayTreeItemIds"}) -> w(" rt.endList(~s.GetCount());~n",[Name]); build_ret(Name,_,#type{base=float,single=true}) -> -%% w(" double Temp~s = ~s;~n", [Name,Name]), w(" rt.addFloat(~s);~n",[Name]); build_ret(Name,_,#type{base=double,single=true}) -> w(" rt.addFloat(~s);~n",[Name]); diff --git a/lib/wx/c_src/gen/wxe_funcs.cpp b/lib/wx/c_src/gen/wxe_funcs.cpp index 3b11c0642e..01a7ad7f70 100644 --- a/lib/wx/c_src/gen/wxe_funcs.cpp +++ b/lib/wx/c_src/gen/wxe_funcs.cpp @@ -60,13 +60,13 @@ void WxeApp::wxe_dispatch(wxeCommand& Ecmd) break; } case WXE_BIN_INCR: - driver_binary_inc_refc(Ecmd.bin[0]->bin); + driver_binary_inc_refc(Ecmd.bin[0].bin); break; case WXE_BIN_DECR: - driver_binary_dec_refc(Ecmd.bin[0]->bin); + driver_binary_dec_refc(Ecmd.bin[0].bin); break; case WXE_INIT_OPENGL: - wxe_initOpenGL(rt, bp); + wxe_initOpenGL(&rt, bp); break; case 100: { // wxEvtHandler::Connect @@ -81,7 +81,7 @@ case 100: { // wxEvtHandler::Connect int * class_nameLen = (int *) bp; bp += 4; if(*haveUserData) { - userData = new wxeErlTerm(Ecmd.bin[0]); + userData = new wxeErlTerm(&Ecmd.bin[0]); } int eventType = wxeEventTypeFromAtom(bp); bp += *eventTypeLen; @@ -5533,6 +5533,7 @@ case wxDC_GetUserScale: { // wxDC::GetUserScale wxDC *This = (wxDC *) getPtr(bp,memenv); bp += 4; if(!This) throw wxe_badarg(0); This->GetUserScale(&x,&y); + rt.ensureFloatCount(3); rt.addFloat(x); rt.addFloat(y); rt.addTupleCount(2); @@ -6430,6 +6431,7 @@ case wxGraphicsContext_GetTextExtent: { // wxGraphicsContext::GetTextExtent bp += *textLen+((8-((0+ *textLen) & 7)) & 7); if(!This) throw wxe_badarg(0); This->GetTextExtent(text,&width,&height,&descent,&externalLeading); + rt.ensureFloatCount(5); rt.addFloat(width); rt.addFloat(height); rt.addFloat(descent); @@ -6575,6 +6577,7 @@ case wxGraphicsMatrix_Get: { // wxGraphicsMatrix::Get wxGraphicsMatrix *This = (wxGraphicsMatrix *) getPtr(bp,memenv); bp += 4; if(!This) throw wxe_badarg(0); This->Get(&a,&b,&c,&d,&tx,&ty); + rt.ensureFloatCount(7); rt.addFloat(a); rt.addFloat(b); rt.addFloat(c); @@ -6676,6 +6679,7 @@ case wxGraphicsMatrix_TransformPoint: { // wxGraphicsMatrix::TransformPoint wxGraphicsMatrix *This = (wxGraphicsMatrix *) getPtr(bp,memenv); bp += 4; if(!This) throw wxe_badarg(0); This->TransformPoint(&x,&y); + rt.ensureFloatCount(3); rt.addFloat(x); rt.addFloat(y); rt.addTupleCount(2); @@ -6687,6 +6691,7 @@ case wxGraphicsMatrix_TransformDistance: { // wxGraphicsMatrix::TransformDistanc wxGraphicsMatrix *This = (wxGraphicsMatrix *) getPtr(bp,memenv); bp += 4; if(!This) throw wxe_badarg(0); This->TransformDistance(&dx,&dy); + rt.ensureFloatCount(3); rt.addFloat(dx); rt.addFloat(dy); rt.addTupleCount(2); @@ -7348,7 +7353,7 @@ case wxControlWithItems_Append_2: { // wxControlWithItems::Append int * itemLen = (int *) bp; bp += 4; wxString item = wxString(bp, wxConvUTF8); bp += *itemLen+((8-((0+ *itemLen) & 7)) & 7); - wxeErlTerm * clientData = new wxeErlTerm(Ecmd.bin[0]); + wxeErlTerm * clientData = new wxeErlTerm(&Ecmd.bin[0]); if(!This) throw wxe_badarg(0); int Result = This->Append(item,clientData); rt.addInt(Result); @@ -7410,7 +7415,7 @@ case wxControlWithItems_getClientData: { // wxControlWithItems::GetClientObject case wxControlWithItems_setClientData: { // wxControlWithItems::SetClientObject wxControlWithItems *This = (wxControlWithItems *) getPtr(bp,memenv); bp += 4; unsigned int * n = (unsigned int *) bp; bp += 4; - wxeErlTerm * clientData = new wxeErlTerm(Ecmd.bin[0]); + wxeErlTerm * clientData = new wxeErlTerm(&Ecmd.bin[0]); if(!This) throw wxe_badarg(0); This->SetClientObject(*n,clientData); break; @@ -7461,7 +7466,7 @@ case wxControlWithItems_Insert_3: { // wxControlWithItems::Insert wxString item = wxString(bp, wxConvUTF8); bp += *itemLen+((8-((0+ *itemLen) & 7)) & 7); unsigned int * pos = (unsigned int *) bp; bp += 4; - wxeErlTerm * clientData = new wxeErlTerm(Ecmd.bin[0]); + wxeErlTerm * clientData = new wxeErlTerm(&Ecmd.bin[0]); if(!This) throw wxe_badarg(0); int Result = This->Insert(item,*pos,clientData); rt.addInt(Result); @@ -8985,7 +8990,7 @@ case wxBitmap_new_3: { // wxBitmap::wxBitmap } case wxBitmap_new_4: { // wxBitmap::wxBitmap int depth=1; - const char * bits = (const char*) Ecmd.bin[0]->base; + const char * bits = (const char*) Ecmd.bin[0].base; int * width = (int *) bp; bp += 4; int * height = (int *) bp; bp += 4; while( * (int*) bp) { switch (* (int*) bp) { @@ -9325,7 +9330,7 @@ case wxCursor_new_1_1: { // wxCursor::wxCursor case wxCursor_new_4: { // wxCursor::wxCursor int hotSpotX=-1; int hotSpotY=-1; - const char * bits = (const char*) Ecmd.bin[0]->base; + const char * bits = (const char*) Ecmd.bin[0].base; int * width = (int *) bp; bp += 4; int * height = (int *) bp; bp += 4; while( * (int*) bp) { switch (* (int*) bp) { @@ -9436,13 +9441,13 @@ case wxImage_new_4: { // wxImage::wxImage bool static_data=false; int * width = (int *) bp; bp += 4; int * height = (int *) bp; bp += 4; - unsigned char * data = (unsigned char*) Ecmd.bin[0]->base; + unsigned char * data = (unsigned char*) Ecmd.bin[0].base; while( * (int*) bp) { switch (* (int*) bp) { case 1: {bp += 4; static_data = *(bool *) bp; bp += 4; } break; }}; - if(!static_data) {data = (unsigned char *) malloc(Ecmd.bin[0]->size);memcpy(data,Ecmd.bin[0]->base,Ecmd.bin[0]->size);}; + if(!static_data) {data = (unsigned char *) malloc(Ecmd.bin[0].size);memcpy(data,Ecmd.bin[0].base,Ecmd.bin[0].size);}; wxImage * Result = new EwxImage(*width,*height,data,static_data); newPtr((void *) Result, 1, memenv); rt.addRef(getRef((void *)Result,memenv), "wxImage"); @@ -9452,14 +9457,14 @@ case wxImage_new_5: { // wxImage::wxImage bool static_data=false; int * width = (int *) bp; bp += 4; int * height = (int *) bp; bp += 4; - unsigned char * data = (unsigned char*) Ecmd.bin[0]->base; - unsigned char * alpha = (unsigned char*) Ecmd.bin[1]->base; + unsigned char * data = (unsigned char*) Ecmd.bin[0].base; + unsigned char * alpha = (unsigned char*) Ecmd.bin[1].base; while( * (int*) bp) { switch (* (int*) bp) { case 1: {bp += 4; static_data = *(bool *) bp; bp += 4; } break; }}; - if(!static_data) { data = (unsigned char *) malloc(Ecmd.bin[0]->size); alpha = (unsigned char *) malloc(Ecmd.bin[1]->size); memcpy(data,Ecmd.bin[0]->base,Ecmd.bin[0]->size); memcpy(alpha,Ecmd.bin[1]->base,Ecmd.bin[1]->size);}; + if(!static_data) { data = (unsigned char *) malloc(Ecmd.bin[0].size); alpha = (unsigned char *) malloc(Ecmd.bin[1].size); memcpy(data,Ecmd.bin[0].base,Ecmd.bin[0].size); memcpy(alpha,Ecmd.bin[1].base,Ecmd.bin[1].size);}; wxImage * Result = new EwxImage(*width,*height,data,alpha,static_data); newPtr((void *) Result, 1, memenv); rt.addRef(getRef((void *)Result,memenv), "wxImage"); @@ -9603,14 +9608,14 @@ case wxImage_Create_4: { // wxImage::Create wxImage *This = (wxImage *) getPtr(bp,memenv); bp += 4; int * width = (int *) bp; bp += 4; int * height = (int *) bp; bp += 4; - unsigned char * data = (unsigned char*) Ecmd.bin[0]->base; + unsigned char * data = (unsigned char*) Ecmd.bin[0].base; bp += 4; /* Align */ while( * (int*) bp) { switch (* (int*) bp) { case 1: {bp += 4; static_data = *(bool *) bp; bp += 4; } break; }}; - if(!static_data) {data = (unsigned char *) malloc(Ecmd.bin[0]->size);memcpy(data,Ecmd.bin[0]->base,Ecmd.bin[0]->size);}; + if(!static_data) {data = (unsigned char *) malloc(Ecmd.bin[0].size);memcpy(data,Ecmd.bin[0].base,Ecmd.bin[0].size);}; if(!This) throw wxe_badarg(0); bool Result = This->Create(*width,*height,data,static_data); rt.addBool(Result); @@ -9621,15 +9626,15 @@ case wxImage_Create_5: { // wxImage::Create wxImage *This = (wxImage *) getPtr(bp,memenv); bp += 4; int * width = (int *) bp; bp += 4; int * height = (int *) bp; bp += 4; - unsigned char * data = (unsigned char*) Ecmd.bin[0]->base; - unsigned char * alpha = (unsigned char*) Ecmd.bin[1]->base; + unsigned char * data = (unsigned char*) Ecmd.bin[0].base; + unsigned char * alpha = (unsigned char*) Ecmd.bin[1].base; bp += 4; /* Align */ while( * (int*) bp) { switch (* (int*) bp) { case 1: {bp += 4; static_data = *(bool *) bp; bp += 4; } break; }}; - if(!static_data) { data = (unsigned char *) malloc(Ecmd.bin[0]->size); alpha = (unsigned char *) malloc(Ecmd.bin[1]->size); memcpy(data,Ecmd.bin[0]->base,Ecmd.bin[0]->size); memcpy(alpha,Ecmd.bin[1]->base,Ecmd.bin[1]->size);}; + if(!static_data) { data = (unsigned char *) malloc(Ecmd.bin[0].size); alpha = (unsigned char *) malloc(Ecmd.bin[1].size); memcpy(data,Ecmd.bin[0].base,Ecmd.bin[0].size); memcpy(alpha,Ecmd.bin[1].base,Ecmd.bin[1].size);}; if(!This) throw wxe_badarg(0); bool Result = This->Create(*width,*height,data,alpha,static_data); rt.addBool(Result); @@ -10142,14 +10147,14 @@ case wxImage_SetAlpha_3: { // wxImage::SetAlpha case wxImage_SetAlpha_2: { // wxImage::SetAlpha bool static_data=false; wxImage *This = (wxImage *) getPtr(bp,memenv); bp += 4; - unsigned char * alpha = (unsigned char*) Ecmd.bin[0]->base; + unsigned char * alpha = (unsigned char*) Ecmd.bin[0].base; bp += 4; /* Align */ while( * (int*) bp) { switch (* (int*) bp) { case 1: {bp += 4; static_data = *(bool *) bp; bp += 4; } break; }}; - if(!static_data) {alpha = (unsigned char *) malloc(Ecmd.bin[0]->size);memcpy(alpha,Ecmd.bin[0]->base,Ecmd.bin[0]->size);}; + if(!static_data) {alpha = (unsigned char *) malloc(Ecmd.bin[0].size);memcpy(alpha,Ecmd.bin[0].base,Ecmd.bin[0].size);}; if(!This) throw wxe_badarg(0); This->SetAlpha(alpha,static_data); break; @@ -10157,14 +10162,14 @@ case wxImage_SetAlpha_2: { // wxImage::SetAlpha case wxImage_SetData_2: { // wxImage::SetData bool static_data=false; wxImage *This = (wxImage *) getPtr(bp,memenv); bp += 4; - unsigned char * data = (unsigned char*) Ecmd.bin[0]->base; + unsigned char * data = (unsigned char*) Ecmd.bin[0].base; bp += 4; /* Align */ while( * (int*) bp) { switch (* (int*) bp) { case 1: {bp += 4; static_data = *(bool *) bp; bp += 4; } break; }}; - if(!static_data) {data = (unsigned char *) malloc(Ecmd.bin[0]->size);memcpy(data,Ecmd.bin[0]->base,Ecmd.bin[0]->size);}; + if(!static_data) {data = (unsigned char *) malloc(Ecmd.bin[0].size);memcpy(data,Ecmd.bin[0].base,Ecmd.bin[0].size);}; if(!This) throw wxe_badarg(0); This->SetData(data,static_data); break; @@ -10172,7 +10177,7 @@ case wxImage_SetData_2: { // wxImage::SetData case wxImage_SetData_4: { // wxImage::SetData bool static_data=false; wxImage *This = (wxImage *) getPtr(bp,memenv); bp += 4; - unsigned char * data = (unsigned char*) Ecmd.bin[0]->base; + unsigned char * data = (unsigned char*) Ecmd.bin[0].base; int * new_width = (int *) bp; bp += 4; int * new_height = (int *) bp; bp += 4; bp += 4; /* Align */ @@ -10181,7 +10186,7 @@ case wxImage_SetData_4: { // wxImage::SetData static_data = *(bool *) bp; bp += 4; } break; }}; - if(!static_data) {data = (unsigned char *) malloc(Ecmd.bin[0]->size);memcpy(data,Ecmd.bin[0]->base,Ecmd.bin[0]->size);}; + if(!static_data) {data = (unsigned char *) malloc(Ecmd.bin[0].size);memcpy(data,Ecmd.bin[0].base,Ecmd.bin[0].size);}; if(!This) throw wxe_badarg(0); This->SetData(data,*new_width,*new_height,static_data); break; @@ -18546,7 +18551,7 @@ case wxTreeCtrl_AddRoot: { // wxTreeCtrl::AddRoot selectedImage = (int)*(int *) bp; bp += 4; } break; case 3: {bp += 4; - data = new wxETreeItemData(Ecmd.bin[0]->size, Ecmd.bin[0]->base); + data = new wxETreeItemData(Ecmd.bin[0].size, Ecmd.bin[0].base); bp += 4; /* Align */ } break; }}; @@ -18573,7 +18578,7 @@ case wxTreeCtrl_AppendItem: { // wxTreeCtrl::AppendItem selectedImage = (int)*(int *) bp; bp += 4; } break; case 3: {bp += 4; - data = new wxETreeItemData(Ecmd.bin[0]->size, Ecmd.bin[0]->base); + data = new wxETreeItemData(Ecmd.bin[0].size, Ecmd.bin[0].base); bp += 4; /* Align */ } break; }}; @@ -18975,7 +18980,7 @@ case wxTreeCtrl_InsertItem: { // wxTreeCtrl::InsertItem selImage = (int)*(int *) bp; bp += 4; } break; case 3: {bp += 4; - data = new wxETreeItemData(Ecmd.bin[0]->size, Ecmd.bin[0]->base); + data = new wxETreeItemData(Ecmd.bin[0].size, Ecmd.bin[0].base); bp += 4; /* Align */ } break; }}; @@ -19054,7 +19059,7 @@ case wxTreeCtrl_PrependItem: { // wxTreeCtrl::PrependItem selectedImage = (int)*(int *) bp; bp += 4; } break; case 3: {bp += 4; - data = new wxETreeItemData(Ecmd.bin[0]->size, Ecmd.bin[0]->base); + data = new wxETreeItemData(Ecmd.bin[0].size, Ecmd.bin[0].base); bp += 4; /* Align */ } break; }}; @@ -19138,7 +19143,7 @@ case wxTreeCtrl_SetItemData: { // wxTreeCtrl::SetItemData wxTreeCtrl *This = (wxTreeCtrl *) getPtr(bp,memenv); bp += 4; bp += 4; /* Align */ wxTreeItemId item = wxTreeItemId((void *) *(wxUint64 *) bp); bp += 8; - wxETreeItemData * data = new wxETreeItemData(Ecmd.bin[0]->size, Ecmd.bin[0]->base); + wxETreeItemData * data = new wxETreeItemData(Ecmd.bin[0].size, Ecmd.bin[0].base); if(!This) throw wxe_badarg(0); This->SetItemData(item,data); break; @@ -20593,21 +20598,21 @@ case wxPalette_new_0: { // wxPalette::wxPalette break; } case wxPalette_new_4: { // wxPalette::wxPalette - const unsigned char * red = (const unsigned char*) Ecmd.bin[0]->base; - const unsigned char * green = (const unsigned char*) Ecmd.bin[1]->base; - const unsigned char * blue = (const unsigned char*) Ecmd.bin[2]->base; - wxPalette * Result = new EwxPalette(Ecmd.bin[0]->size,red,green,blue); + const unsigned char * red = (const unsigned char*) Ecmd.bin[0].base; + const unsigned char * green = (const unsigned char*) Ecmd.bin[1].base; + const unsigned char * blue = (const unsigned char*) Ecmd.bin[2].base; + wxPalette * Result = new EwxPalette(Ecmd.bin[0].size,red,green,blue); newPtr((void *) Result, 1, memenv); rt.addRef(getRef((void *)Result,memenv), "wxPalette"); break; } case wxPalette_Create: { // wxPalette::Create wxPalette *This = (wxPalette *) getPtr(bp,memenv); bp += 4; - const unsigned char * red = (const unsigned char*) Ecmd.bin[0]->base; - const unsigned char * green = (const unsigned char*) Ecmd.bin[1]->base; - const unsigned char * blue = (const unsigned char*) Ecmd.bin[2]->base; + const unsigned char * red = (const unsigned char*) Ecmd.bin[0].base; + const unsigned char * green = (const unsigned char*) Ecmd.bin[1].base; + const unsigned char * blue = (const unsigned char*) Ecmd.bin[2].base; if(!This) throw wxe_badarg(0); - bool Result = This->Create(Ecmd.bin[0]->size,red,green,blue); + bool Result = This->Create(Ecmd.bin[0].size,red,green,blue); rt.addBool(Result); break; } @@ -23746,6 +23751,7 @@ case wxAuiManager_GetDockSizeConstraint: { // wxAuiManager::GetDockSizeConstrain wxAuiManager *This = (wxAuiManager *) getPtr(bp,memenv); bp += 4; if(!This) throw wxe_badarg(0); This->GetDockSizeConstraint(&width_pct,&height_pct); + rt.ensureFloatCount(3); rt.addFloat(width_pct); rt.addFloat(height_pct); rt.addTupleCount(2); @@ -30254,7 +30260,7 @@ case wxStyledTextCtrl_GetUseAntiAliasing: { // wxStyledTextCtrl::GetUseAntiAlias } case wxStyledTextCtrl_AddTextRaw: { // wxStyledTextCtrl::AddTextRaw wxStyledTextCtrl *This = (wxStyledTextCtrl *) getPtr(bp,memenv); bp += 4; - const char * text = (const char*) Ecmd.bin[0]->base; + const char * text = (const char*) Ecmd.bin[0].base; if(!This) throw wxe_badarg(0); This->AddTextRaw(text); break; @@ -30262,7 +30268,7 @@ case wxStyledTextCtrl_AddTextRaw: { // wxStyledTextCtrl::AddTextRaw case wxStyledTextCtrl_InsertTextRaw: { // wxStyledTextCtrl::InsertTextRaw wxStyledTextCtrl *This = (wxStyledTextCtrl *) getPtr(bp,memenv); bp += 4; int * pos = (int *) bp; bp += 4; - const char * text = (const char*) Ecmd.bin[0]->base; + const char * text = (const char*) Ecmd.bin[0].base; if(!This) throw wxe_badarg(0); This->InsertTextRaw(*pos,text); break; @@ -30311,7 +30317,7 @@ case wxStyledTextCtrl_GetTextRangeRaw: { // wxStyledTextCtrl::GetTextRangeRaw } case wxStyledTextCtrl_SetTextRaw: { // wxStyledTextCtrl::SetTextRaw wxStyledTextCtrl *This = (wxStyledTextCtrl *) getPtr(bp,memenv); bp += 4; - const char * text = (const char*) Ecmd.bin[0]->base; + const char * text = (const char*) Ecmd.bin[0].base; if(!This) throw wxe_badarg(0); This->SetTextRaw(text); break; @@ -30327,7 +30333,7 @@ case wxStyledTextCtrl_GetTextRaw: { // wxStyledTextCtrl::GetTextRaw } case wxStyledTextCtrl_AppendTextRaw: { // wxStyledTextCtrl::AppendTextRaw wxStyledTextCtrl *This = (wxStyledTextCtrl *) getPtr(bp,memenv); bp += 4; - const char * text = (const char*) Ecmd.bin[0]->base; + const char * text = (const char*) Ecmd.bin[0].base; if(!This) throw wxe_badarg(0); This->AppendTextRaw(text); break; diff --git a/lib/wx/c_src/wxe_driver.c b/lib/wx/c_src/wxe_driver.c index ea52737fa2..3b71f49196 100644 --- a/lib/wx/c_src/wxe_driver.c +++ b/lib/wx/c_src/wxe_driver.c @@ -118,7 +118,11 @@ wxe_driver_start(ErlDrvPort port, char *buff) ErlDrvTermData term_port = driver_mk_port(port); set_port_control_flags(port, PORT_CONTROL_FLAG_BINARY); data->driver_data = NULL; - data->bin = NULL; + data->bin = (WXEBinRef*) driver_alloc(sizeof(WXEBinRef)*DEF_BINS); + data->bin[0].from = 0; + data->bin[1].from = 0; + data->bin[2].from = 0; + data->max_bins = DEF_BINS; data->port_handle = port; data->port = term_port; data->pdl = driver_pdl_create(port); @@ -146,7 +150,12 @@ wxe_driver_stop(ErlDrvData handle) if(sd->port_handle != WXE_DRV_PORT_HANDLE) { // fprintf(stderr, "%s:%d: STOP \r\n", __FILE__,__LINE__); meta_command(DELETE_PORT,sd); - free(handle); + } else { + // fprintf(stderr, "%s:%d: STOP \r\n", __FILE__,__LINE__); + stop_native_gui(wxe_master); + unload_native_gui(); + free(wxe_master); + wxe_master = NULL; } } @@ -154,10 +163,6 @@ static void wxe_driver_unload(void) { // fprintf(stderr, "%s:%d: UNLOAD \r\n", __FILE__,__LINE__); - stop_native_gui(wxe_master); - unload_native_gui(); - free(wxe_master); - wxe_master = NULL; } static ErlDrvSSizeT @@ -207,26 +212,40 @@ static void standard_outputv(ErlDrvData drv_data, ErlIOVec* ev) { wxe_data* sd = (wxe_data *) drv_data; - WXEBinRef * binref; + WXEBinRef * binref = NULL; ErlDrvBinary* bin; - + int i, max; + + for(i = 0; i < sd->max_bins; i++) { + if(sd->bin[i].from == 0) { + binref = &sd->bin[i]; + break; + } + } + + if(binref == NULL) { /* realloc */ + max = sd->max_bins + DEF_BINS; + driver_realloc(sd->bin, sizeof(WXEBinRef)*max); + for(i=sd->max_bins; i < max; i++) { + sd->bin[i].from = 0; + } + binref = &sd->bin[sd->max_bins]; + sd->max_bins = max; + } + if(ev->vsize == 2) { - binref = driver_alloc(sizeof(WXEBinRef)); binref->base = ev->iov[1].iov_base; binref->size = ev->iov[1].iov_len; binref->from = driver_caller(sd->port_handle); bin = ev->binv[1]; driver_binary_inc_refc(bin); /* Otherwise it could get deallocated */ binref->bin = bin; - binref->next = sd->bin; - sd->bin = binref; - } else { /* Empty binary (becomes NULL) */ - binref = driver_alloc(sizeof(WXEBinRef)); + sd->bin = binref; + } else { /* Empty binary (becomes NULL) */ binref->base = NULL; binref->size = 0; binref->from = driver_caller(sd->port_handle); binref->bin = NULL; - binref->next = sd->bin; sd->bin = binref; } } diff --git a/lib/wx/c_src/wxe_driver.h b/lib/wx/c_src/wxe_driver.h index e35bbe2118..9682f33e95 100644 --- a/lib/wx/c_src/wxe_driver.h +++ b/lib/wx/c_src/wxe_driver.h @@ -37,12 +37,12 @@ typedef struct wxe_bin_ref { size_t size; ErlDrvBinary* bin; ErlDrvTermData from; - WXEBinRefptr next; } WXEBinRef; -typedef struct wxe_data_def { +typedef struct wxe_data_def { void * driver_data; WXEBinRef * bin; /* Argument binaries */ + Uint32 max_bins; ErlDrvPort port_handle; ErlDrvTermData port; int is_cbport; @@ -50,6 +50,9 @@ typedef struct wxe_data_def { } wxe_data; +/* Number of bins per port should be small */ +#define DEF_BINS 3 + void init_glexts(wxe_data*); int load_native_gui(); diff --git a/lib/wx/c_src/wxe_gl.cpp b/lib/wx/c_src/wxe_gl.cpp index 26b45d219e..347718ab14 100644 --- a/lib/wx/c_src/wxe_gl.cpp +++ b/lib/wx/c_src/wxe_gl.cpp @@ -67,7 +67,7 @@ void dlclose(HMODULE Lib) { typedef void * DL_LIB_P; #endif -void wxe_initOpenGL(wxeReturn rt, char *bp) { +void wxe_initOpenGL(wxeReturn *rt, char *bp) { DL_LIB_P LIBhandle; int (*init_opengl)(void *); #ifdef _WIN32 @@ -82,9 +82,9 @@ void wxe_initOpenGL(wxeReturn rt, char *bp) { wxe_gl_dispatch = (WXE_GL_DISPATCH) dlsym(LIBhandle, "egl_dispatch"); if(init_opengl && wxe_gl_dispatch) { (*init_opengl)(erlCallbacks); - rt.addAtom((char *) "ok"); - rt.add(wxString::FromAscii("initiated")); - rt.addTupleCount(2); + rt->addAtom((char *) "ok"); + rt->add(wxString::FromAscii("initiated")); + rt->addTupleCount(2); erl_gl_initiated = TRUE; } else { wxString msg; @@ -95,24 +95,24 @@ void wxe_initOpenGL(wxeReturn rt, char *bp) { msg += wxT("egl_init_opengl "); if(!wxe_gl_dispatch) msg += wxT("egl_dispatch "); - rt.addAtom((char *) "error"); - rt.add(msg); - rt.addTupleCount(2); + rt->addAtom((char *) "error"); + rt->add(msg); + rt->addTupleCount(2); } } else { wxString msg; msg.Printf(wxT("Could not load dll: ")); msg += wxString::FromAscii(bp); - rt.addAtom((char *) "error"); - rt.add(msg); - rt.addTupleCount(2); + rt->addAtom((char *) "error"); + rt->add(msg); + rt->addTupleCount(2); } } else { - rt.addAtom((char *) "ok"); - rt.add(wxString::FromAscii("already initilized")); - rt.addTupleCount(2); + rt->addAtom((char *) "ok"); + rt->add(wxString::FromAscii("already initilized")); + rt->addTupleCount(2); } - rt.send(); + rt->send(); } void setActiveGL(ErlDrvTermData caller, wxGLCanvas *canvas) @@ -132,7 +132,7 @@ void deleteActiveGL(wxGLCanvas *canvas) } } -void gl_dispatch(int op, char *bp,ErlDrvTermData caller,WXEBinRef *bins[]){ +void gl_dispatch(int op, char *bp,ErlDrvTermData caller,WXEBinRef *bins){ if(caller != gl_active) { wxGLCanvas * current = glc[caller]; if(current) { @@ -153,12 +153,12 @@ void gl_dispatch(int op, char *bp,ErlDrvTermData caller,WXEBinRef *bins[]){ char * bs[3]; int bs_sz[3]; for(int i=0; i<3; i++) { - if(bins[i]) { - bs[i] = bins[i]->base; - bs_sz[i] = bins[i]->size; + if(bins[i].from) { + bs[i] = bins[i].base; + bs_sz[i] = bins[i].size; } - else - bs[i] = NULL; + else + break; } wxe_gl_dispatch(op, bp, WXE_DRV_PORT_HANDLE, caller, bs, bs_sz); } diff --git a/lib/wx/c_src/wxe_gl.h b/lib/wx/c_src/wxe_gl.h index dc117bf610..69095036a0 100644 --- a/lib/wx/c_src/wxe_gl.h +++ b/lib/wx/c_src/wxe_gl.h @@ -26,8 +26,8 @@ void activateGL(ErlDrvTermData caller); void setActiveGL(ErlDrvTermData caller, wxGLCanvas *canvas); void deleteActiveGL(wxGLCanvas *canvas); -void wxe_initOpenGL(wxeReturn, char*); -void gl_dispatch(int op, char *bp, ErlDrvTermData caller, WXEBinRef *bins[]); +void wxe_initOpenGL(wxeReturn *, char*); +void gl_dispatch(int op, char *bp, ErlDrvTermData caller, WXEBinRef *bins); WX_DECLARE_HASH_MAP(ErlDrvTermData, wxGLCanvas*, wxIntegerHash, wxIntegerEqual, wxeGLC); extern wxeGLC glc; diff --git a/lib/wx/c_src/wxe_helpers.cpp b/lib/wx/c_src/wxe_helpers.cpp index 120919e7aa..528c541403 100644 --- a/lib/wx/c_src/wxe_helpers.cpp +++ b/lib/wx/c_src/wxe_helpers.cpp @@ -38,10 +38,10 @@ void wxeCommand::Delete() int n = 0; if(buffer) { - while(bin[n]) { - if(bin[n]->bin) - driver_free_binary(bin[n]->bin); - driver_free(bin[n++]); + while(bin[n].from) { + if(bin[n].bin) + driver_free_binary(bin[n].bin); + n++; } if(len > 64) driver_free(buffer); @@ -89,7 +89,6 @@ void wxeFifo::Add(int fc, char * cbuf,int buflen, wxe_data *sd) unsigned int pos; wxeCommand *curr; - WXEBinRef *temp, *start, *prev; int n = 0; if(m_n == (m_max-1)) { // resize @@ -104,9 +103,9 @@ void wxeFifo::Add(int fc, char * cbuf,int buflen, wxe_data *sd) curr->port = sd->port; curr->op = fc; curr->len = buflen; - curr->bin[0] = NULL; - curr->bin[1] = NULL; - curr->bin[2] = NULL; + curr->bin[0].from = 0; + curr->bin[1].from = 0; + curr->bin[2].from = 0; if(cbuf) { if(buflen > 64) @@ -115,26 +114,16 @@ void wxeFifo::Add(int fc, char * cbuf,int buflen, wxe_data *sd) curr->buffer = curr->c_buf; memcpy((void *) curr->buffer, (void *) cbuf, buflen); - temp = sd->bin; - - prev = NULL; - start = temp; - - while(temp) { - if(curr->caller == temp->from) { - curr->bin[n++] = temp; - if(prev) { - prev->next = temp->next; - } else { - start = temp->next; - } - temp = temp->next; - } else { - prev = temp; - temp = temp->next; + for(unsigned int i=0; i<sd->max_bins; i++) { + if(curr->caller == sd->bin[i].from) { + sd->bin[i].from = 0; // Mark copied + curr->bin[n].bin = sd->bin[i].bin; + curr->bin[n].base = sd->bin[i].base; + curr->bin[n].size = sd->bin[i].size; + curr->bin[n].from = 1; + n++; } } - sd->bin = start; } else { // No-op only PING currently curr->buffer = NULL; } @@ -167,7 +156,7 @@ void wxeFifo::Append(wxeCommand *orig) } orig->op = -1; orig->buffer = NULL; - orig->bin[0] = NULL; + orig->bin[0].from = 0; } void wxeFifo::Realloc() diff --git a/lib/wx/c_src/wxe_helpers.h b/lib/wx/c_src/wxe_helpers.h index ec3a5debdb..61d385641f 100644 --- a/lib/wx/c_src/wxe_helpers.h +++ b/lib/wx/c_src/wxe_helpers.h @@ -50,7 +50,7 @@ class wxeCommand ErlDrvTermData caller; ErlDrvTermData port; - WXEBinRef * bin[3]; + WXEBinRef bin[3]; char * buffer; int len; int op; diff --git a/lib/wx/c_src/wxe_impl.cpp b/lib/wx/c_src/wxe_impl.cpp index ef648e008c..b75775ff34 100644 --- a/lib/wx/c_src/wxe_impl.cpp +++ b/lib/wx/c_src/wxe_impl.cpp @@ -58,7 +58,7 @@ extern int wxe_status; wxeFifo * wxe_queue = NULL; wxeFifo * wxe_queue_cb_saved = NULL; -int wxe_batch_caller = 0; // inside batch if larger than 0 +unsigned int wxe_needs_signal = 0; // inside batch if larger than 0 /* ************************************************************ * Commands from erlang @@ -72,35 +72,34 @@ void push_command(int op,char * buf,int len, wxe_data *sd) erl_drv_mutex_lock(wxe_batch_locker_m); wxe_queue->Add(op, buf, len, sd); - if(wxe_batch_caller > 0) { + if(wxe_needs_signal) { // wx-thread is waiting on batch end in cond_wait erl_drv_cond_signal(wxe_batch_locker_c); erl_drv_mutex_unlock(wxe_batch_locker_m); } else { // wx-thread is waiting gui-events - if(op == WXE_BATCH_BEGIN) { - wxe_batch_caller = 1; - } - erl_drv_cond_signal(wxe_batch_locker_c); erl_drv_mutex_unlock(wxe_batch_locker_m); wxWakeUpIdle(); } - } void meta_command(int what, wxe_data *sd) { - if(what == PING_PORT) { + if(what == PING_PORT && wxe_status == WXE_INITIATED) { erl_drv_mutex_lock(wxe_batch_locker_m); - if(wxe_batch_caller > 0) { + if(wxe_needs_signal) { wxe_queue->Add(WXE_DEBUG_PING, NULL, 0, sd); erl_drv_cond_signal(wxe_batch_locker_c); } wxWakeUpIdle(); erl_drv_mutex_unlock(wxe_batch_locker_m); } else { - if(sd) { + if(sd && wxe_status == WXE_INITIATED) { wxeMetaCommand Cmd(sd, what); wxTheApp->AddPendingEvent(Cmd); + if(what == DELETE_PORT) { + driver_free(sd->bin); + free(sd); + } } } } @@ -169,6 +168,7 @@ void WxeApp::MacOpenFile(const wxString &filename) { #endif void WxeApp::shutdown(wxeMetaCommand& Ecmd) { + wxe_status = WXE_EXITING; ExitMainLoop(); delete wxe_queue; delete wxe_queue_cb_saved; @@ -200,34 +200,31 @@ void handle_event_callback(ErlDrvPort port, ErlDrvTermData process) { WxeApp * app = (WxeApp *) wxTheApp; ErlDrvMonitor monitor; + + if(wxe_status != WXE_INITIATED) + return; + // Is thread safe if pdl have been incremented if(driver_monitor_process(port, process, &monitor) == 0) { // Should we be able to handle commands when recursing? probably - erl_drv_mutex_lock(wxe_batch_locker_m); // fprintf(stderr, "\r\nCB EV Start %lu \r\n", process);fflush(stderr); app->recurse_level++; app->dispatch_cb(wxe_queue, wxe_queue_cb_saved, process); app->recurse_level--; // fprintf(stderr, "CB EV done %lu \r\n", process);fflush(stderr); - wxe_batch_caller = 0; - erl_drv_mutex_unlock(wxe_batch_locker_m); driver_demonitor_process(port, &monitor); } } void WxeApp::dispatch_cmds() { - erl_drv_mutex_lock(wxe_batch_locker_m); + if(wxe_status != WXE_INITIATED) + return; recurse_level++; int level = dispatch(wxe_queue_cb_saved, 0, WXE_STORED); dispatch(wxe_queue, level, WXE_NORMAL); recurse_level--; - wxe_batch_caller = 0; - if(wxe_queue->m_old) { - driver_free(wxe_queue->m_old); - wxe_queue->m_old = NULL; - } - erl_drv_mutex_unlock(wxe_batch_locker_m); + // Cleanup old memenv's and deleted objects if(recurse_level == 0) { wxeCommand *curr; @@ -255,16 +252,14 @@ void WxeApp::dispatch_cmds() } } -// Should have erl_drv_mutex_lock(wxe_batch_locker_m); -// when entering this function and it should be released -// afterwards int WxeApp::dispatch(wxeFifo * batch, int blevel, int list_type) { int ping = 0; - // erl_drv_mutex_lock(wxe_batch_locker_m); must be locked already wxeCommand *event; + if(list_type == WXE_NORMAL) erl_drv_mutex_lock(wxe_batch_locker_m); while(true) { while((event = batch->Get()) != NULL) { + if(list_type == WXE_NORMAL) erl_drv_mutex_unlock(wxe_batch_locker_m); switch(event->op) { case -1: break; @@ -282,8 +277,6 @@ int WxeApp::dispatch(wxeFifo * batch, int blevel, int list_type) blevel = 0; break; case WXE_CB_RETURN: - // erl_drv_mutex_unlock(wxe_batch_locker_m); should be called after - // whatever cleaning is necessary if(event->len > 0) { cb_buff = (char *) driver_alloc(event->len); memcpy(cb_buff, event->buffer, event->len); @@ -291,36 +284,43 @@ int WxeApp::dispatch(wxeFifo * batch, int blevel, int list_type) event->Delete(); return blevel; default: - erl_drv_mutex_unlock(wxe_batch_locker_m); if(event->op < OPENGL_START) { // fprintf(stderr, " c %d (%d) \r\n", event->op, blevel); wxe_dispatch(*event); } else { gl_dispatch(event->op,event->buffer,event->caller,event->bin); } - erl_drv_mutex_lock(wxe_batch_locker_m); break; } event->Delete(); + if(list_type == WXE_NORMAL) erl_drv_mutex_lock(wxe_batch_locker_m); } - if((list_type == WXE_STORED) || (blevel <= 0 && list_type == WXE_NORMAL)) { - // erl_drv_mutex_unlock(wxe_batch_locker_m); should be called after - // whatever cleaning is necessary + if(list_type == WXE_STORED) + return blevel; + if(blevel <= 0) { // list_type == WXE_NORMAL + if(wxe_queue->m_old) { + driver_free(wxe_queue->m_old); + wxe_queue->m_old = NULL; + } + erl_drv_mutex_unlock(wxe_batch_locker_m); return blevel; } // sleep until something happens - //fprintf(stderr, "%s:%d sleep %d %d\r\n", __FILE__, __LINE__, batch->size(), blevel);fflush(stderr); - wxe_batch_caller++; + //fprintf(stderr, "%s:%d sleep %d %d\r\n", __FILE__, __LINE__, batch->m_n, blevel);fflush(stderr); + wxe_needs_signal = 1; while(batch->m_n == 0) { erl_drv_cond_wait(wxe_batch_locker_c, wxe_batch_locker_m); } + wxe_needs_signal = 0; } } void WxeApp::dispatch_cb(wxeFifo * batch, wxeFifo * temp, ErlDrvTermData process) { wxeCommand *event; + erl_drv_mutex_lock(wxe_batch_locker_m); while(true) { while((event = batch->Get()) != NULL) { + erl_drv_mutex_unlock(wxe_batch_locker_m); wxeMemEnv *memenv = getMemEnv(event->port); // fprintf(stderr, " Ev %d %lu\r\n", event->op, event->caller); if(event->caller == process || // Callbacks from CB process only @@ -347,7 +347,6 @@ void WxeApp::dispatch_cb(wxeFifo * batch, wxeFifo * temp, ErlDrvTermData process process = event->caller; break; default: - erl_drv_mutex_unlock(wxe_batch_locker_m); size_t start=temp->m_n; if(event->op < OPENGL_START) { // fprintf(stderr, " cb %d \r\n", event->op); @@ -355,8 +354,8 @@ void WxeApp::dispatch_cb(wxeFifo * batch, wxeFifo * temp, ErlDrvTermData process } else { gl_dispatch(event->op,event->buffer,event->caller,event->bin); } - erl_drv_mutex_lock(wxe_batch_locker_m); if(temp->m_n > start) { + erl_drv_mutex_lock(wxe_batch_locker_m); // We have recursed dispatch_cb and messages for this // callback may be saved on temp list move them // to orig list @@ -366,6 +365,7 @@ void WxeApp::dispatch_cb(wxeFifo * batch, wxeFifo * temp, ErlDrvTermData process batch->Append(ev); } } + erl_drv_mutex_unlock(wxe_batch_locker_m); } break; } @@ -374,13 +374,16 @@ void WxeApp::dispatch_cb(wxeFifo * batch, wxeFifo * temp, ErlDrvTermData process // fprintf(stderr, " save %d %lu\r\n", event->op, event->caller); temp->Append(event); } + erl_drv_mutex_lock(wxe_batch_locker_m); } // sleep until something happens // fprintf(stderr, "%s:%d sleep %d %d\r\n", __FILE__, __LINE__, // batch->m_n, temp->m_n);fflush(stderr); + wxe_needs_signal = 1; while(batch->m_n == 0) { erl_drv_cond_wait(wxe_batch_locker_c, wxe_batch_locker_m); } + wxe_needs_signal = 0; } } /* Memory handling */ diff --git a/lib/wx/c_src/wxe_impl.h b/lib/wx/c_src/wxe_impl.h index a0a1c84718..b251d5f0f9 100644 --- a/lib/wx/c_src/wxe_impl.h +++ b/lib/wx/c_src/wxe_impl.h @@ -46,7 +46,8 @@ typedef wxString wxeLocaleC; #define WXE_NOT_INITIATED 0 #define WXE_INITIATED 1 -#define WXE_EXITED 2 +#define WXE_EXITING 2 +#define WXE_EXITED 3 #define WXE_ERROR -1 void send_msg(const char *, const wxString *); // For debugging and error msgs diff --git a/lib/wx/c_src/wxe_return.cpp b/lib/wx/c_src/wxe_return.cpp index aebf6bae1b..f29c8eff4a 100644 --- a/lib/wx/c_src/wxe_return.cpp +++ b/lib/wx/c_src/wxe_return.cpp @@ -19,11 +19,6 @@ #include "wxe_return.h" -// see http://docs.wxwidgets.org/stable/wx_wxarray.html#arraymacros -// this is a magic incantation which must be done! -#include <wx/arrimpl.cpp> -WX_DEFINE_OBJARRAY(wxErlDrvTermDataArray); - #define INLINE wxeReturn::wxeReturn (ErlDrvTermData _port, @@ -31,79 +26,87 @@ wxeReturn::wxeReturn (ErlDrvTermData _port, bool _isResult) { port = _port; caller = _caller; - + isResult = _isResult; - - if (isResult) { - addAtom("_wxe_result_"); - } + rtb = buff; + rt_max = RT_BUFF_SZ; + rt_n = 0; + if (isResult) { + addAtom("_wxe_result_"); + } +} + +//clear everything so we can re-use if we want +void wxeReturn::reset() { + rt_n = 0; + temp_float.empty(); } wxeReturn::~wxeReturn () { - //depending on which version of wxArray we use, we may have to clear it ourselves. + if(rtb != buff) + driver_free(rtb); } -int wxeReturn::send() { - if ((rt.GetCount() == 2 && isResult) || rt.GetCount() == 0) - return 1; // not a call bail out - - if (isResult) { - addTupleCount(2); - } +int wxeReturn::send() { + if ((rt_n == 2 && isResult) || rt_n == 0) + return 1; // not a call bail out - // rt to array - unsigned int rtLength = rt.GetCount(); //signed int + if (isResult) { + addTupleCount(2); + } - size_t size = sizeof(ErlDrvTermData)*(rtLength); - - ErlDrvTermData* rtData = (ErlDrvTermData *) driver_alloc(size); - for (unsigned int i=0; i < rtLength; i++) { - rtData[i] = rt[i]; - } - - int res = erl_drv_send_term(port, caller, rtData, rtLength); - driver_free(rtData); + int res = erl_drv_send_term(port, caller, rtb, rt_n); #ifdef DEBUG - if(res == -1) { - fprintf(stderr, "Failed to send return or event msg\r\n"); - } + if(res == -1) { + fprintf(stderr, "Failed to send return or event msg\r\n"); + } #endif - reset(); - return res; + reset(); + return res; } -//clear everything so we can re-use if we want - void wxeReturn::reset() { - rt.empty(); - temp_float.empty(); +INLINE +unsigned int wxeReturn::size() { + return rt_n; } + INLINE -unsigned int wxeReturn::size() { - return rt.GetCount(); - } - +void wxeReturn::ensureFloatCount(size_t n) { + temp_float.Alloc(n); +} + INLINE -void wxeReturn::add(ErlDrvTermData type, ErlDrvTermData data) { - rt.Add(type); - rt.Add(data); +void wxeReturn::do_add(ErlDrvTermData val) { + if(rt_n >= rt_max) { // realloc + rt_max += RT_BUFF_SZ; + if(rtb == buff) { + rtb = (ErlDrvTermData *) driver_alloc(rt_max * sizeof(ErlDrvTermData)); + for(int i = 0; i < RT_BUFF_SZ; i++) + rtb[i] = buff[i]; + } else { + rtb = (ErlDrvTermData *) driver_realloc(rtb, rt_max * sizeof(ErlDrvTermData)); + } + } + rtb[rt_n++] = val; } -// INLINE -// void wxeReturn::addRef(const void *ptr, const char* className) { -// unsigned int ref_idx = wxe_app->getRef((void *)ptr, memEnv); -// addRef(ref_idx, className); -// } +INLINE +void wxeReturn::add(ErlDrvTermData type, ErlDrvTermData data) { + do_add(type); + do_add(data); +} + INLINE void wxeReturn::addRef(const unsigned int ref, const char* className) { addAtom("wx_ref"); addUint(ref); addAtom(className); - rt.Add(ERL_DRV_NIL); + do_add(ERL_DRV_NIL); addTupleCount(4); } @@ -115,30 +118,30 @@ void wxeReturn::addAtom(const char* atomName) { INLINE void wxeReturn::addBinary(const char* buf, const size_t size) { - rt.Add(ERL_DRV_BUF2BINARY); - rt.Add((ErlDrvTermData)buf); - rt.Add((ErlDrvTermData)size); + do_add(ERL_DRV_BUF2BINARY); + do_add((ErlDrvTermData)buf); + do_add((ErlDrvTermData)size); } INLINE void wxeReturn::addExt2Term(wxeErlTerm *term) { if(term) { - rt.Add(ERL_DRV_EXT2TERM); - rt.Add((ErlDrvTermData)term->bin); - rt.Add((ErlDrvTermData)term->size); + do_add(ERL_DRV_EXT2TERM); + do_add((ErlDrvTermData)term->bin); + do_add((ErlDrvTermData)term->size); } else { - rt.Add(ERL_DRV_NIL); + do_add(ERL_DRV_NIL); } } INLINE void wxeReturn::addExt2Term(wxETreeItemData *val) { if(val) { - rt.Add(ERL_DRV_EXT2TERM); - rt.Add((ErlDrvTermData)(val->bin)); - rt.Add((ErlDrvTermData)(val->size)); + do_add(ERL_DRV_EXT2TERM); + do_add((ErlDrvTermData)(val->bin)); + do_add((ErlDrvTermData)(val->size)); } else - rt.Add(ERL_DRV_NIL); + do_add(ERL_DRV_NIL); } INLINE @@ -168,8 +171,8 @@ void wxeReturn::addTupleCount(unsigned int n) { INLINE void wxeReturn::endList(unsigned int n) { - rt.Add(ERL_DRV_NIL); - add(ERL_DRV_LIST, (ErlDrvTermData)(n+1)); + do_add(ERL_DRV_NIL); + add(ERL_DRV_LIST, (ErlDrvTermData)(n+1)); } INLINE @@ -222,6 +225,7 @@ INLINE void wxeReturn::add(wxArrayDouble val) { unsigned int len = val.GetCount(); + temp_float.Alloc(len); for (unsigned int i = 0; i< len; i++) { addFloat(val[i]); } diff --git a/lib/wx/c_src/wxe_return.h b/lib/wx/c_src/wxe_return.h index 80946e2dc6..6729789116 100644 --- a/lib/wx/c_src/wxe_return.h +++ b/lib/wx/c_src/wxe_return.h @@ -40,10 +40,7 @@ extern "C" { #include <wx/html/htmlcell.h> -// #define send() send_term(__FILE__, __LINE__) - -// see http://docs.wxwidgets.org/stable/wx_wxarray.html -WX_DECLARE_OBJARRAY(ErlDrvTermData, wxErlDrvTermDataArray); +#define RT_BUFF_SZ 64 class wxeReturn { @@ -57,7 +54,6 @@ public: void add(ErlDrvTermData type, ErlDrvTermData data); - // void addRef(const void *ptr, const char* className); void addRef(const unsigned int ref, const char* className); void addAtom(const char* atomName); @@ -65,8 +61,8 @@ public: void addExt2Term(wxeErlTerm * term); void addExt2Term(wxETreeItemData * term); - void addNil() { rt.Add(ERL_DRV_NIL); }; - + void addNil() { do_add(ERL_DRV_NIL); }; + void addUint(unsigned int n); void addInt(int n); @@ -116,6 +112,10 @@ public: void add(const wxHtmlLinkInfo &val); + void do_add(ErlDrvTermData val); + + void ensureFloatCount(size_t n); + int send(); void reset(); @@ -127,15 +127,17 @@ private: inline void addDate(wxDateTime dateTime); inline void addTime(wxDateTime dateTime); - -// WxeApp* wxe_app; + ErlDrvTermData caller; ErlDrvTermData port; -// wxeMemEnv *memEnv; - wxErlDrvTermDataArray rt; wxArrayDouble temp_float; wxMBConvUTF32 utfConverter; bool isResult; + + unsigned int rt_max; + unsigned int rt_n; + ErlDrvTermData *rtb; + ErlDrvTermData buff[RT_BUFF_SZ]; }; #endif /* _WXE_RETURN_H */ diff --git a/lib/wx/doc/src/notes.xml b/lib/wx/doc/src/notes.xml index d31e927458..682ab48ca0 100644 --- a/lib/wx/doc/src/notes.xml +++ b/lib/wx/doc/src/notes.xml @@ -31,55 +31,6 @@ <p>This document describes the changes made to the wxErlang application.</p> -<section><title>Wx 1.4</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> - The undocumented option <c>generic_debug</c> for - <c>gen_server</c> has been removed.</p> - <p> - Own Id: OTP-12183</p> - </item> - </list> - </section> - - - <section><title>Improvements and New Features</title> - <list> - <item> - <p> - Use wxWidgets-3.0, if found, as default backend on - windows.</p> - <p> - Own Id: OTP-12632</p> - </item> - <item> - <p> - Add missing fields in some events records. May require a - recompilation of user applications.</p> - <p> - Own Id: OTP-12660</p> - </item> - </list> - </section> - - - <section><title>Known Bugs and Problems</title> - <list> - <item> - <p> - Remove raise condition where <c>wx</c> could crash during - emulator stoppage.</p> - <p> - Own Id: OTP-12734</p> - </item> - </list> - </section> - -</section> - <section><title>Wx 1.3.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/wx/test/wx_basic_SUITE.erl b/lib/wx/test/wx_basic_SUITE.erl index e3bbb21a23..2a17cc3ab9 100644 --- a/lib/wx/test/wx_basic_SUITE.erl +++ b/lib/wx/test/wx_basic_SUITE.erl @@ -271,13 +271,19 @@ wx_misc(_Config) -> wx:destroy(). -%% Check that all the data_types works in communication +%% Check that all the data_types works in communication %% between erlang and c++ thread. data_types(TestInfo) when is_atom(TestInfo) -> wx_test_lib:tc_info(TestInfo); data_types(_Config) -> Wx = ?mr(wx_ref, wx:new()), - + Frame = wxFrame:new(Wx, 1, "Data Types"), + wxFrame:connect(Frame, show), + wxFrame:show(Frame), + receive #wx{event=#wxShow{}} -> ok + after 1000 -> exit(show_timeout) + end, + CDC = wxClientDC:new(Frame), %% From wx.erl @@ -292,16 +298,31 @@ data_types(_Config) -> ?m(ok, wxDC:setUserScale(CDC, 123.45, 234.67)), ?m({123.45,234.67}, wxDC:getUserScale(CDC)), + %% Array of doubles + try wxGraphicsContext:create(CDC) of + GC -> + wxGraphicsContext:setFont(GC, ?wxITALIC_FONT, {0, 0, 50}), + Ws = wxGraphicsContext:getPartialTextExtents(GC, "a String With More Than 16 Characters"), + _ = lists:foldl(fun(Width, {Index, Acc}) -> + if Width >= Acc, Width < 500 -> {Index+1, Width}; + true -> throw({bad_float, Width, Index, Acc}) + end + end, {0,0.0}, Ws), + ok + catch _:_ -> %% GC not supported on this platform + ok + end, + %% Colors input is 3 or 4 tuple, returns are 4 tuples ?m(ok, wxDC:setTextForeground(CDC, {100,10,1})), ?m({100,10,1,255}, wxDC:getTextForeground(CDC)), ?m(ok, wxDC:setTextForeground(CDC, {100,10,1,43})), ?m({100,10,1,43}, wxDC:getTextForeground(CDC)), - %% Bool + %% Bool ?m(ok, wxDC:setAxisOrientation(CDC, true, false)), ?m(true, is_boolean(wxDC:isOk(CDC))), - + %% wxCoord ?m(true, is_integer(wxDC:maxX(CDC))), @@ -309,7 +330,7 @@ data_types(_Config) -> ?m({_,_}, wxWindow:getSize(Frame)), %% DateTime - DateTime = {Date, _Time} = calendar:now_to_datetime(erlang:now()), + DateTime = {Date, _Time} = calendar:now_to_datetime(os:timestamp()), io:format("DateTime ~p ~n",[DateTime]), Cal = ?mt(wxCalendarCtrl, wxCalendarCtrl:new(Frame, ?wxID_ANY, [{date,DateTime}])), ?m({Date,_}, wxCalendarCtrl:getDate(Cal)), diff --git a/lib/wx/test/wx_class_SUITE.erl b/lib/wx/test/wx_class_SUITE.erl index 45ab0f3a32..50b045a30b 100644 --- a/lib/wx/test/wx_class_SUITE.erl +++ b/lib/wx/test/wx_class_SUITE.erl @@ -71,7 +71,7 @@ calendarCtrl(Config) -> Panel = wxPanel:new(Frame), Sz = wxBoxSizer:new(?wxVERTICAL), - {YMD={_,_,Day},_} = DateTime = calendar:now_to_datetime(erlang:now()), + {YMD={_,_,Day},_} = DateTime = calendar:now_to_datetime(os:timestamp()), Cal = ?mt(wxCalendarCtrl, wxCalendarCtrl:new(Panel, ?wxID_ANY, [{date,DateTime} ])), @@ -287,10 +287,13 @@ helpFrame(Config) -> MFrame = wx:batch(fun() -> MFrame = wxFrame:new(Wx, ?wxID_ANY, "Main Frame"), wxPanel:new(MFrame, [{size, {600,400}}]), + wxFrame:connect(MFrame, show), wxWindow:show(MFrame), MFrame end), - timer:sleep(9), + receive #wx{event=#wxShow{}} -> ok + after 1000 -> exit(show_timeout) + end, {X0, Y0} = wxWindow:getScreenPosition(MFrame), {X, Y, W,H} = wxWindow:getScreenRect(MFrame), @@ -441,15 +444,16 @@ radioBox(Config) -> Frame = wxFrame:new(Wx, ?wxID_ANY, "Frame"), TrSortRadioBox = wxRadioBox:new(Frame, ?wxID_ANY, "Sort by:", - {100, 100},{100, 100}, ["Timestamp"]), + {100, 100},{100, 100}, + ["Timestamp", "Session", "FooBar"]), io:format("TrSortRadioBox ~p ~n", [TrSortRadioBox]), - %% If I uncomment any of these lines, it will crash - - io:format("~p~n", [catch wxControlWithItems:setClientData(TrSortRadioBox, 0, timestamp)]), - %?m(_, wxListBox:append(TrSortRadioBox, "Session Id", session_id)), - %?m(_, wxListBox:insert(TrSortRadioBox, "Session Id", 0, session_id)), - + wxRadioBox:setSelection(TrSortRadioBox, 2), + wxRadioBox:setItemToolTip(TrSortRadioBox, 2, "Test"), + TT0 = ?mt(wxToolTip,wxRadioBox:getItemToolTip(TrSortRadioBox, 0)), + TT1 = ?mt(wxToolTip,wxRadioBox:getItemToolTip(TrSortRadioBox, 2)), + ?m(true, wx:is_null(TT0)), + ?m("Test", wxToolTip:getTip(TT1)), wxWindow:show(Frame), wx_test_lib:wx_destroy(Frame,Config). @@ -530,7 +534,7 @@ popup(Config) -> [{shortHelp, "Press Me"}]), Log = fun(#wx{id=Id, event=Ev}, Obj) -> - io:format("Got ~p from ~p~n", [Id, Ev]), + io:format("Got ~p from ~p~n", [Ev, Id]), wxEvent:skip(Obj) end, CreatePopup = fun() -> @@ -553,7 +557,11 @@ popup(Config) -> Pop end, wxFrame:connect(Frame, command_menu_selected, [{id, 747}]), + wxFrame:connect(Frame, show), wxFrame:show(Frame), + receive #wx{event=#wxShow{}} -> ok + after 1000 -> exit(show_timeout) + end, Pop = CreatePopup(), Scale = case wx_test_lib:user_available(Config) of diff --git a/lib/xmerl/doc/src/notes.xml b/lib/xmerl/doc/src/notes.xml index b9cc2bd329..3fa1f01a79 100644 --- a/lib/xmerl/doc/src/notes.xml +++ b/lib/xmerl/doc/src/notes.xml @@ -31,20 +31,6 @@ <p>This document describes the changes made to the Xmerl application.</p> -<section><title>Xmerl 1.3.8</title> - - <section><title>Fixed Bugs and Malfunctions</title> - <list> - <item> - <p> Remove compiler warnings in xmerl. </p> - <p> - Own Id: OTP-12689</p> - </item> - </list> - </section> - -</section> - <section><title>Xmerl 1.3.7</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/xmerl/src/xmerl.app.src b/lib/xmerl/src/xmerl.app.src index 45cfe9d250..aed9cf176f 100644 --- a/lib/xmerl/src/xmerl.app.src +++ b/lib/xmerl/src/xmerl.app.src @@ -40,5 +40,5 @@ {registered, []}, {env, []}, {applications, [kernel, stdlib]}, - {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]} + {runtime_dependencies, ["stdlib-2.5","kernel-3.0","erts-6.0"]} ]}. diff --git a/lib/xmerl/src/xmerl.erl b/lib/xmerl/src/xmerl.erl index 88eaefc492..d27e101fe2 100644 --- a/lib/xmerl/src/xmerl.erl +++ b/lib/xmerl/src/xmerl.erl @@ -40,6 +40,7 @@ callbacks/1]). -include("xmerl.hrl"). +-include("xmerl_internal.hrl"). %% @spec export(Content, Callback) -> ExportedFormat @@ -273,7 +274,7 @@ tagdef(Tag,Pos,Parents,Args,CBs) -> callbacks(Module) -> Result = check_inheritance(Module, []), -%%% io:format("callbacks = ~p~n", [lists:reverse(Result)]), +%%% ?dbg("callbacks = ~p~n", [lists:reverse(Result)]), lists:reverse(Result). callbacks([M|Mods], Visited) -> @@ -288,7 +289,7 @@ callbacks([], Visited) -> Visited. check_inheritance(M, Visited) -> -%%% io:format("calling ~p:'#xml-inheritance#'()~n", [M]), +%%% ?dbg("calling ~p:'#xml-inheritance#'()~n", [M]), case M:'#xml-inheritance#'() of [] -> [M|Visited]; diff --git a/lib/xmerl/src/xmerl_eventp.erl b/lib/xmerl/src/xmerl_eventp.erl index ad5c3cbc47..beeab3fa5c 100644 --- a/lib/xmerl/src/xmerl_eventp.erl +++ b/lib/xmerl/src/xmerl_eventp.erl @@ -80,17 +80,17 @@ stream_sax(Fname, CallBack, UserState,Options) -> HookF= fun(ParsedEntity, S) -> {CBs,Arg}=xmerl_scan:user_state(S), -% io:format("stream_sax Arg=~p~n",[Arg]), +% ?dbg("stream_sax Arg=~p~n",[Arg]), case ParsedEntity of #xmlComment{} -> % Toss away comments... {[],S}; _ -> % Use callback module for the rest -% io:format("stream_sax ParsedEntity=~p~n",[ParsedEntity]), +% ?dbg("stream_sax ParsedEntity=~p~n",[ParsedEntity]), case xmerl:export_element(ParsedEntity,CBs,Arg) of {error,Reason} -> throw({error,Reason}); Resp -> -% io:format("stream_sax Resp=~p~n",[Resp]), +% ?dbg("stream_sax Resp=~p~n",[Resp]), {Resp,xmerl_scan:user_state({CBs,Resp},S)} end end diff --git a/lib/xmerl/src/xmerl_otpsgml.erl b/lib/xmerl/src/xmerl_otpsgml.erl index 38688e788f..b9649ecbad 100644 --- a/lib/xmerl/src/xmerl_otpsgml.erl +++ b/lib/xmerl/src/xmerl_otpsgml.erl @@ -34,6 +34,7 @@ export_text/1]). -include("xmerl.hrl"). +-include("xmerl_internal.hrl"). '#xml-inheritance#'() -> [xmerl_sgml]. @@ -58,7 +59,7 @@ %% the scope of a markup is not extended by mistake.) '#element#'(Tag, Data, Attrs, _Parents, _E) -> -% io:format("parents:\n~p\n",[_Parents]), +% ?dbg("parents:\n~p\n",[_Parents]), case convert_tag(Tag,Attrs) of {false,NewTag,NewAttrs} -> markup(NewTag, NewAttrs, Data); @@ -108,7 +109,7 @@ convert_aref([#xmlAttribute{name = href, value = V}|_Rest]) -> seealso end; convert_aref([#xmlAttribute{name = K}|Rest]) -> - io:format("Warning: ignoring attribute \'~p\' for tag \'a\'\n",[K]), + error_logger:warning_msg("ignoring attribute \'~p\' for tag \'a\'\n",[K]), convert_aref(Rest). convert_aref_attrs(url,Attrs) -> Attrs; @@ -130,7 +131,7 @@ html_content([_H|T]) -> % convert_seealso_attrs([#xmlAttribute{name = href, value = V} = A|Rest]) -> % [A#xmlAttribute{name=marker,value=normalize_web_ref(V)}|convert_seealso_attrs(Rest)]; % convert_seealso_attrs([#xmlAttribute{name = K}|Rest]) -> -% io:format("Warning: ignoring attribute \'~p\' for tag \'a\'\n",[K]), +% error_logger:warning_msg("ignoring attribute \'~p\' for tag \'a\'\n",[K]), % convert_seealso_attrs(Rest); % convert_seealso_attrs([]) -> % []. diff --git a/lib/xmerl/src/xmerl_regexp.erl b/lib/xmerl/src/xmerl_regexp.erl index 9303bdb125..b41f55ec3d 100644 --- a/lib/xmerl/src/xmerl_regexp.erl +++ b/lib/xmerl/src/xmerl_regexp.erl @@ -41,6 +41,8 @@ -export([setup/1,compile_proc/2]). +-include("xmerl_internal.hrl"). + setup(RE0) -> RE = setup(RE0, [$^]), Pid = spawn(?MODULE,compile_proc,[self(),RE]), @@ -844,7 +846,7 @@ parse_error(E) -> throw({error,E}). re_apply(S, St, {RE,Sc}) -> Subs = erlang:make_tuple(Sc, none), %Make a sub-regexp table. Res = re_apply(RE, [], S, St, Subs), - %% io:format("~p x ~p -> ~p\n", [RE,S,Res]), + %% ?dbg("~p x ~p -> ~p\n", [RE,S,Res]), Res. re_apply(epsilon, More, S, P, Subs) -> %This always matches @@ -900,7 +902,7 @@ re_apply({comp_class,Cc}, More, [C|S], P, Subs) -> re_apply(C, More, [C|S], P, Subs) when is_integer(C) -> re_apply_more(More, S, P+1, Subs); re_apply(_RE, _More, _S, _P, _Subs) -> - %% io:format("~p : ~p\n", [_RE,_S]), + %% ?dbg("~p : ~p\n", [_RE,_S]), nomatch. %% re_apply_more([RegExp], String, Length, SubsExprs) -> @@ -1121,7 +1123,7 @@ build_nfa(C, N, S, NFA) when is_integer(C) -> nfa_char_class(Cc) -> Crs = lists:foldl(fun({C1,C2}, Set) -> add_element({C1,C2}, Set); (C, Set) -> add_element({C,C}, Set) end, [], Cc), - %% io:fwrite("cc: ~p\n", [Crs]), + %% ?dbg("cc: ~p\n", [Crs]), pack_crs(Crs). pack_crs([{C1,C2}=Cr,{C3,C4}|Crs]) when C1 =< C3, C2 >= C4 -> @@ -1141,7 +1143,7 @@ pack_crs([]) -> []. nfa_comp_class(Cc) -> Crs = nfa_char_class(Cc), - %% io:fwrite("comp: ~p\n", [Crs]), + %% ?dbg("comp: ~p\n", [Crs]), comp_crs(Crs, 0). comp_crs([{C1,C2}|Crs], Last) -> @@ -1192,7 +1194,7 @@ build_dfa(Set, Us, N, Ts, Ms, NFA) -> Crs1 = lists:usort(Crs0), %Must remove duplicates! %% Build list of disjoint test ranges. Test = disjoint_crs(Crs1), - %% io:fwrite("bd: ~p\n ~p\n ~p\n ~p\n", [Set,Crs0,Crs1,Test]), + %% ?dbg("bd: ~p\n ~p\n ~p\n ~p\n", [Set,Crs0,Crs1,Test]), build_dfa(Test, Set, Us, N, Ts, Ms, NFA). %% disjoint_crs([CharRange]) -> [CharRange]. @@ -1263,7 +1265,7 @@ move(Sts, Cr, NFA) -> {Crs,St} <- (element(N, NFA))#nfa_state.edges, is_list(Crs), %% begin -%% io:fwrite("move1: ~p\n", [{Sts,Cr,Crs,in_crs(Cr,Crs)}]), +%% ?dbg("move1: ~p\n", [{Sts,Cr,Crs,in_crs(Cr,Crs)}]), %% true %% end, in_crs(Cr, Crs) ]. @@ -1413,7 +1415,7 @@ build_trans(Ts0, NoAccept) -> %% Have transitions, convert to tuple. Ts2 = keysort(1, Ts1), {Tmin,Smin,Ts3} = min_trans(Ts2, NoAccept), - %% io:fwrite("exptr: ~p\n", [{Ts3,Tmin}]), + %% ?dbg("exptr: ~p\n", [{Ts3,Tmin}]), {Trans,Tmax,Smax} = expand_trans(Ts3, Tmin, NoAccept), {list_to_tuple(Trans),Tmin,Smin,Tmax,Smax,Sp1} end. diff --git a/lib/xmerl/src/xmerl_sax_old_dom.erl b/lib/xmerl/src/xmerl_sax_old_dom.erl index c357816a1e..08b20fffcd 100644 --- a/lib/xmerl/src/xmerl_sax_old_dom.erl +++ b/lib/xmerl/src/xmerl_sax_old_dom.erl @@ -28,6 +28,7 @@ %% Include files %%---------------------------------------------------------------------- -include("xmerl_sax_old_dom.hrl"). +-include("xmerl_internal.hrl"). %%---------------------------------------------------------------------- %% External exports @@ -126,7 +127,7 @@ build_dom(endDocument, content=lists:reverse(C) }]}; _ -> - io:format("~p\n", [D]), + %%?dbg("~p\n", [D]), ?error("we're not at end the document when endDocument event is encountered.") end; diff --git a/lib/xmerl/src/xmerl_sax_simple_dom.erl b/lib/xmerl/src/xmerl_sax_simple_dom.erl index 58a11f70fe..4fcd6b2372 100644 --- a/lib/xmerl/src/xmerl_sax_simple_dom.erl +++ b/lib/xmerl/src/xmerl_sax_simple_dom.erl @@ -28,6 +28,7 @@ %% Include files %%---------------------------------------------------------------------- -include("xmerl_sax_old_dom.hrl"). +-include("xmerl_internal.hrl"). %%---------------------------------------------------------------------- %% External exports @@ -127,7 +128,7 @@ build_dom(endDocument, State#xmerl_sax_simple_dom_state{dom=[Decl, {Tag, Attributes, lists:reverse(Content)}]}; _ -> - io:format("~p\n", [D]), + ?dbg("~p\n", [D]), ?error("we're not at end the document when endDocument event is encountered.") end; diff --git a/lib/xmerl/src/xmerl_scan.erl b/lib/xmerl/src/xmerl_scan.erl index 8dfbc2b89e..c15188191a 100644 --- a/lib/xmerl/src/xmerl_scan.erl +++ b/lib/xmerl/src/xmerl_scan.erl @@ -147,7 +147,8 @@ S#xmerl_scanner.quiet -> ok; true -> - ok=io:format("~p- fatal: ~p~n", [?LINE, Reason]) + error_logger:error_msg("~p- fatal: ~p~n", [?LINE, Reason]), + ok end, fatal(Reason, S)). @@ -255,7 +256,7 @@ file(F, Options) -> end. int_file(F, Options,_ExtCharset) -> - %%io:format("int_file F=~p~n",[F]), + %%?dbg("int_file F=~p~n",[F]), case file:read_file(F) of {ok, Bin} -> int_string(binary_to_list(Bin), Options, filename:dirname(F),F); @@ -264,7 +265,7 @@ int_file(F, Options,_ExtCharset) -> end. int_file_decl(F, Options,_ExtCharset) -> -% io:format("int_file_decl F=~p~n",[F]), +% ?dbg("int_file_decl F=~p~n",[F]), case file:read_file(F) of {ok, Bin} -> int_string_decl(binary_to_list(Bin), Options, filename:dirname(F),F); @@ -294,7 +295,7 @@ int_string(Str, Options,FileName) -> int_string(Str, Options, XMLBase, FileName) -> S0=initial_state0(Options,XMLBase), S = S0#xmerl_scanner{filename=FileName}, - %%io:format("int_string1, calling xmerl_lib:detect_charset~n",[]), + %%?dbg("int_string1, calling xmerl_lib:detect_charset~n",[]), %% In case of no encoding attribute in document utf-8 is default, but %% another character set may be detected with help of Byte Order Marker or @@ -559,20 +560,20 @@ scan_document(Str0, S=#xmerl_scanner{event_fun = Event, Str0 end, %% M1 = erlang:memory(), -%% io:format("Memory status before prolog: ~p~n",[M1]), +%% ?dbg("Memory status before prolog: ~p~n",[M1]), {Prolog, Pos, T1, S2} = scan_prolog(Str, S1, _StartPos = 1), %% M2 = erlang:memory(), -%% io:format("Memory status after prolog: ~p~n",[M2]), - %%io:format("scan_document 2, prolog parsed~n",[]), +%% ?dbg("Memory status after prolog: ~p~n",[M2]), + %%?dbg("scan_document 2, prolog parsed~n",[]), T2 = scan_mandatory("<", T1, 1, S2, expected_element_start_tag), %% M3 = erlang:memory(), -%% io:format("Memory status before element: ~p~n",[M3]), +%% ?dbg("Memory status before element: ~p~n",[M3]), {Res, T3, S3} = scan_element(T2,S2,Pos), %% M4 = erlang:memory(), -%% io:format("Memory status after element: ~p~n",[M4]), +%% ?dbg("Memory status after element: ~p~n",[M4]), {Misc, _Pos1, Tail, S4}=scan_misc(T3, S3, Pos + 1), %% M5 = erlang:memory(), -%% io:format("Memory status after misc: ~p~n",[M5]), +%% ?dbg("Memory status after misc: ~p~n",[M5]), S5 = #xmerl_scanner{} = Event(#xmerl_event{event = ended, line = S4#xmerl_scanner.line, @@ -604,7 +605,7 @@ scan_document(Str0, S=#xmerl_scanner{event_fun = Event, case schemaLocations(Res, S5) of {ok, Schemas} -> cleanup(S5), - %%io:format("Schemas: ~p~nRes: ~p~ninhertih_options(S): ~p~n", + %%?dbg("Schemas: ~p~nRes: ~p~ninhertih_options(S): ~p~n", %% [Schemas,Res,inherit_options(S5)]), XSDRes = xmerl_xsd:process_validate(Schemas, Res, inherit_options(S5)), @@ -1373,7 +1374,7 @@ fetch_not_parse(ExtSpec,S=#xmerl_scanner{fetch_fun=Fetch}) -> end. get_file(F,S) -> -% io:format("get_file F=~p~n",[F]), +% ?dbg("get_file F=~p~n",[F]), case file:read_file(F) of {ok,Bin} -> binary_to_list(Bin); @@ -4088,7 +4089,7 @@ schemaLocations(#xmlElement{attributes=Atts,xmlbase=_Base}) -> end. inherit_options(S) -> - %%io:format("xsdbase: ~p~n",[S#xmerl_scanner.xmlbase]), + %%?dbg("xsdbase: ~p~n",[S#xmerl_scanner.xmlbase]), [{xsdbase,S#xmerl_scanner.xmlbase}]. handle_schema_result({XSDRes=#xmlElement{},_},S5) -> @@ -4227,7 +4228,7 @@ string_to_char_set(_,Str) -> %% NewTot = %% case {lists:keysearch(total,1,Mem),OldTot*1.1} of %% {{_,{_,Tot}},Tot110} when Tot > Tot110 -> -%% io:format("From ~p to ~p, total memory: ~p (~p)~n",[OldLine,Line,Tot,OldTot]), +%% ?dbg("From ~p to ~p, total memory: ~p (~p)~n",[OldLine,Line,Tot,OldTot]), %% Tot; %% {{_,{_,Tot}},_} -> %% Tot diff --git a/lib/xmerl/src/xmerl_ucs.erl b/lib/xmerl/src/xmerl_ucs.erl index 6550a9d954..48ae24b1de 100644 --- a/lib/xmerl/src/xmerl_ucs.erl +++ b/lib/xmerl/src/xmerl_ucs.erl @@ -227,7 +227,7 @@ from_ucs4be(<<Ch:32/big-signed-integer, Rest/binary>>,Acc,Tail) -> from_ucs4be(<<>>,Acc,Tail) -> lists:reverse(Acc,Tail); from_ucs4be(Bin,Acc,Tail) -> - io:format("ucs Error: Bin=~p~n Acc=~p~n Tail=~p~n",[Bin,Acc,Tail]), + ucs_error(Bin,Acc,Tail), {error,not_ucs4be}. char_to_ucs4le(Ch) -> @@ -247,7 +247,7 @@ from_ucs4le(<<Ch:32/little-signed-integer, Rest/binary>>,Acc,Tail) -> from_ucs4le(<<>>,Acc,Tail) -> lists:reverse(Acc,Tail); from_ucs4le(Bin,Acc,Tail) -> - io:format("ucs Error: Bin=~p~n Acc=~p~n Tail=~p~n",[Bin,Acc,Tail]), + ucs_error(Bin,Acc,Tail), {error,not_ucs4le}. @@ -269,7 +269,7 @@ from_ucs2be(<<Ch:16/big-signed-integer, Rest/binary>>,Acc,Tail) -> from_ucs2be(<<>>,Acc,Tail) -> lists:reverse(Acc,Tail); from_ucs2be(Bin,Acc,Tail) -> - io:format("ucs Error: Bin=~p~n Acc=~p~n Tail=~p~n",[Bin,Acc,Tail]), + ucs_error(Bin,Acc,Tail), {error,not_ucs2be}. char_to_ucs2le(Ch) -> @@ -287,7 +287,7 @@ from_ucs2le(<<Ch:16/little-signed-integer, Rest/binary>>,Acc,Tail) -> from_ucs2le(<<>>,Acc,Tail) -> lists:reverse(Acc,Tail); from_ucs2le(Bin,Acc,Tail) -> - io:format("ucs Error: Bin=~p~n Acc=~p~n Tail=~p~n",[Bin,Acc,Tail]), + ucs_error(Bin,Acc,Tail), {error,not_ucs2le}. @@ -331,7 +331,7 @@ from_utf16be(<<Hi:16/big-unsigned-integer, Lo:16/big-unsigned-integer, from_utf16be(<<>>,Acc,Tail) -> lists:reverse(Acc,Tail); from_utf16be(Bin,Acc,Tail) -> - io:format("ucs Error: Bin=~p~n Acc=~p~n Tail=~p~n",[Bin,Acc,Tail]), + ucs_error(Bin,Acc,Tail), {error,not_utf16be}. char_to_utf16le(Ch) when is_integer(Ch), Ch >= 0 -> @@ -363,7 +363,7 @@ from_utf16le(<<Hi:16/little-unsigned-integer, Lo:16/little-unsigned-integer, from_utf16le(<<>>,Acc,Tail) -> lists:reverse(Acc,Tail); from_utf16le(Bin,Acc,Tail) -> - io:format("ucs Error: Bin=~p~n Acc=~p~n Tail=~p~n",[Bin,Acc,Tail]), + ucs_error(Bin,Acc,Tail), {error,not_utf16le}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -571,3 +571,6 @@ test_charset(Fun,Input) -> false end. +ucs_error(Bin,Acc,Tail) -> + error_logger:error_msg("~w: Bin=~p~n Acc=~p~n Tail=~p~n", + [?MODULE,Bin,Acc,Tail]). diff --git a/lib/xmerl/src/xmerl_validate.erl b/lib/xmerl/src/xmerl_validate.erl index 60f228474b..e1d71aa818 100644 --- a/lib/xmerl/src/xmerl_validate.erl +++ b/lib/xmerl/src/xmerl_validate.erl @@ -23,7 +23,7 @@ -include("xmerl.hrl"). % record def, macros - +-include("xmerl_internal.hrl"). %% +type validate(xmerl_scanner(),xmlElement())-> @@ -300,7 +300,7 @@ test_attribute_value('NMTOKEN',#xmlAttribute{name=Name,value=V}=Attr, true-> ok; false-> - %%io:format("Warning*** nmtoken,value_incorrect: ~p~n",[V]), + %%?dbg("nmtoken,value_incorrect: ~p~n",[V]), exit({error,{invalid_value_nmtoken,Name,V}}) end end, @@ -381,7 +381,7 @@ test_attribute_value({Type,L},#xmlAttribute{value=Value}=Attr,Default,_S) exit({error,{duplicate_tokens_not_allowed,{list,L}}}) end; test_attribute_value(_Rule,Attr,_,_) -> -% io:format("Attr Value*****~nRule~p~nValue~p~n",[Rule,Attr]), +% ?dbg("Attr Value*****~nRule~p~nValue~p~n",[Rule,Attr]), Attr. @@ -423,11 +423,11 @@ parse({'+',SubRule}, XMLS, Rules, WSaction, S) -> parse({choice,CHOICE}, XMLS, Rules, WSaction, S)-> % case XMLS of % [] -> -% io:format("~p~n",[{choice,CHOICE,[]}]); +% ?dbg("~p~n",[{choice,CHOICE,[]}]); % [#xmlElement{name=Name,pos=Pos}|_] -> -% io:format("~p~n",[{choice,CHOICE,{Name,Pos}}]); +% ?dbg("~p~n",[{choice,CHOICE,{Name,Pos}}]); % [#xmlText{value=V}|_] -> -% io:format("~p~n",[{choice,CHOICE,{text,V}}]) +% ?dbg("~p~n",[{choice,CHOICE,{text,V}}]) % end, choice(CHOICE, XMLS, Rules, WSaction, S); parse(empty, [], _Rules, _WSaction, _S) -> @@ -550,10 +550,10 @@ star(Rule,XMLS,Rules,WSaction,Tree,S) -> {WS,XMLS1} = whitespace_action(XMLS,WSaction), case parse(Rule,XMLS1,Rules,WSaction,S) of {error, _E, {{next,N},{act,A}}}-> - %%io:format("Error~p~n",[_E]), + %%?dbg("Error~p~n",[_E]), {WS++Tree++A,N}; {error, _E}-> - %%io:format("Error~p~n",[_E]), + %%?dbg("Error~p~n",[_E]), % {WS++[Tree],[]}; case whitespace_action(XMLS,ws_action(WSaction,remove)) of {[],_} -> diff --git a/lib/xmerl/src/xmerl_xml.erl b/lib/xmerl/src/xmerl_xml.erl index 702a654629..3354592cf1 100644 --- a/lib/xmerl/src/xmerl_xml.erl +++ b/lib/xmerl/src/xmerl_xml.erl @@ -31,6 +31,7 @@ -import(xmerl_lib, [markup/3, empty_tag/2, export_text/1]). -include("xmerl.hrl"). +-include("xmerl_internal.hrl"). '#xml-inheritance#'() -> []. @@ -39,7 +40,7 @@ %% The '#text#' function is called for every text segment. '#text#'(Text) -> -%io:format("Text=~p~n",[Text]), +%?dbg("Text=~p~n",[Text]), export_text(Text). @@ -55,8 +56,8 @@ %% The '#element#' function is the default handler for XML elements. '#element#'(Tag, [], Attrs, _Parents, _E) -> -%io:format("Empty Tag=~p~n",[Tag]), +%?dbg("Empty Tag=~p~n",[Tag]), empty_tag(Tag, Attrs); '#element#'(Tag, Data, Attrs, _Parents, _E) -> -%io:format("Tag=~p~n",[Tag]), +%?dbg("Tag=~p~n",[Tag]), markup(Tag, Attrs, Data). diff --git a/lib/xmerl/src/xmerl_xpath.erl b/lib/xmerl/src/xmerl_xpath.erl index be0e863ce4..bce2a199f4 100644 --- a/lib/xmerl/src/xmerl_xpath.erl +++ b/lib/xmerl/src/xmerl_xpath.erl @@ -128,18 +128,18 @@ string(Str, Node, Parents, Doc, Options) -> [{H, P}|_] when is_atom(H), is_integer(P) -> full_parents(Parents, Doc) end, -%io:format("string FullParents=~p~n",[FullParents]), +%?dbg("string FullParents=~p~n",[FullParents]), ContextNode=#xmlNode{type = node_type(Node), node = Node, parents = FullParents}, -%io:format("string ContextNode=~p~n",[ContextNode]), +%?dbg("string ContextNode=~p~n",[ContextNode]), WholeDoc = whole_document(Doc), -%io:format("string WholeDoc=~p~n",[WholeDoc]), +%?dbg("string WholeDoc=~p~n",[WholeDoc]), Context=(new_context(Options))#xmlContext{context_node = ContextNode, whole_document = WholeDoc}, -%io:format("string Context=~p~n",[Context]), +%?dbg("string Context=~p~n",[Context]), #state{context = NewContext} = match(Str, #state{context = Context}), -%io:format("string NewContext=~p~n",[NewContext]), +%?dbg("string NewContext=~p~n",[NewContext]), case NewContext#xmlContext.nodeset of ScalObj = #xmlObj{type=Scalar} when Scalar == boolean; Scalar == number; Scalar == string -> @@ -274,7 +274,7 @@ eval_pred(Predicate, S = #state{context = C = NewNodeSet = lists:filter( fun(Node) -> - %io:format("current node: ~p~n", [write_node(Node)]), + %?dbg("current node: ~p~n", [write_node(Node)]), ThisContext = C#xmlContext{context_node = Node}, xmerl_xpath_pred:eval(Predicate, ThisContext) end, NodeSet), @@ -461,7 +461,7 @@ match_descendant_or_self(Tok, N, Acc, Context) -> match_child(Tok, N, Acc, Context) -> - %io:format("match_child(~p)~n", [write_node(N)]), + %?dbg("match_child(~p)~n", [write_node(N)]), #xmlNode{parents = Ps, node = Node, type = Type} = N, case Type of El when El == element; El == root_node -> @@ -738,7 +738,7 @@ node_test({prefix_test, Prefix}, #xmlNode{node = N}, Context) -> end; node_test({name, {Tag, _Prefix, _Local}}, #xmlNode{node = #xmlElement{name = Tag}}=_N, _Context) -> - %io:format("node_test({tag, ~p}, ~p) -> true.~n", [Tag, write_node(_N)]), + %?dbg("node_test({tag, ~p}, ~p) -> true.~n", [Tag, write_node(_N)]), true; node_test({name, {Tag, Prefix, Local}}, #xmlNode{node = #xmlElement{name = Name, @@ -816,7 +816,7 @@ node_test({processing_instruction, Name1}, #xmlNode{node = #xmlPI{name = Name2}}, _Context) -> Name1 == atom_to_list(Name2); node_test(_Other, _N, _Context) -> - %io:format("node_test(~p, ~p) -> false.~n", [_Other, write_node(_N)]), + %?dbg("node_test(~p, ~p) -> false.~n", [_Other, write_node(_N)]), false. diff --git a/lib/xmerl/src/xmerl_xpath_pred.erl b/lib/xmerl/src/xmerl_xpath_pred.erl index b94f3bb14d..acefa68f7e 100644 --- a/lib/xmerl/src/xmerl_xpath_pred.erl +++ b/lib/xmerl/src/xmerl_xpath_pred.erl @@ -58,6 +58,7 @@ -export([core_function/1]). -include("xmerl.hrl"). +-include("xmerl_internal.hrl"). -include("xmerl_xpath.hrl"). %% -record(obj, {type, @@ -88,7 +89,7 @@ eval(Expr, C = #xmlContext{context_node = #xmlNode{pos = Pos}}) -> _ -> mk_boolean(C, Obj) end, -% io:format("eval(~p, ~p) -> ~p~n", [Expr, Pos, Res]), +% ?dbg("eval(~p, ~p) -> ~p~n", [Expr, Pos, Res]), Res. diff --git a/lib/xmerl/src/xmerl_xsd.erl b/lib/xmerl/src/xmerl_xsd.erl index 16d02f571d..c84cb93bb8 100644 --- a/lib/xmerl/src/xmerl_xsd.erl +++ b/lib/xmerl/src/xmerl_xsd.erl @@ -381,7 +381,7 @@ initiate_state2(S,[{target_namespace,_NS}|T]) -> %% initiate_state2(S#xsd_state{targetNamespace=if_list_to_atom(NS)},T); initiate_state2(S,T); %% used in validation phase initiate_state2(S,[H|T]) -> - error_msg("Invalid option: ~p~n",[H]), + error_msg("~w: invalid option: ~p~n",[?MODULE, H]), initiate_state2(S,T). validation_options(S,[{target_namespace,NS}|T]) -> @@ -5391,7 +5391,7 @@ search_attribute(_,{Name,_,_},SchemaAtts) -> end. error_msg(Format,Args) -> - io:format(Format,Args). + error_logger:error_msg(Format,Args). add_once(El,L) -> @@ -5425,7 +5425,7 @@ add_key_once(Key,N,El,L) -> %% "/"++filename:join(L). %% mk_xml_path(Parents,Type,Pos) -> -%% %% io:format("mk_xml_path: Parents = ~p~n",[Parents]), +%% %% ?dbg("mk_xml_path: Parents = ~p~n",[Parents]), %% {filename:join([[io_lib:format("/~w(~w)",[X,Y])||{X,Y}<-Parents],Type]),Pos}. %% @spec format_error(Errors) -> Result diff --git a/lib/xmerl/src/xmerl_xsd_type.erl b/lib/xmerl/src/xmerl_xsd_type.erl index 0f46b1f9aa..acb988b9bc 100644 --- a/lib/xmerl/src/xmerl_xsd_type.erl +++ b/lib/xmerl/src/xmerl_xsd_type.erl @@ -29,6 +29,7 @@ -export([compare_durations/2,compare_dateTime/2]). -include("xmerl.hrl"). +-include("xmerl_internal.hrl"). -include("xmerl_xsd.hrl"). @@ -687,7 +688,8 @@ facet_fun(Type,{fractionDigits,V}) -> fractionDigits_fun(Type,list_to_integer(V)); facet_fun(Type,F) -> fun(_X_) -> - io:format("Warning: not valid facet on ~p ~p~n",[Type,F]) + error_logger:warning_msg("~w: not valid facet on ~p ~p~n", + [?MODULE,Type,F]) end. @@ -1075,7 +1077,7 @@ compare_floats(F1,F2) when F1=="-INF";F2=="INF" -> compare_floats(Str1,Str2) -> F1={S1,_B1,_D1,_E1} = str_to_float(Str1), F2={S2,_B2,_D2,_E2} = str_to_float(Str2), -% io:format("F1: ~p~nF2: ~p~n",[F1,F2]), +% ?dbg("F1: ~p~nF2: ~p~n",[F1,F2]), if S1=='-',S2=='+' -> lt; S1=='+',S2=='-' -> gt; diff --git a/otp_versions.table b/otp_versions.table index 12790c88a8..50a77237ff 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,6 @@ +OTP-17.5.6 : inets-5.10.9 ssh-3.2.4 ssl-6.0.1 # asn1-3.0.4 common_test-1.10.1 compiler-5.0.4 cosEvent-2.1.15 cosEventDomain-1.1.14 cosFileTransfer-1.1.16 cosNotification-1.1.21 cosProperty-1.1.17 cosTime-1.1.14 cosTransactions-1.2.14 crypto-3.5 debugger-4.0.3 dialyzer-2.7.4 diameter-1.9.2 edoc-0.7.16 eldap-1.1.1 erl_docgen-0.3.7 erl_interface-3.7.20 erts-6.4.1 et-1.5 eunit-2.2.9 gs-1.5.16 hipe-3.11.3 ic-4.3.6 jinterface-1.5.12 kernel-3.2 megaco-3.17.3 mnesia-4.12.5 observer-2.0.4 odbc-2.10.22 orber-3.7.1 os_mon-2.3.1 ose-1.0.2 otp_mibs-1.0.10 parsetools-2.0.12 percept-0.8.10 public_key-0.23 reltool-0.6.6 runtime_tools-1.8.16 sasl-2.4.1 snmp-5.1.2 stdlib-2.4 syntax_tools-1.6.18 test_server-3.8.1 tools-2.7.2 typer-0.9.8 webtool-0.8.10 wx-1.3.3 xmerl-1.3.7 : +OTP-17.5.5 : diameter-1.9.2 # asn1-3.0.4 common_test-1.10.1 compiler-5.0.4 cosEvent-2.1.15 cosEventDomain-1.1.14 cosFileTransfer-1.1.16 cosNotification-1.1.21 cosProperty-1.1.17 cosTime-1.1.14 cosTransactions-1.2.14 crypto-3.5 debugger-4.0.3 dialyzer-2.7.4 edoc-0.7.16 eldap-1.1.1 erl_docgen-0.3.7 erl_interface-3.7.20 erts-6.4.1 et-1.5 eunit-2.2.9 gs-1.5.16 hipe-3.11.3 ic-4.3.6 inets-5.10.8 jinterface-1.5.12 kernel-3.2 megaco-3.17.3 mnesia-4.12.5 observer-2.0.4 odbc-2.10.22 orber-3.7.1 os_mon-2.3.1 ose-1.0.2 otp_mibs-1.0.10 parsetools-2.0.12 percept-0.8.10 public_key-0.23 reltool-0.6.6 runtime_tools-1.8.16 sasl-2.4.1 snmp-5.1.2 ssh-3.2.3 ssl-6.0 stdlib-2.4 syntax_tools-1.6.18 test_server-3.8.1 tools-2.7.2 typer-0.9.8 webtool-0.8.10 wx-1.3.3 xmerl-1.3.7 : +OTP-17.5.4 : inets-5.10.8 ssh-3.2.3 # asn1-3.0.4 common_test-1.10.1 compiler-5.0.4 cosEvent-2.1.15 cosEventDomain-1.1.14 cosFileTransfer-1.1.16 cosNotification-1.1.21 cosProperty-1.1.17 cosTime-1.1.14 cosTransactions-1.2.14 crypto-3.5 debugger-4.0.3 dialyzer-2.7.4 diameter-1.9.1 edoc-0.7.16 eldap-1.1.1 erl_docgen-0.3.7 erl_interface-3.7.20 erts-6.4.1 et-1.5 eunit-2.2.9 gs-1.5.16 hipe-3.11.3 ic-4.3.6 jinterface-1.5.12 kernel-3.2 megaco-3.17.3 mnesia-4.12.5 observer-2.0.4 odbc-2.10.22 orber-3.7.1 os_mon-2.3.1 ose-1.0.2 otp_mibs-1.0.10 parsetools-2.0.12 percept-0.8.10 public_key-0.23 reltool-0.6.6 runtime_tools-1.8.16 sasl-2.4.1 snmp-5.1.2 ssl-6.0 stdlib-2.4 syntax_tools-1.6.18 test_server-3.8.1 tools-2.7.2 typer-0.9.8 webtool-0.8.10 wx-1.3.3 xmerl-1.3.7 : OTP-17.5.3 : common_test-1.10.1 diameter-1.9.1 erts-6.4.1 snmp-5.1.2 test_server-3.8.1 # asn1-3.0.4 compiler-5.0.4 cosEvent-2.1.15 cosEventDomain-1.1.14 cosFileTransfer-1.1.16 cosNotification-1.1.21 cosProperty-1.1.17 cosTime-1.1.14 cosTransactions-1.2.14 crypto-3.5 debugger-4.0.3 dialyzer-2.7.4 edoc-0.7.16 eldap-1.1.1 erl_docgen-0.3.7 erl_interface-3.7.20 et-1.5 eunit-2.2.9 gs-1.5.16 hipe-3.11.3 ic-4.3.6 inets-5.10.7 jinterface-1.5.12 kernel-3.2 megaco-3.17.3 mnesia-4.12.5 observer-2.0.4 odbc-2.10.22 orber-3.7.1 os_mon-2.3.1 ose-1.0.2 otp_mibs-1.0.10 parsetools-2.0.12 percept-0.8.10 public_key-0.23 reltool-0.6.6 runtime_tools-1.8.16 sasl-2.4.1 ssh-3.2.2 ssl-6.0 stdlib-2.4 syntax_tools-1.6.18 tools-2.7.2 typer-0.9.8 webtool-0.8.10 wx-1.3.3 xmerl-1.3.7 : OTP-17.5.2 : inets-5.10.7 ssh-3.2.2 # asn1-3.0.4 common_test-1.10 compiler-5.0.4 cosEvent-2.1.15 cosEventDomain-1.1.14 cosFileTransfer-1.1.16 cosNotification-1.1.21 cosProperty-1.1.17 cosTime-1.1.14 cosTransactions-1.2.14 crypto-3.5 debugger-4.0.3 dialyzer-2.7.4 diameter-1.9 edoc-0.7.16 eldap-1.1.1 erl_docgen-0.3.7 erl_interface-3.7.20 erts-6.4 et-1.5 eunit-2.2.9 gs-1.5.16 hipe-3.11.3 ic-4.3.6 jinterface-1.5.12 kernel-3.2 megaco-3.17.3 mnesia-4.12.5 observer-2.0.4 odbc-2.10.22 orber-3.7.1 os_mon-2.3.1 ose-1.0.2 otp_mibs-1.0.10 parsetools-2.0.12 percept-0.8.10 public_key-0.23 reltool-0.6.6 runtime_tools-1.8.16 sasl-2.4.1 snmp-5.1.1 ssl-6.0 stdlib-2.4 syntax_tools-1.6.18 test_server-3.8 tools-2.7.2 typer-0.9.8 webtool-0.8.10 wx-1.3.3 xmerl-1.3.7 : OTP-17.5.1 : ssh-3.2.1 # asn1-3.0.4 common_test-1.10 compiler-5.0.4 cosEvent-2.1.15 cosEventDomain-1.1.14 cosFileTransfer-1.1.16 cosNotification-1.1.21 cosProperty-1.1.17 cosTime-1.1.14 cosTransactions-1.2.14 crypto-3.5 debugger-4.0.3 dialyzer-2.7.4 diameter-1.9 edoc-0.7.16 eldap-1.1.1 erl_docgen-0.3.7 erl_interface-3.7.20 erts-6.4 et-1.5 eunit-2.2.9 gs-1.5.16 hipe-3.11.3 ic-4.3.6 inets-5.10.6 jinterface-1.5.12 kernel-3.2 megaco-3.17.3 mnesia-4.12.5 observer-2.0.4 odbc-2.10.22 orber-3.7.1 os_mon-2.3.1 ose-1.0.2 otp_mibs-1.0.10 parsetools-2.0.12 percept-0.8.10 public_key-0.23 reltool-0.6.6 runtime_tools-1.8.16 sasl-2.4.1 snmp-5.1.1 ssl-6.0 stdlib-2.4 syntax_tools-1.6.18 test_server-3.8 tools-2.7.2 typer-0.9.8 webtool-0.8.10 wx-1.3.3 xmerl-1.3.7 : diff --git a/system/doc/efficiency_guide/binaryhandling.xml b/system/doc/efficiency_guide/binaryhandling.xml index 0ac1a7ee32..1d247bcdd5 100644 --- a/system/doc/efficiency_guide/binaryhandling.xml +++ b/system/doc/efficiency_guide/binaryhandling.xml @@ -416,6 +416,7 @@ non_opt_eq([], <<>>) -> how you can find out whether your code can be optimized.</p> <section> + <marker id="bin_opt_info"></marker> <title>Option bin_opt_info</title> <p>Use the <c>bin_opt_info</c> option to have the compiler print a lot of |