diff options
Diffstat (limited to 'lib')
269 files changed, 9438 insertions, 3191 deletions
diff --git a/lib/.gitignore b/lib/.gitignore index 4125111ebd..58c49adce0 100644 --- a/lib/.gitignore +++ b/lib/.gitignore @@ -546,6 +546,8 @@ snmp/doc/intex.html /syntax_tools/doc/src/erl_syntax.xml /syntax_tools/doc/src/erl_syntax_lib.xml /syntax_tools/doc/src/erl_tidy.xml +/syntax_tools/doc/src/merl.xml +/syntax_tools/doc/src/merl_transform.xml /syntax_tools/doc/src/igor.xml /syntax_tools/doc/src/prettypr.xml diff --git a/lib/asn1/doc/src/asn1ct.xml b/lib/asn1/doc/src/asn1ct.xml index 4e0bf055fc..30808a5ead 100644 --- a/lib/asn1/doc/src/asn1ct.xml +++ b/lib/asn1/doc/src/asn1ct.xml @@ -371,6 +371,15 @@ File3.asn</pre> representation of a value of the <c>ASN.1</c> type <c>Type</c>. The value is a random value and subsequent calls to this function will for most types return different values.</p> + <note> + <p>Currently, the <c>value</c> function has many limitations. + Essentially, it will mostly work for old specifications based + on the 1997 standard for ASN.1, but not for most modern-style + applications. Another limitation is that the <c>value</c> function + may not work if options that change code generations strategies + such as the options <c>macro_name_prefix</c> and + <c>record_name_prefix</c> have been used.</p> + </note> </desc> </func> @@ -391,6 +400,15 @@ File3.asn</pre> This function is useful during test to secure that the generated encode and decode functions as well as the general runtime support work as expected.</p> + <note> + <p>Currently, the <c>test</c> functions have many limitations. + Essentially, they will mostly work for old specifications based + on the 1997 standard for ASN.1, but not for most modern-style + applications. Another limitation is that the <c>test</c> functions + may not work if options that change code generations strategies + such as the options <c>macro_name_prefix</c> and + <c>record_name_prefix</c> have been used.</p> + </note> <list type="bulleted"> <item> <p><c>test/1</c> iterates over all types in <c>Module</c>.</p> diff --git a/lib/asn1/vsn.mk b/lib/asn1/vsn.mk index c909c908d6..d4c46863a3 100644 --- a/lib/asn1/vsn.mk +++ b/lib/asn1/vsn.mk @@ -1 +1 @@ -ASN1_VSN = 3.0.4 +ASN1_VSN = 4.0 diff --git a/lib/common_test/src/common_test.app.src b/lib/common_test/src/common_test.app.src index 580d5dbd7b..0be1466fc9 100644 --- a/lib/common_test/src/common_test.app.src +++ b/lib/common_test/src/common_test.app.src @@ -63,9 +63,10 @@ ct_master_logs]}, {applications, [kernel,stdlib]}, {env, []}, - {runtime_dependencies,["xmerl-1.3.7","webtool-0.8.10","tools-2.6.14", - "test_server-3.7.1","stdlib-2.0","ssh-3.0.1", - "snmp-4.25.1","sasl-2.4","runtime_tools-1.8.14", - "kernel-3.0","inets-5.10","erts-6.0", - "debugger-4.0","crypto-3.3","compiler-5.0"]}]}. + {runtime_dependencies,["xmerl-1.3.8","tools-2.8", + "test_server-3.9","stdlib-2.5","ssh-4.0", + "snmp-5.1.2","sasl-2.4.2","runtime_tools-1.8.16", + "kernel-4.0","inets-6.0","erts-7.0", + "debugger-4.1","crypto-3.6","compiler-6.0", + "observer-2.1"]}]}. diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index e2d921729c..ff2bd20ab3 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.10.1 +COMMON_TEST_VSN = 1.11 diff --git a/lib/compiler/doc/src/compile.xml b/lib/compiler/doc/src/compile.xml index 5fccdcdcb5..a271729c82 100644 --- a/lib/compiler/doc/src/compile.xml +++ b/lib/compiler/doc/src/compile.xml @@ -32,15 +32,15 @@ <modulesummary>Erlang Compiler</modulesummary> <description> <p>This module provides an interface to the standard Erlang - compiler. It can generate either a new file which contains - the object code, or return a binary which can be loaded directly. + compiler. It can generate either a new file, which contains + the object code, or return a binary, which can be loaded directly. </p> </description> <funcs> <func> <name>file(File)</name> - <fsummary>Compile a file</fsummary> + <fsummary>Compiles a file.</fsummary> <desc> <p>Is the same as <c>file(File, [verbose,report_errors,report_warnings])</c>. @@ -50,7 +50,7 @@ <func> <name>file(File, Options) -> CompRet</name> - <fsummary>Compile a file</fsummary> + <fsummary>Compiles a file.</fsummary> <type> <v>CompRet = ModRet | BinRet | ErrRet</v> <v>ModRet = {ok,ModuleName} | {ok,ModuleName,Warnings}</v> @@ -64,39 +64,38 @@ <p>Returns <c>{ok,ModuleName}</c> if successful, or <c>error</c> if there are errors. An object code file is created if - the compilation succeeds with no errors. It is considered + the compilation succeeds without errors. It is considered to be an error if the module name in the source code is not the same as the basename of the output file.</p> - <p><marker id="type-option"/>Here follows first all elements of <c>Options</c> that in - some way control the behavior of the compiler.</p> + <p><marker id="type-option"/>Available options:</p> <taglist> <tag><c>basic_validation</c></tag> <item> - <p>This option is fast way to test whether a module will - compile successfully (mainly useful for code generators - that want to verify the code they emit). No code will + <p>This option is a fast way to test whether a module will + compile successfully. This is useful for code generators + that want to verify the code that they emit. No code is generated. If warnings are enabled, warnings generated by the <c>erl_lint</c> module (such as warnings for unused - variables and functions) will be returned too.</p> + variables and functions) are also returned.</p> - <p>Use the <c>strong_validation</c> option to generate all + <p>Use option <c>strong_validation</c> to generate all warnings that the compiler would generate.</p> </item> <tag><c>strong_validation</c></tag> <item> - <p>Similar to the <c>basic_validation</c> option, no code - will be generated, but more compiler passes will be run - to ensure also warnings generated by the optimization - passes are generated (such as clauses that will not match + <p>Similar to option <c>basic_validation</c>. No code + is generated, but more compiler passes are run + to ensure that warnings generated by the optimization + passes are generated (such as clauses that will not match, or expressions that are guaranteed to fail with an - exception at run-time).</p> + exception at runtime).</p> </item> <tag><c>binary</c></tag> <item> - <p>Causes the compiler to return the object code in a + <p>The compiler returns the object code in a binary instead of creating an object file. If successful, the compiler returns <c>{ok,ModuleName,Binary}</c>.</p> </item> @@ -105,7 +104,9 @@ <item> <p>The compiler will emit informational warnings about binary matching optimizations (both successful and unsuccessful). - See the <em>Efficiency Guide</em> for further information.</p> + For more information, see the section about + <seealso marker="doc/efficiency_guide:binaryhandling#bin_opt_info">bin_opt_info</seealso> + in the Efficiency Guide.</p> </item> <tag><c>compressed</c></tag> @@ -117,20 +118,19 @@ <tag><c>debug_info</c></tag> <item> <marker id="debug_info"></marker> - <p>Include debug information in the form of abstract code + <p>Includes debug information in the form of abstract code (see <seealso marker="erts:absform">The Abstract Format</seealso> in ERTS User's Guide) in the compiled beam module. Tools - such as Debugger, Xref and Cover require the debug - information to be included.</p> + such as <c>Debugger</c>, <c>Xref</c>, and <c>Cover</c> require + the debug information to be included.</p> <p><em>Warning</em>: Source code can be reconstructed from the debug information. Use encrypted debug information - (see below) to prevent this.</p> + (<c>encrypt_debug_info</c>) to prevent this.</p> - <p>See - <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso> - for details.</p> + <p>For details, see + <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso>.</p> </item> <tag><c>{debug_info_key,KeyString}</c></tag> @@ -138,65 +138,61 @@ <tag><c>{debug_info_key,{Mode,KeyString}}</c></tag> <item> <marker id="debug_info_key"></marker> - <p>Include debug information, but encrypt it, so that it + <p>Includes debug information, but encrypts it so that it cannot be accessed without supplying the key. (To give - the <c>debug_info</c> option as well is allowed, but is + option <c>debug_info</c> as well is allowed, but not necessary.) Using this option is a good way to always have the debug information available during testing, yet - protect the source code.</p> + protecting the source code.</p> <p><c>Mode</c> is the type of crypto algorithm to be used - for encrypting the debug information. The default type -- - and currently the only type -- is <c>des3_cbc</c>.</p> - <p>See - <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso> - for details.</p> + for encrypting the debug information. The default + (and currently the only) type is <c>des3_cbc</c>.</p> + <p>For details, see + <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso>.</p> </item> <tag><c>encrypt_debug_info</c></tag> <item> <marker id="encrypt_debug_info"></marker> - <p>Like the <c>debug_info_key</c> option above, except that - the key will be read from an <c>.erlang.crypt</c> file. + <p>Similar to the <c>debug_info_key</c> option, but + the key is read from an <c>.erlang.crypt</c> file. </p> - <p>See - <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso> - for details.</p> + <p>For details, see + <seealso marker="stdlib:beam_lib#debug_info">beam_lib(3)</seealso>.</p> </item> <tag><c>makedep</c></tag> <item> - <p>Produce a Makefile rule to track headers dependencies. + <p>Produces a Makefile rule to track headers dependencies. No object file is produced. </p> <p>By default, this rule is written to - <c><![CDATA[<File>.Pbeam]]></c>. However, if the option + <c><![CDATA[<File>.Pbeam]]></c>. However, if option <c>binary</c> is set, nothing is written and the rule is returned in <c>Binary</c>. </p> - <p>For instance, if one has the following module: + <p>For example, if you have the following module: </p> <code> -module(module). -include_lib("eunit/include/eunit.hrl"). --include("header.hrl"). - </code> - <p>Here is the Makefile rule generated by this option: +-include("header.hrl").</code> + <p>The Makefile rule generated by this option looks as follows: </p> <code> module.beam: module.erl \ /usr/local/lib/erlang/lib/eunit/include/eunit.hrl \ - header.hrl - </code> + header.hrl</code> </item> <tag><c>{makedep_output, Output}</c></tag> <item> - <p>Write generated rule(s) to <c>Output</c> instead of the + <p>Writes generated rules to <c>Output</c> instead of the default <c><![CDATA[<File>.Pbeam]]></c>. <c>Output</c> can be a filename or an <c>io_device()</c>. To write to - stdout, use <c>standard_io</c>. However if <c>binary</c> + stdout, use <c>standard_io</c>. However, if <c>binary</c> is set, nothing is written to <c>Output</c> and the result is returned to the caller with <c>{ok, ModuleName, Binary}</c>. @@ -205,7 +201,7 @@ module.beam: module.erl \ <tag><c>{makedep_target, Target}</c></tag> <item> - <p>Change the name of the rule emitted to <c>Target</c>. + <p>Changes the name of the rule emitted to <c>Target</c>. </p> </item> @@ -217,20 +213,20 @@ module.beam: module.erl \ <tag><c>makedep_add_missing</c></tag> <item> - <p>Consider missing headers as generated files and add them to the + <p>Considers missing headers as generated files and adds them to the dependencies. </p> </item> <tag><c>makedep_phony</c></tag> <item> - <p>Add a phony target for each dependency. + <p>Adds a phony target for each dependency. </p> </item> <tag><c>'P'</c></tag> <item> - <p>Produces a listing of the parsed code after preprocessing + <p>Produces a listing of the parsed code, after preprocessing and parse transforms, in the file <c><![CDATA[<File>.P]]></c>. No object file is produced. </p> @@ -238,7 +234,7 @@ module.beam: module.erl \ <tag><c>'E'</c></tag> <item> - <p>Produces a listing of the code after all source code + <p>Produces a listing of the code, after all source code transformations have been performed, in the file <c><![CDATA[<File>.E]]></c>. No object file is produced. </p> @@ -258,21 +254,21 @@ module.beam: module.erl \ <tag><c>report</c></tag> <item> - <p>This is a short form for both <c>report_errors</c> and + <p>A short form for both <c>report_errors</c> and <c>report_warnings</c>.</p> </item> <tag><c>return_errors</c></tag> <item> - <p>If this flag is set, then + <p>If this flag is set, <c>{error,ErrorList,WarningList}</c> is returned when there are errors.</p> </item> <tag><c>return_warnings</c></tag> <item> - <p>If this flag is set, then an extra field containing - <c>WarningList</c> is added to the tuples returned on + <p>If this flag is set, an extra field, containing + <c>WarningList</c>, is added to the tuples returned on success.</p> </item> @@ -284,13 +280,13 @@ module.beam: module.erl \ <tag><c>return</c></tag> <item> - <p>This is a short form for both <c>return_errors</c> and + <p>A short form for both <c>return_errors</c> and <c>return_warnings</c>.</p> </item> <tag><c>verbose</c></tag> <item> - <p>Causes more verbose information from the compiler + <p>Causes more verbose information from the compiler, describing what it is doing.</p> </item> @@ -314,7 +310,7 @@ module.beam: module.erl \ <tag><c>{i,Dir}</c></tag> <item> - <p>Add <c>Dir</c> to the list of directories to be searched + <p>Adds <c>Dir</c> to the list of directories to be searched when including a file. When encountering an <c>-include</c> or <c>-include_lib</c> directive, the compiler searches for header files in the following @@ -322,14 +318,14 @@ module.beam: module.erl \ <list type="ordered"> <item> <p><c>"."</c>, the current working directory of - the file server;</p> + the file server</p> </item> <item> - <p>the base name of the compiled file;</p> + <p>The base name of the compiled file</p> </item> <item> - <p>the directories specified using the <c>i</c> option. - The directory specified last is searched first.</p> + <p>The directories specified using option <c>i</c>; + the directory specified last is searched first</p> </item> </list> </item> @@ -353,15 +349,15 @@ module.beam: module.erl \ <tag><c>from_asm</c></tag> <item> <p>The input file is expected to be assembler code (default - file suffix ".S"). Note that the format of assembler files - is not documented, and may change between releases.</p> + file suffix ".S"). Notice that the format of assembler files + is not documented, and can change between releases.</p> </item> <tag><c>from_core</c></tag> <item> <p>The input file is expected to be core code (default - file suffix ".core"). Note that the format of core files - is not documented, and may change between releases.</p> + file suffix ".core"). Notice that the format of core files + is not documented, and can change between releases.</p> </item> <tag><c>no_strict_record_tests</c></tag> @@ -369,9 +365,9 @@ module.beam: module.erl \ <p>This option is not recommended.</p> <p>By default, the generated code for - the <c>Record#record_tag.field</c> operation verifies that - the tuple <c>Record</c> is of the correct size for - the record and that the first element is the tag + operation <c>Record#record_tag.field</c> verifies that + the tuple <c>Record</c> has the correct size for + the record, and that the first element is the tag <c>record_tag</c>. Use this option to omit the verification code.</p> </item> @@ -390,79 +386,87 @@ module.beam: module.erl \ <tag><c>{no_auto_import,[{F,A}, ...]}</c></tag> <item> <p>Makes the function <c>F/A</c> no longer being - auto-imported from the module <c>erlang</c>, which resolves - BIF name clashes. This option has to be used to resolve name - clashes with BIFs auto-imported before R14A, if one wants to + auto-imported from the <c>erlang</c> module, which resolves + BIF name clashes. This option must be used to resolve name + clashes with BIFs auto-imported before R14A, if it is needed to call the local function with the same name as an auto-imported BIF without module prefix.</p> <note> - <p>From R14A and forward, the compiler resolves calls + <p>As from R14A and forward, the compiler resolves calls without module prefix to local or imported functions before - trying auto-imported BIFs. If the BIF is to be + trying with auto-imported BIFs. If the BIF is to be called, use the <c>erlang</c> module prefix in the call, not - <c>{ no_auto_import,[{F,A}, ...]}</c></p> + <c>{ no_auto_import,[{F,A}, ...]}</c>.</p> </note> <p>If this option is written in the source code, as a <c>-compile</c> directive, the syntax <c>F/A</c> can be used instead - of <c>{F,A}</c>. Example:</p> + of <c>{F,A}</c>, for example:</p> <code>-compile({no_auto_import,[error/1]}).</code> </item> <tag><c>no_auto_import</c></tag> <item> - <p>Do not auto import any functions from the module <c>erlang</c>.</p> + <p>Do not auto-import any functions from <c>erlang</c> module.</p> </item> <tag><c>no_line_info</c></tag> <item> - <p>Omit line number information in order to produce a slightly + <p>Omits line number information to produce a slightly smaller output file. </p> </item> </taglist> - <p>If warnings are turned on (the <c>report_warnings</c> option - described above), the following options control what type of - warnings that will be generated. + <p>If warnings are turned on (option <c>report_warnings</c> + described earlier), the following options control what type of + warnings that are generated. <marker id="erl_lint_options"></marker> - With the exception of <c>{warn_format,Verbosity}</c> all - options below have two forms; one <c>warn_xxx</c> form to - turn on the warning and one <c>nowarn_xxx</c> form to turn off - the warning. In the description that follows, the form that - is used to change the default value is listed.</p> + Except from <c>{warn_format,Verbosity}</c>, the following options + have two forms:</p> + <list type="bulleted"> + <item>A <c>warn_xxx</c> form, to turn on the warning.</item> + <item>A <c>nowarn_xxx</c> form, to turn off the warning.</item> + </list> + <p>In the descriptions that follow, the form that is used to change + the default value are listed.</p> <taglist> <tag><c>{warn_format, Verbosity}</c></tag> <item> <p>Causes warnings to be emitted for malformed format strings as arguments to <c>io:format</c> and similar - functions. <c>Verbosity</c> selects the amount of - warnings: 0 = no warnings; 1 = warnings for invalid - format strings and incorrect number of arguments; 2 = - warnings also when the validity could not be checked - (for example, when the format string argument is a - variable). The default verbosity is 1. Verbosity 0 can - also be selected by the option <c>nowarn_format</c>.</p> + functions.</p> + <p><c>Verbosity</c> selects the number of warnings:</p> + <list type="bulleted"> + <item><c>0</c> = No warnings</item> + <item><c>1</c> = Warnings for invalid format strings and incorrect + number of arguments</item> + <item><c>2</c> = Warnings also when the validity cannot + be checked, for example, when the format string argument is a + variable.</item> + </list> + <p>The default verbosity is <c>1</c>. Verbosity <c>0</c> can + also be selected by option <c>nowarn_format</c>.</p> </item> <tag><c>nowarn_bif_clash</c></tag> <item> - <p>This option is removed, it will generate a fatal error if used.</p> + <p>This option is removed, it generates a fatal error if used.</p> <warning> - <p>Beginning with R14A, the compiler no longer calls the + <p>As from beginning with R14A, the compiler no longer calls the auto-imported BIF if the name clashes with a local or - explicitly imported function and a call without explicit - module name is issued. Instead the local or imported - function is called. Still accepting <c>nowarn_bif_clash</c> would makes a - module calling functions clashing with autoimported BIFs + explicitly imported function, and a call without explicit + module name is issued. Instead, the local or imported + function is called. Still accepting <c>nowarn_bif_clash</c> would + make a module calling functions clashing with auto-imported BIFs compile with both the old and new compilers, but with - completely different semantics, why the option was removed.</p> + completely different semantics. This is why the option is removed.</p> - <p>The use of this option has always been strongly discouraged. - From OTP R14A and forward it's an error to use it.</p> + <p>The use of this option has always been discouraged. + As from R14A, it is an error to use it.</p> <p>To resolve BIF clashes, use explicit module names or the <c>{no_auto_import,[F/A]}</c> compiler directive.</p> </warning> @@ -470,11 +474,11 @@ module.beam: module.erl \ <tag><c>{nowarn_bif_clash, FAs}</c></tag> <item> - <p>This option is removed, it will generate a fatal error if used.</p> + <p>This option is removed, it generates a fatal error if used.</p> <warning> - <p>The use of this option has always been strongly discouraged. - From OTP R14A and forward it's an error to use it.</p> + <p>The use of this option has always been discouraged. + As from R14A, it is an error to use it.</p> <p>To resolve BIF clashes, use explicit module names or the <c>{no_auto_import,[F/A]}</c> compiler directive.</p> </warning> @@ -482,35 +486,29 @@ module.beam: module.erl \ <tag><c>warn_export_all</c></tag> <item> - <p>Causes a warning to be emitted if the <c>export_all</c> - option has also been given.</p> + <p>Emits a warning if option <c>export_all</c> is also given.</p> </item> <tag><c>warn_export_vars</c></tag> <item> - <p>Causes warnings to be emitted for all implicitly - exported variables referred to after the primitives - where they were first defined. No warnings for exported - variables unless they are referred to in some pattern, - which is the default, can be selected by the option - <c>nowarn_export_vars</c>.</p> + <p>Emits warnings for all implicitly exported variables + referred to after the primitives where they were first defined. + By default, the compiler only emits warnings for exported + variables referred to in a pattern.</p> </item> - <tag><c>warn_shadow_vars</c></tag> + <tag><c>nowarn_shadow_vars</c></tag> <item> - <p>Causes warnings to be emitted for "fresh" variables - in functional objects or list comprehensions with the same - name as some already defined variable. The default is to - warn for such variables. No warnings for shadowed - variables can be selected by the option - <c>nowarn_shadow_vars</c>.</p> + <p>Turns off warnings for "fresh" variables + in functional objects or list comprehensions with the same + name as some already defined variable. Default is to + emit warnings for such variables.</p> </item> <tag><c>nowarn_unused_function</c></tag> <item> - <p>Turns off warnings for unused local functions. - By default (<c>warn_unused_function</c>), warnings are - emitted for all local functions that are not called + <p>Turns off warnings for unused local functions. Default + is to emit warnings for all local functions that are not called directly or indirectly by an exported function. The compiler does not include unused local functions in the generated beam file, but the warning is still useful @@ -519,148 +517,142 @@ module.beam: module.erl \ <tag><c>{nowarn_unused_function, FAs}</c></tag> <item> - <p>Turns off warnings for unused local functions as - <c>nowarn_unused_function</c> but only for the mentioned + <p>Turns off warnings for unused local functions like + <c>nowarn_unused_function</c> does, but only for the mentioned local functions. <c>FAs</c> is a tuple <c>{Name,Arity}</c> or a list of such tuples.</p> </item> <tag><c>nowarn_deprecated_function</c></tag> <item> - <p>Turns off warnings for calls to deprecated functions. By - default (<c>warn_deprecated_function</c>), warnings are - emitted for every call to a function known by the compiler - to be deprecated. Note that the compiler does not know - about the <c>-deprecated()</c> attribute but uses an + <p>Turns off warnings for calls to deprecated functions. Default + is to emit warnings for every call to a function known by the + compiler to be deprecated. Notice that the compiler does not know + about attribute <c>-deprecated()</c>, but uses an assembled list of deprecated functions in Erlang/OTP. To - do a more general check the <c>Xref</c> tool can be used. + do a more general check, the <c>Xref</c> tool can be used. See also <seealso marker="tools:xref#deprecated_function">xref(3)</seealso> and the function - <seealso marker="tools:xref#m/1">xref:m/1</seealso> also - accessible through - the <seealso marker="stdlib:c#xm/1">c:xm/1</seealso> - function.</p> + <seealso marker="tools:xref#m/1">xref:m/1</seealso>, also + accessible through the function + <seealso marker="stdlib:c#xm/1">c:xm/1</seealso>.</p> </item> <tag><c>{nowarn_deprecated_function, MFAs}</c></tag> <item> - <p>Turns off warnings for calls to deprecated functions as - <c>nowarn_deprecated_function</c> but only for + <p>Turns off warnings for calls to deprecated functions like + <c>nowarn_deprecated_function</c> does, but only for the mentioned functions. <c>MFAs</c> is a tuple <c>{Module,Name,Arity}</c> or a list of such tuples.</p> </item> <tag><c>nowarn_deprecated_type</c></tag> <item> - <p>Turns off warnings for uses of deprecated types. By - default (<c>warn_deprecated_type</c>), warnings are - emitted for every use of a type known by the compiler - to be deprecated.</p> + <p>Turns off warnings for use of deprecated types. Default + is to emit warnings for every use of a type known by the compiler + to be deprecated.</p> </item> <tag><c>warn_obsolete_guard</c></tag> <item> - <p>Causes warnings to be emitted for calls to old type - testing BIFs such as <c>pid/1</c> and <c>list/1</c>. See - the - <seealso marker="doc/reference_manual:expressions#guards">Erlang Reference Manual</seealso> + <p>Emits warnings for calls to old type testing BIFs, + such as <c>pid/1</c> and <c>list/1</c>. See the + <seealso marker="doc/reference_manual:expressions#guards">Erlang Reference Manual</seealso> for a complete list of type testing BIFs and their old - equivalents. No warnings for calls to old type testing - BIFs, which is the default, can be selected by the option - <c>nowarn_obsolete_guard</c>.</p> + equivalents. Default is to emit no warnings for calls to + old type testing BIFs.</p> </item> <tag><c>warn_unused_import</c></tag> <item> - <p>Causes warnings to be emitted for unused imported - functions. No warnings for unused imported functions, - which is the default, can be selected by the option - <c>nowarn_unused_import</c>. </p> + <p>Emits warnings for unused imported functions. + Default is to emit no warnings for unused imported functions.</p> </item> <tag><c>nowarn_unused_vars</c></tag> <item> - <p>By default, warnings are emitted for variables which - are not used, with the exception of variables beginning - with an underscore ("Prolog style warnings"). + <p>By default, warnings are emitted for unused variables, + except for variables beginning with an underscore + ("Prolog style warnings"). Use this option to turn off this kind of warnings.</p> </item> <tag><c>nowarn_unused_record</c></tag> <item> - <p>Turns off warnings for unused record types. By - default (<c>warn_unused_records</c>), warnings are - emitted for unused locally defined record types.</p> + <p>Turns off warnings for unused record types. Default is to + emit warnings for unused locally defined record types.</p> </item> </taglist> <p>Another class of warnings is generated by the compiler during optimization and code generation. They warn about patterns that will never match (such as <c>a=b</c>), guards - that will always evaluate to false, and expressions that will + that always evaluate to false, and expressions that always fail (such as <c>atom+42</c>).</p> - - <p>Note that the compiler does not warn for expressions that it - does not attempt to optimize. For instance, the compiler tries - to evaluate <c>1/0</c>, notices that it will cause an - exception and emits a warning. On the other hand, - the compiler is silent about the similar expression - <c>X/0</c>; because of the variable in it, the compiler does - not even try to evaluate and therefore it emits no warnings. - </p> - - <p>Currently, those warnings cannot be disabled (except by + <p>Those warnings cannot be disabled (except by disabling all warnings).</p> + <note> + <p>The compiler does not warn for expressions that it + does not attempt to optimize. For example, the compiler tries + to evaluate <c>1/0</c>, detects that it will cause an + exception, and emits a warning. However, + the compiler is silent about the similar expression, + <c>X/0</c>, because of the variable in it. Thus, the compiler does + not even try to evaluate and therefore it emits no warnings.</p> + </note> + <warning> - <p>Obviously, the absence of warnings does not mean that + <p>The absence of warnings does not mean that there are no remaining errors in the code.</p> </warning> - - <p>Note that all the options except the include path - (<c>{i,Dir}</c>) can also be given in the file with a - <c>-compile([Option,...])</c>. attribute. - The <c>-compile()</c> attribute is allowed after function + + <note> + <p>All options, except the include path + (<c>{i,Dir}</c>), can also be given in the file with attribute + <c>-compile([Option,...])</c>. + Attribute <c>-compile()</c> is allowed after the function definitions.</p> - - <p>Note also that the <c>{nowarn_unused_function, FAs}</c>, + </note> + + <note> + <p>The options <c>{nowarn_unused_function, FAs}</c>, <c>{nowarn_bif_clash, FAs}</c>, and - <c>{nowarn_deprecated_function, MFAs}</c> options are only + <c>{nowarn_deprecated_function, MFAs}</c> are only recognized when given in files. They are not affected by - the <c>warn_unused_function</c>, <c>warn_bif_clash</c>, or - <c>warn_deprecated_function</c> options.</p> + options <c>warn_unused_function</c>, <c>warn_bif_clash</c>, or + <c>warn_deprecated_function</c>.</p> + </note> <p>For debugging of the compiler, or for pure curiosity, the intermediate code generated by each compiler pass can be inspected. - A complete list of the options to produce list files can be - printed by typing <c>compile:options()</c> at the Erlang - shell prompt. - The options will be printed in order that the passes are + To print a complete list of the options to produce list files, + type <c>compile:options()</c> at the Erlang shell prompt. + The options are printed in the order that the passes are executed. If more than one listing option is used, the one representing the earliest pass takes effect.</p> - <p><em>Unrecognized options are ignored.</em></p> + <p>Unrecognized options are ignored.</p> <p>Both <c>WarningList</c> and <c>ErrorList</c> have the following format:</p> <code> -[{FileName,[ErrorInfo]}]. - </code> - - <p><c>ErrorInfo</c> is described below. The file name has been - included here as the compiler uses the Erlang pre-processor - <c>epp</c>, which allows the code to be included in other - files. For this reason, it is important to know to - <em>which</em> file an error or warning line number refers. +[{FileName,[ErrorInfo]}].</code> + + <p><c>ErrorInfo</c> is described later in this section. + The filename is included here, as the compiler uses the + Erlang pre-processor <c>epp</c>, which allows the code to be + included in other files. It is therefore important to know to + <em>which</em> file the line number of an error or a warning refers. </p> </desc> </func> <func> <name>forms(Forms)</name> - <fsummary>Compile a list of forms</fsummary> + <fsummary>Compiles a list of forms.</fsummary> <desc> <p>Is the same as <c>forms(File, [verbose,report_errors,report_warnings])</c>. @@ -670,7 +662,7 @@ module.beam: module.erl \ <func> <name>forms(Forms, Options) -> CompRet</name> - <fsummary>Compile a list of forms</fsummary> + <fsummary>Compiles a list of forms.</fsummary> <type> <v>Forms = [Form]</v> <v>CompRet = BinRet | ErrRet</v> @@ -681,48 +673,49 @@ module.beam: module.erl \ <desc> <p>Analogous to <c>file/1</c>, but takes a list of forms (in the Erlang abstract format representation) as first argument. - The option <c>binary</c> is implicit; i.e., no object code - file is produced. Options that would ordinarily produce a - listing file, such as 'E', will instead cause the internal - format for that compiler pass (an Erlang term; usually not a - binary) to be returned instead of a binary.</p> + Option <c>binary</c> is implicit, that is, no object code + file is produced. For options that normally produce a listing + file, such as 'E', the internal format for that compiler pass + (an Erlang term, usually not a binary) is returned instead of + a binary.</p> </desc> </func> <func> <name>format_error(ErrorDescriptor) -> chars()</name> - <fsummary>Format an error descriptor</fsummary> + <fsummary>Formats an error descriptor.</fsummary> <type> <v>ErrorDescriptor = errordesc()</v> </type> <desc> <p>Uses an <c>ErrorDescriptor</c> and returns a deep list of - characters which describes the error. This function is - usually called implicitly when an <c>ErrorInfo</c> structure - is processed. See below.</p> + characters that describes the error. This function is + usually called implicitly when an <c>ErrorInfo</c> structure + (described in section + <seealso marker="#error_information">Error Information</seealso>) is processed.</p> </desc> </func> <func> <name>output_generated(Options) -> true | false</name> - <fsummary>Determine whether the compile will generate an output file</fsummary> + <fsummary>Determines whether the compiler generates an output file.</fsummary> <type> <v>Options = [term()]</v> </type> <desc> - <p>Determines whether the compiler would generate a <c>beam</c> + <p>Determines whether the compiler generates a <c>beam</c> file with the given options. <c>true</c> means that a <c>beam</c> - file would be generated; <c>false</c> means that the compiler - would generate some listing file, return a binary, or merely - check the syntax of the source code.</p> + file is generated. <c>false</c> means that the compiler + generates some listing file, returns a binary, or merely + checks the syntax of the source code.</p> </desc> </func> <func> <name>noenv_file(File, Options) -> CompRet</name> - <fsummary>Compile a file (ignoring ERL_COMPILER_OPTIONS)</fsummary> + <fsummary>Compiles a file (ignoring <c>ERL_COMPILER_OPTIONS)</c>.</fsummary> <desc> - <p>Works exactly like <seealso marker="#file/2">file/2</seealso>, + <p>Works like <seealso marker="#file/2">file/2</seealso>, except that the environment variable <c>ERL_COMPILER_OPTIONS</c> is not consulted.</p> </desc> @@ -730,9 +723,9 @@ module.beam: module.erl \ <func> <name>noenv_forms(Forms, Options) -> CompRet</name> - <fsummary>Compile a list of forms (ignoring ERL_COMPILER_OPTIONS)</fsummary> + <fsummary>Compiles a list of forms (ignoring <c>ERL_COMPILER_OPTIONS)</c>.</fsummary> <desc> - <p>Works exactly like <seealso marker="#forms/2">forms/2</seealso>, + <p>Works like <seealso marker="#forms/2">forms/2</seealso>, except that the environment variable <c>ERL_COMPILER_OPTIONS</c> is not consulted.</p> </desc> @@ -740,12 +733,13 @@ module.beam: module.erl \ <func> <name>noenv_output_generated(Options) -> true | false</name> - <fsummary>Determine whether the compile will generate an output file (ignoring ERL_COMPILER_OPTIONS)</fsummary> + <fsummary>Determines whether the compiler generates an output file + (ignoring <c>ERL_COMPILER_OPTIONS)</c>.</fsummary> <type> <v>Options = [term()]</v> </type> <desc> - <p>Works exactly like + <p>Works like <seealso marker="#output_generated/1">output_generated/1</seealso>, except that the environment variable <c>ERL_COMPILER_OPTIONS</c> is not consulted.</p> @@ -755,14 +749,14 @@ module.beam: module.erl \ </funcs> <section> - <title>Default compiler options</title> + <title>Default Compiler Options</title> <p>The (host operating system) environment variable <c>ERL_COMPILER_OPTIONS</c> can be used to give default compiler options. Its value must be a valid Erlang term. If the value is a - list, it will be used as is. If it is not a list, it will be put + list, it is used as is. If it is not a list, it is put into a list.</p> - <p>The list will be appended to any options given to + <p>The list is appended to any options given to <seealso marker="#file/2">file/2</seealso>, <seealso marker="#forms/2">forms/2</seealso>, and <seealso marker="#output_generated/1">output_generated/2</seealso>. @@ -770,9 +764,9 @@ module.beam: module.erl \ <seealso marker="#noenv_file/2">noenv_file/2</seealso>, <seealso marker="#noenv_forms/2">noenv_forms/2</seealso>, or <seealso marker="#noenv_output_generated/1">noenv_output_generated/2</seealso> - if you don't want the environment variable to be consulted - (for instance, if you are calling the compiler recursively from - inside a parse transform).</p> + if you do not want the environment variable to be consulted, + for example, if you are calling the compiler recursively from + inside a parse transform.</p> </section> <section> @@ -781,31 +775,31 @@ module.beam: module.erl \ module. Inlining means that a call to a function is replaced with the function body with the arguments replaced with the actual values. The semantics are preserved, except if exceptions are - generated in the inlined code. Exceptions will be reported as + generated in the inlined code. Exceptions are reported as occurring in the function the body was inlined into. Also, - <c>function_clause</c> exceptions will be converted to similar + <c>function_clause</c> exceptions are converted to similar <c>case_clause</c> exceptions.</p> - <p>When a function is inlined, the original function will be + <p>When a function is inlined, the original function is kept if it is exported (either by an explicit export or if the - <c>export_all</c> option was given) or if not all calls to the - function were inlined.</p> + option <c>export_all</c> was given) or if not all calls to the + function are inlined.</p> <p>Inlining does not necessarily improve running time. - For instance, inlining may increase Beam stack usage which will - probably be detrimental to performance for recursive functions. + For example, inlining can increase Beam stack use, which + probably is detrimental to performance for recursive functions. </p> - <p>Inlining is never default; it must be explicitly enabled with a + <p>Inlining is never default. It must be explicitly enabled with a compiler option or a <c>-compile()</c> attribute in the source module.</p> - <p>To enable inlining, either use the <c>inline</c> option to - let the compiler decide which functions to inline or + <p>To enable inlining, either use the option <c>inline</c> to + let the compiler decide which functions to inline, or <c>{inline,[{Name,Arity},...]}</c> to have the compiler inline all calls to the given functions. If the option is given inside a <c>compile</c> directive in an Erlang module, <c>{Name,Arity}</c> - may be written as <c>Name/Arity</c>.</p> + can be written as <c>Name/Arity</c>.</p> <p>Example of explicit inlining:</p> @@ -817,33 +811,30 @@ pi() -> 3.1416. <p>Example of implicit inlining:</p> <pre> --compile(inline). - </pre> +-compile(inline).</pre> - <p>The <c>{inline_size,Size}</c> option controls how large functions - that are allowed to be inlined. Default is <c>24</c>, which will - keep the size of the inlined code roughly the same as - the un-inlined version (only relatively small functions will be + <p>The option <c>{inline_size,Size}</c> controls how large functions + that are allowed to be inlined. Default is <c>24</c>, which + keeps the size of the inlined code roughly the same as + the un-inlined version (only relatively small functions are inlined).</p> <p>Example:</p> <pre> %% Aggressive inlining - will increase code size. -compile(inline). --compile({inline_size,100}). - </pre> +-compile({inline_size,100}).</pre> </section> <section> - <title>Inlining of list functions</title> - <p>The compiler can also inline a variety of list manipulation functions - from the stdlib's lists module.</p> + <title>Inlining of List Functions</title> + <p>The compiler can also inline various list manipulation functions + from the module <c>list</c> in <c>STDLIB</c>.</p> <p>This feature must be explicitly enabled with a compiler option or a <c>-compile()</c> attribute in the source module.</p> - <p>To enable inlining of list functions, use the <c>inline_list_funcs</c> - option.</p> + <p>To enable inlining of list functions, use option <c>inline_list_funcs</c>.</p> <p>The following functions are inlined:</p> <list type="bulleted"> @@ -869,24 +860,23 @@ pi() -> 3.1416. </section> <section> + <marker id="error_information"></marker> <title>Error Information</title> - <p>The <c>ErrorInfo</c> mentioned above is the standard - <c>ErrorInfo</c> structure which is returned from all IO modules. + <p>The <c>ErrorInfo</c> mentioned earlier is the standard + <c>ErrorInfo</c> structure, which is returned from all I/O modules. It has the following format:</p> <code> -{ErrorLine, Module, ErrorDescriptor} - </code> +{ErrorLine, Module, ErrorDescriptor}</code> - <p><c>ErrorLine</c> will be the atom <c>none</c> if the error does - not correspond to a specific line (e.g. if the source file does - not exist).</p> + <p><c>ErrorLine</c> is the atom <c>none</c> if the error does + not correspond to a specific line, for example, if the source file does + not exist.</p> <p>A string describing the error is obtained with the following call:</p> <code> -Module:format_error(ErrorDescriptor) - </code> +Module:format_error(ErrorDescriptor)</code> </section> <section> diff --git a/lib/compiler/doc/src/ref_man.xml b/lib/compiler/doc/src/ref_man.xml index 6478ad4b11..6584e79c4e 100644 --- a/lib/compiler/doc/src/ref_man.xml +++ b/lib/compiler/doc/src/ref_man.xml @@ -29,7 +29,7 @@ <file>application.sgml</file> </header> <description> - <p>The <em>Compiler</em> application compiles Erlang + <p>The <c>Compiler</c> application compiles Erlang code to byte-code. The highly compact byte-code is executed by the Erlang emulator.</p> </description> diff --git a/lib/compiler/src/Makefile b/lib/compiler/src/Makefile index 7c4cebdc28..78efc8dff0 100644 --- a/lib/compiler/src/Makefile +++ b/lib/compiler/src/Makefile @@ -70,6 +70,7 @@ MODULES = \ cerl \ cerl_clauses \ cerl_inline \ + cerl_sets \ cerl_trees \ compile \ core_lib \ diff --git a/lib/compiler/src/beam_bsm.erl b/lib/compiler/src/beam_bsm.erl index 2a15c1ddf3..ee3e88959d 100644 --- a/lib/compiler/src/beam_bsm.erl +++ b/lib/compiler/src/beam_bsm.erl @@ -242,6 +242,12 @@ btb_reaches_match_2([{bif,_,{f,F},Ss,Dst}=I|Is], Regs0, D0) -> Regs = btb_kill([Dst], Regs0), D = btb_follow_branch(F, Regs, D0), btb_reaches_match_1(Is, Regs, D); +btb_reaches_match_2([{get_map_elements,{f,F},Src,{list,Ls}}=I|Is], Regs0, D0) -> + {Ss,Ds} = beam_utils:split_even(Ls), + btb_ensure_not_used([Src|Ss], I, Regs0), + Regs = btb_kill(Ds, Regs0), + D = btb_follow_branch(F, Regs, D0), + btb_reaches_match_1(Is, Regs, D); btb_reaches_match_2([{test,bs_start_match2,{f,F},Live,[Ctx,_],Ctx}=I|Is], Regs0, D0) -> CtxRegs = btb_context_regs(Regs0), diff --git a/lib/compiler/src/beam_dead.erl b/lib/compiler/src/beam_dead.erl index 5932d8ce1d..bbe607cf19 100644 --- a/lib/compiler/src/beam_dead.erl +++ b/lib/compiler/src/beam_dead.erl @@ -96,7 +96,7 @@ move_move_into_block([], Acc) -> reverse(Acc). %%% forward(Is, Lc) -> - forward(Is, gb_trees:empty(), Lc, []). + forward(Is, #{}, Lc, []). forward([{move,_,_}=Move|[{label,L}|_]=Is], D, Lc, Acc) -> %% move/2 followed by jump/1 is optimized by backward/3. @@ -115,19 +115,20 @@ forward([{label,Lbl}=LblI,{block,[{set,[Dst],[Lit],move}|BlkIs]}=Blk|Is], D, Lc, %% cannot be reached in any other way than through the select_val/3 %% instruction (i.e. there can be no fallthrough to such label and %% it cannot be referenced by, for example, a jump/1 instruction). - Block = case gb_trees:lookup({Lbl,Dst}, D) of - {value,Lit} -> {block,BlkIs}; %Safe to remove move instruction. - _ -> Blk %Must keep move instruction. - end, + Key = {Lbl,Dst}, + Block = case D of + #{Key := Lit} -> {block,BlkIs}; %Safe to remove move instruction. + _ -> Blk %Must keep move instruction. + end, forward([Block|Is], D, Lc, [LblI|Acc]); forward([{label,Lbl}=LblI|[{move,Lit,Dst}|Is1]=Is0], D, Lc, Acc) -> %% Assumption: The target labels in a select_val/3 instruction %% cannot be reached in any other way than through the select_val/3 %% instruction (i.e. there can be no fallthrough to such label and %% it cannot be referenced by, for example, a jump/1 instruction). - Is = case gb_trees:lookup({Lbl,Dst}, D) of - {value,Lit} -> Is1; %Safe to remove move instruction. - _ -> Is0 %Keep move instruction. + Is = case maps:find({Lbl,Dst}, D) of + {ok,Lit} -> Is1; %Safe to remove move instruction. + _ -> Is0 %Keep move instruction. end, forward(Is, D, Lc, [LblI|Acc]); forward([{test,is_eq_exact,_,[Same,Same]}|Is], D, Lc, Acc) -> @@ -156,11 +157,11 @@ forward([], _, Lc, Acc) -> {Acc,Lc}. update_value_dict([Lit,{f,Lbl}|T], Reg, D0) -> Key = {Lbl,Reg}, - D = case gb_trees:lookup(Key, D0) of - none -> gb_trees:insert(Key, Lit, D0); %New. - {value,inconsistent} -> D0; %Inconsistent. - {value,_} -> gb_trees:update(Key, inconsistent, D0) - end, + D = case D0 of + #{Key := inconsistent} -> D0; + #{Key := _} -> D0#{Key := inconsistent}; + _ -> D0#{Key => Lit} + end, update_value_dict(T, Reg, D); update_value_dict([], _, D) -> D. diff --git a/lib/compiler/src/beam_dict.erl b/lib/compiler/src/beam_dict.erl index 68dc104dd3..b1aa98278e 100644 --- a/lib/compiler/src/beam_dict.erl +++ b/lib/compiler/src/beam_dict.erl @@ -31,22 +31,22 @@ -type index() :: non_neg_integer(). --type atom_tab() :: gb_trees:tree(atom(), index()). +-type atom_tab() :: #{atom() => index()}. -type import_tab() :: gb_trees:tree(mfa(), index()). --type fname_tab() :: gb_trees:tree(Name :: term(), index()). --type line_tab() :: gb_trees:tree({Fname :: index(), Line :: term()}, index()). +-type fname_tab() :: #{Name :: term() => index()}. +-type line_tab() :: #{{Fname :: index(), Line :: term()} => index()}. -type literal_tab() :: dict:dict(Literal :: term(), index()). -record(asm, - {atoms = gb_trees:empty() :: atom_tab(), + {atoms = #{} :: atom_tab(), exports = [] :: [{label(), arity(), label()}], locals = [] :: [{label(), arity(), label()}], imports = gb_trees:empty() :: import_tab(), strings = <<>> :: binary(), %String pool lambdas = [], %[{...}] literals = dict:new() :: literal_tab(), - fnames = gb_trees:empty() :: fname_tab(), - lines = gb_trees:empty() :: line_tab(), + fnames = #{} :: fname_tab(), + lines = #{} :: line_tab(), num_lines = 0 :: non_neg_integer(), %Number of line instructions next_import = 0 :: non_neg_integer(), string_offset = 0 :: non_neg_integer(), @@ -77,14 +77,12 @@ highest_opcode(#asm{highest_opcode=Op}) -> Op. %% atom(Atom, Dict) -> {Index,Dict'} -spec atom(atom(), bdict()) -> {pos_integer(), bdict()}. -atom(Atom, #asm{atoms=Atoms0}=Dict) when is_atom(Atom) -> - case gb_trees:lookup(Atom, Atoms0) of - {value,Index} -> - {Index,Dict}; - none -> - NextIndex = gb_trees:size(Atoms0) + 1, - Atoms = gb_trees:insert(Atom, NextIndex, Atoms0), - {NextIndex,Dict#asm{atoms=Atoms}} +atom(Atom, #asm{atoms=Atoms}=Dict) when is_atom(Atom) -> + case Atoms of + #{ Atom := Index} -> {Index,Dict}; + _ -> + NextIndex = maps:size(Atoms) + 1, + {NextIndex,Dict#asm{atoms=Atoms#{Atom=>NextIndex}}} end. %% Remembers an exported function. @@ -177,26 +175,22 @@ line([], #asm{num_lines=N}=Dict) -> %% No location available. Return the special pre-defined %% index 0. {0,Dict#asm{num_lines=N+1}}; -line([{location,Name,Line}], #asm{lines=Lines0,num_lines=N}=Dict0) -> +line([{location,Name,Line}], #asm{lines=Lines,num_lines=N}=Dict0) -> {FnameIndex,Dict1} = fname(Name, Dict0), - case gb_trees:lookup({FnameIndex,Line}, Lines0) of - {value,Index} -> - {Index,Dict1#asm{num_lines=N+1}}; - none -> - Index = gb_trees:size(Lines0) + 1, - Lines = gb_trees:insert({FnameIndex,Line}, Index, Lines0), - Dict = Dict1#asm{lines=Lines,num_lines=N+1}, - {Index,Dict} + Key = {FnameIndex,Line}, + case Lines of + #{Key := Index} -> {Index,Dict1#asm{num_lines=N+1}}; + _ -> + Index = maps:size(Lines) + 1, + {Index, Dict1#asm{lines=Lines#{Key=>Index},num_lines=N+1}} end. -fname(Name, #asm{fnames=Fnames0}=Dict) -> - case gb_trees:lookup(Name, Fnames0) of - {value,Index} -> - {Index,Dict}; - none -> - Index = gb_trees:size(Fnames0), - Fnames = gb_trees:insert(Name, Index, Fnames0), - {Index,Dict#asm{fnames=Fnames}} +fname(Name, #asm{fnames=Fnames}=Dict) -> + case Fnames of + #{Name := Index} -> {Index,Dict}; + _ -> + Index = maps:size(Fnames), + {Index,Dict#asm{fnames=Fnames#{Name=>Index}}} end. %% Returns the atom table. @@ -204,14 +198,12 @@ fname(Name, #asm{fnames=Fnames0}=Dict) -> -spec atom_table(bdict()) -> {non_neg_integer(), [[non_neg_integer(),...]]}. atom_table(#asm{atoms=Atoms}) -> - NumAtoms = gb_trees:size(Atoms), - Sorted = lists:keysort(2, gb_trees:to_list(Atoms)), - Fun = fun({A,_}) -> - L = atom_to_list(A), - [length(L)|L] - end, - AtomTab = lists:map(Fun, Sorted), - {NumAtoms,AtomTab}. + NumAtoms = maps:size(Atoms), + Sorted = lists:keysort(2, maps:to_list(Atoms)), + {NumAtoms,[begin + L = atom_to_list(A), + [length(L)|L] + end || {A,_} <- Sorted]}. %% Returns the table of local functions. %% local_table(Dict) -> {NumLocals, [{Function, Arity, Label}...]} @@ -273,11 +265,11 @@ my_term_to_binary(Term) -> non_neg_integer(),[{non_neg_integer(),non_neg_integer()}]}. line_table(#asm{fnames=Fnames0,lines=Lines0,num_lines=NumLineInstrs}) -> - NumFnames = gb_trees:size(Fnames0), - Fnames1 = lists:keysort(2, gb_trees:to_list(Fnames0)), + NumFnames = maps:size(Fnames0), + Fnames1 = lists:keysort(2, maps:to_list(Fnames0)), Fnames = [Name || {Name,_} <- Fnames1], - NumLines = gb_trees:size(Lines0), - Lines1 = lists:keysort(2, gb_trees:to_list(Lines0)), + NumLines = maps:size(Lines0), + Lines1 = lists:keysort(2, maps:to_list(Lines0)), Lines = [L || {L,_} <- Lines1], {NumLineInstrs,NumFnames,Fnames,NumLines,Lines}. diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl index 52b6464c7f..80b2998ddc 100644 --- a/lib/compiler/src/beam_jump.erl +++ b/lib/compiler/src/beam_jump.erl @@ -152,14 +152,14 @@ function({function,Name,Arity,CLabel,Asm0}) -> share(Is0) -> %% We will get more sharing if we never fall through to a label. Is = eliminate_fallthroughs(Is0, []), - share_1(Is, dict:new(), [], []). + share_1(Is, #{}, [], []). share_1([{label,_}=Lbl|Is], Dict, [], Acc) -> share_1(Is, Dict, [], [Lbl|Acc]); share_1([{label,L}=Lbl|Is], Dict0, Seq, Acc) -> - case dict:find(Seq, Dict0) of + case maps:find(Seq, Dict0) of error -> - Dict = dict:store(Seq, L, Dict0), + Dict = maps:put(Seq, L, Dict0), share_1(Is, Dict, [], [Lbl|Seq ++ Acc]); {ok,Label} -> share_1(Is, Dict0, [], [Lbl,{jump,{f,Label}}|Acc]) @@ -188,7 +188,7 @@ clean_non_sharable(Dict) -> %% a sequence inside the 'try' block is a sequence that ends %% with an instruction that causes an exception. Any sequence %% that causes an exception must contain a line/1 instruction. - dict:filter(fun(K, _V) -> sharable_with_try(K) end, Dict). + maps:filter(fun(K, _V) -> sharable_with_try(K) end, Dict). sharable_with_try([{line,_}|_]) -> %% This sequence may cause an exception and may potentially @@ -268,13 +268,13 @@ extract_seq_1(_, _) -> no. -record(st, {fc, %Label for function class errors. entry, %Entry label (must not be moved). mlbl, %Moved labels. - labels %Set of referenced labels. + labels :: cerl_sets:set() %Set of referenced labels. }). opt([{label,Fc}|_]=Is0, CLabel) -> Lbls = initial_labels(Is0), find_fixpoint(fun(Is) -> - St = #st{fc=Fc,entry=CLabel,mlbl=dict:new(), + St = #st{fc=Fc,entry=CLabel,mlbl=#{}, labels=Lbls}, opt(Is, [], St) end, Is0). @@ -320,11 +320,11 @@ opt([{test,_,{f,_}=Lbl,_,_,_}=I|Is], Acc, St) -> opt([{select,_,_R,Fail,Vls}=I|Is], Acc, St) -> skip_unreachable(Is, [I|Acc], label_used([Fail|Vls], St)); opt([{label,Lbl}=I|Is], Acc, #st{mlbl=Mlbl}=St0) -> - case dict:find(Lbl, Mlbl) of + case maps:find(Lbl, Mlbl) of {ok,Lbls} -> %% Essential to remove the list of labels from the dictionary, %% since we will rescan the inserted labels. We MUST rescan. - St = St0#st{mlbl=dict:erase(Lbl, Mlbl)}, + St = St0#st{mlbl=maps:remove(Lbl, Mlbl)}, insert_labels([Lbl|Lbls], Is, Acc, St); error -> opt(Is, [I|Acc], St0) end; @@ -339,7 +339,7 @@ opt([{jump,{f,L}=Lbl}=I|Is], Acc0, #st{mlbl=Mlbl0}=St0) -> St = case Lbls of [] -> St0; [_|_] -> - Mlbl = dict:append_list(L, Lbls, Mlbl0), + Mlbl = maps_append_list(L, Lbls, Mlbl0), St0#st{mlbl=Mlbl} end, skip_unreachable(Is, [I|Acc], label_used(Lbl, St)); @@ -363,14 +363,20 @@ opt([I|Is], Acc, #st{labels=Used0}=St0) -> end; opt([], Acc, #st{fc=Fc,mlbl=Mlbl}) -> Code = reverse(Acc), - case dict:find(Fc, Mlbl) of + case maps:find(Fc, Mlbl) of {ok,Lbls} -> insert_fc_labels(Lbls, Mlbl, Code); error -> Code end. +maps_append_list(K,Vs,M) -> + case M of + #{K:=Vs0} -> M#{K:=Vs0++Vs}; % same order as dict + _ -> M#{K => Vs} + end. + insert_fc_labels([L|Ls], Mlbl, Acc0) -> Acc = [{label,L}|Acc0], - case dict:find(L, Mlbl) of + case maps:find(L, Mlbl) of error -> insert_fc_labels(Ls, Mlbl, Acc); {ok,Lbls} -> @@ -434,7 +440,7 @@ skip_unreachable([], Acc, St) -> %% Add one or more label to the set of used labels. -label_used({f,L}, St) -> St#st{labels=gb_sets:add(L, St#st.labels)}; +label_used({f,L}, St) -> St#st{labels=cerl_sets:add_element(L,St#st.labels)}; label_used([H|T], St0) -> label_used(T, label_used(H, St0)); label_used([], St) -> St; label_used(_Other, St) -> St. @@ -442,7 +448,7 @@ label_used(_Other, St) -> St. %% Test if label is used. is_label_used(L, St) -> - gb_sets:is_member(L, St#st.labels). + cerl_sets:is_element(L, St#st.labels). %% is_unreachable_after(Instruction) -> boolean() %% Test whether the code after Instruction is unreachable. @@ -472,14 +478,14 @@ is_exit_instruction(_) -> false. %% (including inside blocks). is_label_used_in(Lbl, Is) -> - is_label_used_in_1(Is, Lbl, gb_sets:empty()). + is_label_used_in_1(Is, Lbl, cerl_sets:new()). is_label_used_in_1([{block,Block}|Is], Lbl, Empty) -> lists:any(fun(I) -> is_label_used_in_block(I, Lbl) end, Block) orelse is_label_used_in_1(Is, Lbl, Empty); is_label_used_in_1([I|Is], Lbl, Empty) -> Used = ulbl(I, Empty), - gb_sets:is_member(Lbl, Used) orelse is_label_used_in_1(Is, Lbl, Empty); + cerl_sets:is_element(Lbl, Used) orelse is_label_used_in_1(Is, Lbl, Empty); is_label_used_in_1([], _, _) -> false. is_label_used_in_block({set,_,_,Info}, Lbl) -> @@ -506,7 +512,7 @@ remove_unused_labels(Is) -> rem_unused(Is, Used, []). rem_unused([{label,Lbl}=I|Is0], Used, [Prev|_]=Acc) -> - case gb_sets:is_member(Lbl, Used) of + case cerl_sets:is_element(Lbl, Used) of false -> Is = case is_unreachable_after(Prev) of true -> drop_upto_label(Is0); @@ -528,7 +534,7 @@ initial_labels([{line,_}|Is], Acc) -> initial_labels([{label,Lbl}|Is], Acc) -> initial_labels(Is, [Lbl|Acc]); initial_labels([{func_info,_,_,_},{label,Lbl}|_], Acc) -> - gb_sets:from_list([Lbl|Acc]). + cerl_sets:from_list([Lbl|Acc]). drop_upto_label([{label,_}|_]=Is) -> Is; drop_upto_label([_|Is]) -> drop_upto_label(Is); @@ -576,10 +582,10 @@ ulbl({get_map_elements,Lbl,_Src,_List}, Used) -> ulbl(_, Used) -> Used. mark_used({f,0}, Used) -> Used; -mark_used({f,L}, Used) -> gb_sets:add(L, Used). +mark_used({f,L}, Used) -> cerl_sets:add_element(L, Used). mark_used_list([{f,L}|T], Used) -> - mark_used_list(T, gb_sets:add(L, Used)); + mark_used_list(T, cerl_sets:add_element(L, Used)); mark_used_list([_|T], Used) -> mark_used_list(T, Used); mark_used_list([], Used) -> Used. diff --git a/lib/compiler/src/beam_type.erl b/lib/compiler/src/beam_type.erl index 4731b5e78e..7ab548152e 100644 --- a/lib/compiler/src/beam_type.erl +++ b/lib/compiler/src/beam_type.erl @@ -554,10 +554,10 @@ flush(Rs, [{set,[_],[],{put_tuple,_}}|_]=Is0, Acc0) -> Acc = flush_all(Rs, Is0, Acc0), {[],Acc}; flush(Rs0, [{set,Ds,Ss,_Op}|_], Acc0) -> - Save = gb_sets:from_list(Ss), + Save = cerl_sets:from_list(Ss), Acc = save_regs(Rs0, Save, Acc0), Rs1 = foldl(fun(S, A) -> mark(S, A, clean) end, Rs0, Ss), - Kill = gb_sets:from_list(Ds), + Kill = cerl_sets:from_list(Ds), Rs = kill_regs(Rs1, Kill), {Rs,Acc}; flush(Rs0, Is, Acc0) -> @@ -580,7 +580,7 @@ save_regs(Rs, Save, Acc) -> foldl(fun(R, A) -> save_reg(R, Save, A) end, Acc, Rs). save_reg({I,V,dirty}, Save, Acc) -> - case gb_sets:is_member(V, Save) of + case cerl_sets:is_element(V, Save) of true -> [{set,[V],[{fr,I}],fmove}|checkerror(Acc)]; false -> Acc end; @@ -590,7 +590,7 @@ kill_regs(Rs, Kill) -> [kill_reg(R, Kill) || R <- Rs]. kill_reg({_,V,_}=R, Kill) -> - case gb_sets:is_member(V, Kill) of + case cerl_sets:is_element(V, Kill) of true -> free; false -> R end; diff --git a/lib/compiler/src/cerl_sets.erl b/lib/compiler/src/cerl_sets.erl new file mode 100644 index 0000000000..4df78dc432 --- /dev/null +++ b/lib/compiler/src/cerl_sets.erl @@ -0,0 +1,206 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2000-2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(cerl_sets). + +%% Standard interface. +-export([new/0,is_set/1,size/1,to_list/1,from_list/1]). +-export([is_element/2,add_element/2,del_element/2]). +-export([union/2,union/1,intersection/2,intersection/1]). +-export([is_disjoint/2]). +-export([subtract/2,is_subset/2]). +-export([fold/3,filter/2]). + +-export_type([set/0, set/1]). + +%%------------------------------------------------------------------------------ + +-type set() :: set(_). +-opaque set(Element) :: #{Element => 'ok'}. + +%%------------------------------------------------------------------------------ + +%% new() -> Set +-spec new() -> set(). + +new() -> #{}. + +%% is_set(Set) -> boolean(). +%% Return 'true' if Set is a set of elements, else 'false'. +-spec is_set(Set) -> boolean() when + Set :: term(). + +is_set(S) when is_map(S) -> true; +is_set(_) -> false. + +%% size(Set) -> int(). +%% Return the number of elements in Set. +-spec size(Set) -> non_neg_integer() when + Set :: set(). + +size(S) -> maps:size(S). + +%% to_list(Set) -> [Elem]. +%% Return the elements in Set as a list. +-spec to_list(Set) -> List when + Set :: set(Element), + List :: [Element]. + +to_list(S) -> maps:keys(S). + +%% from_list([Elem]) -> Set. +%% Build a set from the elements in List. +-spec from_list(List) -> Set when + List :: [Element], + Set :: set(Element). +from_list(Ls) -> maps:from_list([{K,ok}||K<-Ls]). + +%% is_element(Element, Set) -> boolean(). +%% Return 'true' if Element is an element of Set, else 'false'. +-spec is_element(Element, Set) -> boolean() when + Set :: set(Element). + +is_element(E,S) -> + case S of + #{E := _} -> true; + _ -> false + end. + +%% add_element(Element, Set) -> Set. +%% Return Set with Element inserted in it. +-spec add_element(Element, Set1) -> Set2 when + Set1 :: set(Element), + Set2 :: set(Element). + +add_element(E,S) -> S#{E=>ok}. + +-spec del_element(Element, Set1) -> Set2 when + Set1 :: set(Element), + Set2 :: set(Element). + +%% del_element(Element, Set) -> Set. +%% Return Set but with Element removed. +del_element(E,S) -> maps:remove(E,S). + +%% union(Set1, Set2) -> Set +%% Return the union of Set1 and Set2. +-spec union(Set1, Set2) -> Set3 when + Set1 :: set(Element), + Set2 :: set(Element), + Set3 :: set(Element). + +union(S1,S2) -> maps:merge(S1,S2). + +%% union([Set]) -> Set +%% Return the union of the list of sets. +-spec union(SetList) -> Set when + SetList :: [set(Element)], + Set :: set(Element). + +union([S1,S2|Ss]) -> + union1(union(S1, S2), Ss); +union([S]) -> S; +union([]) -> new(). + +union1(S1, [S2|Ss]) -> + union1(union(S1, S2), Ss); +union1(S1, []) -> S1. + +%% intersection(Set1, Set2) -> Set. +%% Return the intersection of Set1 and Set2. +-spec intersection(Set1, Set2) -> Set3 when + Set1 :: set(Element), + Set2 :: set(Element), + Set3 :: set(Element). + +intersection(S1, S2) -> + filter(fun (E) -> is_element(E, S1) end, S2). + +%% intersection([Set]) -> Set. +%% Return the intersection of the list of sets. +-spec intersection(SetList) -> Set when + SetList :: [set(Element),...], + Set :: set(Element). + +intersection([S1,S2|Ss]) -> + intersection1(intersection(S1, S2), Ss); +intersection([S]) -> S. + +intersection1(S1, [S2|Ss]) -> + intersection1(intersection(S1, S2), Ss); +intersection1(S1, []) -> S1. + +%% is_disjoint(Set1, Set2) -> boolean(). +%% Check whether Set1 and Set2 are disjoint. +-spec is_disjoint(Set1, Set2) -> boolean() when + Set1 :: set(Element), + Set2 :: set(Element). + +is_disjoint(S1, S2) when map_size(S1) < map_size(S2) -> + fold(fun (_, false) -> false; + (E, true) -> not is_element(E, S2) + end, true, S1); +is_disjoint(S1, S2) -> + fold(fun (_, false) -> false; + (E, true) -> not is_element(E, S1) + end, true, S2). + +%% subtract(Set1, Set2) -> Set. +%% Return all and only the elements of Set1 which are not also in +%% Set2. +-spec subtract(Set1, Set2) -> Set3 when + Set1 :: set(Element), + Set2 :: set(Element), + Set3 :: set(Element). + +subtract(S1, S2) -> + filter(fun (E) -> not is_element(E, S2) end, S1). + +%% is_subset(Set1, Set2) -> boolean(). +%% Return 'true' when every element of Set1 is also a member of +%% Set2, else 'false'. +-spec is_subset(Set1, Set2) -> boolean() when + Set1 :: set(Element), + Set2 :: set(Element). + +is_subset(S1, S2) -> + fold(fun (E, Sub) -> Sub andalso is_element(E, S2) end, true, S1). + +%% fold(Fun, Accumulator, Set) -> Accumulator. +%% Fold function Fun over all elements in Set and return Accumulator. +-spec fold(Function, Acc0, Set) -> Acc1 when + Function :: fun((Element, AccIn) -> AccOut), + Set :: set(Element), + Acc0 :: Acc, + Acc1 :: Acc, + AccIn :: Acc, + AccOut :: Acc. + +fold(F, Init, D) -> + lists:foldl(fun(E,Acc) -> F(E,Acc) end,Init,maps:keys(D)). + +%% filter(Fun, Set) -> Set. +%% Filter Set with Fun. +-spec filter(Pred, Set1) -> Set2 when + Pred :: fun((Element) -> boolean()), + Set1 :: set(Element), + Set2 :: set(Element). + +filter(F, D) -> + maps:from_list(lists:filter(fun({K,_}) -> F(K) end, maps:to_list(D))). diff --git a/lib/compiler/src/compile.erl b/lib/compiler/src/compile.erl index 22810c910c..0158cf64db 100644 --- a/lib/compiler/src/compile.erl +++ b/lib/compiler/src/compile.erl @@ -342,10 +342,10 @@ run_tc({Name,Fun}, St) -> run_eprof({Name,Fun}, Name, St) -> io:format("~p: Running eprof\n", [Name]), - eprof:start_profiling([self()]), + c:appcall(tools, eprof, start_profiling, [[self()]]), Val = (catch Fun(St)), - eprof:stop_profiling(), - eprof:analyze(), + c:appcall(tools, eprof, stop_profiling, []), + c:appcall(tools, eprof, analyze, []), Val; run_eprof({_,Fun}, _, St) -> catch Fun(St). diff --git a/lib/compiler/src/compiler.app.src b/lib/compiler/src/compiler.app.src index 2a40c1c379..0bfd998301 100644 --- a/lib/compiler/src/compiler.app.src +++ b/lib/compiler/src/compiler.app.src @@ -45,6 +45,7 @@ cerl, cerl_clauses, cerl_inline, + cerl_sets, cerl_trees, compile, core_scan, @@ -69,5 +70,5 @@ {registered, []}, {applications, [kernel, stdlib]}, {env, []}, - {runtime_dependencies, ["stdlib-2.0","kernel-3.0","hipe-3.10.3","erts-7.0", - "crypto-3.3"]}]}. + {runtime_dependencies, ["stdlib-2.5","kernel-4.0","hipe-3.12","erts-7.0", + "crypto-3.6"]}]}. diff --git a/lib/compiler/src/sys_core_fold.erl b/lib/compiler/src/sys_core_fold.erl index 6f8279f65e..102a6951e8 100644 --- a/lib/compiler/src/sys_core_fold.erl +++ b/lib/compiler/src/sys_core_fold.erl @@ -92,10 +92,10 @@ -endif. %% Variable value info. --record(sub, {v=[], %Variable substitutions - s=[], %Variables in scope - t=[], %Types - in_guard=false}). %In guard or not. +-record(sub, {v=[], %Variable substitutions + s=cerl_sets:new() :: cerl_sets:set(), %Variables in scope + t=#{} :: map(), %Types + in_guard=false}). %In guard or not. -type type_info() :: cerl:cerl() | 'bool' | 'integer'. -type yes_no_maybe() :: 'yes' | 'no' | 'maybe'. @@ -1123,7 +1123,7 @@ let_substs(Vs0, As0, Sub0) -> {Vs2,As1,Ss} = let_substs_1(Vs1, As0, Sub1), Sub2 = sub_add_scope([V || #c_var{name=V} <- Vs2], Sub1), {Vs2,As1, - foldl(fun ({V,S}, Sub) -> sub_set_name(V, S, Sub) end, Sub2, Ss)}. + foldl(fun ({V,S}, Sub) -> sub_set_name(V, S, Sub) end, Sub2, Ss)}. let_substs_1(Vs, #c_values{es=As}, Sub) -> let_subst_list(Vs, As, Sub); @@ -1242,10 +1242,10 @@ is_subst(_) -> false. %% to force renaming if variables in the scope occurs as pattern %% variables. -sub_new() -> #sub{v=orddict:new(),s=gb_trees:empty(),t=[]}. +sub_new() -> #sub{v=orddict:new(),s=cerl_sets:new(),t=#{}}. sub_new(#sub{}=Sub) -> - Sub#sub{v=orddict:new(),t=[]}. + Sub#sub{v=orddict:new(),t=#{}}. sub_new_preserve_types(#sub{}=Sub) -> Sub#sub{v=orddict:new()}. @@ -1262,16 +1262,16 @@ sub_set_var(#c_var{name=V}, Val, Sub) -> sub_set_name(V, Val, #sub{v=S,s=Scope,t=Tdb0}=Sub) -> Tdb1 = kill_types(V, Tdb0), Tdb = copy_type(V, Val, Tdb1), - Sub#sub{v=orddict:store(V, Val, S),s=gb_sets:add(V, Scope),t=Tdb}. + Sub#sub{v=orddict:store(V, Val, S),s=cerl_sets:add_element(V, Scope),t=Tdb}. sub_del_var(#c_var{name=V}, #sub{v=S,s=Scope,t=Tdb}=Sub) -> %% Profiling shows that for programs with many record operations, %% sub_del_var/2 is a bottleneck. Since the scope contains all %% variables that are live, we know that V cannot be present in S %% if it is not in the scope. - case gb_sets:is_member(V, Scope) of + case cerl_sets:is_element(V, Scope) of false -> - Sub#sub{s=gb_sets:insert(V, Scope)}; + Sub#sub{s=cerl_sets:add_element(V, Scope)}; true -> Sub#sub{v=orddict:erase(V, S),t=kill_types(V, Tdb)} end. @@ -1282,12 +1282,12 @@ sub_subst_var(#c_var{name=V}, Val, #sub{v=S0}) -> sub_add_scope(Vs, #sub{s=Scope0}=Sub) -> Scope = foldl(fun(V, S) when is_integer(V); is_atom(V) -> - gb_sets:add(V, S) + cerl_sets:add_element(V, S) end, Scope0, Vs), Sub#sub{s=Scope}. sub_subst_scope(#sub{v=S0,s=Scope}=Sub) -> - S = [{-1,#c_var{name=Sv}} || Sv <- gb_sets:to_list(Scope)]++S0, + S = [{-1,#c_var{name=Sv}} || Sv <- cerl_sets:to_list(Scope)]++S0, Sub#sub{v=S}. sub_is_val(#c_var{name=V}, #sub{v=S,s=Scope}) -> @@ -1295,7 +1295,7 @@ sub_is_val(#c_var{name=V}, #sub{v=S,s=Scope}) -> %% became the new bottleneck. Since the scope contains all %% live variables, a variable V can only be the target for %% a substitution if it is in the scope. - gb_sets:is_member(V, Scope) andalso v_is_value(V, S). + cerl_sets:is_element(V, Scope) andalso v_is_value(V, S). v_is_value(Var, [{_,#c_var{name=Var}}|_]) -> true; v_is_value(Var, [_|T]) -> v_is_value(Var, T); @@ -1760,8 +1760,9 @@ case_opt_compiler_generated(Core) -> %% return Expr0 unchanged. %% case_expand_var(E, #sub{t=Tdb}) -> - case orddict:find(cerl:var_name(E), Tdb) of - {ok,T0} -> + Key = cerl:var_name(E), + case Tdb of + #{Key:=T0} -> case cerl:is_c_tuple(T0) of false -> E; @@ -1785,7 +1786,7 @@ case_expand_var(E, #sub{t=Tdb}) -> E end end; - error -> + _ -> E end. @@ -2147,7 +2148,7 @@ is_bool_expr_list([], _) -> true. %% functions, or is_record/2). %% is_safe_bool_expr(Core, Sub) -> - is_safe_bool_expr_1(Core, Sub, gb_sets:empty()). + is_safe_bool_expr_1(Core, Sub, cerl_sets:new()). is_safe_bool_expr_1(#c_call{module=#c_literal{val=erlang}, name=#c_literal{val=is_record}, @@ -2193,7 +2194,7 @@ is_safe_bool_expr_1(#c_let{vars=Vars,arg=Arg,body=B}, Sub, BoolVars) -> true -> case {is_safe_bool_expr_1(Arg, Sub, BoolVars),Vars} of {true,[#c_var{name=V}]} -> - is_safe_bool_expr_1(B, Sub, gb_sets:add(V, BoolVars)); + is_safe_bool_expr_1(B, Sub, cerl_sets:add_element(V, BoolVars)); {false,_} -> is_safe_bool_expr_1(B, Sub, BoolVars) end; @@ -2202,7 +2203,7 @@ is_safe_bool_expr_1(#c_let{vars=Vars,arg=Arg,body=B}, Sub, BoolVars) -> is_safe_bool_expr_1(#c_literal{val=Val}, _Sub, _) -> is_boolean(Val); is_safe_bool_expr_1(#c_var{name=V}, _Sub, BoolVars) -> - gb_sets:is_element(V, BoolVars); + cerl_sets:is_element(V, BoolVars); is_safe_bool_expr_1(_, _, _) -> false. is_safe_bool_expr_list([C|Cs], Sub, BoolVars) -> @@ -2236,7 +2237,7 @@ move_let_into_expr(#c_let{vars=InnerVs0,body=InnerBody0}=Inner, %% in <InnerBody> %% Arg = body(Arg0, Sub0), - ScopeSub0 = sub_subst_scope(Sub0#sub{t=[]}), + ScopeSub0 = sub_subst_scope(Sub0#sub{t=#{}}), {OuterVs,ScopeSub} = pattern_list(OuterVs0, ScopeSub0), OuterBody = body(OuterBody0, ScopeSub), @@ -2275,15 +2276,15 @@ move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let, CaVars0 = Ca0#c_clause.pats, G0 = Ca0#c_clause.guard, B0 = Ca0#c_clause.body, - ScopeSub0 = sub_subst_scope(Sub0#sub{t=[]}), + ScopeSub0 = sub_subst_scope(Sub0#sub{t=#{}}), {CaVars,ScopeSub} = pattern_list(CaVars0, ScopeSub0), G = guard(G0, ScopeSub), B1 = body(B0, ScopeSub), {Lvs,B2,Sub1} = let_substs(Lvs0, B1, Sub0), - Sub2 = Sub1#sub{s=gb_sets:union(ScopeSub#sub.s, - Sub1#sub.s)}, + Sub2 = Sub1#sub{s=cerl_sets:union(ScopeSub#sub.s, + Sub1#sub.s)}, Lbody = body(Lbody0, Sub2), B = Let#c_let{vars=Lvs,arg=core_lib:make_values(B2),body=Lbody}, @@ -2574,7 +2575,7 @@ move_case_into_arg(#c_case{arg=#c_let{vars=OuterVars0,arg=OuterArg, %% let <OuterVars> = <OuterArg> %% in case <InnerArg> of <InnerClauses> end %% - ScopeSub0 = sub_subst_scope(Sub#sub{t=[]}), + ScopeSub0 = sub_subst_scope(Sub#sub{t=#{}}), {OuterVars,ScopeSub} = pattern_list(OuterVars0, ScopeSub0), InnerArg = body(InnerArg0, ScopeSub), Outer#c_let{vars=OuterVars,arg=OuterArg, @@ -2603,7 +2604,7 @@ move_case_into_arg(#c_case{arg=#c_case{arg=OuterArg, %% <OuterCb> %% end %% - ScopeSub0 = sub_subst_scope(Sub#sub{t=[]}), + ScopeSub0 = sub_subst_scope(Sub#sub{t=#{}}), {OuterPats,ScopeSub} = pattern_list(OuterPats0, ScopeSub0), OuterGuard = guard(OuterGuard0, ScopeSub), InnerArg = body(InnerArg0, ScopeSub), @@ -2688,9 +2689,9 @@ is_any_var_used([], _) -> false. -spec get_type(cerl:cerl(), #sub{}) -> type_info() | 'none'. get_type(#c_var{name=V}, #sub{t=Tdb}) -> - case orddict:find(V, Tdb) of - {ok,Type} -> Type; - error -> none + case Tdb of + #{V:=Type} -> Type; + _ -> none end; get_type(C, _) -> case cerl:type(C) of @@ -2805,35 +2806,38 @@ update_types_1(#c_var{name=V,anno=Anno}, Pat, Types) -> update_types_1(_, _, Types) -> Types. update_types_2(V, [#c_tuple{}=P], Types) -> - orddict:store(V, P, Types); + Types#{V=>P}; update_types_2(V, [#c_literal{val=Bool}], Types) when is_boolean(Bool) -> - orddict:store(V, bool, Types); + Types#{V=>bool}; update_types_2(V, [Type], Types) when is_atom(Type) -> - orddict:store(V, Type, Types); + Types#{V=>Type}; update_types_2(_, _, Types) -> Types. %% kill_types(V, Tdb) -> Tdb' %% Kill any entries that references the variable, %% either in the key or in the value. -kill_types(V, [{V,_}|Tdb]) -> - kill_types(V, Tdb); -kill_types(V, [{_,#c_tuple{}=Tuple}=Entry|Tdb]) -> +kill_types(V, Tdb) -> + maps:from_list(kill_types2(V,maps:to_list(Tdb))). + +kill_types2(V, [{V,_}|Tdb]) -> + kill_types2(V, Tdb); +kill_types2(V, [{_,#c_tuple{}=Tuple}=Entry|Tdb]) -> case core_lib:is_var_used(V, Tuple) of - false -> [Entry|kill_types(V, Tdb)]; - true -> kill_types(V, Tdb) + false -> [Entry|kill_types2(V, Tdb)]; + true -> kill_types2(V, Tdb) end; -kill_types(V, [{_,Atom}=Entry|Tdb]) when is_atom(Atom) -> - [Entry|kill_types(V, Tdb)]; -kill_types(_, []) -> []. +kill_types2(V, [{_,Atom}=Entry|Tdb]) when is_atom(Atom) -> + [Entry|kill_types2(V, Tdb)]; +kill_types2(_, []) -> []. %% copy_type(DestVar, SrcVar, Tdb) -> Tdb' %% If the SrcVar has a type, assign it to DestVar. %% copy_type(V, #c_var{name=Src}, Tdb) -> - case orddict:find(Src, Tdb) of - {ok,Type} -> orddict:store(V, Type, Tdb); - error -> Tdb + case Tdb of + #{Src:=Type} -> Tdb#{V=>Type}; + _ -> Tdb end; copy_type(_, _, Tdb) -> Tdb. @@ -3237,12 +3241,12 @@ format_error(bin_var_used_in_guard) -> verify_scope(E, #sub{s=Scope}) -> Free0 = cerl_trees:free_variables(E), Free = [V || V <- Free0, not is_tuple(V)], %Ignore function names. - case ordsets:is_subset(Free, gb_sets:to_list(Scope)) of + case ordsets:is_subset(Free, cerl_sets:to_list(Scope)) of true -> true; false -> io:format("~p\n", [E]), io:format("~p\n", [Free]), - io:format("~p\n", [gb_sets:to_list(Scope)]), + io:format("~p\n", [cerl_sets:to_list(Scope)]), false end. -endif. diff --git a/lib/compiler/src/v3_codegen.erl b/lib/compiler/src/v3_codegen.erl index aa2ebc0f85..c9b1a45cfc 100644 --- a/lib/compiler/src/v3_codegen.erl +++ b/lib/compiler/src/v3_codegen.erl @@ -43,7 +43,7 @@ -export([module/2]). -import(lists, [member/2,keymember/3,keysort/2,keydelete/3, - append/1,map/2,flatmap/2,filter/2,foldl/3,foldr/3,mapfoldl/3, + append/1,flatmap/2,filter/2,foldl/3,foldr/3,mapfoldl/3, sort/1,reverse/1,reverse/2]). -import(v3_life, [vdb_find/2]). @@ -57,8 +57,7 @@ break, %Break label recv, %Receive label is_top_block, %Boolean: top block or not - functable=gb_trees:empty(), %Gb tree of local functions: - % {{Name,Arity},Label} + functable=#{}, %Map of local functions: {Name,Arity}=>Label in_catch=false, %Inside a catch or not. need_frame, %Need a stack frame. ultimate_failure %Label for ultimate match failure. @@ -673,9 +672,7 @@ select_val_cg(Type, R, [Val, {f,Lbl}], Tf, Vf, [{label,Lbl}|Sis]) -> [{test,select_type_test(Type),{f,Tf},[R]}, {test,is_eq_exact,{f,Vf},[R,{Type,Val}]}|Sis]; select_val_cg(Type, R, Vls0, Tf, Vf, Sis) -> - Vls1 = map(fun ({f,_Lbl} = F) -> F; - (Value) -> {Type,Value} - end, Vls0), + Vls1 = [case Value of {f,_Lbl} -> Value; _ -> {Type,Value} end || Value <- Vls0], [{test,select_type_test(Type),{f,Tf},[R]}, {select_val,R,{f,Vf},{list,Vls1}}|Sis]. select_type_test(integer) -> is_integer; @@ -1080,7 +1077,7 @@ protected_cg(Ts, Rs, _Fail, I, Vdb, Bef, St0) -> St2#cg{bfail=Pfail}), %%ok = io:fwrite("cg ~w: ~p~n", [?LINE,{Rs,I,Vdb,Aft}]), %% Set return values to false. - Mis = map(fun ({var,V}) -> {move,{atom,false},fetch_var(V, Aft)} end, Rs), + Mis = [{move,{atom,false},fetch_var(V,Aft)}||{var,V} <- Rs], {Tis ++ [{jump,{f,Psucc}}, {label,Pfail}] ++ Mis ++ [{label,Psucc}], Aft,St3#cg{bfail=St0#cg.bfail}}. @@ -1263,13 +1260,12 @@ enter_line(_, _, _) -> local_func_label(Name, Arity, St) -> local_func_label({Name,Arity}, St). -local_func_label(Key, #cg{functable=Tab}=St0) -> - case gb_trees:lookup(Key, Tab) of - {value,Label} -> - {Label,St0}; - none -> +local_func_label(Key, #cg{functable=Map}=St0) -> + case Map of + #{Key := Label} -> {Label,St0}; + _ -> {Label,St} = new_label(St0), - {Label,St#cg{functable=gb_trees:insert(Key, Label, Tab)}} + {Label,St#cg{functable=Map#{Key => Label}}} end. %% need_stack_frame(State) -> State' @@ -1992,25 +1988,28 @@ clear_dead(Sr, Until, Vdb) -> stk=clear_dead_stk(Sr#sr.stk, Until, Vdb)}. clear_dead_reg(Sr, Until, Vdb) -> - Reg = map(fun ({_I,V} = IV) -> - case vdb_find(V, Vdb) of - {V,_,L} when L > Until -> IV; - _ -> free %Remove anything else - end; - ({reserved,_I,_V} = Reserved) -> Reserved; - (free) -> free - end, Sr#sr.reg), + Reg = [case R of + {_I,V} = IV -> + case vdb_find(V, Vdb) of + {V,_,L} when L > Until -> IV; + _ -> free %Remove anything else + end; + {reserved,_I,_V} = Reserved -> Reserved; + free -> free + end || R <- Sr#sr.reg], reserve(Sr#sr.res, Reg, Sr#sr.stk). clear_dead_stk(Stk, Until, Vdb) -> - map(fun ({V} = T) -> - case vdb_find(V, Vdb) of - {V,_,L} when L > Until -> T; - _ -> dead %Remove anything else - end; - (free) -> free; - (dead) -> dead - end, Stk). + [case S of + {V} = T -> + case vdb_find(V, Vdb) of + {V,_,L} when L > Until -> T; + _ -> dead %Remove anything else + end; + free -> free; + dead -> dead + end || S <- Stk]. + %% sr_merge(Sr1, Sr2) -> Sr. %% Merge two stack/register states keeping the longest of both stack diff --git a/lib/compiler/src/v3_kernel.erl b/lib/compiler/src/v3_kernel.erl index 7dff58582e..c21b2a1505 100644 --- a/lib/compiler/src/v3_kernel.erl +++ b/lib/compiler/src/v3_kernel.erl @@ -114,7 +114,7 @@ copy_anno(Kdst, Ksrc) -> ff, %Current function vcount=0, %Variable counter fcount=0, %Fun counter - ds=[], %Defined variables + ds=cerl_sets:new() :: cerl_sets:set(), %Defined variables funs=[], %Fun functions free=[], %Free variables ws=[] :: [warning()], %Warnings. @@ -148,7 +148,7 @@ include_attribute(_) -> true. function({#c_var{name={F,Arity}=FA},Body}, St0) -> try - St1 = St0#kern{func=FA,ff=undefined,vcount=0,fcount=0,ds=sets:new()}, + St1 = St0#kern{func=FA,ff=undefined,vcount=0,fcount=0,ds=cerl_sets:new()}, {#ifun{anno=Ab,vars=Kvs,body=B0},[],St2} = expr(Body, new_sub(), St1), {B1,_,St3} = ubody(B0, return, St2), %%B1 = B0, St3 = St2, %Null second pass @@ -715,15 +715,15 @@ force_variable(Ke, St0) -> %% handling. pattern(#c_var{anno=A,name=V}, _Isub, Osub, St0) -> - case sets:is_element(V, St0#kern.ds) of + case cerl_sets:is_element(V, St0#kern.ds) of true -> {New,St1} = new_var_name(St0), {#k_var{anno=A,name=New}, set_vsub(V, New, Osub), - St1#kern{ds=sets:add_element(New, St1#kern.ds)}}; + St1#kern{ds=cerl_sets:add_element(New, St1#kern.ds)}}; false -> {#k_var{anno=A,name=V},Osub, - St0#kern{ds=sets:add_element(V, St0#kern.ds)}} + St0#kern{ds=cerl_sets:add_element(V, St0#kern.ds)}} end; pattern(#c_literal{anno=A,val=Val}, _Isub, Osub, St) -> {#k_literal{anno=A,val=Val},Osub,St}; @@ -897,7 +897,7 @@ new_vars(0, St, Vs) -> {Vs,St}. make_vars(Vs) -> [ #k_var{name=V} || V <- Vs ]. add_var_def(V, St) -> - St#kern{ds=sets:add_element(V#k_var.name, St#kern.ds)}. + St#kern{ds=cerl_sets:add_element(V#k_var.name, St#kern.ds)}. %%add_vars_def(Vs, St) -> %% Ds = foldl(fun (#k_var{name=V}, Ds) -> add_element(V, Ds) end, diff --git a/lib/compiler/src/v3_life.erl b/lib/compiler/src/v3_life.erl index 4b1f1c3f71..ee0565efb6 100644 --- a/lib/compiler/src/v3_life.erl +++ b/lib/compiler/src/v3_life.erl @@ -411,7 +411,7 @@ is_gc_bif(Bif, Arity) -> %% must be sorted. init_vars(Vs) -> - sort([{V,0,0} || {var,V} <- Vs]). + vdb_new(Vs). new_vars([], _, Vdb) -> Vdb; new_vars([V], I, Vdb) -> vdb_store_new(V, {V,I,I}, Vdb); @@ -430,6 +430,16 @@ use_vars(Vs, I, Vdb) -> vdb_update_vars(Vs, Vdb, I). add_var(V, F, L, Vdb) -> vdb_store_new(V, {V,F,L}, Vdb). +%% is_in_guard() -> true|false. + +is_in_guard() -> + get(guard_refc) > 0. + +%% vdb + +vdb_new(Vs) -> + sort([{V,0,0} || {var,V} <- Vs]). + vdb_find(V, Vdb) -> case lists:keyfind(V, 1, Vdb) of false -> error; @@ -471,8 +481,3 @@ vdb_sub(Min, Max, Vdb) -> [ if L >= Max -> {V,F,locked}; true -> Vd end || {V,F,L}=Vd <- Vdb, F < Min, L >= Min ]. - -%% is_in_guard() -> true|false. - -is_in_guard() -> - get(guard_refc) > 0. diff --git a/lib/compiler/test/warnings_SUITE.erl b/lib/compiler/test/warnings_SUITE.erl index f6ba75577d..5742b7e6cf 100644 --- a/lib/compiler/test/warnings_SUITE.erl +++ b/lib/compiler/test/warnings_SUITE.erl @@ -38,7 +38,8 @@ -export([pattern/1,pattern2/1,pattern3/1,pattern4/1, guard/1,bad_arith/1,bool_cases/1,bad_apply/1, files/1,effect/1,bin_opt_info/1,bin_construction/1, - comprehensions/1,maps/1,redundant_boolean_clauses/1, + comprehensions/1,maps/1,maps_bin_opt_info/1, + redundant_boolean_clauses/1, latin1_fallback/1,underscore/1,no_warnings/1]). % Default timetrap timeout (set in init_per_testcase). @@ -64,6 +65,7 @@ groups() -> [pattern,pattern2,pattern3,pattern4,guard, bad_arith,bool_cases,bad_apply,files,effect, bin_opt_info,bin_construction,comprehensions,maps, + maps_bin_opt_info, redundant_boolean_clauses,latin1_fallback, underscore,no_warnings]}]. @@ -619,6 +621,19 @@ maps(Config) when is_list(Config) -> run(Config, Ts), ok. +maps_bin_opt_info(Config) when is_list(Config) -> + Ts = [{map_bsm, + <<" + t1(<<0:8,7:8,T/binary>>,#{val := I}=M) -> + t1(T, M#{val := I+1}); + t1(<<_:8>>,M) -> + M. + ">>, + [bin_opt_info], + {warnings,[{2,beam_bsm,bin_opt}]}}], + [] = run(Config, Ts), + ok. + redundant_boolean_clauses(Config) when is_list(Config) -> Ts = [{redundant_boolean_clauses, <<" diff --git a/lib/compiler/vsn.mk b/lib/compiler/vsn.mk index 05e682c893..69f71ba5dd 100644 --- a/lib/compiler/vsn.mk +++ b/lib/compiler/vsn.mk @@ -1 +1 @@ -COMPILER_VSN = 5.0.4 +COMPILER_VSN = 6.0 diff --git a/lib/cosNotification/src/cosNotification.app.src b/lib/cosNotification/src/cosNotification.app.src index 09bf8f01fc..52ce164d46 100644 --- a/lib/cosNotification/src/cosNotification.app.src +++ b/lib/cosNotification/src/cosNotification.app.src @@ -117,6 +117,6 @@ {applications, [orber, stdlib, kernel]}, {env, []}, {mod, {cosNotificationApp, []}}, - {runtime_dependencies, ["stdlib-2.0","orber-3.6.27","kernel-3.0","erts-7.0", + {runtime_dependencies, ["stdlib-2.5","orber-3.6.27","kernel-3.0","erts-7.0", "cosTime-1.1.14","cosEvent-2.1.15"]} ]}. diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index 22c430bcd3..adacdcbc73 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -3749,7 +3749,7 @@ out: static ERL_NIF_TERM ec_key_generate(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { #if defined(HAVE_EC) - EC_KEY *key; + EC_KEY *key = NULL; const EC_GROUP *group; const EC_POINT *public_key; ERL_NIF_TERM priv_key; diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index 72944eea8e..ff7af1f2c1 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -1884,8 +1884,9 @@ dss_params() -> 18320614775012672475365915366944922415598782131828709277168615511695849821411624805195787607930033958243224786899641459701930253094446221381818858674389863050420226114787005820357372837321561754462061849169568607689530279303056075793886577588606958623645901271866346406773590024901668622321064384483571751669]. ec_key_named() -> - {D2_pub, D2_priv} = crypto:generate_key(ecdh, sect113r2), - {[D2_priv, sect113r2], [D2_pub, sect113r2]}. + Curve = secp112r2, + {D2_pub, D2_priv} = crypto:generate_key(ecdh, Curve), + {[D2_priv, Curve], [D2_pub, Curve]}. ec_msg() -> <<99,234,6,64,190,237,201,99,80,248,58,40,70,45,149,218,5,246,242,63>>. diff --git a/lib/crypto/test/old_crypto_SUITE.erl b/lib/crypto/test/old_crypto_SUITE.erl index 040edbf092..80306927c5 100644 --- a/lib/crypto/test/old_crypto_SUITE.erl +++ b/lib/crypto/test/old_crypto_SUITE.erl @@ -1887,9 +1887,9 @@ ec(Config) when is_list(Config) -> ec_do() -> %% test for a name curve - {D2_pub, D2_priv} = crypto:generate_key(ecdh, sect113r2), - PrivECDH = [D2_priv, sect113r2], - PubECDH = [D2_pub, sect113r2], + {D2_pub, D2_priv} = crypto:generate_key(ecdh, secp112r2), + PrivECDH = [D2_priv, secp112r2], + PubECDH = [D2_pub, secp112r2], %%TODO: find a published test case for a EC key %% test for a full specified curve and public key, diff --git a/lib/crypto/vsn.mk b/lib/crypto/vsn.mk index 8489b59562..55b1b3e8c4 100644 --- a/lib/crypto/vsn.mk +++ b/lib/crypto/vsn.mk @@ -1 +1 @@ -CRYPTO_VSN = 3.5 +CRYPTO_VSN = 3.6 diff --git a/lib/debugger/src/debugger.app.src b/lib/debugger/src/debugger.app.src index f102385d39..a013c5c11f 100644 --- a/lib/debugger/src/debugger.app.src +++ b/lib/debugger/src/debugger.app.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -47,5 +47,5 @@ ]}, {registered, [dbg_iserver, dbg_wx_mon, dbg_wx_winman]}, {applications, [kernel, stdlib]}, - {runtime_dependencies, ["wx-1.2","stdlib-2.0","kernel-3.0","erts-6.0", + {runtime_dependencies, ["wx-1.2","stdlib-2.5","kernel-3.0","erts-6.0", "compiler-5.0"]}]}. diff --git a/lib/debugger/test/map_SUITE.erl b/lib/debugger/test/map_SUITE.erl index 12fdd184b8..74847e161f 100644 --- a/lib/debugger/test/map_SUITE.erl +++ b/lib/debugger/test/map_SUITE.erl @@ -1308,7 +1308,7 @@ t_guard_receive(Config) when is_list(Config) -> done = call(Pid, done), ok. --define(t_guard_receive_large_procs, 150). +-define(t_guard_receive_large_procs, 50). t_guard_receive_large(Config) when is_list(Config) -> M = lists:foldl(fun(_,#{procs := Ps } = M) -> @@ -1326,7 +1326,7 @@ guard_receive_large_loop(M) -> receive #{pid := Pid, msg := hello} -> case M of - #{done := Count, procs := #{Pid := 150}} -> + #{done := Count, procs := #{Pid := 15}} -> Pid ! {self(), done}, guard_receive_large_loop(M#{done := Count + 1}); #{procs := #{Pid := Count} = Ps} -> diff --git a/lib/debugger/vsn.mk b/lib/debugger/vsn.mk index b82f0f4e37..b6fd4e8e44 100644 --- a/lib/debugger/vsn.mk +++ b/lib/debugger/vsn.mk @@ -1 +1 @@ -DEBUGGER_VSN = 4.0.3 +DEBUGGER_VSN = 4.1 diff --git a/lib/dialyzer/doc/src/dialyzer.xml b/lib/dialyzer/doc/src/dialyzer.xml index 2a8bf6edcc..fc076c24a6 100644 --- a/lib/dialyzer/doc/src/dialyzer.xml +++ b/lib/dialyzer/doc/src/dialyzer.xml @@ -70,7 +70,7 @@ [--build_plt] [--add_to_plt] [--remove_from_plt] [--check_plt] [--no_check_plt] [--plt_info] [--get_warnings] [--dump_callgraph file] [--no_native] [--fullpath] - [--statistics]</code> + [--statistics] [--no_native_cache]</code> <p>Options:</p> <taglist> <tag><c><![CDATA[files_or_dirs]]></c> (for backwards compatibility also @@ -198,6 +198,11 @@ heuristically performs when dialyzing many files; this avoids the compilation time but it may result in (much) longer analysis time.</item> + <tag><c><![CDATA[--no_native_cache]]></c></tag> + <item>By default, Dialyzer caches the results of native compilation in the + <c>$XDG_CACHE_HOME/erlang/dialyzer_hipe_cache</c> directory. + <c>XDG_CACHE_HOME</c> defaults to <c>$HOME/.cache</c>. + Use this option to disable caching.</item> <tag><c><![CDATA[--fullpath]]></c></tag> <item>Display the full path names of files for which warnings are emitted.</item> <tag><c><![CDATA[--gui]]></c></tag> @@ -368,6 +373,7 @@ Option :: {files, [Filename :: string()]} | {include_dirs, [DirName :: string()]} | {output_file, FileName :: string()} | {output_plt, FileName :: string()} + | {check_plt, boolean()}, | {analysis_type, 'succ_typings' | 'plt_add' | 'plt_build' | diff --git a/lib/dialyzer/src/dialyzer.app.src b/lib/dialyzer/src/dialyzer.app.src index 7b2e1d4a9d..b6b9173a84 100644 --- a/lib/dialyzer/src/dialyzer.app.src +++ b/lib/dialyzer/src/dialyzer.app.src @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2014. All Rights Reserved. +%% Copyright Ericsson AB 2006-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -45,6 +45,6 @@ {registered, []}, {applications, [compiler, gs, hipe, kernel, stdlib, wx]}, {env, []}, - {runtime_dependencies, ["wx-1.2","syntax_tools-1.6.14","stdlib-2.0", + {runtime_dependencies, ["wx-1.2","syntax_tools-1.6.14","stdlib-2.5", "kernel-3.0","hipe-3.10.3","erts-7.0", "compiler-5.0"]}]}. diff --git a/lib/dialyzer/src/dialyzer.erl b/lib/dialyzer/src/dialyzer.erl index c9e7da9ef0..c8537e3bd8 100644 --- a/lib/dialyzer/src/dialyzer.erl +++ b/lib/dialyzer/src/dialyzer.erl @@ -2,7 +2,7 @@ %%----------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2014. All Rights Reserved. +%% Copyright Ericsson AB 2006-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -162,14 +162,7 @@ run(Opts) -> {error, Msg} -> throw({dialyzer_error, Msg}); OptsRecord -> - case OptsRecord#options.check_plt of - true -> - case cl_check_init(OptsRecord) of - {ok, ?RET_NOTHING_SUSPICIOUS} -> ok; - {error, ErrorMsg1} -> throw({dialyzer_error, ErrorMsg1}) - end; - false -> ok - end, + ok = check_init(OptsRecord), case dialyzer_cl:start(OptsRecord) of {?RET_DISCREPANCIES, Warnings} -> Warnings; {?RET_NOTHING_SUSPICIOUS, _} -> [] @@ -179,6 +172,16 @@ run(Opts) -> erlang:error({dialyzer_error, lists:flatten(ErrorMsg)}) end. +check_init(#options{analysis_type = plt_check}) -> + ok; +check_init(#options{check_plt = true} = OptsRecord) -> + case cl_check_init(OptsRecord) of + {ok, _} -> ok; + {error, Msg} -> throw({dialyzer_error, Msg}) + end; +check_init(#options{check_plt = false}) -> + ok. + internal_gui(OptsRecord) -> F = fun() -> dialyzer_gui_wx:start(OptsRecord), @@ -199,17 +202,13 @@ gui(Opts) -> throw({dialyzer_error, Msg}); OptsRecord -> ok = check_gui_options(OptsRecord), - case cl_check_init(OptsRecord) of - {ok, ?RET_NOTHING_SUSPICIOUS} -> - F = fun() -> - dialyzer_gui_wx:start(OptsRecord) - end, - case doit(F) of - {ok, _} -> ok; - {error, Msg} -> throw({dialyzer_error, Msg}) - end; - {error, ErrorMsg1} -> - throw({dialyzer_error, ErrorMsg1}) + ok = check_init(OptsRecord), + F = fun() -> + dialyzer_gui_wx:start(OptsRecord) + end, + case doit(F) of + {ok, _} -> ok; + {error, Msg} -> throw({dialyzer_error, Msg}) end catch throw:{dialyzer_error, ErrorMsg} -> diff --git a/lib/dialyzer/src/dialyzer_cl.erl b/lib/dialyzer/src/dialyzer_cl.erl index 4386a8d52a..55fcd15641 100644 --- a/lib/dialyzer/src/dialyzer_cl.erl +++ b/lib/dialyzer/src/dialyzer_cl.erl @@ -512,32 +512,82 @@ hipe_compile(Files, #options{erlang_mode = ErlangMode} = Options) -> dialyzer_worker], report_native_comp(Options), {T1, _} = statistics(wall_clock), - native_compile(Mods), + Cache = (get(dialyzer_options_native_cache) =/= false), + native_compile(Mods, Cache), {T2, _} = statistics(wall_clock), report_elapsed_time(T1, T2, Options) end end. -native_compile(Mods) -> +native_compile(Mods, Cache) -> case dialyzer_utils:parallelism() > ?MIN_PARALLELISM of true -> Parent = self(), - Pids = [spawn(fun () -> Parent ! {self(), hc(M)} end) || M <- Mods], + Pids = [spawn(fun () -> Parent ! {self(), hc(M, Cache)} end) || M <- Mods], lists:foreach(fun (Pid) -> receive {Pid, Res} -> Res end end, Pids); false -> - lists:foreach(fun (Mod) -> hc(Mod) end, Mods) + lists:foreach(fun (Mod) -> hc(Mod, Cache) end, Mods) end. -hc(Mod) -> +hc(Mod, Cache) -> {module, Mod} = code:ensure_loaded(Mod), case code:is_module_native(Mod) of true -> ok; false -> %% io:format(" ~w", [Mod]), - {ok, Mod} = hipe:c(Mod), - ok + case Cache of + false -> + {ok, Mod} = hipe:c(Mod), + ok; + true -> + hc_cache(Mod) + end end. +hc_cache(Mod) -> + CacheBase = cache_base_dir(), + %% Use HiPE architecture and version in directory name, to avoid + %% clashes between incompatible binaries. + HipeArchVersion = + lists:concat( + [erlang:system_info(hipe_architecture), "-", + hipe:version(), "-", + hipe_bifs:system_crc()]), + CacheDir = filename:join(CacheBase, HipeArchVersion), + OrigBeamFile = code:which(Mod), + {ok, {Mod, <<Checksum:128>>}} = beam_lib:md5(OrigBeamFile), + CachedBeamFile = filename:join(CacheDir, lists:concat([Mod, "-", Checksum, ".beam"])), + ok = filelib:ensure_dir(CachedBeamFile), + ModBin = + case filelib:is_file(CachedBeamFile) of + true -> + {ok, BinFromFile} = file:read_file(CachedBeamFile), + BinFromFile; + false -> + {ok, Mod, CompiledBin} = compile:file(OrigBeamFile, [from_beam, native, binary]), + ok = file:write_file(CachedBeamFile, CompiledBin), + CompiledBin + end, + code:unstick_dir(filename:dirname(OrigBeamFile)), + {module, Mod} = code:load_binary(Mod, CachedBeamFile, ModBin), + true = code:is_module_native(Mod), + ok. + +cache_base_dir() -> + %% http://standards.freedesktop.org/basedir-spec/basedir-spec-0.7.html + %% If XDG_CACHE_HOME is set to an absolute path, use it as base. + XdgCacheHome = os:getenv("XDG_CACHE_HOME"), + CacheHome = + case is_list(XdgCacheHome) andalso filename:pathtype(XdgCacheHome) =:= absolute of + true -> + XdgCacheHome; + false -> + %% Otherwise, the default is $HOME/.cache. + {ok, [[Home]]} = init:get_argument(home), + filename:join(Home, ".cache") + end, + filename:join([CacheHome, "dialyzer_hipe_cache"]). + new_state() -> #cl_state{}. diff --git a/lib/dialyzer/src/dialyzer_cl_parse.erl b/lib/dialyzer/src/dialyzer_cl_parse.erl index 21fc424a1b..fae88ed6e8 100644 --- a/lib/dialyzer/src/dialyzer_cl_parse.erl +++ b/lib/dialyzer/src/dialyzer_cl_parse.erl @@ -75,6 +75,9 @@ cl(["-nn"|T]) -> cl(["--no_native"|T]) -> put(dialyzer_options_native, false), cl(T); +cl(["--no_native_cache"|T]) -> + put(dialyzer_options_native_cache, false), + cl(T); cl(["--plt_info"|T]) -> put(dialyzer_options_analysis_type, plt_info), cl(T); @@ -363,7 +366,7 @@ help_message() -> [--build_plt] [--add_to_plt] [--remove_from_plt] [--check_plt] [--no_check_plt] [--plt_info] [--get_warnings] [--dump_callgraph file] [--no_native] [--fullpath] - [--statistics] + [--statistics] [--no_native_cache] Options: files_or_dirs (for backwards compatibility also as: -c files_or_dirs) Use Dialyzer from the command line to detect defects in the @@ -468,6 +471,11 @@ Options: Bypass the native code compilation of some key files that Dialyzer heuristically performs when dialyzing many files; this avoids the compilation time but it may result in (much) longer analysis time. + --no_native_cache + By default, Dialyzer caches the results of native compilation in the + $XDG_CACHE_HOME/erlang/dialyzer_hipe_cache directory. + XDG_CACHE_HOME defaults to $HOME/.cache. Use this option to disable + caching. --fullpath Display the full path names of files for which warnings are emitted. --gui diff --git a/lib/dialyzer/test/plt_SUITE.erl b/lib/dialyzer/test/plt_SUITE.erl index ef4cdc57f0..ecbac14e5d 100644 --- a/lib/dialyzer/test/plt_SUITE.erl +++ b/lib/dialyzer/test/plt_SUITE.erl @@ -6,12 +6,13 @@ -include_lib("common_test/include/ct.hrl"). -include("dialyzer_test_constants.hrl"). --export([suite/0, all/0, build_plt/1, beam_tests/1, update_plt/1]). +-export([suite/0, all/0, build_plt/1, beam_tests/1, update_plt/1, + run_plt_check/1, run_succ_typings/1]). suite() -> [{timetrap, ?plt_timeout}]. -all() -> [build_plt, beam_tests, update_plt]. +all() -> [build_plt, beam_tests, update_plt, run_plt_check, run_succ_typings]. build_plt(Config) -> OutDir = ?config(priv_dir, Config), @@ -37,14 +38,76 @@ beam_tests(Config) when is_list(Config) -> ">>, Opts = [no_auto_import], {ok, BeamFile} = compile(Config, Prog, no_auto_import, Opts), - [] = run_dialyzer([BeamFile]), + [] = run_dialyzer(plt_build, [BeamFile], []), ok. -run_dialyzer(Files) -> - dialyzer:run([{analysis_type, plt_build}, - {files, Files}, - {from, byte_code}, - {check_plt, false}]). +run_plt_check(Config) when is_list(Config) -> + Mod1 = <<" + -module(run_plt_check1). + ">>, + + Mod2A = <<" + -module(run_plt_check2). + ">>, + + {ok, BeamFile1} = compile(Config, Mod1, run_plt_check1, []), + {ok, BeamFile2} = compile(Config, Mod2A, run_plt_check2, []), + [] = run_dialyzer(plt_build, [BeamFile1, BeamFile2], []), + + Mod2B = <<" + -module(run_plt_check2). + + -export([call/1]). + + call(X) -> run_plt_check1:call(X). + ">>, + + {ok, BeamFile2} = compile(Config, Mod2B, run_plt_check2, []), + + % callgraph warning as run_plt_check2:call/1 makes a call to unexported + % function run_plt_check1:call/1. + [_] = run_dialyzer(plt_check, [], []), + + ok. + +run_succ_typings(Config) when is_list(Config) -> + Mod1A = <<" + -module(run_succ_typings1). + + -export([call/0]). + + call() -> a. + ">>, + + {ok, BeamFile1} = compile(Config, Mod1A, run_succ_typings1, []), + [] = run_dialyzer(plt_build, [BeamFile1], []), + + Mod1B = <<" + -module(run_succ_typings1). + + -export([call/0]). + + call() -> b. + ">>, + + Mod2 = <<" + -module(run_succ_typings2). + + -export([call/0]). + + -spec call() -> b. + call() -> run_succ_typings1:call(). + ">>, + + {ok, BeamFile1} = compile(Config, Mod1B, run_succ_typings1, []), + {ok, BeamFile2} = compile(Config, Mod2, run_succ_typings2, []), + % contract types warning as run_succ_typings2:call/0 makes a call to + % run_succ_typings1:call/0, which returns a (not b) in the PLT. + [_] = run_dialyzer(succ_typings, [BeamFile2], [{check_plt, false}]), + % warning not returned as run_succ_typings1 is updated in the PLT. + [] = run_dialyzer(succ_typings, [BeamFile2], [{check_plt, true}]), + + ok. %%% [James Fish:] %%% If a function is removed from a module and the module has previously @@ -103,3 +166,9 @@ compile(Config, Prog, Module, CompileOpts) -> Opts = [{outdir, PrivDir}, debug_info | CompileOpts], {ok, Module} = compile:file(Filename, Opts), {ok, filename:join([PrivDir, lists:concat([Module, ".beam"])])}. + +run_dialyzer(Analysis, Files, Opts) -> + dialyzer:run([{analysis_type, Analysis}, + {files, Files}, + {from, byte_code} | + Opts]). diff --git a/lib/dialyzer/vsn.mk b/lib/dialyzer/vsn.mk index 527afaf4ef..48e0830109 100644 --- a/lib/dialyzer/vsn.mk +++ b/lib/dialyzer/vsn.mk @@ -1 +1 @@ -DIALYZER_VSN = 2.7.4 +DIALYZER_VSN = 2.8 diff --git a/lib/diameter/doc/src/notes.xml b/lib/diameter/doc/src/notes.xml index 6931788c83..c5df63a7f0 100644 --- a/lib/diameter/doc/src/notes.xml +++ b/lib/diameter/doc/src/notes.xml @@ -42,6 +42,36 @@ first.</p> <!-- ===================================================================== --> +<section><title>diameter 1.9.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix broken relay counters.</p> + <p> + OTP-12654 in OTP 17.5.3 broke counters in the case of + answer messages received in the relay application. + Counters were accumulated as unknown messages or + no_result_code instead of as relayed messages on the + intended Result-Code and 'Experimental-Result' tuples.</p> + <p> + Own Id: OTP-12741</p> + </item> + <item> + <p> + Fix diameter_sctp listener race.</p> + <p> + An oversight in OTP-12428 made it possible to start a + transport process that could not establish associations.</p> + <p> + Own Id: OTP-12744</p> + </item> + </list> + </section> + +</section> + <section><title>diameter 1.9.1</title> <section><title>Known Bugs and Problems</title> @@ -65,7 +95,7 @@ first.</p> received in an answer not setting the E-bit. The correct AVP is now extracted from the incoming message.</p> <p> - Own Id: OTP-12654 Aux Id: seq12851 </p> + Own Id: OTP-12654</p> </item> <item> <p> diff --git a/lib/diameter/src/base/diameter_traffic.erl b/lib/diameter/src/base/diameter_traffic.erl index ffd2c0afa2..eb4bbae931 100644 --- a/lib/diameter/src/base/diameter_traffic.erl +++ b/lib/diameter/src/base/diameter_traffic.erl @@ -131,11 +131,11 @@ peer_down(TPid) -> %% incr/4 %% --------------------------------------------------------------------------- -incr(Dir, #diameter_packet{header = H}, TPid, Dict) -> - incr(Dir, H, TPid, Dict); +incr(Dir, #diameter_packet{header = H}, TPid, AppDict) -> + incr(Dir, H, TPid, AppDict); -incr(Dir, #diameter_header{} = H, TPid, Dict) -> - incr(TPid, {msg_id(H, Dict), Dir}). +incr(Dir, #diameter_header{} = H, TPid, AppDict) -> + incr(TPid, {msg_id(H, AppDict), Dir}). %% --------------------------------------------------------------------------- %% incr_error/4 @@ -143,26 +143,26 @@ incr(Dir, #diameter_header{} = H, TPid, Dict) -> %% Identify messages using the application dictionary, not the encode %% dictionary, which may differ in the case of answer-message. -incr_error(Dir, T, Pid, {_Dict, AppDict}) -> +incr_error(Dir, T, Pid, {_MsgDict, AppDict}) -> incr_error(Dir, T, Pid, AppDict); %% Decoded message without errors. incr_error(recv, #diameter_packet{errors = []}, _, _) -> ok; -incr_error(recv = D, #diameter_packet{header = H}, TPid, Dict) -> - incr_error(D, H, TPid, Dict); +incr_error(recv = D, #diameter_packet{header = H}, TPid, AppDict) -> + incr_error(D, H, TPid, AppDict); %% Encoded message with errors and an identifiable header ... -incr_error(send = D, {_, _, #diameter_header{} = H}, TPid, Dict) -> - incr_error(D, H, TPid, Dict); +incr_error(send = D, {_, _, #diameter_header{} = H}, TPid, AppDict) -> + incr_error(D, H, TPid, AppDict); %% ... or not. incr_error(send = D, {_,_}, TPid, _) -> incr_error(D, unknown, TPid); -incr_error(Dir, #diameter_header{} = H, TPid, Dict) -> - incr_error(Dir, msg_id(H, Dict), TPid); +incr_error(Dir, #diameter_header{} = H, TPid, AppDict) -> + incr_error(Dir, msg_id(H, AppDict), TPid); incr_error(Dir, Id, TPid, _) -> incr_error(Dir, Id, TPid). @@ -179,18 +179,20 @@ incr_error(Dir, Id, TPid) -> | Reason when Pkt :: #diameter_packet{}, TPid :: pid(), - DictT :: module() | {module(), module(), module()}, + DictT :: module() | {MsgDict :: module(), + AppDict :: module(), + CommonDict:: module()}, Counter :: {'Result-Code', integer()} | {'Experimental-Result', integer(), integer()}, Reason :: atom(). -incr_rc(Dir, Pkt, TPid, {Dict, _, _} = DictT) -> +incr_rc(Dir, Pkt, TPid, {_, AppDict, _} = DictT) -> try incr_result(Dir, Pkt, TPid, DictT) catch exit: {E,_} when E == no_result_code; E == invalid_error_bit -> - incr(TPid, {msg_id(Pkt#diameter_packet.header, Dict), Dir, E}), + incr(TPid, {msg_id(Pkt#diameter_packet.header, AppDict), Dir, E}), E end; @@ -259,7 +261,8 @@ recv(false, #request{ref = Ref, handler = Pid} = Req, _, Pkt, Dict0, _) -> %% any others are discarded. %% ... or not. -recv(false, false, _, _, _, _) -> +recv(false, false, TPid, _, _, _) -> + incr(TPid, {{unknown, 0}, recv, discarded}), ok. %% spawn_request/4 @@ -307,14 +310,14 @@ recv_request(TPid, Pkt, Dict0, RecvData) -> %% from old code %% recv_R/5 -recv_R({#diameter_app{id = Id, dictionary = Dict} = App, Caps}, +recv_R({#diameter_app{id = Id, dictionary = AppDict} = App, Caps}, TPid, Pkt0, Dict0, RecvData) -> - incr(recv, Pkt0, TPid, Dict), - Pkt = errors(Id, diameter_codec:decode(Id, Dict, Pkt0)), - incr_error(recv, Pkt, TPid, Dict), + incr(recv, Pkt0, TPid, AppDict), + Pkt = errors(Id, diameter_codec:decode(Id, AppDict, Pkt0)), + incr_error(recv, Pkt, TPid, AppDict), {Caps, Pkt, App, recv_R(App, TPid, Dict0, Caps, RecvData, Pkt)}; %% Note that the decode is different depending on whether or not Id is %% ?APP_ID_RELAY. @@ -522,14 +525,17 @@ send_A(_, _, _, _) -> %% send_A/6 -send_A(T, TPid, DictT, ReqPkt, EvalPktFs, EvalFs) -> - reply(T, TPid, DictT, EvalPktFs, ReqPkt), +send_A(T, TPid, {AppDict, Dict0} = DictT0, ReqPkt, EvalPktFs, EvalFs) -> + {MsgDict, Pkt} = reply(T, TPid, DictT0, EvalPktFs, ReqPkt), + incr(send, Pkt, TPid, AppDict), + incr_rc(send, Pkt, TPid, {MsgDict, AppDict, Dict0}), %% count outgoing + send(TPid, Pkt), lists:foreach(fun diameter_lib:eval/1, EvalFs). %% answer/6 answer({reply, Ans}, _Caps, _Pkt, App, Dict0, _RecvData) -> - {dict(App#diameter_app.dictionary, Dict0, Ans), Ans}; + {msg_dict(App#diameter_app.dictionary, Dict0, Ans), Ans}; answer({call, Opts}, Caps, Pkt, App, Dict0, RecvData) -> #diameter_caps{origin_host = {OH,_}} @@ -552,27 +558,37 @@ answer({answer_message, RC} = T, Caps, Pkt, App, Dict0, _RecvData) -> orelse ?ERROR({invalid_return, T, handle_request, App}), answer_message(RC, Caps, Dict0, Pkt). -%% dict/3 +%% msg_dict/3 +%% +%% Return the dictionary defining the message grammar in question: the +%% application dictionary or the common dictionary. + +msg_dict(AppDict, Dict0, [Msg]) + when is_list(Msg); + is_tuple(Msg) -> + msg_dict(AppDict, Dict0, Msg); -%% An incoming answer, not yet decoded. -dict(Dict, Dict0, #diameter_packet{header - = #diameter_header{is_request = false, - is_error = E}, - msg = undefined}) -> - if E -> Dict0; true -> Dict end; +msg_dict(AppDict, Dict0, Msg) -> + choose(is_answer_message(Msg, Dict0), Dict0, AppDict). -dict(Dict, Dict0, [Msg]) -> - dict(Dict, Dict0, Msg); +%% Incoming, not yet decoded. +is_answer_message(#diameter_packet{header = #diameter_header{} = H, + msg = undefined}, + Dict0) -> + is_answer_message([H], Dict0); -dict(Dict, Dict0, #diameter_packet{msg = Msg}) -> - dict(Dict, Dict0, Msg); +is_answer_message(#diameter_packet{msg = Msg}, Dict0) -> + is_answer_message(Msg, Dict0); -dict(Dict, Dict0, Msg) -> - choose(is_answer_message(Msg, Dict0), Dict0, Dict). +%% Message sent as a header/avps list. +is_answer_message([#diameter_header{is_request = R, is_error = E} | _], _) -> + E andalso not R; +%% Message sent as a tagged avp/value list. is_answer_message([Name | _], _) -> Name == 'answer-message'; +%% Message sent as a record. is_answer_message(Rec, Dict) -> try 'answer-message' == Dict:rec2msg(element(1,Rec)) @@ -642,7 +658,7 @@ resend(false, %% %% Relay a reply to a relayed request. -%% Answer from the peer: reset the hop by hop identifier and send. +%% Answer from the peer: reset the hop by hop identifier. resend(#diameter_packet{bin = B} = Pkt, _Caps, @@ -681,13 +697,13 @@ is_loop(Code, Vid, OH, Dict0, Avps) -> %% reply/5 %% Local answer ... -reply({Dict, Ans}, TPid, {AppDict, Dict0}, Fs, ReqPkt) -> - local(Ans, TPid, {Dict, AppDict, Dict0}, Fs, ReqPkt); +reply({MsgDict, Ans}, TPid, {AppDict, Dict0}, Fs, ReqPkt) -> + local(Ans, TPid, {MsgDict, AppDict, Dict0}, Fs, ReqPkt); %% ... or relayed. -reply(#diameter_packet{} = Pkt, TPid, _Dict0, Fs, _ReqPkt) -> +reply(#diameter_packet{} = Pkt, _TPid, {AppDict, Dict0}, Fs, _ReqPkt) -> eval_packet(Pkt, Fs), - send(TPid, Pkt). + {msg_dict(AppDict, Dict0, Pkt), Pkt}. %% local/5 %% @@ -700,14 +716,12 @@ local([Msg], TPid, DictT, Fs, ReqPkt) is_tuple(Msg) -> local(Msg, TPid, DictT, Fs, ReqPkt#diameter_packet{errors = []}); -local(Msg, TPid, {Dict, AppDict, Dict0} = DictT, Fs, ReqPkt) -> - Pkt = encode({Dict, AppDict}, +local(Msg, TPid, {MsgDict, AppDict, Dict0}, Fs, ReqPkt) -> + Pkt = encode({MsgDict, AppDict}, TPid, - reset(make_answer_packet(Msg, ReqPkt), Dict, Dict0), + reset(make_answer_packet(Msg, ReqPkt), MsgDict, Dict0), Fs), - incr(send, Pkt, TPid, AppDict), - incr_rc(send, Pkt, TPid, DictT), %% count outgoing - send(TPid, Pkt). + {MsgDict, Pkt}. %% reset/3 @@ -1067,54 +1081,75 @@ find_avp(Code, VId, [_ | Avps]) -> %% Increment a stats counter for result codes in incoming and outgoing %% answers. +%% Message sent as a header/avps list. +incr_result(send = Dir, + #diameter_packet{msg = [#diameter_header{} = H | _]} + = Pkt, + TPid, + DictT) -> + incr_res(Dir, Pkt#diameter_packet{header = H}, TPid, DictT); + %% Outgoing message as binary: don't count. (Sending binaries is only %% partially supported.) -incr_result(_, #diameter_packet{msg = undefined = No}, _, _) -> +incr_result(send, #diameter_packet{header = undefined = No}, _, _) -> No; %% Incoming or outgoing. Outgoing with encode errors never gets here %% since encode fails. -incr_result(Dir, Pkt, TPid, {Dict, AppDict, Dict0}) -> - #diameter_packet{header = #diameter_header{is_error = E} - = Hdr, - errors = Es} - = Pkt, +incr_result(Dir, Pkt, TPid, DictT) -> + incr_res(Dir, Pkt, TPid, DictT). + +incr_res(Dir, + #diameter_packet{header = #diameter_header{is_error = E} + = Hdr, + errors = Es} + = Pkt, + TPid, + DictT) -> + {MsgDict, AppDict, Dict0} = DictT, Id = msg_id(Hdr, AppDict), + %% Could be {relay, 0}, in which case the R-bit is redundant since + %% only answers are being counted. Let it be however, so that the + %% same tuple is in both send/recv and result code counters. %% Count incoming decode errors. recv /= Dir orelse [] == Es orelse incr_error(Dir, Id, TPid, AppDict), %% Exit on a missing result code. - T = rc_counter(Dict, Dir, Pkt), - T == false andalso ?LOGX(no_result_code, {Dict, Dir, Hdr}), + T = rc_counter(MsgDict, Dir, Pkt), + T == false andalso ?LOGX(no_result_code, {MsgDict, Dir, Hdr}), {Ctr, RC, Avp} = T, %% Or on an inappropriate value. is_result(RC, E, Dict0) - orelse ?LOGX(invalid_error_bit, {Dict, Dir, Hdr, Avp}), + orelse ?LOGX(invalid_error_bit, {MsgDict, Dir, Hdr, Avp}), incr(TPid, {Id, Dir, Ctr}), Ctr. %% msg_id/2 -msg_id(#diameter_packet{header = H}, Dict) -> - msg_id(H, Dict); +msg_id(#diameter_packet{header = H}, AppDict) -> + msg_id(H, AppDict); %% Only count on known keys so as not to be vulnerable to attack: %% there are 2^32 (application ids) * 2^24 (command codes) = 2^56 %% pairs for an attacker to choose from. -msg_id(Hdr, Dict) -> +msg_id(Hdr, AppDict) -> {Aid, Code, R} = Id = diameter_codec:msg_id(Hdr), - if Aid == ?APP_ID_RELAY -> + case AppDict:id() of + ?APP_ID_RELAY -> {relay, R}; - true -> - choose(Aid /= Dict:id() orelse '' == Dict:msg_name(Code, 0 == R), - unknown, - Id) + A -> + unknown(A /= Aid orelse '' == AppDict:msg_name(Code, 0 == R), Id) end. +unknown(true, {_, _, R}) -> + {unknown, R}; +unknown(false, Id) -> + Id. + %% No E-bit: can't be 3xxx. is_result(RC, false, _Dict0) -> RC < 3000 orelse 4000 =< RC; @@ -1142,7 +1177,11 @@ incr(TPid, Counter) -> %% applications MUST include either one Result-Code AVP or one %% Experimental-Result AVP. -rc_counter(Dict, recv, #diameter_packet{header = H, avps = As}) -> +rc_counter(Dict, Dir, #diameter_packet{header = H, + avps = As, + msg = Msg}) + when Dir == recv; %% decoded incoming + Msg == undefined -> %% relayed outgoing rc_counter(Dict, [H|As]); rc_counter(Dict, _, #diameter_packet{msg = Msg}) -> @@ -1434,12 +1473,12 @@ fold_record(Rec, R) -> %% send_R/6 send_R(Pkt0, - {TPid, Caps, #diameter_app{dictionary = Dict} = App}, + {TPid, Caps, #diameter_app{dictionary = AppDict} = App}, Opts, {Pid, Ref}, SvcName, Fs) -> - Pkt = encode(Dict, TPid, Pkt0, Fs), + Pkt = encode(AppDict, TPid, Pkt0, Fs), #options{timeout = Timeout} = Opts, @@ -1452,7 +1491,7 @@ send_R(Pkt0, packet = Pkt0}, try - incr(send, Pkt, TPid, Dict), + incr(send, Pkt, TPid, AppDict), TRef = send_request(TPid, Pkt, Req, SvcName, Timeout), Pid ! Ref, %% tell caller a send has been attempted handle_answer(SvcName, @@ -1492,10 +1531,10 @@ handle_answer(SvcName, id = Id} = App, {answer, Req, Dict0, Pkt}) -> - Dict = dict(AppDict, Dict0, Pkt), - handle_A(errors(Id, diameter_codec:decode({Dict, AppDict}, Pkt)), + MsgDict = msg_dict(AppDict, Dict0, Pkt), + handle_A(errors(Id, diameter_codec:decode({MsgDict, AppDict}, Pkt)), SvcName, - Dict, + MsgDict, Dict0, App, Req). @@ -1765,19 +1804,19 @@ retransmit(T, {_, _, App}, _, _, _, _) -> ?ERROR({invalid_return, T, prepare_retransmit, App}). resend_request(Pkt0, - {TPid, Caps, #diameter_app{dictionary = Dict}}, + {TPid, Caps, #diameter_app{dictionary = AppDict}}, Req0, SvcName, Tmo, Fs) -> - Pkt = encode(Dict, TPid, Pkt0, Fs), + Pkt = encode(AppDict, TPid, Pkt0, Fs), Req = Req0#request{transport = TPid, packet = Pkt0, caps = Caps}, ?LOG(retransmission, Pkt#diameter_packet.header), - incr(TPid, {msg_id(Pkt, Dict), send, retransmission}), + incr(TPid, {msg_id(Pkt, AppDict), send, retransmission}), TRef = send_request(TPid, Pkt, Req, SvcName, Tmo), {TRef, Req}. @@ -1887,7 +1926,7 @@ get_avp(Dict, Name, [#diameter_header{} | Avps]) -> find_avp(Code, VId, Avps) of A -> - avp_decode(Dict, Name, ungroup(A)) + (avp_decode(Dict, Name, ungroup(A)))#diameter_avp{name = Name} catch error: _ -> undefined diff --git a/lib/diameter/src/diameter.appup.src b/lib/diameter/src/diameter.appup.src index 0ef0fd35a9..b89859ed24 100644 --- a/lib/diameter/src/diameter.appup.src +++ b/lib/diameter/src/diameter.appup.src @@ -65,11 +65,14 @@ {update, diameter_sup, supervisor}]}, {"1.9", [{load_module, diameter_codec}, %% 17.5 {load_module, diameter_traffic}, + {load_module, diameter_sctp}, {load_module, diameter_gen_base_rfc6733}, {load_module, diameter_gen_acct_rfc6733}, {load_module, diameter_gen_base_rfc3588}, {load_module, diameter_gen_base_accounting}, - {load_module, diameter_gen_relay}]} + {load_module, diameter_gen_relay}]}, + {"1.9.1", [{load_module, diameter_traffic}, %% 17.5.3 + {load_module, diameter_sctp}]} ], [ {"0.9", [{restart_application, diameter}]}, @@ -120,7 +123,10 @@ {load_module, diameter_gen_base_rfc3588}, {load_module, diameter_gen_acct_rfc6733}, {load_module, diameter_gen_base_rfc6733}, + {load_module, diameter_sctp}, {load_module, diameter_traffic}, - {load_module, diameter_codec}]} + {load_module, diameter_codec}]}, + {"1.9.1", [{load_module, diameter_sctp}, + {load_module, diameter_traffic}]} ] }. diff --git a/lib/diameter/src/transport/diameter_sctp.erl b/lib/diameter/src/transport/diameter_sctp.erl index 2c8d6f0a14..f80de0a816 100644 --- a/lib/diameter/src/transport/diameter_sctp.erl +++ b/lib/diameter/src/transport/diameter_sctp.erl @@ -223,9 +223,9 @@ init(T) -> i({listen, Ref, {Opts, Addrs}}) -> {[Matches], Rest} = proplists:split(Opts, [accept]), {LAs, Sock} = AS = open(Addrs, Rest, ?DEFAULT_PORT), - proc_lib:init_ack({ok, self(), LAs}), ok = gen_sctp:listen(Sock, true), true = diameter_reg:add_new({?MODULE, listener, {Ref, AS}}), + proc_lib:init_ack({ok, self(), LAs}), start_timer(#listener{ref = Ref, socket = Sock, accept = accept(Matches)}); diff --git a/lib/diameter/test/diameter_3xxx_SUITE.erl b/lib/diameter/test/diameter_3xxx_SUITE.erl index 44fc3a60aa..44cb0cc484 100644 --- a/lib/diameter/test/diameter_3xxx_SUITE.erl +++ b/lib/diameter/test/diameter_3xxx_SUITE.erl @@ -195,13 +195,13 @@ counters(_, _, _, _) -> stats(?CLIENT, E, rfc3588, L) when E == answer; E == answer_3xxx -> - [{{unknown,recv},2}, + [{{{unknown,0},recv},2}, {{{0,257,0},recv},1}, {{{0,257,1},send},1}, {{{0,275,0},recv},6}, {{{0,275,1},send},10}, - {{unknown,recv,{'Result-Code',3001}},1}, - {{unknown,recv,{'Result-Code',3007}},1}, + {{{unknown,0},recv,{'Result-Code',3001}},1}, + {{{unknown,0},recv,{'Result-Code',3007}},1}, {{{0,257,0},recv,{'Result-Code',2001}},1}, {{{0,275,0},recv,{'Result-Code',2001}},1}, {{{0,275,0},recv,{'Result-Code',3008}},2}, @@ -213,15 +213,15 @@ stats(?CLIENT, E, rfc3588, L) stats(?SERVER, E, rfc3588, L) when E == answer; E == answer_3xxx -> - [{{unknown,recv},1}, - {{unknown,send},2}, + [{{{unknown,0},send},2}, + {{{unknown,1},recv},1}, {{{0,257,0},send},1}, {{{0,257,1},recv},1}, {{{0,275,0},send},6}, {{{0,275,1},recv},8}, - {{unknown,recv,error},1}, - {{unknown,send,{'Result-Code',3001}},1}, - {{unknown,send,{'Result-Code',3007}},1}, + {{{unknown,0},send,{'Result-Code',3001}},1}, + {{{unknown,0},send,{'Result-Code',3007}},1}, + {{{unknown,1},recv,error},1}, {{{0,257,0},send,{'Result-Code',2001}},1}, {{{0,275,0},send,{'Result-Code',2001}},1}, {{{0,275,0},send,{'Result-Code',3008}},2}, @@ -232,13 +232,13 @@ stats(?SERVER, E, rfc3588, L) = L; stats(?CLIENT, answer, rfc6733, L) -> - [{{unknown,recv},2}, + [{{{unknown,0},recv},2}, {{{0,257,0},recv},1}, {{{0,257,1},send},1}, {{{0,275,0},recv},8}, {{{0,275,1},send},10}, - {{unknown,recv,{'Result-Code',3001}},1}, - {{unknown,recv,{'Result-Code',3007}},1}, + {{{unknown,0},recv,{'Result-Code',3001}},1}, + {{{unknown,0},recv,{'Result-Code',3007}},1}, {{{0,257,0},recv,{'Result-Code',2001}},1}, {{{0,275,0},recv,{'Result-Code',3008}},2}, {{{0,275,0},recv,{'Result-Code',3999}},1}, @@ -248,15 +248,15 @@ stats(?CLIENT, answer, rfc6733, L) -> = L; stats(?SERVER, answer, rfc6733, L) -> - [{{unknown,recv},1}, - {{unknown,send},2}, + [{{{unknown,0},send},2}, + {{{unknown,1},recv},1}, {{{0,257,0},send},1}, {{{0,257,1},recv},1}, {{{0,275,0},send},8}, {{{0,275,1},recv},8}, - {{unknown,recv,error},1}, - {{unknown,send,{'Result-Code',3001}},1}, - {{unknown,send,{'Result-Code',3007}},1}, + {{{unknown,0},send,{'Result-Code',3001}},1}, + {{{unknown,0},send,{'Result-Code',3007}},1}, + {{{unknown,1},recv,error},1}, {{{0,257,0},send,{'Result-Code',2001}},1}, {{{0,275,0},send,{'Result-Code',3008}},2}, {{{0,275,0},send,{'Result-Code',3999}},1}, @@ -267,13 +267,13 @@ stats(?SERVER, answer, rfc6733, L) -> = L; stats(?CLIENT, answer_3xxx, rfc6733, L) -> - [{{unknown,recv},2}, + [{{{unknown,0},recv},2}, {{{0,257,0},recv},1}, {{{0,257,1},send},1}, {{{0,275,0},recv},8}, {{{0,275,1},send},10}, - {{unknown,recv,{'Result-Code',3001}},1}, - {{unknown,recv,{'Result-Code',3007}},1}, + {{{unknown,0},recv,{'Result-Code',3001}},1}, + {{{unknown,0},recv,{'Result-Code',3007}},1}, {{{0,257,0},recv,{'Result-Code',2001}},1}, {{{0,275,0},recv,{'Result-Code',2001}},1}, {{{0,275,0},recv,{'Result-Code',3008}},2}, @@ -284,15 +284,15 @@ stats(?CLIENT, answer_3xxx, rfc6733, L) -> = L; stats(?SERVER, answer_3xxx, rfc6733, L) -> - [{{unknown,recv},1}, - {{unknown,send},2}, + [{{{unknown,0},send},2}, + {{{unknown,1},recv},1}, {{{0,257,0},send},1}, {{{0,257,1},recv},1}, {{{0,275,0},send},8}, {{{0,275,1},recv},8}, - {{unknown,recv,error},1}, - {{unknown,send,{'Result-Code',3001}},1}, - {{unknown,send,{'Result-Code',3007}},1}, + {{{unknown,0},send,{'Result-Code',3001}},1}, + {{{unknown,0},send,{'Result-Code',3007}},1}, + {{{unknown,1},recv,error},1}, {{{0,257,0},send,{'Result-Code',2001}},1}, {{{0,275,0},send,{'Result-Code',2001}},1}, {{{0,275,0},send,{'Result-Code',3008}},2}, @@ -304,12 +304,12 @@ stats(?SERVER, answer_3xxx, rfc6733, L) -> = L; stats(?CLIENT, callback, rfc3588, L) -> - [{{unknown,recv},1}, + [{{{unknown,0},recv},1}, {{{0,257,0},recv},1}, {{{0,257,1},send},1}, {{{0,275,0},recv},6}, {{{0,275,1},send},10}, - {{unknown,recv,{'Result-Code',3007}},1}, + {{{unknown,0},recv,{'Result-Code',3007}},1}, {{{0,257,0},recv,{'Result-Code',2001}},1}, {{{0,275,0},recv,{'Result-Code',2001}},2}, {{{0,275,0},recv,{'Result-Code',3999}},1}, @@ -318,14 +318,14 @@ stats(?CLIENT, callback, rfc3588, L) -> = L; stats(?SERVER, callback, rfc3588, L) -> - [{{unknown,recv},1}, - {{unknown,send},1}, + [{{{unknown,0},send},1}, + {{{unknown,1},recv},1}, {{{0,257,0},send},1}, {{{0,257,1},recv},1}, {{{0,275,0},send},6}, {{{0,275,1},recv},8}, - {{unknown,recv,error},1}, - {{unknown,send,{'Result-Code',3007}},1}, + {{{unknown,0},send,{'Result-Code',3007}},1}, + {{{unknown,1},recv,error},1}, {{{0,257,0},send,{'Result-Code',2001}},1}, {{{0,275,0},send,{'Result-Code',2001}},2}, {{{0,275,0},send,{'Result-Code',3999}},1}, @@ -335,12 +335,12 @@ stats(?SERVER, callback, rfc3588, L) -> = L; stats(?CLIENT, callback, rfc6733, L) -> - [{{unknown,recv},1}, + [{{{unknown,0},recv},1}, {{{0,257,0},recv},1}, {{{0,257,1},send},1}, {{{0,275,0},recv},8}, {{{0,275,1},send},10}, - {{unknown,recv,{'Result-Code',3007}},1}, + {{{unknown,0},recv,{'Result-Code',3007}},1}, {{{0,257,0},recv,{'Result-Code',2001}},1}, {{{0,275,0},recv,{'Result-Code',2001}},2}, {{{0,275,0},recv,{'Result-Code',3999}},1}, @@ -350,14 +350,14 @@ stats(?CLIENT, callback, rfc6733, L) -> = L; stats(?SERVER, callback, rfc6733, L) -> - [{{unknown,recv},1}, - {{unknown,send},1}, + [{{{unknown,0},send},1}, + {{{unknown,1},recv},1}, {{{0,257,0},send},1}, {{{0,257,1},recv},1}, {{{0,275,0},send},8}, {{{0,275,1},recv},8}, - {{unknown,recv,error},1}, - {{unknown,send,{'Result-Code',3007}},1}, + {{{unknown,0},send,{'Result-Code',3007}},1}, + {{{unknown,1},recv,error},1}, {{{0,257,0},send,{'Result-Code',2001}},1}, {{{0,275,0},send,{'Result-Code',2001}},2}, {{{0,275,0},send,{'Result-Code',3999}},1}, diff --git a/lib/diameter/test/diameter_config_SUITE.erl b/lib/diameter/test/diameter_config_SUITE.erl index bbdf672291..4bcaa8119f 100644 --- a/lib/diameter/test/diameter_config_SUITE.erl +++ b/lib/diameter/test/diameter_config_SUITE.erl @@ -50,7 +50,7 @@ {request_errors, RE}, {call_mutates_state, C}]] || D <- [diameter_gen_base_rfc3588, diameter_gen_base_rfc6733], - M <- [?MODULE, [?MODULE, now()]], + M <- [?MODULE, [?MODULE, diameter_lib:now()]], A <- [0, common, make_ref()], S <- [[], make_ref()], AE <- [report, callback, discard], diff --git a/lib/diameter/test/diameter_relay_SUITE.erl b/lib/diameter/test/diameter_relay_SUITE.erl index 735a908d97..7142239bbb 100644 --- a/lib/diameter/test/diameter_relay_SUITE.erl +++ b/lib/diameter/test/diameter_relay_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2013. All Rights Reserved. +%% Copyright Ericsson AB 2010-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -49,6 +49,7 @@ send_timeout_1/1, send_timeout_2/1, info/1, + counters/1, disconnect/1, stop_services/1, stop/1]). @@ -120,6 +121,7 @@ all() -> start_services, connect, {group, all}, + counters, {group, all, [parallel]}, disconnect, stop_services, @@ -201,8 +203,8 @@ send3(_Config) -> send4(_Config) -> call(?SERVER4). -%% Send an ASR that loops between the relays and expect the loop to -%% be detected. +%% Send an ASR that loops between the relays (RELAY1 -> RELAY2 -> +%% RELAY1) and expect the loop to be detected. send_loop(_Config) -> Req = ['ASR', {'Destination-Realm', realm(?SERVER1)}, {'Destination-Host', ?SERVER1}, @@ -227,8 +229,103 @@ send_timeout(Tmo) -> call(Req, [{filter, realm}, {timeout, Tmo}]). info(_Config) -> + %% Wait for RELAY1 to have answered all requests, so that the + %% suite doesn't end before all answers are sent and counted. + receive after 6000 -> ok end, [] = ?util:info(). +counters(_Config) -> + [] = ?util:run([[fun counters/2, K, S] + || K <- [statistics, transport, connections], + S <- ?SERVICES]). + +counters(Key, Svc) -> + counters(Key, Svc, [_|_] = diameter:service_info(Svc, Key)). + +counters(statistics, Svc, Stats) -> + stats(Svc, lists:foldl(fun({K,N},D) -> orddict:update_counter(K, N, D) end, + orddict:new(), + lists:append([L || {P,L} <- Stats, is_pid(P)]))); + +counters(_, _, _) -> + todo. + +stats(?CLIENT, L) -> + [{{{0,257,0},recv},2}, %% CEA + {{{0,257,1},send},2}, %% CER + {{{0,258,0},recv},1}, %% RAA (send_timeout_1) + {{{0,258,1},send},2}, %% RAR (send_timeout_[12]) + {{{0,274,0},recv},1}, %% ASA (send_loop) + {{{0,274,1},send},1}, %% ASR (send_loop) + {{{0,275,0},recv},4}, %% STA (send[1-4]) + {{{0,275,1},send},4}, %% STR (send[1-4]) + {{{unknown,0},recv,discarded},1}, %% RAR (send_timeout_2) + {{{0,257,0},recv,{'Result-Code',2001}},2}, %% CEA + {{{0,258,0},recv,{'Result-Code',3002}},1}, %% RAA (send_timeout_1) + {{{0,274,0},recv,{'Result-Code',3005}},1}, %% ASA (send_loop) + {{{0,275,0},recv,{'Result-Code',2001}},4}] %% STA (send[1-4]) + = L; + +stats(S, L) + when S == ?SERVER1; + S == ?SERVER2; + S == ?SERVER3; + S == ?SERVER4 -> + [{{{0,257,0},send},1}, %% CEA + {{{0,257,1},recv},1}, %% CER + {{{0,275,0},send},1}, %% STA (send[1-4]) + {{{0,275,1},recv},1}, %% STR (send[1-4]) + {{{0,257,0},send,{'Result-Code',2001}},1}, %% CEA + {{{0,275,0},send,{'Result-Code',2001}},1}] %% STA (send[1-4]) + = L; + +stats(?RELAY1, L) -> + [{{{relay,0},recv},3}, %% STA x 2 (send[12]) + %% ASA (send_loop) + {{{relay,0},send},6}, %% STA x 2 (send[12]) + %% ASA x 2 (send_loop) + %% RAA x 2 (send_timeout_[12]) + {{{relay,1},recv},6}, %% STR x 2 (send[12]) + %% ASR x 2 (send_loop) + %% RAR x 2 (send_timeout_[12]) + {{{relay,1},send},5}, %% STR x 2 (send[12]) + %% ASR (send_loop) + %% RAR x 2 (send_timeout_[12]) + {{{0,257,0},recv},3}, %% CEA + {{{0,257,0},send},1}, %% " + {{{0,257,1},recv},1}, %% CER + {{{0,257,1},send},3}, %% " + {{{relay,0},recv,{'Result-Code',2001}},2}, %% STA x 2 (send[34]) + {{{relay,0},recv,{'Result-Code',3005}},1}, %% ASA (send_loop) + {{{relay,0},send,{'Result-Code',2001}},2}, %% STA x 2 (send[34]) + {{{relay,0},send,{'Result-Code',3002}},2}, %% RAA (send_timeout_[12]) + {{{relay,0},send,{'Result-Code',3005}},2}, %% ASA (send_loop) + {{{0,257,0},recv,{'Result-Code',2001}},3}, %% CEA + {{{0,257,0},send,{'Result-Code',2001}},1}] %% " + = L; + +stats(?RELAY2, L) -> + [{{{relay,0},recv},3}, %% STA x 2 (send[34]) + %% ASA (send_loop) + {{{relay,0},send},3}, %% STA x 2 (send[34]) + %% ASA (send_loop) + {{{relay,1},recv},5}, %% STR x 2 (send[34]) + %% RAR x 2 (send_timeout_[12]) + %% ASR (send_loop) + {{{relay,1},send},3}, %% STR x 2 (send[34]) + %% ASR (send_loop) + {{{0,257,0},recv},2}, %% CEA + {{{0,257,0},send},2}, %% " + {{{0,257,1},recv},2}, %% CER + {{{0,257,1},send},2}, %% " + {{{relay,0},recv,{'Result-Code',2001}},2}, %% STA x 2 (send[34]) + {{{relay,0},recv,{'Result-Code',3005}},1}, %% ASA (send_loop) + {{{relay,0},send,{'Result-Code',2001}},2}, %% STA x 2 (send[34]) + {{{relay,0},send,{'Result-Code',3005}},1}, %% ASA (send_loop) + {{{0,257,0},recv,{'Result-Code',2001}},2}, %% CEA + {{{0,257,0},send,{'Result-Code',2001}},2}] %% " + = L. + %% =========================================================================== realm(Host) -> @@ -303,18 +400,24 @@ handle_request(Pkt, OH, {_Ref, #diameter_caps{origin_host = {OH,_}} = Caps}) when OH /= ?CLIENT -> request(Pkt, Caps). -%% RELAY1 routes any ASR or RAR to RELAY2 ... +%% RELAY1 answers ACR after it's timed out at the client. +request(#diameter_packet{header = #diameter_header{cmd_code = 271}}, + #diameter_caps{origin_host = {?RELAY1, _}}) -> + receive after 1000 -> {answer_message, 3004} end; %% TOO_BUSY + +%% RELAY1 routes any ASR or RAR to RELAY2. request(#diameter_packet{header = #diameter_header{cmd_code = C}}, #diameter_caps{origin_host = {?RELAY1, _}}) when C == 274; %% ASR C == 258 -> %% RAR {relay, [{filter, {realm, realm(?RELAY2)}}]}; -%% ... which in turn routes it back. Expect diameter to either answer -%% either with DIAMETER_LOOP_DETECTED/DIAMETER_UNABLE_TO_COMPLY. +%% RELAY2 routes ASR back to RELAY1 to induce DIAMETER_LOOP_DETECTED. request(#diameter_packet{header = #diameter_header{cmd_code = 274}}, #diameter_caps{origin_host = {?RELAY2, _}}) -> {relay, [{filter, {host, ?RELAY1}}]}; + +%% RELAY2 discards RAR to induce DIAMETER_UNABLE_TO_DELIVER. request(#diameter_packet{header = #diameter_header{cmd_code = 258}}, #diameter_caps{origin_host = {?RELAY2, _}}) -> discard; diff --git a/lib/diameter/test/diameter_tls_SUITE.erl b/lib/diameter/test/diameter_tls_SUITE.erl index 55565692ec..e5bbda9c91 100644 --- a/lib/diameter/test/diameter_tls_SUITE.erl +++ b/lib/diameter/test/diameter_tls_SUITE.erl @@ -319,19 +319,19 @@ make_cert(Dir, Base) -> make_cert(Dir, Base ++ "_key.pem", Base ++ "_ca.pem"). make_cert(Dir, Keyfile, Certfile) -> - [K,C] = Paths = [filename:join([Dir, F]) || F <- [Keyfile, Certfile]], + [KP,CP] = [filename:join([Dir, F]) || F <- [Keyfile, Certfile]], - KCmd = join(["openssl genrsa -out", K, "2048"]), - CCmd = join(["openssl req -new -x509 -key", K, "-out", C, "-days 7", - "-subj /C=SE/ST=./L=Stockholm/CN=www.erlang.org"]), + KC = join(["openssl genrsa -out", KP, "2048"]), + CC = join(["openssl req -new -x509 -key", KP, "-out", CP, "-days 7", + "-subj /C=SE/ST=./L=Stockholm/CN=www.erlang.org"]), %% Hope for the best and only check that files are written. - os:cmd(KCmd), - os:cmd(CCmd), + [{_, _, {ok,_}},{_, _, {ok,_}}] + = [{P,O,T} || {P,C} <- [{KP,KC}, {CP,CC}], + O <- [os:cmd(C)], + T <- [file:read_file_info(P)]], - [_,_] = [T || P <- Paths, {ok, T} <- [file:read_file_info(P)]], - - {K,C}. + {KP,CP}. join(Strs) -> string:join(Strs, " "). diff --git a/lib/diameter/test/diameter_traffic_SUITE.erl b/lib/diameter/test/diameter_traffic_SUITE.erl index 7ff6ba7ab9..17faf30a9b 100644 --- a/lib/diameter/test/diameter_traffic_SUITE.erl +++ b/lib/diameter/test/diameter_traffic_SUITE.erl @@ -145,8 +145,12 @@ %% them as binary. -define(STRING_DECODES, [true, false]). +%% Which transport protocol to use. +-define(TRANSPORTS, [tcp, sctp]). + -record(group, - {client_service, + {transport, + client_service, client_encoding, client_dict0, client_strings, @@ -234,19 +238,20 @@ %% =========================================================================== suite() -> - [{timetrap, {seconds, 60}}]. + [{timetrap, {seconds, 10}}]. all() -> [start, result_codes, {group, traffic}, outstanding, empty, stop]. groups() -> Ts = tc(), + Sctp = ?util:have_sctp(), [{?util:name([R,D,A,C]), [parallel], Ts} || R <- ?ENCODINGS, D <- ?RFCS, A <- ?ENCODINGS, C <- ?CONTAINERS] ++ - [{?util:name([R,D,A,C,SD,CD]), + [{?util:name([T,R,D,A,C,SD,CD]), [], [start_services, add_transports, @@ -254,15 +259,19 @@ groups() -> {group, ?util:name([R,D,A,C])}, remove_transports, stop_services]} - || R <- ?ENCODINGS, + || T <- ?TRANSPORTS, + T /= sctp orelse Sctp, + R <- ?ENCODINGS, D <- ?RFCS, A <- ?ENCODINGS, C <- ?CONTAINERS, SD <- ?STRING_DECODES, CD <- ?STRING_DECODES] ++ - [{traffic, [parallel], [{group, ?util:name([R,D,A,C,SD,CD])} - || R <- ?ENCODINGS, + [{traffic, [parallel], [{group, ?util:name([T,R,D,A,C,SD,CD])} + || T <- ?TRANSPORTS, + T /= sctp orelse Sctp, + R <- ?ENCODINGS, D <- ?RFCS, A <- ?ENCODINGS, C <- ?CONTAINERS, @@ -271,8 +280,9 @@ groups() -> init_per_group(Name, Config) -> case ?util:name(Name) of - [R,D,A,C,SD,CD] -> - G = #group{client_service = [$C|?util:unique_string()], + [T,R,D,A,C,SD,CD] -> + G = #group{transport = T, + client_service = [$C|?util:unique_string()], client_encoding = R, client_dict0 = dict0(D), client_strings = CD, @@ -288,8 +298,18 @@ init_per_group(Name, Config) -> end_per_group(_, _) -> ok. +%% Skip testcases that can reasonably fail under SCTP. init_per_testcase(Name, Config) -> - [{testcase, Name} | Config]. + case [skip || #group{transport = sctp} + <- [proplists:get_value(group, Config)], + send_maxlen == Name + orelse send_long == Name] + of + [skip] -> + {skip, sctp}; + [] -> + [{testcase, Name} | Config] + end. end_per_testcase(_, _) -> ok. @@ -367,16 +387,18 @@ start_services(Config) -> | ?SERVICE(CN, CD)]). add_transports(Config) -> - #group{client_service = CN, + #group{transport = T, + client_service = CN, server_service = SN} = group(Config), LRef = ?util:listen(SN, - tcp, + T, [{capabilities_cb, fun capx/2}, + {pool_size, 8}, {spawn_opt, [{min_heap_size, 8096}]}, {applications, apps(rfc3588)}]), Cs = [?util:connect(CN, - tcp, + T, LRef, [{id, Id}, {capabilities, [{'Origin-State-Id', origin(Id)}]}, diff --git a/lib/diameter/test/diameter_transport_SUITE.erl b/lib/diameter/test/diameter_transport_SUITE.erl index f098851bea..78bddbd1cf 100644 --- a/lib/diameter/test/diameter_transport_SUITE.erl +++ b/lib/diameter/test/diameter_transport_SUITE.erl @@ -64,7 +64,7 @@ = #diameter_caps{host_ip_address = Addrs}}). -%% The term we register after open a listening port with gen_tcp. +%% The term we register after open a listening port with gen_{tcp,sctp}. -define(TEST_LISTENER(Ref, PortNr), {?MODULE, listen, Ref, PortNr}). @@ -85,7 +85,7 @@ %% =========================================================================== suite() -> - [{timetrap, {minutes, 2}}]. + [{timetrap, {seconds, 15}}]. all() -> [start, @@ -401,12 +401,13 @@ gen_listen(tcp) -> %% gen_accept/2 gen_accept(sctp, Sock) -> - Assoc = ?RECV(?SCTP(Sock, {_, #sctp_assoc_change{state = comm_up, - outbound_streams = O, - inbound_streams = I, - assoc_id = A}}), - {O, I, A}), - putr(assoc, Assoc), + #sctp_assoc_change{state = comm_up, + outbound_streams = OS, + inbound_streams = IS, + assoc_id = Id} + = ?RECV(?SCTP(Sock, {_, #sctp_assoc_change{} = S}), S), + + putr(assoc, {OS, IS, Id}), {ok, Sock}; gen_accept(tcp, LSock) -> gen_tcp:accept(LSock). diff --git a/lib/diameter/test/diameter_util.erl b/lib/diameter/test/diameter_util.erl index c496876ee1..df7d268429 100644 --- a/lib/diameter/test/diameter_util.erl +++ b/lib/diameter/test/diameter_util.erl @@ -204,13 +204,14 @@ seed() -> %% unique_string/0 unique_string() -> - us(diameter_lib:now()). - -us({M,S,U}) -> - tl(lists:append(["-" ++ integer_to_list(N) || N <- [M,S,U]])); - -us(MonoT) -> - integer_to_list(MonoT). + try erlang:unique_integer() of + N -> + integer_to_list(N) + catch + error: undef -> %% OTP < 18 + {M,S,U} = timestamp(), + tl(lists:append(["-" ++ integer_to_list(N) || N <- [M,S,U]])) + end. %% --------------------------------------------------------------------------- %% have_sctp/0 diff --git a/lib/diameter/test/diameter_watchdog_SUITE.erl b/lib/diameter/test/diameter_watchdog_SUITE.erl index 5a3ff2c92f..f39e12686e 100644 --- a/lib/diameter/test/diameter_watchdog_SUITE.erl +++ b/lib/diameter/test/diameter_watchdog_SUITE.erl @@ -673,8 +673,7 @@ jitter(T,D) -> %% Generate a unique hostname for the faked peer. hostname() -> - {M,S,U} = diameter_util:timestamp(), - lists:flatten(io_lib:format("~p-~p-~p", [M,S,U])). + ?util:unique_string(). putr(Key, Val) -> put({?MODULE, Key}, Val). diff --git a/lib/diameter/vsn.mk b/lib/diameter/vsn.mk index db7f72c44e..c278e74dca 100644 --- a/lib/diameter/vsn.mk +++ b/lib/diameter/vsn.mk @@ -16,5 +16,5 @@ # %CopyrightEnd% APPLICATION = diameter -DIAMETER_VSN = 1.9.1 +DIAMETER_VSN = 1.9.2 APP_VSN = $(APPLICATION)-$(DIAMETER_VSN)$(PRE_VSN) diff --git a/lib/edoc/src/edoc.app.src b/lib/edoc/src/edoc.app.src index 9e1155d3e8..e4b9040c78 100644 --- a/lib/edoc/src/edoc.app.src +++ b/lib/edoc/src/edoc.app.src @@ -23,5 +23,5 @@ {registered,[]}, {applications, [compiler,kernel,stdlib,syntax_tools]}, {env, []}, - {runtime_dependencies, ["xmerl-1.3.7","syntax_tools-1.6.14","stdlib-2.0", + {runtime_dependencies, ["xmerl-1.3.7","syntax_tools-1.6.14","stdlib-2.5", "kernel-3.0","inets-5.10","erts-6.0"]}]}. diff --git a/lib/edoc/vsn.mk b/lib/edoc/vsn.mk index 24cfbf16d5..49a73331c6 100644 --- a/lib/edoc/vsn.mk +++ b/lib/edoc/vsn.mk @@ -1 +1 @@ -EDOC_VSN = 0.7.16 +EDOC_VSN = 0.7.17 diff --git a/lib/eldap/doc/src/eldap.xml b/lib/eldap/doc/src/eldap.xml index ed35ee3a9c..253ba7c2ff 100644 --- a/lib/eldap/doc/src/eldap.xml +++ b/lib/eldap/doc/src/eldap.xml @@ -121,7 +121,7 @@ filter() See present/1, substrings/2, <item>Any error responded from ssl:connect/3</item> </taglist> <p>The <c>Timeout</c> parameter is for the actual tls upgrade (phase 2) while the timeout in - <seealso marker="#open/2">erl_tar:open/2</seealso> is used for the initial negotiation about + <seealso marker="#open/2">eldap:open/2</seealso> is used for the initial negotiation about upgrade (phase 1). </p> </desc> @@ -298,7 +298,7 @@ filter() See present/1, substrings/2, search(Handle, [{base, "dc=example, dc=com"}, {filter, Filter}, {attributes, ["cn"]}]), </pre> <p>The <c>timeout</c> option in the <c>SearchOptions</c> is for the ldap server, while - the timeout in <seealso marker="#open/2">erl_tar:open/2</seealso> is used for each + the timeout in <seealso marker="#open/2">eldap:open/2</seealso> is used for each individual request in the search operation. </p> </desc> @@ -403,7 +403,11 @@ filter() See present/1, substrings/2, <v>OptionalAttrs = [Attr]</v> <v>Attr = {matchingRule,string()} | {type,string()} | {dnAttributes,boolean()}</v> </type> - <desc> <p>Creates an extensible match filter. For example, <c>eldap:extensibleMatch("Bar",[{type,"sn"},{matchingRule,"caseExactMatch"}]))</c> creates a filter which performs a <c>caseExactMatch</c> on the attribute <c>sn</c> and matches with the value <c>"Bar"</c>. The default value of <c>dnAttributes</c> is <c>false</c>.</p> </desc> + <desc> <p>Creates an extensible match filter. For example, </p> + <code> + eldap:extensibleMatch("Bar", [{type,"sn"}, {matchingRule,"caseExactMatch"}])) + </code> + <p>creates a filter which performs a <c>caseExactMatch</c> on the attribute <c>sn</c> and matches with the value <c>"Bar"</c>. The default value of <c>dnAttributes</c> is <c>false</c>.</p> </desc> </func> <func> <name>'and'([Filter]) -> filter()</name> diff --git a/lib/eldap/test/eldap_basic_SUITE.erl b/lib/eldap/test/eldap_basic_SUITE.erl index 137c61b2d9..8d754e934c 100644 --- a/lib/eldap/test/eldap_basic_SUITE.erl +++ b/lib/eldap/test/eldap_basic_SUITE.erl @@ -896,9 +896,9 @@ client_timeout(Fun, Config) -> T = 1000, case eldap:open([Host], [{timeout,T},{port,Port}|Opts]) of {ok,H} -> - T0 = now(), + T0 = erlang:monotonic_time(), {error,{gen_tcp_error,timeout}} = Fun(H), - T_op = diff(T0,now()), + T_op = ms_passed(T0), ct:log("Time = ~p, Timeout spec = ~p",[T_op,T]), if T_op < T -> @@ -910,8 +910,12 @@ client_timeout(Fun, Config) -> Other -> ct:fail("eldap:open failed: ~p",[Other]) end. -diff({M1,S1,U1},{M2,S2,U2}) -> - ( ((M2-M1)*1000 + (S2-S1))*1000 + (U2-U1) ). +%% Help function, elapsed milliseconds since T0 +ms_passed(T0) -> + %% OTP 18 + erlang:convert_time_unit(erlang:monotonic_time() - T0, + native, + micro_seconds) / 1000. %%%---------------------------------------------------------------- init_ssl_certs_et_al(Config) -> diff --git a/lib/erl_docgen/src/erl_docgen.app.src b/lib/erl_docgen/src/erl_docgen.app.src index e2830b2692..d63d880d89 100644 --- a/lib/erl_docgen/src/erl_docgen.app.src +++ b/lib/erl_docgen/src/erl_docgen.app.src @@ -9,6 +9,6 @@ {registered,[]}, {applications, [kernel,stdlib]}, {env, []}, - {runtime_dependencies, ["xmerl-1.3.7","stdlib-2.0","edoc-0.7.13","erts-6.0"]} + {runtime_dependencies, ["xmerl-1.3.7","stdlib-2.5","edoc-0.7.13","erts-6.0"]} ] }. diff --git a/lib/erl_docgen/vsn.mk b/lib/erl_docgen/vsn.mk index 5823c96253..2abd3d2b7e 100644 --- a/lib/erl_docgen/vsn.mk +++ b/lib/erl_docgen/vsn.mk @@ -1 +1 @@ -ERL_DOCGEN_VSN = 0.3.8 +ERL_DOCGEN_VSN = 0.4 diff --git a/lib/et/src/Makefile b/lib/et/src/Makefile index 377e593712..b6873371ed 100644 --- a/lib/et/src/Makefile +++ b/lib/et/src/Makefile @@ -65,7 +65,7 @@ APPUP_TARGET = $(EBIN)/$(APPUP_FILE) # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- -ERL_COMPILE_FLAGS += -pa $(ERL_TOP)/lib/et/ebin -I../include +ERL_COMPILE_FLAGS += -pa $(ERL_TOP)/lib/et/ebin -I../include -Werror # ---------------------------------------------------- # Special Build Targets diff --git a/lib/et/src/et_collector.erl b/lib/et/src/et_collector.erl index e05c67be60..1f60dee8ca 100644 --- a/lib/et/src/et_collector.erl +++ b/lib/et/src/et_collector.erl @@ -64,6 +64,8 @@ -export([init/1,terminate/2, code_change/3, handle_call/3, handle_cast/2, handle_info/2]). +-compile([{nowarn_deprecated_function,[{erlang,now,0}]}]). + -include("et_internal.hrl"). -include("../include/et.hrl"). diff --git a/lib/et/src/et_selector.erl b/lib/et/src/et_selector.erl index c8e9c907b2..5497096377 100644 --- a/lib/et/src/et_selector.erl +++ b/lib/et/src/et_selector.erl @@ -28,6 +28,8 @@ parse_event/2 ]). +-compile([{nowarn_deprecated_function,[{erlang,now,0}]}]). + -include("../include/et.hrl"). %%---------------------------------------------------------------------- diff --git a/lib/eunit/doc/overview.edoc b/lib/eunit/doc/overview.edoc index 872a017440..df716cdeea 100644 --- a/lib/eunit/doc/overview.edoc +++ b/lib/eunit/doc/overview.edoc @@ -569,6 +569,9 @@ Examples: ```?assertMatch({found, {fred, _}}, lookup(bloggs, Table))''' ```?assertMatch([X|_] when X > 0, binary_to_list(B))''' </dd> +<dt>`assertNotMatch(GuardedPattern, Expr)'</dt> +<dd>The inverse case of assertMatch, for convenience. +</dd> <dt>`assertEqual(Expect, Expr)'</dt> <dd>Evaluates the expressions `Expect' and `Expr' and compares the results for equality, if testing is enabled. If the values are not @@ -583,6 +586,9 @@ Examples: ```?assertEqual("b" ++ "a", lists:reverse("ab"))''' ```?assertEqual(foo(X), bar(Y))''' </dd> +<dt>`assertNotEqual(Unexpected, Expr)'</dt> +<dd>The inverse case of assertEqual, for convenience. +</dd> <dt>`assertException(ClassPattern, TermPattern, Expr)'</dt> <dt>`assertError(TermPattern, Expr)'</dt> <dt>`assertExit(TermPattern, Expr)'</dt> diff --git a/lib/eunit/include/eunit.hrl b/lib/eunit/include/eunit.hrl index 9e8d34567a..53d291430d 100644 --- a/lib/eunit/include/eunit.hrl +++ b/lib/eunit/include/eunit.hrl @@ -414,7 +414,7 @@ -else. -define(debugMsg(S), begin - io:fwrite(user, <<"~s:~w:~w: ~s\n">>, + io:fwrite(user, <<"~ts:~w:~w: ~ts\n">>, [?FILE, ?LINE, self(), S]), ok end). @@ -423,7 +423,7 @@ -define(debugVal(E), begin ((fun (__V) -> - ?debugFmt(<<"~s = ~P">>, [(??E), __V, 15]), + ?debugFmt(<<"~ts = ~tP">>, [(??E), __V, 15]), __V end)(E)) end). @@ -433,7 +433,7 @@ {__T0, _} = statistics(wall_clock), __V = (E), {__T1, _} = statistics(wall_clock), - ?debugFmt(<<"~s: ~.3f s">>, [(S), (__T1-__T0)/1000]), + ?debugFmt(<<"~ts: ~.3f s">>, [(S), (__T1-__T0)/1000]), __V end)()) end). diff --git a/lib/eunit/src/eunit.app.src b/lib/eunit/src/eunit.app.src index 7a3978e200..b4ff6c9242 100644 --- a/lib/eunit/src/eunit.app.src +++ b/lib/eunit/src/eunit.app.src @@ -19,4 +19,4 @@ {registered,[]}, {applications, [kernel,stdlib]}, {env, []}, - {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]}]}. + {runtime_dependencies, ["stdlib-2.5","kernel-3.0","erts-6.0"]}]}. diff --git a/lib/eunit/src/eunit.erl b/lib/eunit/src/eunit.erl index 9c589dfa86..fbfd123c43 100644 --- a/lib/eunit/src/eunit.erl +++ b/lib/eunit/src/eunit.erl @@ -231,7 +231,7 @@ event_logger(LogFile) -> event_logger_loop(Reference, FD) -> receive {status, _Id, _Info}=Msg -> - io:fwrite(FD, "~p.\n", [Msg]), + io:fwrite(FD, "~tp.\n", [Msg]), event_logger_loop(Reference, FD); {stop, Reference, _ReplyTo} -> %% no need to reply, just exit diff --git a/lib/eunit/src/eunit_data.erl b/lib/eunit/src/eunit_data.erl index cbbc6fbc15..8b53a3681d 100644 --- a/lib/eunit/src/eunit_data.erl +++ b/lib/eunit/src/eunit_data.erl @@ -391,7 +391,7 @@ parse({with, X, As}=T) when is_list(As) -> parse({S, T1} = T) when is_list(S) -> case eunit_lib:is_string(S) of true -> - group(#group{tests = T1, desc = list_to_binary(S)}); + group(#group{tests = T1, desc = unicode:characters_to_binary(S)}); false -> bad_test(T) end; diff --git a/lib/eunit/src/eunit_internal.hrl b/lib/eunit/src/eunit_internal.hrl index 92694ec39b..8e1e27811f 100644 --- a/lib/eunit/src/eunit_internal.hrl +++ b/lib/eunit/src/eunit_internal.hrl @@ -14,8 +14,8 @@ -define(DEFAULT_MODULE_WRAPPER_NAME, eunit_wrapper_). -ifdef(DEBUG). --define(debugmsg(S),io:fwrite("\n* ~s: ~s\n", [?MODULE,S])). --define(debugmsg1(S,As),io:fwrite("\n* ~s: " ++ S ++ "\n", [?MODULE] ++ As)). +-define(debugmsg(S),io:fwrite("\n* ~ts: ~ts\n", [?MODULE,S])). +-define(debugmsg1(S,As),io:fwrite("\n* ~ts: " ++ S ++ "\n", [?MODULE] ++ As)). -else. -define(debugmsg(S),ok). -define(debugmsg1(S,As),ok). diff --git a/lib/eunit/src/eunit_lib.erl b/lib/eunit/src/eunit_lib.erl index 40bae93298..d8f98cffa5 100644 --- a/lib/eunit/src/eunit_lib.erl +++ b/lib/eunit/src/eunit_lib.erl @@ -57,7 +57,7 @@ format_exception({Class,Term,Trace}, Depth) when is_atom(Class), is_list(Trace) -> case is_stacktrace(Trace) of true -> - io_lib:format("~s**~w:~s", + io_lib:format("~ts**~w:~ts", [format_stacktrace(Trace), Class, format_term(Term, Depth)]); false -> @@ -67,11 +67,11 @@ format_exception(Term, Depth) -> format_term(Term, Depth). format_term(Term, Depth) -> - io_lib:format("~P\n", [Term, Depth]). + io_lib:format("~tP\n", [Term, Depth]). format_exit_term(Term) -> {Reason, Trace} = analyze_exit_term(Term), - io_lib:format("~P~s", [Reason, 15, Trace]). + io_lib:format("~tP~ts", [Reason, 15, Trace]). analyze_exit_term({Reason, [_|_]=Trace}=Term) -> case is_stacktrace(Trace) of @@ -102,7 +102,7 @@ format_stacktrace(Trace) -> format_stacktrace(Trace, "in function", "in call from"). format_stacktrace([{M,F,A,L}|Fs], Pre, Pre1) when is_integer(A) -> - [io_lib:fwrite("~s ~w:~w/~w~s\n", + [io_lib:fwrite("~ts ~w:~w/~w~ts\n", [Pre, M, F, A, format_stacktrace_location(L)]) | format_stacktrace(Fs, Pre1, Pre1)]; format_stacktrace([{M,F,As,L}|Fs], Pre, Pre1) when is_list(As) -> @@ -110,15 +110,15 @@ format_stacktrace([{M,F,As,L}|Fs], Pre, Pre1) when is_list(As) -> C = case is_op(M,F,A) of true when A =:= 1 -> [A1] = As, - io_lib:fwrite("~s ~s", [F,format_arg(A1)]); + io_lib:fwrite("~ts ~ts", [F,format_arg(A1)]); true when A =:= 2 -> [A1, A2] = As, - io_lib:fwrite("~s ~s ~s", + io_lib:fwrite("~ts ~ts ~ts", [format_arg(A1),F,format_arg(A2)]); false -> - io_lib:fwrite("~w(~s)", [F,format_arglist(As)]) + io_lib:fwrite("~w(~ts)", [F,format_arglist(As)]) end, - [io_lib:fwrite("~s ~w:~w/~w~s\n called as ~s\n", + [io_lib:fwrite("~ts ~w:~w/~w~ts\n called as ~ts\n", [Pre,M,F,A,format_stacktrace_location(L),C]) | format_stacktrace(Fs,Pre1,Pre1)]; format_stacktrace([{M,F,As}|Fs], Pre, Pre1) -> @@ -130,18 +130,18 @@ format_stacktrace_location(Location) -> File = proplists:get_value(file, Location), Line = proplists:get_value(line, Location), if File =/= undefined, Line =/= undefined -> - io_lib:format(" (~s, line ~w)", [File, Line]); + io_lib:format(" (~ts, line ~w)", [File, Line]); true -> "" end. format_arg(A) -> - io_lib:format("~P",[A,15]). + io_lib:format("~tP",[A,15]). format_arglist([A]) -> format_arg(A); format_arglist([A|As]) -> - [io_lib:format("~P,",[A,15]) | format_arglist(As)]; + [io_lib:format("~tP,",[A,15]) | format_arglist(As)]; format_arglist([]) -> "". @@ -155,41 +155,41 @@ is_op(_M, _F, _A) -> false. format_error({bad_test, Term}) -> - error_msg("bad test descriptor", "~P", [Term, 15]); + error_msg("bad test descriptor", "~tP", [Term, 15]); format_error({bad_generator, {{M,F,A}, Term}}) -> error_msg(io_lib:format("result from generator ~w:~w/~w is not a test", [M,F,A]), - "~P", [Term, 15]); + "~tP", [Term, 15]); format_error({generator_failed, {{M,F,A}, Exception}}) -> error_msg(io_lib:format("test generator ~w:~w/~w failed",[M,F,A]), - "~s", [format_exception(Exception)]); + "~ts", [format_exception(Exception)]); format_error({no_such_function, {M,F,A}}) when is_atom(M), is_atom(F), is_integer(A) -> error_msg(io_lib:format("no such function: ~w:~w/~w", [M,F,A]), "", []); format_error({module_not_found, M}) -> - error_msg("test module not found", "~p", [M]); + error_msg("test module not found", "~tp", [M]); format_error({application_not_found, A}) when is_atom(A) -> error_msg("application not found", "~w", [A]); format_error({file_read_error, {_R, Msg, F}}) -> - error_msg("error reading file", "~s: ~s", [Msg, F]); + error_msg("error reading file", "~ts: ~ts", [Msg, F]); format_error({setup_failed, Exception}) -> - error_msg("context setup failed", "~s", + error_msg("context setup failed", "~ts", [format_exception(Exception)]); format_error({cleanup_failed, Exception}) -> - error_msg("context cleanup failed", "~s", + error_msg("context cleanup failed", "~ts", [format_exception(Exception)]); format_error({{bad_instantiator, {{M,F,A}, Term}}, _DummyException}) -> error_msg(io_lib:format("result from instantiator ~w:~w/~w is not a test", [M,F,A]), - "~P", [Term, 15]); + "~tP", [Term, 15]); format_error({instantiation_failed, Exception}) -> - error_msg("instantiation of subtests failed", "~s", + error_msg("instantiation of subtests failed", "~ts", [format_exception(Exception)]). error_msg(Title, Fmt, Args) -> Msg = io_lib:format("**"++Fmt, Args), % gets indentation right - io_lib:fwrite("*** ~s ***\n~s\n\n", [Title, Msg]). + io_lib:fwrite("*** ~ts ***\n~ts\n\n", [Title, Msg]). -ifdef(TEST). format_exception_test_() -> diff --git a/lib/eunit/src/eunit_proc.erl b/lib/eunit/src/eunit_proc.erl index 03d1a18321..98ae31d54b 100644 --- a/lib/eunit/src/eunit_proc.erl +++ b/lib/eunit/src/eunit_proc.erl @@ -230,7 +230,7 @@ insulator_wait(Child, Parent, Buf, St) -> message_super(Id, {progress, 'begin', {Type, Data}}, St), insulator_wait(Child, Parent, [[] | Buf], St); {child, Child, Id, {'end', Status, Time}} -> - Data = [{time, Time}, {output, buffer_to_binary(hd(Buf))}], + Data = [{time, Time}, {output, lists:reverse(hd(Buf))}], message_super(Id, {progress, 'end', {Status, Data}}, St), insulator_wait(Child, Parent, tl(Buf), St); {child, Child, Id, {skipped, Reason}} -> @@ -272,9 +272,6 @@ kill_task(Child, St) -> exit(Child, kill), terminate_insulator(St). -buffer_to_binary([B]) when is_binary(B) -> B; % avoid unnecessary copying -buffer_to_binary(Buf) -> list_to_binary(lists:reverse(Buf)). - %% Unlinking before exit avoids polluting the parent process with exit %% signals from the insulator. The child process is already dead here. @@ -597,7 +594,7 @@ group_leader_loop(Runner, Wait, Buf) -> %% no more messages and nothing to wait for; we ought to %% have collected all immediately pending output now process_flag(priority, normal), - Runner ! {self(), buffer_to_binary(Buf)} + Runner ! {self(), lists:reverse(Buf)} end. group_leader_sync(G) -> diff --git a/lib/eunit/src/eunit_surefire.erl b/lib/eunit/src/eunit_surefire.erl index 2d1f0b1497..d6684f33cb 100644 --- a/lib/eunit/src/eunit_surefire.erl +++ b/lib/eunit/src/eunit_surefire.erl @@ -206,6 +206,7 @@ handle_cancel(test, Data, St) -> format_name({Module, Function, Arity}, Line) -> lists:flatten([atom_to_list(Module), ":", atom_to_list(Function), "/", integer_to_list(Arity), "_", integer_to_list(Line)]). + format_desc(undefined) -> ""; format_desc(Desc) when is_binary(Desc) -> @@ -279,7 +280,7 @@ write_report_to(TestSuite, FileDescriptor) -> %% Write the XML header. %% ---------------------------------------------------------------------------- write_header(FileDescriptor) -> - file:write(FileDescriptor, [<<"<?xml version=\"1.0\" encoding=\"UTF-8\" ?>">>, ?NEWLINE]). + io:format(FileDescriptor, "~ts~ts", [<<"<?xml version=\"1.0\" encoding=\"UTF-8\" ?>">>, ?NEWLINE]). %% ---------------------------------------------------------------------------- %% Write the testsuite start tag, with attributes describing the statistics @@ -303,7 +304,7 @@ write_start_tag( <<"\" time=\"">>, format_time(Time), <<"\" name=\"">>, escape_attr(Name), <<"\">">>, ?NEWLINE], - file:write(FileDescriptor, StartTag). + io:format(FileDescriptor, "~ts", [StartTag]). %% ---------------------------------------------------------------------------- %% Recursive function to write the test cases. @@ -317,7 +318,7 @@ write_testcases([TestCase| Tail], FileDescriptor) -> %% Write the testsuite end tag. %% ---------------------------------------------------------------------------- write_end_tag(FileDescriptor) -> - file:write(FileDescriptor, [<<"</testsuite>">>, ?NEWLINE]). + io:format(FileDescriptor, "~ts~ts", [<<"</testsuite>">>, ?NEWLINE]). %% ---------------------------------------------------------------------------- %% Write a test case, as a testcase tag. @@ -344,7 +345,7 @@ write_testcase( {ok, <<>>} -> [<<"/>">>, ?NEWLINE]; _ -> [<<">">>, ?NEWLINE, format_testcase_result(Result), format_testcase_output(Output), ?INDENT, <<"</testcase>">>, ?NEWLINE] end, - file:write(FileDescriptor, [StartTag, ContentAndEndTag]). + io:format(FileDescriptor, "~ts~ts", [StartTag, ContentAndEndTag]). %% ---------------------------------------------------------------------------- %% Format the result of the test. @@ -427,7 +428,7 @@ escape_suitename([Char | Tail], Acc) -> escape_suitename(Tail, [Char | Acc]). %% Replace < with <, > with > and & with & %% ---------------------------------------------------------------------------- escape_text(Text) when is_binary(Text) -> escape_text(binary_to_list(Text)); -escape_text(Text) -> escape_xml(lists:flatten(Text), [], false). +escape_text(Text) -> escape_xml(to_utf8(lists:flatten(Text)), [], false). %% ---------------------------------------------------------------------------- @@ -435,7 +436,7 @@ escape_text(Text) -> escape_xml(lists:flatten(Text), [], false). %% Replace < with <, > with > and & with & %% ---------------------------------------------------------------------------- escape_attr(Text) when is_binary(Text) -> escape_attr(binary_to_list(Text)); -escape_attr(Text) -> escape_xml(lists:flatten(Text), [], true). +escape_attr(Text) -> escape_xml(to_utf8(lists:flatten(Text)), [], true). escape_xml([], Acc, _ForAttr) -> lists:reverse(Acc); escape_xml([$< | Tail], Acc, ForAttr) -> escape_xml(Tail, [$;, $t, $l, $& | Acc], ForAttr); @@ -443,3 +444,17 @@ escape_xml([$> | Tail], Acc, ForAttr) -> escape_xml(Tail, [$;, $t, $g, $& | Acc] escape_xml([$& | Tail], Acc, ForAttr) -> escape_xml(Tail, [$;, $p, $m, $a, $& | Acc], ForAttr); escape_xml([$" | Tail], Acc, true) -> escape_xml(Tail, [$;, $t, $o, $u, $q, $& | Acc], true); % " escape_xml([Char | Tail], Acc, ForAttr) when is_integer(Char) -> escape_xml(Tail, [Char | Acc], ForAttr). + +%% the input may be utf8 or latin1; the resulting list is unicode +to_utf8(Desc) when is_binary(Desc) -> + case unicode:characters_to_list(Desc) of + {_,_,_} -> unicode:characters_to_list(Desc, latin1); + X -> X + end; +to_utf8(Desc) when is_list(Desc) -> + try + to_utf8(list_to_binary(Desc)) + catch + _:_ -> + Desc + end. diff --git a/lib/eunit/src/eunit_tty.erl b/lib/eunit/src/eunit_tty.erl index f21b2da3d3..699d2adaca 100644 --- a/lib/eunit/src/eunit_tty.erl +++ b/lib/eunit/src/eunit_tty.erl @@ -83,7 +83,7 @@ terminate({ok, Data}, St) -> sync_end(error) end; terminate({error, Reason}, _St) -> - fwrite("Internal error: ~P.\n", [Reason, 25]), + fwrite("Internal error: ~tP.\n", [Reason, 25]), sync_end(error). sync_end(Result) -> @@ -177,7 +177,7 @@ indent(_N) -> print_group_start(I, Desc) -> indent(I), - fwrite("~s\n", [Desc]). + fwrite("~ts\n", [Desc]). print_group_end(I, Time) -> if Time > 0 -> @@ -195,13 +195,13 @@ print_test_begin(I, Data) -> true -> io_lib:fwrite("~w:", [Line]) end, D = if Desc =:= "" ; Desc =:= undefined -> ""; - true -> io_lib:fwrite(" (~s)", [Desc]) + true -> io_lib:fwrite(" (~ts)", [Desc]) end, case proplists:get_value(source, Data) of {Module, Name, _Arity} -> - fwrite("~s:~s ~s~s...", [Module, L, Name, D]); + fwrite("~ts:~ts ~ts~ts...", [Module, L, Name, D]); _ -> - fwrite("~s~s...", [L, D]) + fwrite("~ts~ts...", [L, D]) end. print_test_end(Data) -> @@ -209,21 +209,21 @@ print_test_end(Data) -> T = if Time > 0 -> io_lib:fwrite("[~.3f s] ", [Time/1000]); true -> "" end, - fwrite("~sok\n", [T]). + fwrite("~tsok\n", [T]). print_test_error({error, Exception}, Data) -> Output = proplists:get_value(output, Data), - fwrite("*failed*\n~s", [eunit_lib:format_exception(Exception)]), + fwrite("*failed*\n~ts", [eunit_lib:format_exception(Exception)]), case Output of <<>> -> fwrite("\n\n"); <<Text:800/binary, _:1/binary, _/binary>> -> - fwrite(" output:<<\"~s\">>...\n\n", [Text]); + fwrite(" output:<<\"~ts\">>...\n\n", [Text]); _ -> - fwrite(" output:<<\"~s\">>\n\n", [Output]) + fwrite(" output:<<\"~ts\">>\n\n", [Output]) end; print_test_error({skipped, Reason}, _) -> - fwrite("*did not run*\n::~s\n", [format_skipped(Reason)]). + fwrite("*did not run*\n::~ts\n", [format_skipped(Reason)]). format_skipped({module_not_found, M}) -> io_lib:fwrite("missing module: ~w", [M]); @@ -244,12 +244,12 @@ format_cancel(undefined) -> format_cancel(timeout) -> "*timed out*\n"; format_cancel({startup, Reason}) -> - io_lib:fwrite("*could not start test process*\n::~P\n\n", + io_lib:fwrite("*could not start test process*\n::~tP\n\n", [Reason, 15]); format_cancel({blame, _SubId}) -> "*cancelled because of subtask*\n"; format_cancel({exit, Reason}) -> - io_lib:fwrite("*unexpected termination of test process*\n::~P\n\n", + io_lib:fwrite("*unexpected termination of test process*\n::~tP\n\n", [Reason, 15]); format_cancel({abort, Reason}) -> eunit_lib:format_error(Reason). diff --git a/lib/eunit/test/Makefile b/lib/eunit/test/Makefile index e4ddf4e42c..b0dde64c67 100644 --- a/lib/eunit/test/Makefile +++ b/lib/eunit/test/Makefile @@ -20,7 +20,9 @@ include $(ERL_TOP)/make/target.mk include $(ERL_TOP)/make/$(TARGET)/otp.mk MODULES = \ - eunit_SUITE + eunit_SUITE \ + tlatin \ + tutf8 ERL_FILES= $(MODULES:%=%.erl) diff --git a/lib/eunit/test/eunit_SUITE.erl b/lib/eunit/test/eunit_SUITE.erl index d13dc73923..2ac6fafe5d 100644 --- a/lib/eunit/test/eunit_SUITE.erl +++ b/lib/eunit/test/eunit_SUITE.erl @@ -1,35 +1,35 @@ %% %% %CopyrightBegin% -%% +%% %% Copyright Ericsson AB 2010-2011. All Rights Reserved. -%% +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% -module(eunit_SUITE). --export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, - app_test/1,appup_test/1,eunit_test/1]). - + app_test/1,appup_test/1,eunit_test/1,surefire_utf8_test/1,surefire_latin_test/1]). + -include_lib("common_test/include/ct.hrl"). suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> - [app_test, appup_test, eunit_test]. +all() -> + [app_test, appup_test, eunit_test, surefire_utf8_test, surefire_latin_test]. -groups() -> +groups() -> []. init_per_suite(Config) -> @@ -54,3 +54,21 @@ eunit_test(Config) when is_list(Config) -> ok = file:set_cwd(code:lib_dir(eunit)), ok = eunit:test(eunit). +surefire_latin_test(Config) when is_list(Config) -> + ok = file:set_cwd(proplists:get_value(priv_dir, Config, ".")), + check_surefire(tlatin), + ok. + +surefire_utf8_test(Config) when is_list(Config) -> + ok = file:set_cwd(proplists:get_value(priv_dir, Config, ".")), + check_surefire(tutf8), + ok. + +check_surefire(Module) -> + File = "TEST-"++atom_to_list(Module)++".xml", + file:delete(File), + % ignore test result, some fail on purpose + eunit:test(Module, [{report,{eunit_surefire,[{dir,"."}]}}]), + {ok, Bin} = file:read_file(File), + [_|_] = unicode:characters_to_list(Bin, unicode), + ok.
\ No newline at end of file diff --git a/lib/eunit/test/tlatin.erl b/lib/eunit/test/tlatin.erl new file mode 100644 index 0000000000..a42e67d581 --- /dev/null +++ b/lib/eunit/test/tlatin.erl @@ -0,0 +1,15 @@ +% coding: latin-1 + +-module(tlatin). + +-include_lib("eunit/include/eunit.hrl"). + +'foo_�_test_'() -> + [ + {"1�1", fun() -> io:format("1�1 ~s ~w",[<<"a�">>, 'Z�k']), io:format([128,64,255,255]), ?assert("g�"=="g�") end} + ,{<<"2�2">>, fun() -> io:format("2�2 ~s",[<<"b�">>]), io:format([128,64]), ?assert("g�"=="g�") end} + ,{<<"3�3"/utf8>>, fun() -> io:format("3�3 ~ts",[<<"c�"/utf8>>]), io:format([128,64]), ?assert("g�"=="g�") end} + ,{"1�1", fun() -> io:format("1�1 ~s ~w",[<<"a�">>,'Zb�d']), io:format([128,64,255,255]), ?assert("w�"=="w�") end} + ,{<<"2�2">>, fun() -> io:format("2�2 ~s",[<<"b�">>]), io:format([128,64]), ?assert("w�"=="w�") end} + ,{<<"3�3"/utf8>>, fun() -> io:format("3�3 ~ts",[<<"c�"/utf8>>]), io:format([128,64]), ?assert("w�"=="w�") end} + ]. diff --git a/lib/eunit/test/tutf8.erl b/lib/eunit/test/tutf8.erl new file mode 100644 index 0000000000..c902f3ad18 --- /dev/null +++ b/lib/eunit/test/tutf8.erl @@ -0,0 +1,15 @@ +%% coding: utf-8 + +-module(tutf8). + +-include_lib("eunit/include/eunit.hrl"). + +'foo_ö_test_'() -> + [ + {"1ö汉1", fun() -> io:format("1å汉1 ~s ~w",[<<"aö汉">>, 'Zök']), io:format([128,64,255,255]), ?assert("gö汉"=="gö汉") end} + ,{<<"2ö汉2">>, fun() -> io:format("2å汉2 ~s",[<<"bö汉">>]), io:format([128,64]), ?assert("gö汉"=="gö汉") end} + ,{<<"3ö汉3"/utf8>>, fun() -> io:format("3å汉3 ~ts",[<<"cö汉"/utf8>>]), io:format([128,64]), ?assert("gö汉"=="gö汉") end} + ,{"1ä汉1", fun() -> io:format("1ä汉1 ~s ~w",[<<"aä汉">>, 'Zbäd']), io:format([128,64,255,255]), ?assert("wå汉"=="wä汉") end} + ,{<<"2ä汉2">>, fun() -> io:format("2ä汉2 ~s",[<<"bä汉">>]), io:format([128,64]), ?assert("wå汉"=="wä汉") end} + ,{<<"3ä汉"/utf8>>, fun() -> io:format("3ä汉3 ~ts",[<<"cä汉"/utf8>>]), io:format([128,64]), ?assert("wå汉"=="wä汉") end} + ]. diff --git a/lib/eunit/vsn.mk b/lib/eunit/vsn.mk index dca8b3ece0..b551ee6eb6 100644 --- a/lib/eunit/vsn.mk +++ b/lib/eunit/vsn.mk @@ -1 +1 @@ -EUNIT_VSN = 2.2.9 +EUNIT_VSN = 2.3 diff --git a/lib/hipe/cerl/cerl_pmatch.erl b/lib/hipe/cerl/cerl_pmatch.erl index 3bc93e80dd..4f04b0a7ed 100644 --- a/lib/hipe/cerl/cerl_pmatch.erl +++ b/lib/hipe/cerl/cerl_pmatch.erl @@ -31,7 +31,7 @@ -module(cerl_pmatch). --define(NO_UNUSED, true). +%%-define(NO_UNUSED, true). -export([clauses/2]). -ifndef(NO_UNUSED). @@ -59,6 +59,8 @@ %% @see transform/2 -ifndef(NO_UNUSED). +-spec core_transform(cerl:c_module(), [_]) -> cerl:c_module(). + core_transform(M, Opts) -> cerl:to_records(transform(cerl:from_records(M), Opts)). -endif. % NO_UNUSED @@ -76,6 +78,8 @@ core_transform(M, Opts) -> %% @see core_transform/2 -ifndef(NO_UNUSED). +-spec transform(cerl:cerl(), [_]) -> cerl:cerl(). + transform(M, _Opts) -> expr(M, env__empty()). -endif. % NO_UNUSED @@ -109,7 +113,7 @@ transform(M, _Opts) -> %% @see expr/2 %% @see transform/2 --spec clauses([cerl:cerl()], rec_env:environment()) -> +-spec clauses([cerl:cerl(),...], rec_env:environment()) -> {cerl:cerl(), [cerl:cerl()]}. clauses(Cs, Env) -> @@ -406,6 +410,8 @@ make_let(Vs, A, B) -> %% @see rec_env -ifndef(NO_UNUSED). +-spec expr(cerl:cerl(), rec_env:environment()) -> cerl:cerl(). + expr(E, Env) -> case cerl:type(E) of literal -> diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl index 5b1401b34a..ee77d65932 100644 --- a/lib/hipe/cerl/erl_bif_types.erl +++ b/lib/hipe/cerl/erl_bif_types.erl @@ -1113,8 +1113,8 @@ type(hipe_bifs, set_native_address, 3, Xs, Opaques) -> type(hipe_bifs, set_native_address_in_fe, 2, Xs, Opaques) -> strict(hipe_bifs, set_native_address_in_fe, 2, Xs, fun (_) -> t_atom('true') end, Opaques); -type(hipe_bifs, system_crc, 1, Xs, Opaques) -> - strict(hipe_bifs, system_crc, 1, Xs, fun (_) -> t_crc32() end, Opaques); +type(hipe_bifs, system_crc, 0, _, _Opaques) -> + t_crc32(); type(hipe_bifs, term_to_word, 1, Xs, Opaques) -> strict(hipe_bifs, term_to_word, 1, Xs, fun (_) -> t_integer() end, Opaques); @@ -2490,8 +2490,8 @@ arg_types(hipe_bifs, set_native_address, 3) -> [t_mfa(), t_integer(), t_boolean()]; arg_types(hipe_bifs, set_native_address_in_fe, 2) -> [t_integer(), t_integer()]; -arg_types(hipe_bifs, system_crc, 1) -> - [t_crc32()]; +arg_types(hipe_bifs, system_crc, 0) -> + []; arg_types(hipe_bifs, term_to_word, 1) -> [t_any()]; arg_types(hipe_bifs, update_code_size, 3) -> diff --git a/lib/hipe/main/hipe.app.src b/lib/hipe/main/hipe.app.src index 22ea71b4e6..7b6d9e30e3 100644 --- a/lib/hipe/main/hipe.app.src +++ b/lib/hipe/main/hipe.app.src @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2012. All Rights Reserved. +%% Copyright Ericsson AB 2002-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -223,5 +223,5 @@ {registered,[]}, {applications, [kernel,stdlib]}, {env, []}, - {runtime_dependencies, ["syntax_tools-1.6.14","stdlib-2.0","kernel-3.0", + {runtime_dependencies, ["syntax_tools-1.6.14","stdlib-2.5","kernel-3.0", "erts-7.0","compiler-5.0"]}]}. diff --git a/lib/hipe/vsn.mk b/lib/hipe/vsn.mk index 60b4e0559b..e507ae933f 100644 --- a/lib/hipe/vsn.mk +++ b/lib/hipe/vsn.mk @@ -1 +1 @@ -HIPE_VSN = 3.11.3 +HIPE_VSN = 3.12 diff --git a/lib/inets/doc/src/Makefile b/lib/inets/doc/src/Makefile index 1a8e1c7ca8..961bfa838d 100644 --- a/lib/inets/doc/src/Makefile +++ b/lib/inets/doc/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2012. All Rights Reserved. +# Copyright Ericsson AB 1997-2015. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -52,6 +52,7 @@ XML_REF3_FILES = \ httpc.xml\ httpd.xml \ httpd_conf.xml \ + httpd_custom_api.xml \ httpd_socket.xml \ httpd_util.xml \ mod_alias.xml \ diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index 4178cb7d4c..6984408932 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -366,7 +366,7 @@ filename() = string() <tag><c><![CDATA[receiver]]></c></tag> <item> <p>Defines how the client will deliver the result of an - asynchroneous request (<c>sync</c> has the value + asynchronous request (<c>sync</c> has the value <c>false</c>). </p> <taglist> diff --git a/lib/inets/doc/src/httpd.xml b/lib/inets/doc/src/httpd.xml index e40660ab39..435f99ee23 100644 --- a/lib/inets/doc/src/httpd.xml +++ b/lib/inets/doc/src/httpd.xml @@ -204,7 +204,15 @@ <marker id="props_limit"></marker> <p><em>Limit properties</em> </p> - <taglist> + <taglist> + + <marker id="prop_customize"></marker> + <tag>{customize, atom()}</tag> + <item> + <p>A callback module to customize the inets HTTP servers behaviour + see <seealso marker="http_custom_api"> httpd_custom_api</seealso> </p> + </item> + <marker id="prop_disable_chunked_encoding"></marker> <tag>{disable_chunked_transfer_encoding_send, boolean()}</tag> <item> diff --git a/lib/inets/doc/src/httpd_custom_api.xml b/lib/inets/doc/src/httpd_custom_api.xml new file mode 100644 index 0000000000..faf1d277df --- /dev/null +++ b/lib/inets/doc/src/httpd_custom_api.xml @@ -0,0 +1,63 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2015</year><year>2015</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>httpd_custom_api</title> + <file>httpd_custom_api.xml</file> + </header> + <module>httpd_custom_api</module> + <modulesummary>Behaviour with optional callbacks to customize the inets HTTP server.</modulesummary> + <description> + <p> The module implementing this behaviour shall be supplied to to the servers + configuration with the option <seealso marker="httpd:prop_customize"> customize</seealso></p> + + </description> + <funcs> + <func> + <name>response_header({HeaderName, HeaderValue}) -> {true, Header} | false </name> + <fsummary>Filter and possible alter HTTP response headers.</fsummary> + <type> + <v>Header = {HeaderName :: string(), HeaderValue::string()}</v> + <d>The header name will be in lower case and should not be altered.</d> + </type> + <desc> + <p> Filter and possible alter HTTP response headers before they are sent to the client. + </p> + </desc> + </func> + + <func> + <name>request_header({HeaderName, HeaderValue}) -> {true, Header} | false </name> + <fsummary>Filter and possible alter HTTP request headers.</fsummary> + <type> + <v>Header = {HeaderName :: string(), HeaderValue::string()}</v> + <d>The header name will be in lower case and should not be altered.</d> + </type> + <desc> + <p> Filter and possible alter HTTP request headers before they are processed by the server. + </p> + </desc> + </func> + </funcs> +</erlref> + + diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 12bbc2b736..f563a8c4b0 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -32,7 +32,38 @@ <file>notes.xml</file> </header> - <section><title>Inets 5.10.7</title> + <section><title>Inets 5.10.9</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add behaviour with optional callbacks to customize the + inets HTTP server.</p> + <p> + Own Id: OTP-12776</p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 5.10.8</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Reject messages with a Content-Length less than 0</p> + <p> + Own Id: OTP-12739 Aux Id: seq12860 </p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 5.10.7</title> <section><title>Improvements and New Features</title> <list> diff --git a/lib/inets/doc/src/ref_man.xml b/lib/inets/doc/src/ref_man.xml index aaedf330b4..3afb020431 100644 --- a/lib/inets/doc/src/ref_man.xml +++ b/lib/inets/doc/src/ref_man.xml @@ -4,7 +4,7 @@ <application xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>1997</year><year>2013</year> + <year>1997</year><year>2015</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -39,6 +39,7 @@ <xi:include href="httpc.xml"/> <xi:include href="httpd.xml"/> <xi:include href="httpd_conf.xml"/> + <xi:include href="httpd_custom_api.xml"/> <xi:include href="httpd_socket.xml"/> <xi:include href="httpd_util.xml"/> <xi:include href="mod_alias.xml"/> diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index 8f2f11ce8e..f4f0c37570 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -87,7 +87,7 @@ %% block the httpc manager process in odd cases such as trying to call %% a server that does not exist. (See OTP-6735) The only API function %% sending messages to the handler process that can be called before -%% init has compleated is cancel and that is not a problem! (Send and +%% init has completed is cancel and that is not a problem! (Send and %% stream will not be called before the first request has been sent and %% the reply or part of it has arrived.) %%-------------------------------------------------------------------- @@ -392,7 +392,7 @@ handle_call(info, _, State) -> %% When the request in process has been canceled the handler process is %% stopped and the pipelined requests will be reissued or remaining %% requests will be sent on a new connection. This is is -%% based on the assumption that it is proably cheaper to reissue the +%% based on the assumption that it is probably cheaper to reissue the %% requests than to wait for a potentiall large response that we then %% only throw away. This of course is not always true maybe we could %% do something smarter here?! If the request canceled is not @@ -1345,7 +1345,7 @@ handle_empty_queue(Session, ProfileName, TimeOut, State) -> %% closed by the server, the client may want to close it. NewState = activate_queue_timeout(TimeOut, State), update_session(ProfileName, Session, #session.queue_length, 0), - %% Note mfa will be initilized when a new request + %% Note mfa will be initialized when a new request %% arrives. {noreply, NewState#state{request = undefined, diff --git a/lib/inets/src/http_server/Makefile b/lib/inets/src/http_server/Makefile index 51e3dd9212..00bad51ff9 100644 --- a/lib/inets/src/http_server/Makefile +++ b/lib/inets/src/http_server/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2005-2013. All Rights Reserved. +# Copyright Ericsson AB 2005-2015. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -46,6 +46,7 @@ MODULES = \ httpd_connection_sup\ httpd_cgi \ httpd_conf \ + httpd_custom \ httpd_example \ httpd_esi \ httpd_file\ diff --git a/lib/inets/src/http_server/httpd_conf.erl b/lib/inets/src/http_server/httpd_conf.erl index dbdc1be272..a21eb915d4 100644 --- a/lib/inets/src/http_server/httpd_conf.erl +++ b/lib/inets/src/http_server/httpd_conf.erl @@ -785,8 +785,15 @@ fix_mime_types(ConfigList0) -> [{"html","text/html"},{"htm","text/html"}]} | ConfigList0] end; - _ -> - ConfigList0 + MimeTypes -> + case filelib:is_file(MimeTypes) of + true -> + {ok, MimeTypesList} = load_mime_types(MimeTypes), + ConfigList = proplists:delete(mime_types, ConfigList0), + [{mime_types, MimeTypesList} | ConfigList]; + false -> + ConfigList0 + end end. store({mime_types,MimeTypesList},ConfigList) -> diff --git a/lib/inets/src/http_server/httpd_custom.erl b/lib/inets/src/http_server/httpd_custom.erl new file mode 100644 index 0000000000..342469a579 --- /dev/null +++ b/lib/inets/src/http_server/httpd_custom.erl @@ -0,0 +1,69 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015-2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% +-module(httpd_custom). + +-export([response_header/1, request_header/1]). +-export([customize_headers/3]). + +-include_lib("inets/src/inets_app/inets_internal.hrl"). + +response_header(Header) -> + {true, httpify(Header)}. +request_header(Header) -> + {true, Header}. + +customize_headers(?MODULE, Function, Arg) -> + ?MODULE:Function(Arg); +customize_headers(Module, Function, Arg) -> + try Module:Function(Arg) of + {true, Value} -> + ?MODULE:Function(Value); + false -> + false + catch + _:_ -> + ?MODULE:Function(Arg) + end. + +httpify({Key0, Value}) -> + %% make sure first letter is capital (defacto standard) + Words1 = string:tokens(Key0, "-"), + Words2 = upify(Words1, []), + Key = new_key(Words2), + Key ++ ": " ++ Value ++ ?CRLF . + +new_key([]) -> + ""; +new_key([W]) -> + W; +new_key([W1,W2]) -> + W1 ++ "-" ++ W2; +new_key([W|R]) -> + W ++ "-" ++ new_key(R). + +upify([], Acc) -> + lists:reverse(Acc); +upify([Key|Rest], Acc) -> + upify(Rest, [upify2(Key)|Acc]). + +upify2([C|Rest]) when (C >= $a) andalso (C =< $z) -> + [C-($a-$A)|Rest]; +upify2(Str) -> + Str. diff --git a/lib/inets/src/http_server/httpd_request.erl b/lib/inets/src/http_server/httpd_request.erl index 6985065c3e..782120c284 100644 --- a/lib/inets/src/http_server/httpd_request.erl +++ b/lib/inets/src/http_server/httpd_request.erl @@ -42,28 +42,28 @@ %%%========================================================================= %%% Internal application API %%%========================================================================= -parse([Bin, MaxSizes]) -> - ?hdrt("parse", [{bin, Bin}, {max_sizes, MaxSizes}]), - parse_method(Bin, [], 0, proplists:get_value(max_method, MaxSizes), MaxSizes, []); +parse([Bin, Options]) -> + ?hdrt("parse", [{bin, Bin}, {max_sizes, Options}]), + parse_method(Bin, [], 0, proplists:get_value(max_method, Options), Options, []); parse(Unknown) -> ?hdrt("parse", [{unknown, Unknown}]), exit({bad_args, Unknown}). %% Functions that may be returned during the decoding process %% if the input data is incompleate. -parse_method([Bin, Method, Current, Max, MaxSizes, Result]) -> - parse_method(Bin, Method, Current, Max, MaxSizes, Result). +parse_method([Bin, Method, Current, Max, Options, Result]) -> + parse_method(Bin, Method, Current, Max, Options, Result). -parse_uri([Bin, URI, Current, Max, MaxSizes, Result]) -> - parse_uri(Bin, URI, Current, Max, MaxSizes, Result). +parse_uri([Bin, URI, Current, Max, Options, Result]) -> + parse_uri(Bin, URI, Current, Max, Options, Result). -parse_version([Bin, Rest, Version, Current, Max, MaxSizes, Result]) -> - parse_version(<<Rest/binary, Bin/binary>>, Version, Current, Max, MaxSizes, +parse_version([Bin, Rest, Version, Current, Max, Options, Result]) -> + parse_version(<<Rest/binary, Bin/binary>>, Version, Current, Max, Options, Result). -parse_headers([Bin, Rest, Header, Headers, Current, Max, MaxSizes, Result]) -> +parse_headers([Bin, Rest, Header, Headers, Current, Max, Options, Result]) -> parse_headers(<<Rest/binary, Bin/binary>>, - Header, Headers, Current, Max, MaxSizes, Result). + Header, Headers, Current, Max, Options, Result). whole_body([Bin, Body, Length]) -> whole_body(<<Body/binary, Bin/binary>>, Length). @@ -134,13 +134,13 @@ update_mod_data(ModData, Method, RequestURI, HTTPVersion, Headers)-> %%%======================================================================== %%% Internal functions %%%======================================================================== -parse_method(<<>>, Method, Current, Max, MaxSizes, Result) -> - {?MODULE, parse_method, [Method, Current, Max, MaxSizes, Result]}; -parse_method(<<?SP, Rest/binary>>, Method, _Current, _Max, MaxSizes, Result) -> - parse_uri(Rest, [], 0, proplists:get_value(max_uri, MaxSizes), MaxSizes, +parse_method(<<>>, Method, Current, Max, Options, Result) -> + {?MODULE, parse_method, [Method, Current, Max, Options, Result]}; +parse_method(<<?SP, Rest/binary>>, Method, _Current, _Max, Options, Result) -> + parse_uri(Rest, [], 0, proplists:get_value(max_uri, Options), Options, [string:strip(lists:reverse(Method)) | Result]); -parse_method(<<Octet, Rest/binary>>, Method, Current, Max, MaxSizes, Result) when Current =< Max -> - parse_method(Rest, [Octet | Method], Current + 1, Max, MaxSizes, Result); +parse_method(<<Octet, Rest/binary>>, Method, Current, Max, Options, Result) when Current =< Max -> + parse_method(Rest, [Octet | Method], Current + 1, Max, Options, Result); parse_method(_, _, _, Max, _, _) -> %% We do not know the version of the client as it comes after the %% method send the lowest version in the response so that the client @@ -153,30 +153,30 @@ parse_uri(_, _, Current, MaxURI, _, _) %% uri send the lowest version in the response so that the client %% will be able to handle it. {error, {size_error, MaxURI, 414, "URI unreasonably long"},lowest_version()}; -parse_uri(<<>>, URI, Current, Max, MaxSizes, Result) -> - {?MODULE, parse_uri, [URI, Current, Max, MaxSizes, Result]}; -parse_uri(<<?SP, Rest/binary>>, URI, _, _, MaxSizes, Result) -> - parse_version(Rest, [], 0, proplists:get_value(max_version, MaxSizes), MaxSizes, +parse_uri(<<>>, URI, Current, Max, Options, Result) -> + {?MODULE, parse_uri, [URI, Current, Max, Options, Result]}; +parse_uri(<<?SP, Rest/binary>>, URI, _, _, Options, Result) -> + parse_version(Rest, [], 0, proplists:get_value(max_version, Options), Options, [string:strip(lists:reverse(URI)) | Result]); %% Can happen if it is a simple HTTP/0.9 request e.i "GET /\r\n\r\n" -parse_uri(<<?CR, _Rest/binary>> = Data, URI, _, _, MaxSizes, Result) -> - parse_version(Data, [], 0, proplists:get_value(max_version, MaxSizes), MaxSizes, +parse_uri(<<?CR, _Rest/binary>> = Data, URI, _, _, Options, Result) -> + parse_version(Data, [], 0, proplists:get_value(max_version, Options), Options, [string:strip(lists:reverse(URI)) | Result]); -parse_uri(<<Octet, Rest/binary>>, URI, Current, Max, MaxSizes, Result) -> - parse_uri(Rest, [Octet | URI], Current + 1, Max, MaxSizes, Result). +parse_uri(<<Octet, Rest/binary>>, URI, Current, Max, Options, Result) -> + parse_uri(Rest, [Octet | URI], Current + 1, Max, Options, Result). -parse_version(<<>>, Version, Current, Max, MaxSizes, Result) -> - {?MODULE, parse_version, [<<>>, Version, Current, Max, MaxSizes, Result]}; -parse_version(<<?LF, Rest/binary>>, Version, Current, Max, MaxSizes, Result) -> +parse_version(<<>>, Version, Current, Max, Options, Result) -> + {?MODULE, parse_version, [<<>>, Version, Current, Max, Options, Result]}; +parse_version(<<?LF, Rest/binary>>, Version, Current, Max, Options, Result) -> %% If ?CR is is missing RFC2616 section-19.3 - parse_version(<<?CR, ?LF, Rest/binary>>, Version, Current, Max, MaxSizes, Result); -parse_version(<<?CR, ?LF, Rest/binary>>, Version, _, _, MaxSizes, Result) -> - parse_headers(Rest, [], [], 0, proplists:get_value(max_header, MaxSizes), MaxSizes, + parse_version(<<?CR, ?LF, Rest/binary>>, Version, Current, Max, Options, Result); +parse_version(<<?CR, ?LF, Rest/binary>>, Version, _, _, Options, Result) -> + parse_headers(Rest, [], [], 0, proplists:get_value(max_header, Options), Options, [string:strip(lists:reverse(Version)) | Result]); -parse_version(<<?CR>> = Data, Version, Current, Max, MaxSizes, Result) -> - {?MODULE, parse_version, [Data, Version, Current, Max, MaxSizes, Result]}; -parse_version(<<Octet, Rest/binary>>, Version, Current, Max, MaxSizes, Result) when Current =< Max -> - parse_version(Rest, [Octet | Version], Current + 1, Max, MaxSizes, Result); +parse_version(<<?CR>> = Data, Version, Current, Max, Options, Result) -> + {?MODULE, parse_version, [Data, Version, Current, Max, Options, Result]}; +parse_version(<<Octet, Rest/binary>>, Version, Current, Max, Options, Result) when Current =< Max -> + parse_version(Rest, [Octet | Version], Current + 1, Max, Options, Result); parse_version(_, _, _, Max,_,_) -> {error, {size_error, Max, 413, "Version string unreasonably long"}, lowest_version()}. @@ -185,34 +185,42 @@ parse_headers(_, _, _, Current, Max, _, Result) HttpVersion = lists:nth(3, lists:reverse(Result)), {error, {size_error, Max, 413, "Headers unreasonably long"}, HttpVersion}; -parse_headers(<<>>, Header, Headers, Current, Max, MaxSizes, Result) -> +parse_headers(<<>>, Header, Headers, Current, Max, Options, Result) -> {?MODULE, parse_headers, [<<>>, Header, Headers, Current, Max, - MaxSizes, Result]}; -parse_headers(<<?CR,?LF,?LF,Body/binary>>, [], [], Current, Max, MaxSizes, Result) -> + Options, Result]}; +parse_headers(<<?CR,?LF,?LF,Body/binary>>, [], [], Current, Max, Options, Result) -> %% If ?CR is is missing RFC2616 section-19.3 parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, [], [], Current, Max, - MaxSizes, Result); + Options, Result); -parse_headers(<<?LF,?LF,Body/binary>>, [], [], Current, Max, MaxSizes, Result) -> +parse_headers(<<?LF,?LF,Body/binary>>, [], [], Current, Max, Options, Result) -> %% If ?CR is is missing RFC2616 section-19.3 parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, [], [], Current, Max, - MaxSizes, Result); + Options, Result); parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, [], [], _, _, _, Result) -> NewResult = list_to_tuple(lists:reverse([Body, {#http_request_h{}, []} | Result])), {ok, NewResult}; parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, Header, Headers, _, _, - MaxSizes, Result) -> + Options, Result) -> + Customize = proplists:get_value(customize, Options), case http_request:key_value(lists:reverse(Header)) of undefined -> %% Skip headers with missing : - {ok, list_to_tuple(lists:reverse([Body, {http_request:headers(Headers, #http_request_h{}), Headers} | Result]))}; + FinalHeaders = lists:filtermap(fun(H) -> + httpd_custom:customize_headers(Customize, request_header, H) + end, + Headers), + {ok, list_to_tuple(lists:reverse([Body, {http_request:headers(FinalHeaders, #http_request_h{}), FinalHeaders} | Result]))}; NewHeader -> - case check_header(NewHeader, MaxSizes) of + case check_header(NewHeader, Options) of ok -> - {ok, list_to_tuple(lists:reverse([Body, {http_request:headers([NewHeader | Headers], + FinalHeaders = lists:filtermap(fun(H) -> + httpd_custom:customize_headers(Customize, request_header, H) + end, [NewHeader | Headers]), + {ok, list_to_tuple(lists:reverse([Body, {http_request:headers(FinalHeaders, #http_request_h{}), - [NewHeader | Headers]} | Result]))}; + FinalHeaders} | Result]))}; {error, Reason} -> HttpVersion = lists:nth(3, lists:reverse(Result)), @@ -221,12 +229,12 @@ parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, Header, Headers, _, _, end; parse_headers(<<?CR,?LF,?CR>> = Data, Header, Headers, Current, Max, - MaxSizes, Result) -> + Options, Result) -> {?MODULE, parse_headers, [Data, Header, Headers, Current, Max, - MaxSizes, Result]}; -parse_headers(<<?LF>>, [], [], Current, Max, MaxSizes, Result) -> + Options, Result]}; +parse_headers(<<?LF>>, [], [], Current, Max, Options, Result) -> %% If ?CR is is missing RFC2616 section-19.3 - parse_headers(<<?CR,?LF>>, [], [], Current, Max, MaxSizes, Result); + parse_headers(<<?CR,?LF>>, [], [], Current, Max, Options, Result); %% There where no headers, which is unlikely to happen. parse_headers(<<?CR,?LF>>, [], [], _, _, _, Result) -> @@ -235,30 +243,30 @@ parse_headers(<<?CR,?LF>>, [], [], _, _, _, Result) -> {ok, NewResult}; parse_headers(<<?LF>>, Header, Headers, Current, Max, - MaxSizes, Result) -> + Options, Result) -> %% If ?CR is is missing RFC2616 section-19.3 - parse_headers(<<?CR,?LF>>, Header, Headers, Current, Max, MaxSizes, Result); + parse_headers(<<?CR,?LF>>, Header, Headers, Current, Max, Options, Result); parse_headers(<<?CR,?LF>> = Data, Header, Headers, Current, Max, - MaxSizes, Result) -> + Options, Result) -> {?MODULE, parse_headers, [Data, Header, Headers, Current, Max, - MaxSizes, Result]}; + Options, Result]}; parse_headers(<<?LF, Octet, Rest/binary>>, Header, Headers, Current, Max, - MaxSizes, Result) -> + Options, Result) -> %% If ?CR is is missing RFC2616 section-19.3 parse_headers(<<?CR,?LF, Octet, Rest/binary>>, Header, Headers, Current, Max, - MaxSizes, Result); + Options, Result); parse_headers(<<?CR,?LF, Octet, Rest/binary>>, Header, Headers, _, Max, - MaxSizes, Result) -> + Options, Result) -> case http_request:key_value(lists:reverse(Header)) of undefined -> %% Skip headers with missing : parse_headers(Rest, [Octet], Headers, - 0, Max, MaxSizes, Result); + 0, Max, Options, Result); NewHeader -> - case check_header(NewHeader, MaxSizes) of + case check_header(NewHeader, Options) of ok -> parse_headers(Rest, [Octet], [NewHeader | Headers], - 0, Max, MaxSizes, Result); + 0, Max, Options, Result); {error, Reason} -> HttpVersion = lists:nth(3, lists:reverse(Result)), {error, Reason, HttpVersion} @@ -266,19 +274,19 @@ parse_headers(<<?CR,?LF, Octet, Rest/binary>>, Header, Headers, _, Max, end; parse_headers(<<?CR>> = Data, Header, Headers, Current, Max, - MaxSizes, Result) -> + Options, Result) -> {?MODULE, parse_headers, [Data, Header, Headers, Current, Max, - MaxSizes, Result]}; + Options, Result]}; parse_headers(<<?LF>>, Header, Headers, Current, Max, - MaxSizes, Result) -> + Options, Result) -> %% If ?CR is is missing RFC2616 section-19.3 parse_headers(<<?CR, ?LF>>, Header, Headers, Current, Max, - MaxSizes, Result); + Options, Result); parse_headers(<<Octet, Rest/binary>>, Header, Headers, Current, - Max, MaxSizes, Result) -> + Max, Options, Result) -> parse_headers(Rest, [Octet | Header], Headers, Current + 1, Max, - MaxSizes, Result). + Options, Result). whole_body(Body, Length) -> case size(Body) of @@ -417,8 +425,12 @@ check_header({"content-length", Value}, Maxsizes) -> case length(Value) =< MaxLen of true -> try - _ = list_to_integer(Value), - ok + list_to_integer(Value) + of + I when I>= 0 -> + ok; + _ -> + {error, {size_error, Max, 411, "negative content-length"}} catch _:_ -> {error, {size_error, Max, 411, "content-length not an integer"}} end; diff --git a/lib/inets/src/http_server/httpd_request_handler.erl b/lib/inets/src/http_server/httpd_request_handler.erl index f7a9fe5d49..9947e17b47 100644 --- a/lib/inets/src/http_server/httpd_request_handler.erl +++ b/lib/inets/src/http_server/httpd_request_handler.erl @@ -121,13 +121,15 @@ continue_init(Manager, ConfigDB, SocketType, Socket, TimeOut) -> MaxURISize = max_uri_size(ConfigDB), NrOfRequest = max_keep_alive_request(ConfigDB), MaxContentLen = max_content_length(ConfigDB), + Customize = customize(ConfigDB), {_, Status} = httpd_manager:new_connection(Manager), MFA = {httpd_request, parse, [[{max_uri, MaxURISize}, {max_header, MaxHeaderSize}, {max_version, ?HTTP_MAX_VERSION_STRING}, {max_method, ?HTTP_MAX_METHOD_STRING}, - {max_content_length, MaxContentLen} + {max_content_length, MaxContentLen}, + {customize, Customize} ]]}, State = #state{mod = Mod, @@ -550,11 +552,13 @@ handle_next_request(#state{mod = #mod{connection = true} = ModData, MaxHeaderSize = max_header_size(ModData#mod.config_db), MaxURISize = max_uri_size(ModData#mod.config_db), MaxContentLen = max_content_length(ModData#mod.config_db), + Customize = customize(ModData#mod.config_db), MFA = {httpd_request, parse, [[{max_uri, MaxURISize}, {max_header, MaxHeaderSize}, {max_version, ?HTTP_MAX_VERSION_STRING}, {max_method, ?HTTP_MAX_METHOD_STRING}, - {max_content_length, MaxContentLen} + {max_content_length, MaxContentLen}, + {customize, Customize} ]]}, TmpState = State#state{mod = NewModData, mfa = MFA, @@ -640,3 +644,6 @@ max_keep_alive_request(ConfigDB) -> max_content_length(ConfigDB) -> httpd_util:lookup(ConfigDB, max_content_length, ?HTTP_MAX_CONTENT_LENGTH). + +customize(ConfigDB) -> + httpd_util:lookup(ConfigDB, customize, httpd_custom). diff --git a/lib/inets/src/http_server/httpd_response.erl b/lib/inets/src/http_server/httpd_response.erl index 2fa91d47a0..71dc05e46d 100644 --- a/lib/inets/src/http_server/httpd_response.erl +++ b/lib/inets/src/http_server/httpd_response.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -176,7 +176,7 @@ send_header(#mod{socket_type = Type, StatusLine = [NewVer, " ", io_lib:write(NewStatusCode), " ", httpd_util:reason_phrase(NewStatusCode), ?CRLF], ConnectionHeader = get_connection(Conn, NewVer), - Head = list_to_binary([StatusLine, Headers, ConnectionHeader , ?CRLF]), + Head = [StatusLine, Headers, ConnectionHeader , ?CRLF], httpd_socket:deliver(Type, Sock, Head). map_status_code("HTTP/1.0", Code) @@ -286,45 +286,21 @@ create_header(ConfigDb, KeyValueTupleHeaders) -> Date = httpd_util:rfc1123_date(), ContentType = "text/html", Server = server(ConfigDb), - NewHeaders = add_default_headers([{"date", Date}, - {"content-type", ContentType} - | if Server=="" -> []; - true -> [{"server", Server}] - end - ], - KeyValueTupleHeaders), - lists:map(fun fix_header/1, NewHeaders). - - + Headers0 = add_default_headers([{"date", Date}, + {"content-type", ContentType} + | if Server=="" -> []; + true -> [{"server", Server}] + end + ], + KeyValueTupleHeaders), + CustomizeCB = httpd_util:lookup(ConfigDb, customize, httpd_custom), + lists:filtermap(fun(H) -> + httpd_custom:customize_headers(CustomizeCB, response_header, H) + end, + [Header || Header <- Headers0]). server(ConfigDb) -> httpd_util:lookup(ConfigDb, server, ?SERVER_SOFTWARE). -fix_header({Key0, Value}) -> - %% make sure first letter is capital - Words1 = string:tokens(Key0, "-"), - Words2 = upify(Words1, []), - Key = new_key(Words2), - Key ++ ": " ++ Value ++ ?CRLF . - -new_key([]) -> - ""; -new_key([W]) -> - W; -new_key([W1,W2]) -> - W1 ++ "-" ++ W2; -new_key([W|R]) -> - W ++ "-" ++ new_key(R). - -upify([], Acc) -> - lists:reverse(Acc); -upify([Key|Rest], Acc) -> - upify(Rest, [upify2(Key)|Acc]). - -upify2([C|Rest]) when (C >= $a) andalso (C =< $z) -> - [C-($a-$A)|Rest]; -upify2(Str) -> - Str. - add_default_headers([], Headers) -> Headers; diff --git a/lib/inets/src/inets_app/inets.app.src b/lib/inets/src/inets_app/inets.app.src index b7c3e341e8..6ba9795d9e 100644 --- a/lib/inets/src/inets_app/inets.app.src +++ b/lib/inets/src/inets_app/inets.app.src @@ -63,6 +63,7 @@ httpd_cgi, httpd_connection_sup, httpd_conf, + httpd_custom, httpd_esi, httpd_example, httpd_file, diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 0dfc65e8f7..ab7ffadf75 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -1289,7 +1289,9 @@ dummy_server_init(Caller, ip_comm, Inet, _) -> {max_header, ?HTTP_MAX_HEADER_SIZE}, {max_version,?HTTP_MAX_VERSION_STRING}, {max_method, ?HTTP_MAX_METHOD_STRING}, - {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}]]}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}, + {customize, httpd_custom} + ]]}, [], ListenSocket); dummy_server_init(Caller, ssl, Inet, SSLOptions) -> @@ -1305,7 +1307,8 @@ dummy_ssl_server_init(Caller, BaseOpts, Inet) -> {max_method, ?HTTP_MAX_METHOD_STRING}, {max_version,?HTTP_MAX_VERSION_STRING}, {max_method, ?HTTP_MAX_METHOD_STRING}, - {max_content_length, ?HTTP_MAX_CONTENT_LENGTH} + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}, + {customize, httpd_custom} ]]}, [], ListenSocket). @@ -1384,18 +1387,20 @@ handle_request(Module, Function, Args, Socket) -> stop; <<>> -> {httpd_request, parse, [[{max_uri,?HTTP_MAX_URI_SIZE}, - {max_header, ?HTTP_MAX_HEADER_SIZE}, - {max_version,?HTTP_MAX_VERSION_STRING}, - {max_method, ?HTTP_MAX_METHOD_STRING}, - {max_content_length, ?HTTP_MAX_CONTENT_LENGTH} - ]]}; + {max_header, ?HTTP_MAX_HEADER_SIZE}, + {max_version,?HTTP_MAX_VERSION_STRING}, + {max_method, ?HTTP_MAX_METHOD_STRING}, + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}, + {customize, httpd_custom} + ]]}; Data -> handle_request(httpd_request, parse, [Data, [{max_uri, ?HTTP_MAX_URI_SIZE}, - {max_header, ?HTTP_MAX_HEADER_SIZE}, + {max_header, ?HTTP_MAX_HEADER_SIZE}, {max_version,?HTTP_MAX_VERSION_STRING}, {max_method, ?HTTP_MAX_METHOD_STRING}, - {max_content_length, ?HTTP_MAX_CONTENT_LENGTH} + {max_content_length, ?HTTP_MAX_CONTENT_LENGTH}, + {customize, httpd_custom} ]], Socket) end; NewMFA -> diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index 11f2c6f298..bc6b0d5c79 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -53,6 +53,8 @@ all() -> {group, https_basic}, {group, http_limit}, {group, https_limit}, + {group, http_custom}, + {group, https_custom}, {group, http_basic_auth}, {group, https_basic_auth}, {group, http_auth_api}, @@ -66,7 +68,8 @@ all() -> {group, http_security}, {group, https_security}, {group, http_reload}, - {group, https_reload} + {group, https_reload}, + {group, http_mime_types} ]. groups() -> @@ -75,6 +78,8 @@ groups() -> {https_basic, [], basic_groups()}, {http_limit, [], [{group, limit}]}, {https_limit, [], [{group, limit}]}, + {http_custom, [], [{group, custom}]}, + {https_custom, [], [{group, custom}]}, {http_basic_auth, [], [{group, basic_auth}]}, {https_basic_auth, [], [{group, basic_auth}]}, {http_auth_api, [], [{group, auth_api}]}, @@ -89,7 +94,9 @@ groups() -> {https_security, [], [{group, security}]}, {http_reload, [], [{group, reload}]}, {https_reload, [], [{group, reload}]}, + {http_mime_types, [], [alias_1_1, alias_1_0, alias_0_9]}, {limit, [], [max_clients_1_1, max_clients_1_0, max_clients_0_9]}, + {custom, [], [customize]}, {reload, [], [non_disturbing_reconfiger_dies, disturbing_reconfiger_dies, non_disturbing_1_1, @@ -176,6 +183,7 @@ end_per_suite(_Config) -> %%-------------------------------------------------------------------- init_per_group(Group, Config0) when Group == https_basic; Group == https_limit; + Group == https_custom; Group == https_basic_auth; Group == https_auth_api; Group == https_auth_api_dets; @@ -186,12 +194,14 @@ init_per_group(Group, Config0) when Group == https_basic; init_ssl(Group, Config0); init_per_group(Group, Config0) when Group == http_basic; Group == http_limit; + Group == http_custom; Group == http_basic_auth; Group == http_auth_api; Group == http_auth_api_dets; Group == http_auth_api_mnesia; Group == http_security; - Group == http_reload + Group == http_reload; + Group == http_mime_types -> ok = start_apps(Group), init_httpd(Group, [{type, ip_comm} | Config0]); @@ -235,7 +245,8 @@ end_per_group(Group, _Config) when Group == http_basic; Group == http_auth_api_mnesia; Group == http_htaccess; Group == http_security; - Group == http_reload + Group == http_reload; + Group == http_mime_types -> inets:stop(); end_per_group(Group, _Config) when Group == https_basic; @@ -840,6 +851,24 @@ cgi_chunked_encoding_test(Config) when is_list(Config) -> ?config(node, Config), Requests). %%------------------------------------------------------------------------- +alias_1_1() -> + [{doc, "Test mod_alias"}]. + +alias_1_1(Config) when is_list(Config) -> + alias([{http_version, "HTTP/1.1"} | Config]). + +alias_1_0() -> + [{doc, "Test mod_alias"}]. + +alias_1_0(Config) when is_list(Config) -> + alias([{http_version, "HTTP/1.0"} | Config]). + +alias_0_9() -> + [{doc, "Test mod_alias"}]. + +alias_0_9(Config) when is_list(Config) -> + alias([{http_version, "HTTP/0.9"} | Config]). + alias() -> [{doc, "Test mod_alias"}]. @@ -898,7 +927,6 @@ trace(Config) when is_list(Config) -> Cb = ?config(version_cb, Config), Cb:trace(?config(type, Config), ?config(port, Config), ?config(host, Config), ?config(node, Config)). - %%------------------------------------------------------------------------- light() -> ["Test light load"]. @@ -956,6 +984,30 @@ missing_CR(Config) -> {version, Version}]). %%------------------------------------------------------------------------- +customize() -> + [{doc, "Test filtering of headers with custom callback"}]. + +customize(Config) when is_list(Config) -> + Version = "HTTP/1.1", + Host = ?config(host, Config), + Type = ?config(type, Config), + ok = httpd_test_lib:verify_request(?config(type, Config), Host, + ?config(port, Config), + transport_opts(Type, Config), + ?config(node, Config), + http_request("GET /index.html ", Version, Host), + [{statuscode, 200}, + {header, "Content-Type", "text/html"}, + {header, "Date"}, + {no_header, "Server"}, + {version, Version}]). + +response_header({"server", _}) -> + false; +response_header(Header) -> + {true, Header}. + +%%------------------------------------------------------------------------- max_header() -> ["Denial Of Service (DOS) attack, prevented by max_header"]. max_header(Config) when is_list(Config) -> @@ -1259,22 +1311,26 @@ setup_server_dirs(ServerRoot, DocRoot, DataDir) -> CgiDir = filename:join(ServerRoot, "cgi-bin"), AuthDir = filename:join(ServerRoot, "auth"), PicsDir = filename:join(ServerRoot, "icons"), + ConfigDir = filename:join(ServerRoot, "config"), ok = file:make_dir(ServerRoot), ok = file:make_dir(DocRoot), ok = file:make_dir(CgiDir), ok = file:make_dir(AuthDir), ok = file:make_dir(PicsDir), + ok = file:make_dir(ConfigDir), DocSrc = filename:join(DataDir, "server_root/htdocs"), AuthSrc = filename:join(DataDir, "server_root/auth"), CgiSrc = filename:join(DataDir, "server_root/cgi-bin"), PicsSrc = filename:join(DataDir, "server_root/icons"), + ConfigSrc = filename:join(DataDir, "server_root/config"), inets_test_lib:copy_dirs(DocSrc, DocRoot), inets_test_lib:copy_dirs(AuthSrc, AuthDir), inets_test_lib:copy_dirs(CgiSrc, CgiDir), inets_test_lib:copy_dirs(PicsSrc, PicsDir), + inets_test_lib:copy_dirs(ConfigSrc, ConfigDir), Cgi = case test_server:os_type() of {win32, _} -> @@ -1295,24 +1351,27 @@ setup_server_dirs(ServerRoot, DocRoot, DataDir) -> start_apps(Group) when Group == https_basic; Group == https_limit; + Group == https_custom; Group == https_basic_auth; Group == https_auth_api; Group == https_auth_api_dets; Group == https_auth_api_mnesia; - Group == http_htaccess; - Group == http_security; - Group == http_reload + Group == https_htaccess; + Group == https_security; + Group == https_reload -> inets_test_lib:start_apps([inets, asn1, crypto, public_key, ssl]); start_apps(Group) when Group == http_basic; Group == http_limit; + Group == http_custom; Group == http_basic_auth; Group == http_auth_api; Group == http_auth_api_dets; Group == http_auth_api_mnesia; Group == https_htaccess; Group == https_security; - Group == https_reload-> + Group == https_reload; + Group == http_mime_types-> inets_test_lib:start_apps([inets]). server_start(_, HttpdConfig) -> @@ -1364,6 +1423,10 @@ server_config(http_limit, Config) -> [{max_clients, 1}, %% Make sure option checking code is run {max_content_length, 100000002}] ++ server_config(http, Config); +server_config(http_custom, Config) -> + [{custom, ?MODULE}] ++ server_config(http, Config); +server_config(https_custom, Config) -> + [{custom, ?MODULE}] ++ server_config(https, Config); server_config(https_limit, Config) -> [{max_clients, 1}] ++ server_config(https, Config); server_config(http_basic_auth, Config) -> @@ -1400,6 +1463,11 @@ server_config(http_security, Config) -> server_config(https_security, Config) -> ServerRoot = ?config(server_root, Config), tl(auth_conf(ServerRoot)) ++ security_conf(ServerRoot) ++ server_config(https, Config); +server_config(http_mime_types, Config0) -> + Config1 = basic_conf() ++ server_config(http, Config0), + ServerRoot = ?config(server_root, Config0), + MimeTypesFile = filename:join([ServerRoot,"config", "mime.types"]), + [{mime_types, MimeTypesFile} | proplists:delete(mime_types, Config1)]; server_config(http, Config) -> ServerRoot = ?config(server_root, Config), diff --git a/lib/inets/test/httpd_SUITE_data/server_root/config/mime.types b/lib/inets/test/httpd_SUITE_data/server_root/config/mime.types new file mode 100644 index 0000000000..b68cff21a6 --- /dev/null +++ b/lib/inets/test/httpd_SUITE_data/server_root/config/mime.types @@ -0,0 +1,4 @@ +text/html html +text/html htm +text/html shtml +image/gif gif diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index e9ecb2632a..f52347e39e 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -2,7 +2,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2001-2014. All Rights Reserved. +# Copyright Ericsson AB 2001-2015. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -18,6 +18,6 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 5.10.7 +INETS_VSN = 6.0 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" diff --git a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpSelf.java b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpSelf.java index 5b9d13ad81..74afbbcca6 100644 --- a/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpSelf.java +++ b/lib/jinterface/java_src/com/ericsson/otp/erlang/OtpSelf.java @@ -153,9 +153,6 @@ public class OtpSelf extends OtpLocalNode { * the port number you wish to use for incoming connections. * Specifying 0 lets the system choose an available port. * - * @param transportFactory - * the transport factory to use when creating connections. - * * @exception IOException * in case of server transport failure */ diff --git a/lib/kernel/doc/src/error_logger.xml b/lib/kernel/doc/src/error_logger.xml index df2f0b01ee..f49d63b5a6 100644 --- a/lib/kernel/doc/src/error_logger.xml +++ b/lib/kernel/doc/src/error_logger.xml @@ -58,12 +58,11 @@ specific events. (<c>add_report_handler/1,2</c>). Also, there is a useful event handler in STDLIB for multi-file logging of events, see <c>log_mf_h(3)</c>.</p> - <p>Warning events were introduced in Erlang/OTP R9C. To retain - backwards compatibility, these are by default tagged as errors, - thus showing up as error reports in the logs. By using - the command line flag <c><![CDATA[+W <w | i>]]></c>, they can instead - be tagged as warnings or info. Tagging them as warnings may - require rewriting existing user defined event handlers.</p> + <p>Warning events were introduced in Erlang/OTP R9C and are enabled + by default as of 18.0. To retain backwards compatibility with existing + user defined event handlers, these may be tagged as errors or info + using the command line flag <c><![CDATA[+W <e | i | w>]]></c>, thus + showing up as error or info reports in the logs.</p> </description> <datatypes> <datatype> @@ -132,7 +131,7 @@ ok</pre> <desc> <p>Returns the current mapping for warning events. Events sent using <c>warning_msg/1,2</c> or <c>warning_report/1,2</c> - are tagged as errors (default), warnings or info, depending + are tagged as errors, warnings (default) or info, depending on the value of the command line flag <c>+W</c>.</p> <pre> os$ <input>erl</input> @@ -140,25 +139,25 @@ Erlang (BEAM) emulator version 5.4.8 [hipe] [threads:0] [kernel-poll] Eshell V5.4.8 (abort with ^G) 1> <input>error_logger:warning_map().</input> -error -2> <input>error_logger:warning_msg("Warnings tagged as: ~p~n", [error]).</input> +warning +2> <input>error_logger:warning_msg("Warnings tagged as: ~p~n", [warning]).</input> -=ERROR REPORT==== 11-Aug-2005::15:31:23 === -Warnings tagged as: error +=WARNING REPORT==== 11-Aug-2005::15:31:55 === +Warnings tagged as: warning ok 3> User switch command --> q -os$ <input>erl +W w</input> +os$ <input>erl +W e</input> Erlang (BEAM) emulator version 5.4.8 [hipe] [threads:0] [kernel-poll] Eshell V5.4.8 (abort with ^G) 1> <input>error_logger:warning_map().</input> -warning -2> <input>error_logger:warning_msg("Warnings tagged as: ~p~n", [warning]).</input> +error +2> <input>error_logger:warning_msg("Warnings tagged as: ~p~n", [error]).</input> -=WARNING REPORT==== 11-Aug-2005::15:31:55 === -Warnings tagged as: warning +=ERROR REPORT==== 11-Aug-2005::15:31:23 === +Warnings tagged as: error ok</pre> </desc> </func> diff --git a/lib/kernel/doc/src/gen_tcp.xml b/lib/kernel/doc/src/gen_tcp.xml index 820ecd1e30..71ef5cd48f 100644 --- a/lib/kernel/doc/src/gen_tcp.xml +++ b/lib/kernel/doc/src/gen_tcp.xml @@ -347,11 +347,22 @@ do_recv(Sock, Bs) -> </func> <func> <name name="shutdown" arity="2"/> - <fsummary>Immediately close a socket</fsummary> + <fsummary>Asynchronously close a socket</fsummary> <desc> - <p>Immediately close a socket in one or two directions.</p> + <p>Close a socket in one or two directions.</p> <p><c><anno>How</anno> == write</c> means closing the socket for writing, reading from it is still possible.</p> + <p>If <c><anno>How</anno> == read</c>, or there is no outgoing + data buffered in the <c><anno>Socket</anno></c> port, + then the socket is shutdown immediately and any error encountered + is returned in <c><anno>Reason</anno></c>.</p> + <p>If there is data buffered in the socket port, then the attempt + to shutdown the socket is postponed until that data is written to the + kernel socket send buffer. Any errors encountered will result + in the socket being closed and <c>{error, closed}</c> being returned + on the next + <seealso marker="gen_tcp#recv/2">recv/2</seealso> or + <seealso marker="gen_tcp#send/2">send/2</seealso>.</p> <p>To be able to handle that the peer has done a shutdown on the write side, the <c>{exit_on_close, false}</c> option is useful.</p> diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl index 6635885aaf..a1a99a4e18 100644 --- a/lib/kernel/src/application_controller.erl +++ b/lib/kernel/src/application_controller.erl @@ -490,7 +490,8 @@ init(Init, Kernel) -> %% called during start-up of any app. case check_conf_data(ConfData) of ok -> - _ = ets:new(ac_tab, [set, public, named_table]), + _ = ets:new(ac_tab, [set, public, named_table, + {read_concurrency,true}]), S = #state{conf_data = ConfData}, {ok, KAppl} = make_appl(Kernel), case catch load(S, KAppl) of diff --git a/lib/kernel/src/code.erl b/lib/kernel/src/code.erl index d73d1ff281..65045666ec 100644 --- a/lib/kernel/src/code.erl +++ b/lib/kernel/src/code.erl @@ -107,7 +107,7 @@ is_module_native(_) -> -spec make_stub_module(Module, Beam, Info) -> Module when Module :: module(), Beam :: binary(), - Info :: {list(), list()}. + Info :: {list(), list(), binary()}. make_stub_module(_, _, _) -> erlang:nif_error(undef). @@ -560,12 +560,12 @@ load_native_code_for_all_loaded() -> try hipe_unified_loader:chunk_name(Architecture) of ChunkTag -> Loaded = all_loaded(), - spawn(fun() -> load_all_native(Loaded, ChunkTag) end) + _ = spawn(fun() -> load_all_native(Loaded, ChunkTag) end), + ok catch _:_ -> ok - end, - ok. + end. load_all_native(Loaded, ChunkTag) -> catch load_all_native_1(Loaded, ChunkTag). @@ -582,7 +582,8 @@ load_all_native_1([{Mod,BeamFilename}|T], ChunkTag) -> undefined -> ok; NativeCode when is_binary(NativeCode) -> - load_native_partial(Mod, NativeCode) + _ = load_native_partial(Mod, NativeCode), + ok end; true -> ok end, diff --git a/lib/kernel/src/inet.erl b/lib/kernel/src/inet.erl index ec2c350931..d668738109 100644 --- a/lib/kernel/src/inet.erl +++ b/lib/kernel/src/inet.erl @@ -1527,26 +1527,28 @@ tcp_controlling_process(S, NewOwner) when is_port(S), is_pid(NewOwner) -> _ -> case prim_inet:getopt(S, active) of {ok, A0} -> - case A0 of - false -> ok; - _ -> ok = prim_inet:setopt(S, active, false) - end, - case tcp_sync_input(S, NewOwner, false) of - true -> %% socket already closed, + SetOptRes = + case A0 of + false -> ok; + _ -> prim_inet:setopt(S, active, false) + end, + case {tcp_sync_input(S, NewOwner, false), SetOptRes} of + {true, _} -> %% socket already closed ok; - false -> + {false, ok} -> try erlang:port_connect(S, NewOwner) of true -> unlink(S), %% unlink from port case A0 of false -> ok; - _ -> ok = prim_inet:setopt(S, active, A0) - end, - ok + _ -> prim_inet:setopt(S, active, A0) + end catch error:Reason -> {error, Reason} - end + end; + {false, Error} -> + Error end; Error -> Error diff --git a/lib/kernel/src/inet_parse.erl b/lib/kernel/src/inet_parse.erl index a88c94a453..a694642b19 100644 --- a/lib/kernel/src/inet_parse.erl +++ b/lib/kernel/src/inet_parse.erl @@ -675,28 +675,22 @@ ipv6_addr_done(Ar, Br, N) -> ipv6_addr_done(Ar) -> list_to_tuple(lists:reverse(Ar)). -%% Collect Hex digits -hex(Cs) -> hex(Cs, []). -%% -hex([C|Cs], R) when C >= $0, C =< $9 -> - hex(Cs, [C|R]); -hex([C|Cs], R) when C >= $a, C =< $f -> - hex(Cs, [C|R]); -hex([C|Cs], R) when C >= $A, C =< $F -> - hex(Cs, [C|R]); -hex(Cs, [_|_]=R) when is_list(Cs) -> +%% Collect 1-4 Hex digits +hex(Cs) -> hex(Cs, [], 4). +%% +hex([C|Cs], R, N) when C >= $0, C =< $9, N > 0 -> + hex(Cs, [C|R], N-1); +hex([C|Cs], R, N) when C >= $a, C =< $f, N > 0 -> + hex(Cs, [C|R], N-1); +hex([C|Cs], R, N) when C >= $A, C =< $F, N > 0 -> + hex(Cs, [C|R], N-1); +hex(Cs, [_|_]=R, _) when is_list(Cs) -> {lists:reverse(R),Cs}; -hex(_, _) -> +hex(_, _, _) -> erlang:error(badarg). %% Hex string to integer -hex_to_int(Cs0) -> - case strip0(Cs0) of - Cs when length(Cs) =< 4 -> - erlang:list_to_integer("0"++Cs, 16); - _ -> - erlang:error(badarg) - end. +hex_to_int(Cs) -> erlang:list_to_integer(Cs, 16). %% Dup onto head of existing list dup(0, _, L) -> diff --git a/lib/kernel/src/inet_sctp.erl b/lib/kernel/src/inet_sctp.erl index 93528d305d..f0f13c8d4a 100644 --- a/lib/kernel/src/inet_sctp.erl +++ b/lib/kernel/src/inet_sctp.erl @@ -133,15 +133,18 @@ connect_get_assoc(S, Addr, Port, Active, Timer) -> Timeout = inet:timeout(Timer), receive {sctp,S,Addr,Port,{_,#sctp_assoc_change{state=St}=Ev}} -> - case Active of - once -> - ok = prim_inet:setopt(S, active, once); - _ -> ok - end, - if St =:= comm_up -> + SetOptRes = + case Active of + once -> prim_inet:setopt(S, active, once); + _ -> ok + end, + case {St, SetOptRes} of + {comm_up, ok} -> {ok,Ev}; - true -> - {error,Ev} + {_, ok} -> + {error,Ev}; + {_, Error} -> + Error end after Timeout -> {error,timeout} diff --git a/lib/kernel/src/inet_tcp_dist.erl b/lib/kernel/src/inet_tcp_dist.erl index 835dcf2705..fb60a14afb 100644 --- a/lib/kernel/src/inet_tcp_dist.erl +++ b/lib/kernel/src/inet_tcp_dist.erl @@ -112,7 +112,6 @@ listen_options(Opts0) -> end, case application:get_env(kernel, inet_dist_listen_options) of {ok,ListenOpts} -> - erlang:display({inet_dist_listen_options, ListenOpts}), ListenOpts ++ Opts1; _ -> Opts1 @@ -340,7 +339,6 @@ do_setup(Kernel, Node, Type, MyNode, LongOrShortNames,SetupTime) -> connect_options(Opts) -> case application:get_env(kernel, inet_dist_connect_options) of {ok,ConnectOpts} -> - erlang:display({inet_dist_listen_options, ConnectOpts}), ConnectOpts ++ Opts; _ -> Opts diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index 0cb10791d7..9787dca162 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -115,6 +115,6 @@ {applications, []}, {env, [{error_logger, tty}]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-7.0", "stdlib-2.0", "sasl-2.4"]} + {runtime_dependencies, ["erts-7.0", "stdlib-2.5", "sasl-2.4"]} ] }. diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index 1bae762bed..5d3836bad7 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -1,7 +1,7 @@ %% -*- erlang -*- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2014. All Rights Reserved. +%% Copyright Ericsson AB 1999-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -17,7 +17,7 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"3\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-17 + [{<<"3\\.[0-2](\\.[0-9]+)*">>,[restart_new_emulator]}], % OTP-17 %% Down to - max one major revision back - [{<<"3\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-17 + [{<<"3\\.[0-2](\\.[0-9]+)*">>,[restart_new_emulator]}] % OTP-17 }. diff --git a/lib/kernel/src/user_drv.erl b/lib/kernel/src/user_drv.erl index 380c685869..d3deca3a20 100644 --- a/lib/kernel/src/user_drv.erl +++ b/lib/kernel/src/user_drv.erl @@ -135,7 +135,7 @@ server1(Iport, Oport, Shell) -> Iport, Oport), %% Enter the server loop. - server_loop(Iport, Oport, Curr, User, Gr, queue:new()). + server_loop(Iport, Oport, Curr, User, Gr, {false, queue:new()}). rem_sh_opts(Node) -> [{expand_fun,fun(B)-> rpc:call(Node,edlin_expand,expand,[B]) end}]. @@ -165,7 +165,7 @@ server_loop(Iport, Oport, User, Gr, IOQueue) -> put(current_group, Curr), server_loop(Iport, Oport, Curr, User, Gr, IOQueue). -server_loop(Iport, Oport, Curr, User, Gr, IOQueue) -> +server_loop(Iport, Oport, Curr, User, Gr, {Resp, IOQ} = IOQueue) -> receive {Iport,{data,Bs}} -> BsBin = list_to_binary(Bs), @@ -182,9 +182,9 @@ server_loop(Iport, Oport, Curr, User, Gr, IOQueue) -> {Oport,ok} -> %% We get this ok from the port, in io_request we store %% info about where to send reply at head of queue - {{value,{Origin,Reply}},ReplyQ} = queue:out(IOQueue), + {Origin,Reply} = Resp, Origin ! {reply,Reply}, - NewQ = handle_req(next, Iport, Oport, ReplyQ), + NewQ = handle_req(next, Iport, Oport, {false, IOQ}), server_loop(Iport, Oport, Curr, User, Gr, NewQ); {'EXIT',Iport,_R} -> server_loop(Iport, Oport, Curr, User, Gr, IOQueue); @@ -238,28 +238,30 @@ handle_req({Curr,get_unicode_state},Iport,_Oport,IOQueue) -> handle_req({Curr,set_unicode_state, Bool},Iport,_Oport,IOQueue) -> Curr ! {self(),set_unicode_state,set_unicode_state(Iport,Bool)}, IOQueue; -handle_req(next,Iport,Oport,IOQueue) -> - case queue:out(IOQueue) of - {{value,Next},ExecQ} -> - NewQ = handle_req(Next,Iport,Oport,queue:new()), - queue:join(NewQ,ExecQ); +handle_req(next,Iport,Oport,{false,IOQ}=IOQueue) -> + case queue:out(IOQ) of {empty,_} -> - IOQueue - end; -handle_req(Msg,Iport,Oport,IOQueue) -> - case queue:peek(IOQueue) of - empty -> - {Origin,Req} = Msg, + IOQueue; + {{value,{Origin,Req}},ExecQ} -> case io_request(Req, Iport, Oport) of - ok -> IOQueue; + ok -> + handle_req(next,Iport,Oport,{false,ExecQ}); Reply -> - %% Push reply info to front of queue - queue:in_r({Origin,Reply},IOQueue) - end; - _Else -> - %% All requests are queued when we have outstanding sync put_chars - queue:in(Msg,IOQueue) - end. + {{Origin,Reply}, ExecQ} + end + end; +handle_req(Msg,Iport,Oport,{false,IOQ}=IOQueue) -> + empty = queue:peek(IOQ), + {Origin,Req} = Msg, + case io_request(Req, Iport, Oport) of + ok -> + IOQueue; + Reply -> + {{Origin,Reply}, IOQ} + end; +handle_req(Msg,_Iport,_Oport,{Resp, IOQ}) -> + %% All requests are queued when we have outstanding sync put_chars + {Resp, queue:in(Msg,IOQ)}. %% port_bytes(Bytes, InPort, OutPort, CurrentProcess, UserProcess, Group) %% Check the Bytes from the port to see if it contains a ^G. If so, diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index 549c65d034..c82aaf0582 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -1654,9 +1654,7 @@ get_mode(Config) when is_list(Config) -> init(Tester) -> {ok, Tester}. -handle_event({error, _GL, {emulator, _, _}}, Tester) -> - {ok, Tester}; -handle_event({error, _GL, Msg}, Tester) -> +handle_event({warning_msg, _GL, Msg}, Tester) -> Tester ! Msg, {ok, Tester}; handle_event(_Event, State) -> diff --git a/lib/kernel/test/error_logger_SUITE.erl b/lib/kernel/test/error_logger_SUITE.erl index 05bf5aae18..1c2e56f083 100644 --- a/lib/kernel/test/error_logger_SUITE.erl +++ b/lib/kernel/test/error_logger_SUITE.erl @@ -32,7 +32,7 @@ error_report/1, info_report/1, error/1, info/1, emulator/1, tty/1, logfile/1, add/1, delete/1]). --export([generate_error/0]). +-export([generate_error/2]). -export([init/1, handle_event/2, handle_call/2, handle_info/2, @@ -210,13 +210,16 @@ emulator(suite) -> []; emulator(doc) -> []; emulator(Config) when is_list(Config) -> ?line error_logger:add_report_handler(?MODULE, self()), - spawn(?MODULE, generate_error, []), - reported(emulator), + Msg = "Error in process ~p on node ~p with exit value:~n~p~n", + Error = {badmatch,4}, + Stack = [{module, function, 2, []}], + Pid = spawn(?MODULE, generate_error, [Error, Stack]), + reported(error, Msg, [Pid, node(), {Error, Stack}]), ?line my_yes = error_logger:delete_report_handler(?MODULE), ok. -generate_error() -> - erlang:error({badmatch,4}). +generate_error(Error, Stack) -> + erlang:raise(error, Error, Stack). %%----------------------------------------------------------------- %% We don't enables or disables tty error logging here. We do not @@ -283,15 +286,6 @@ reported(Tag, Type, Report) -> test_server:fail(no_report_received) end. -reported(emulator) -> - receive - {error, "~s~n", String} when is_list(String) -> - test_server:messages_get(), - ok - after 1000 -> - test_server:fail(no_report_received) - end. - %%----------------------------------------------------------------- %% The error_logger handler (gen_event behaviour). %% Sends a notification to the Tester process about the events diff --git a/lib/kernel/test/error_logger_warn_SUITE.erl b/lib/kernel/test/error_logger_warn_SUITE.erl index 2bf467610e..fb576d77a3 100644 --- a/lib/kernel/test/error_logger_warn_SUITE.erl +++ b/lib/kernel/test/error_logger_warn_SUITE.erl @@ -21,8 +21,8 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, - basic/1,warnings_info/1,warnings_warnings/1, - rb_basic/1,rb_warnings_info/1,rb_warnings_warnings/1, + basic/1,warnings_info/1,warnings_errors/1, + rb_basic/1,rb_warnings_info/1,rb_warnings_errors/1, rb_trunc/1,rb_utc/1,file_utc/1]). %% Internal exports. @@ -48,8 +48,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [basic, warnings_info, warnings_warnings, rb_basic, - rb_warnings_info, rb_warnings_warnings, rb_trunc, + [basic, warnings_info, warnings_errors, rb_basic, + rb_warnings_info, rb_warnings_errors, rb_trunc, rb_utc, file_utc]. groups() -> @@ -88,11 +88,11 @@ warnings_info(Config) when is_list(Config) -> put(elw_config,Config), warnings_info(). -warnings_warnings(doc) -> - ["Tests mapping warnings to warnings functionality"]; -warnings_warnings(Config) when is_list(Config) -> +warnings_errors(doc) -> + ["Tests mapping warnings to errors functionality"]; +warnings_errors(Config) when is_list(Config) -> put(elw_config,Config), - warnings_warnings(). + warnings_errors(). rb_basic(doc) -> ["Tests basic rb functionality"]; @@ -106,11 +106,11 @@ rb_warnings_info(Config) when is_list(Config) -> put(elw_config,Config), rb_warnings_info(). -rb_warnings_warnings(doc) -> - ["Tests warnings as warnings rb functionality"]; -rb_warnings_warnings(Config) when is_list(Config) -> +rb_warnings_errors(doc) -> + ["Tests warnings as errors rb functionality"]; +rb_warnings_errors(Config) when is_list(Config) -> put(elw_config,Config), - rb_warnings_warnings(). + rb_warnings_errors(). rb_trunc(doc) -> ["Tests rb functionality on truncated data"]; @@ -159,6 +159,9 @@ install_relay(Node) -> rpc:call(Node,error_logger,add_report_handler,[?MODULE,[self()]]). +warning_map(Node) -> + rpc:call(Node,error_logger,warning_map,[]). + format(Node,A,B) -> rpc:call(Node,error_logger,format,[A,B]). error_msg(Node,A,B) -> @@ -185,19 +188,20 @@ basic() -> ?line ok = install_relay(Node), ?line Self = self(), ?line GL = group_leader(), + ?line warning = warning_map(Node), ?line format(Node,"~p~n",[Self]), ?line ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), ?line error_msg(Node,"~p~n",[Self]), ?line ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), ?line warning_msg(Node,"~p~n",[Self]), - ?line ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), + ?line ?EXPECT({handle_event,{warning_msg,GL,{_,"~p~n",[Self]}}}), ?line info_msg(Node,"~p~n",[Self]), ?line ?EXPECT({handle_event,{info_msg,GL,{_,"~p~n",[Self]}}}), ?line Report = [{self,Self},{gl,GL},make_ref()], ?line error_report(Node,Report), ?line ?EXPECT({handle_event,{error_report,GL,{_,std_error,Report}}}), ?line warning_report(Node,Report), - ?line ?EXPECT({handle_event,{error_report,GL,{_,std_error,Report}}}), + ?line ?EXPECT({handle_event,{warning_report,GL,{_,std_warning,Report}}}), ?line info_report(Node,Report), ?line ?EXPECT({handle_event,{info_report,GL,{_,std_info,Report}}}), @@ -209,6 +213,7 @@ warnings_info() -> ?line ok = install_relay(Node), ?line Self = self(), ?line GL = group_leader(), + ?line info = warning_map(Node), ?line Report = [{self,Self},{gl,GL},make_ref()], ?line warning_msg(Node,"~p~n",[Self]), ?line ?EXPECT({handle_event,{info_msg,GL,{_,"~p~n",[Self]}}}), @@ -217,16 +222,17 @@ warnings_info() -> ?line stop_node(Node), ok. -warnings_warnings() -> - ?line Node = start_node(nn(),"+Ww"), +warnings_errors() -> + ?line Node = start_node(nn(),"+We"), ?line ok = install_relay(Node), ?line Self = self(), ?line GL = group_leader(), + ?line error = warning_map(Node), ?line Report = [{self,Self},{gl,GL},make_ref()], ?line warning_msg(Node,"~p~n",[Self]), - ?line ?EXPECT({handle_event,{warning_msg,GL,{_,"~p~n",[Self]}}}), + ?line ?EXPECT({handle_event,{error,GL,{_,"~p~n",[Self]}}}), ?line warning_report(Node,Report), - ?line ?EXPECT({handle_event,{warning_report,GL,{_,std_warning,Report}}}), + ?line ?EXPECT({handle_event,{error_report,GL,{_,std_error,Report}}}), ?line stop_node(Node), ok. @@ -356,6 +362,7 @@ rb_basic() -> "error_logger_mf_maxfiles 5"), ?line Self = self(), ?line GL = group_leader(), + ?line warning = warning_map(Node), ?line Report = [{self,Self},{gl,GL},make_ref()], ?line fake_gl(Node,warning_msg,"~p~n",[Self]), ?line fake_gl(Node,warning_report,Report), @@ -363,10 +370,14 @@ rb_basic() -> ?line application:start(sasl), ?line rb:start([{report_dir, rd()}]), ?line rb:list(), - ?line true = (one_rb_lines([error]) > 1), - ?line true = (one_rb_lines([error_report]) > 1), - ?line 1 = one_rb_findstr([error],pid_to_list(Self)), - ?line 1 = one_rb_findstr([error_report],pid_to_list(Self)), + ?line true = (one_rb_lines([error]) =:= 0), + ?line true = (one_rb_lines([error_report]) =:= 0), + ?line 0 = one_rb_findstr([error],pid_to_list(Self)), + ?line 0 = one_rb_findstr([error_report],pid_to_list(Self)), + ?line 1 = one_rb_findstr([warning_msg],pid_to_list(Self)), + ?line 1 = one_rb_findstr([warning_report],pid_to_list(Self)), + ?line 0 = one_rb_findstr([info_msg],pid_to_list(Self)), + ?line 0 = one_rb_findstr([info_report],pid_to_list(Self)), ?line 2 = one_rb_findstr([],pid_to_list(Self)), ?line true = (one_rb_findstr([progress],"===") > 4), ?line rb:stop(), @@ -381,6 +392,7 @@ rb_warnings_info() -> "error_logger_mf_maxfiles 5"), ?line Self = self(), ?line GL = group_leader(), + ?line info = warning_map(Node), ?line Report = [{self,Self},{gl,GL},make_ref()], ?line fake_gl(Node,warning_msg,"~p~n",[Self]), ?line fake_gl(Node,warning_report,Report), @@ -403,13 +415,14 @@ rb_warnings_info() -> ?line stop_node(Node), ok. -rb_warnings_warnings() -> +rb_warnings_errors() -> ?line clean_rd(), - ?line Node = start_node(nn(),"+W w -boot start_sasl -sasl error_logger_mf_dir "++ + ?line Node = start_node(nn(),"+W e -boot start_sasl -sasl error_logger_mf_dir "++ quote(rd())++" error_logger_mf_maxbytes 5000 " "error_logger_mf_maxfiles 5"), ?line Self = self(), ?line GL = group_leader(), + ?line error = warning_map(Node), ?line Report = [{self,Self},{gl,GL},make_ref()], ?line fake_gl(Node,warning_msg,"~p~n",[Self]), ?line fake_gl(Node,warning_report,Report), @@ -417,12 +430,12 @@ rb_warnings_warnings() -> ?line application:start(sasl), ?line rb:start([{report_dir, rd()}]), ?line rb:list(), - ?line true = (one_rb_lines([error]) =:= 0), - ?line true = (one_rb_lines([error_report]) =:= 0), - ?line 0 = one_rb_findstr([error],pid_to_list(Self)), - ?line 0 = one_rb_findstr([error_report],pid_to_list(Self)), - ?line 1 = one_rb_findstr([warning_msg],pid_to_list(Self)), - ?line 1 = one_rb_findstr([warning_report],pid_to_list(Self)), + ?line true = (one_rb_lines([error]) > 1), + ?line true = (one_rb_lines([error_report]) > 1), + ?line 1 = one_rb_findstr([error],pid_to_list(Self)), + ?line 1 = one_rb_findstr([error_report],pid_to_list(Self)), + ?line 0 = one_rb_findstr([warning_msg],pid_to_list(Self)), + ?line 0 = one_rb_findstr([warning_report],pid_to_list(Self)), ?line 0 = one_rb_findstr([info_msg],pid_to_list(Self)), ?line 0 = one_rb_findstr([info_report],pid_to_list(Self)), ?line 2 = one_rb_findstr([],pid_to_list(Self)), @@ -434,7 +447,7 @@ rb_warnings_warnings() -> rb_trunc() -> ?line clean_rd(), - ?line Node = start_node(nn(),"+W w -boot start_sasl -sasl error_logger_mf_dir "++ + ?line Node = start_node(nn(),"-boot start_sasl -sasl error_logger_mf_dir "++ quote(rd())++" error_logger_mf_maxbytes 5000 " "error_logger_mf_maxfiles 5"), ?line Self = self(), @@ -467,7 +480,7 @@ rb_trunc() -> rb_utc() -> ?line clean_rd(), - ?line Node = start_node(nn(),"+W w -boot start_sasl -sasl error_logger_mf_dir "++ + ?line Node = start_node(nn(),"-boot start_sasl -sasl error_logger_mf_dir "++ quote(rd())++" error_logger_mf_maxbytes 5000 " "error_logger_mf_maxfiles 5 -sasl utc_log true"), ?line Self = self(), @@ -500,7 +513,7 @@ rb_utc() -> file_utc() -> ?line file:delete(lf()), - ?line SS="+W w -stdlib utc_log true -kernel error_logger "++ oquote("{file,"++iquote(lf())++"}"), + ?line SS="-stdlib utc_log true -kernel error_logger "++ oquote("{file,"++iquote(lf())++"}"), %erlang:display(SS), ?line Node = start_node(nn(),SS), %erlang:display(rpc:call(Node,application,get_env,[kernel,error_logger])), diff --git a/lib/kernel/test/gen_tcp_api_SUITE.erl b/lib/kernel/test/gen_tcp_api_SUITE.erl index c27d265550..4a527e2f51 100644 --- a/lib/kernel/test/gen_tcp_api_SUITE.erl +++ b/lib/kernel/test/gen_tcp_api_SUITE.erl @@ -32,6 +32,7 @@ t_connect_bad/1, t_recv_timeout/1, t_recv_eof/1, t_shutdown_write/1, t_shutdown_both/1, t_shutdown_error/1, + t_shutdown_async/1, t_fdopen/1, t_fdconnect/1, t_implicit_inet6/1]). -export([getsockfd/0,closesockfd/1]). @@ -41,7 +42,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [{group, t_accept}, {group, t_connect}, {group, t_recv}, t_shutdown_write, t_shutdown_both, t_shutdown_error, - t_fdopen, t_fdconnect, t_implicit_inet6]. + t_shutdown_async, t_fdopen, t_fdconnect, t_implicit_inet6]. groups() -> [{t_accept, [], [t_accept_timeout]}, @@ -155,7 +156,34 @@ t_shutdown_error(Config) when is_list(Config) -> ?line ok = gen_tcp:close(L), ?line {error, closed} = gen_tcp:shutdown(L, read_write), ok. - + +t_shutdown_async(Config) when is_list(Config) -> + ?line {OS, _} = os:type(), + ?line {ok, L} = gen_tcp:listen(0, [{sndbuf, 4096}]), + ?line {ok, Port} = inet:port(L), + ?line {ok, Client} = gen_tcp:connect(localhost, Port, + [{recbuf, 4096}, + {active, false}]), + ?line {ok, S} = gen_tcp:accept(L), + ?line PayloadSize = 1024 * 1024, + ?line Payload = lists:duplicate(PayloadSize, $.), + ?line ok = gen_tcp:send(S, Payload), + ?line case erlang:port_info(S, queue_size) of + {queue_size, N} when N > 0 -> ok; + {queue_size, 0} when OS =:= win32 -> ok; + {queue_size, 0} = T -> ?t:fail({unexpected, T}) + end, + + ?line ok = gen_tcp:shutdown(S, write), + ?line {ok, Buf} = gen_tcp:recv(Client, PayloadSize), + ?line {error, closed} = gen_tcp:recv(Client, 0), + ?line case length(Buf) of + PayloadSize -> ok; + Sz -> ?t:fail({payload_size, + {expected, PayloadSize}, + {received, Sz}}) + end. + %%% gen_tcp:fdopen/2 diff --git a/lib/kernel/test/inet_SUITE.erl b/lib/kernel/test/inet_SUITE.erl index 44a32fc1ec..c77de9316f 100644 --- a/lib/kernel/test/inet_SUITE.erl +++ b/lib/kernel/test/inet_SUITE.erl @@ -569,8 +569,11 @@ parse_address(Config) when is_list(Config) -> "::-1", "::g", "f:f11::10100:2", + "f:f11::01100:2", "::17000", + "::01700", "10000::", + "01000::", "::8:7:6:5:4:3:2:1", "8:7:6:5:4:3:2:1::", "8:7:6:5:4::3:2:1", diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index e1d447a465..c912da0091 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 3.2 +KERNEL_VSN = 4.0 diff --git a/lib/megaco/doc/src/megaco.xml b/lib/megaco/doc/src/megaco.xml index dff1c3afc6..0a8dfe8a13 100644 --- a/lib/megaco/doc/src/megaco.xml +++ b/lib/megaco/doc/src/megaco.xml @@ -336,7 +336,7 @@ megaco_incr_timer() = #megaco_incr_timer{} <tag><c><![CDATA[request_keep_alive_timeout]]></c></tag> <item> <p>Specifies the timeout time for the request-keep-alive timer. </p> - <p>This timer is started when the <em>first</em> reply to an asynchroneous + <p>This timer is started when the <em>first</em> reply to an asynchronous request (issued using the <seealso marker="megaco#cast">megaco:cast/3</seealso> function) arrives. As long as this timer is running, replies will @@ -837,7 +837,7 @@ megaco_incr_timer() = #megaco_incr_timer{} <tag><c><![CDATA[request_keep_alive_timeout]]></c></tag> <item> <p>Specifies the timeout time for the request-keep-alive timer. </p> - <p>This timer is started when the <em>first</em> reply to an asynchroneous + <p>This timer is started when the <em>first</em> reply to an asynchronous request (issued using the <seealso marker="megaco#cast">megaco:cast/3</seealso> function) arrives. As long as this timer is running, replies will diff --git a/lib/megaco/src/app/megaco.app.src b/lib/megaco/src/app/megaco.app.src index 6ab85a1bbc..573b1857f6 100644 --- a/lib/megaco/src/app/megaco.app.src +++ b/lib/megaco/src/app/megaco.app.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2013. All Rights Reserved. +%% Copyright Ericsson AB 1999-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -113,8 +113,8 @@ {applications, [stdlib, kernel]}, {env, []}, {mod, {megaco_sup, []}}, - {runtime_dependencies, ["stdlib-2.0","runtime_tools-1.8.14","kernel-3.0", - "et-1.5","erts-6.0","debugger-4.0", + {runtime_dependencies, ["stdlib-2.5","runtime_tools-1.8.14","kernel-3.0", + "et-1.5","erts-7.0","debugger-4.0", "asn1-3.0"]} ]}. diff --git a/lib/megaco/src/app/megaco.appup.src b/lib/megaco/src/app/megaco.appup.src index 92504e8e87..1c55a92b55 100644 --- a/lib/megaco/src/app/megaco.appup.src +++ b/lib/megaco/src/app/megaco.appup.src @@ -2,7 +2,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2014. All Rights Reserved. +%% Copyright Ericsson AB 2001-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -183,11 +183,15 @@ %% | %% v %% 3.17.3 +%% | +%% v +%% 3.18 %% %% {"%VSN%", [ + {"3.17.3", [{restart_application,megaco}]}, {"3.17.2", []}, {"3.17.1", [{restart_application,megaco}]}, {"3.17.0.3", [{restart_application,megaco}]}, @@ -202,6 +206,7 @@ } ], [ + {"3.17.3", [{restart_application,megaco}]}, {"3.17.2", []}, {"3.17.1", [{restart_application,megaco}]}, {"3.17.0.3", [{restart_application,megaco}]}, diff --git a/lib/megaco/src/engine/megaco_trans_sender.erl b/lib/megaco/src/engine/megaco_trans_sender.erl index 710fef405a..e07f404289 100644 --- a/lib/megaco/src/engine/megaco_trans_sender.erl +++ b/lib/megaco/src/engine/megaco_trans_sender.erl @@ -672,8 +672,7 @@ to(To, Start) -> %% Time in milli seconds t() -> - {A,B,C} = erlang:now(), - A*1000000000+B*1000+(C div 1000). + erlang:monotonic_time(milli_seconds). warning_msg(F, A) -> ?megaco_warning("Transaction sender: " ++ F, A). diff --git a/lib/megaco/vsn.mk b/lib/megaco/vsn.mk index 8687d622e9..ede36e3fe6 100644 --- a/lib/megaco/vsn.mk +++ b/lib/megaco/vsn.mk @@ -2,7 +2,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2014. All Rights Reserved. +# Copyright Ericsson AB 1997-2015. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -18,6 +18,6 @@ # %CopyrightEnd% APPLICATION = megaco -MEGACO_VSN = 3.17.3 +MEGACO_VSN = 3.18 PRE_VSN = APP_VSN = "$(APPLICATION)-$(MEGACO_VSN)$(PRE_VSN)" diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index f501a4485b..b9c2fd915c 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -306,6 +306,8 @@ ms() -> -spec abort(_) -> no_return(). +abort(Reason = {aborted, _}) -> + exit(Reason); abort(Reason) -> exit({aborted, Reason}). @@ -1626,13 +1628,7 @@ dirty_read(Oid) -> dirty_read(Tab, Key) when is_atom(Tab), Tab /= schema -> -%% case catch ?ets_lookup(Tab, Key) of -%% {'EXIT', _} -> - %% Bad luck, we have to perform a real lookup - dirty_rpc(Tab, mnesia_lib, db_get, [Tab, Key]); -%% Val -> -%% Val -%% end; + dirty_rpc(Tab, mnesia_lib, db_get, [Tab, Key]); dirty_read(Tab, _Key) -> abort({bad_type, Tab}). diff --git a/lib/mnesia/src/mnesia_lib.erl b/lib/mnesia/src/mnesia_lib.erl index 7bd207f816..fc7362a31d 100644 --- a/lib/mnesia/src/mnesia_lib.erl +++ b/lib/mnesia/src/mnesia_lib.erl @@ -411,7 +411,7 @@ pr_other(Var) -> verbose("~p (~p) val(mnesia_gvar, ~w) -> ~p ~n", [self(), process_info(self(), registered_name), Var, Why]), - exit(Why). + mnesia:abort(Why). %% Some functions for list valued variables add(Var, Val) -> diff --git a/lib/mnesia/test/mnesia_evil_coverage_test.erl b/lib/mnesia/test/mnesia_evil_coverage_test.erl index 2d1623b6ca..430c1f1d84 100644 --- a/lib/mnesia/test/mnesia_evil_coverage_test.erl +++ b/lib/mnesia/test/mnesia_evil_coverage_test.erl @@ -1338,11 +1338,11 @@ user_properties(Config) when is_list(Config) -> ?match([], mnesia:table_info(Tab2, user_properties)), ?match([], mnesia:table_info(Tab3, user_properties)), - ?match({'EXIT', {no_exists, {Tab1, user_property, PropKey}}}, + ?match({'EXIT', {aborted, {no_exists, {Tab1, user_property, PropKey}}}}, mnesia:read_table_property(Tab1, PropKey)), - ?match({'EXIT', {no_exists, {Tab2, user_property, PropKey}}}, + ?match({'EXIT', {aborted, {no_exists, {Tab2, user_property, PropKey}}}}, mnesia:read_table_property(Tab2, PropKey)), - ?match({'EXIT', {no_exists, {Tab3, user_property, PropKey}}}, + ?match({'EXIT', {aborted, {no_exists, {Tab3, user_property, PropKey}}}}, mnesia:read_table_property(Tab3, PropKey)), ?match({atomic, ok}, mnesia:write_table_property(Tab1, Prop)), diff --git a/lib/mnesia/vsn.mk b/lib/mnesia/vsn.mk index b23339e408..79dd495c4b 100644 --- a/lib/mnesia/vsn.mk +++ b/lib/mnesia/vsn.mk @@ -1 +1 @@ -MNESIA_VSN = 4.12.5 +MNESIA_VSN = 4.13 diff --git a/lib/observer/vsn.mk b/lib/observer/vsn.mk index 10ed3bdfe5..7e7e32099b 100644 --- a/lib/observer/vsn.mk +++ b/lib/observer/vsn.mk @@ -1 +1 @@ -OBSERVER_VSN = 2.0.4 +OBSERVER_VSN = 2.1 diff --git a/lib/orber/src/orber.app.src b/lib/orber/src/orber.app.src index 5dda63982f..217c1b247f 100644 --- a/lib/orber/src/orber.app.src +++ b/lib/orber/src/orber.app.src @@ -104,7 +104,7 @@ {applications, [stdlib, kernel, mnesia]}, {env, []}, {mod, {orber, []}}, - {runtime_dependencies, ["stdlib-2.0","ssl-5.3.4","mnesia-4.12","kernel-3.0", + {runtime_dependencies, ["stdlib-2.5","ssl-5.3.4","mnesia-4.12","kernel-3.0", "inets-5.10","erts-7.0"]} ]}. diff --git a/lib/os_mon/c_src/cpu_sup.c b/lib/os_mon/c_src/cpu_sup.c index e9fd75a32c..9e217db105 100644 --- a/lib/os_mon/c_src/cpu_sup.c +++ b/lib/os_mon/c_src/cpu_sup.c @@ -31,15 +31,28 @@ #include <unistd.h> #include <string.h> +#if (defined(__APPLE__) && defined(__MACH__)) || defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__DragonFly__) +#include <sys/param.h> +#include <sys/sysctl.h> +#include <limits.h> +#include <fcntl.h> +#endif +#if defined(__FreeBSD__) || defined(__DragonFly__) +#include <kvm.h> +#include <sys/user.h> +#endif + #if defined(__sun__) #include <kstat.h> #endif -#include <sys/sysinfo.h> #include <errno.h> +#if defined(__sun__) || defined(__linux__) +#include <sys/sysinfo.h> +#endif + #if defined(__linux__) -#include <string.h> /* strlen */ #define PROCSTAT "/proc/stat" #define BUFFERSIZE (256) @@ -59,6 +72,13 @@ typedef struct { #endif +#if defined(__FreeBSD__) +#include <sys/resource.h> +#include <sys/sysctl.h> +#define CU_BSD_VALUES (6) +#endif + + #define FD_IN (0) #define FD_OUT (1) #define FD_ERR (2) @@ -124,10 +144,15 @@ static void util_measure(unsigned int **result_vec, int *result_sz); #if defined(__sun__) static unsigned int misc_measure(char* name); #endif -static void send(unsigned int data); +static void sendi(unsigned int data); static void sendv(unsigned int data[], int ints); static void error(char* err_msg); +#if (defined(__APPLE__) && defined(__MACH__)) || defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__DragonFly__) +static void bsd_count_procs(void); +static void bsd_loadavg(int); +#endif + #if defined(__sun__) static kstat_ctl_t *kstat_ctl; #endif @@ -138,12 +163,16 @@ static int processors_online() { } #endif +#if defined(__FreeBSD__) +void getsysctl(const char *, void *, size_t); +#endif + int main(int argc, char** argv) { char cmd; int rc; int sz; unsigned int *rv; -#if defined(__linux__) +#if defined(__linux__) || defined(__FreeBSD__) unsigned int no_of_cpus = 0; #endif @@ -156,7 +185,14 @@ int main(int argc, char** argv) { #if defined(__linux__) no_of_cpus = processors_online(); if ( (rv = (unsigned int*)malloc(sizeof(unsigned int)*(2 + 2*no_of_cpus*CU_VALUES))) == NULL) { - error("cpu_cup: malloc error"); + error("cpu_sup: malloc error"); + } +#endif + +#if defined(__FreeBSD__) + getsysctl("hw.ncpu", &no_of_cpus, sizeof(int)); + if ( (rv = (unsigned int*)malloc(sizeof(unsigned int)*(2 + 2*no_of_cpus*CU_BSD_VALUES))) == NULL) { + error("cpu_sup: malloc error"); } #endif @@ -173,20 +209,104 @@ int main(int argc, char** argv) { error("Erlang has closed"); switch(cmd) { - case PING: send(4711); break; + case PING: sendi(4711); break; #if defined(__sun__) - case NPROCS: send(misc_measure("nproc")); break; - case AVG1: send(misc_measure("avenrun_1min")); break; - case AVG5: send(misc_measure("avenrun_5min")); break; - case AVG15: send(misc_measure("avenrun_15min")); break; + case NPROCS: sendi(misc_measure("nproc")); break; + case AVG1: sendi(misc_measure("avenrun_1min")); break; + case AVG5: sendi(misc_measure("avenrun_5min")); break; + case AVG15: sendi(misc_measure("avenrun_15min")); break; +#elif defined(__OpenBSD__) || (defined(__APPLE__) && defined(__MACH__)) || defined(__FreeBSD__) || defined(__DragonFly__) + case NPROCS: bsd_count_procs(); break; + case AVG1: bsd_loadavg(0); break; + case AVG5: bsd_loadavg(1); break; + case AVG15: bsd_loadavg(2); break; #endif +#if defined(__sun__) || defined(__linux__) || defined(__FreeBSD__) case UTIL: util_measure(&rv,&sz); sendv(rv, sz); break; +#endif case QUIT: free((void*)rv); return 0; default: error("Bad command"); break; } } - return 0; /* supress warnings */ + return 0; /* suppress warnings */ +} + +/* ---------------------------- * + * BSD stat functions * + * ---------------------------- */ +#if defined(__OpenBSD__) || (defined(__APPLE__) && defined(__MACH__)) || defined(__FreeBSD__) || defined(__DragonFly__) + +static void bsd_loadavg(int idx) { + double avgs[3]; + if (getloadavg(avgs, 3) < 0) { + error(strerror(errno)); + return; + } + sendi((unsigned int)(avgs[idx] * 256)); +} + +#endif + +#if defined(__OpenBSD__) + +static void bsd_count_procs(void) { + int err, nproc; + size_t len = sizeof(nproc); + int mib[] = { CTL_KERN, KERN_NPROCS }; + + err = sysctl(mib, sizeof(mib) / sizeof(mib[0]), &nproc, &len, NULL, 0); + if (err) { + error(strerror(errno)); + return; + } + + sendi((unsigned int)nproc); +} + +#elif defined(__FreeBSD__) || defined(__DragonFly__) + +static void bsd_count_procs(void) { + kvm_t *kd; + struct kinfo_proc *kp; + char err[_POSIX2_LINE_MAX]; + int cnt = 0; + + if ((kd = kvm_open(NULL, "/dev/null", NULL, O_RDONLY, err)) == NULL) { + error(err); + return; + } + +#if defined(KERN_PROC_PROC) + if ((kp = kvm_getprocs(kd, KERN_PROC_PROC, 0, &cnt)) == NULL) { +#else + if ((kp = kvm_getprocs(kd, KERN_PROC_ALL, 0, &cnt)) == NULL) { +#endif + error(strerror(errno)); + return; + } + + (void)kvm_close(kd); + sendi((unsigned int)cnt); } + +#elif (defined(__APPLE__) && defined(__MACH__)) + +static void bsd_count_procs(void) { + int err; + size_t len = 0; + int mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_ALL }; + + err = sysctl(mib, sizeof(mib) / sizeof(mib[0]), NULL, &len, NULL, 0); + if (err) { + error(strerror(errno)); + return; + } + + sendi((unsigned int)(len / sizeof(struct kinfo_proc))); +} + +#endif + /* ---------------------------- * * Linux stat functions * * ---------------------------- */ @@ -417,10 +537,75 @@ static void util_measure(unsigned int **result_vec, int *result_sz) { #endif /* ---------------------------- * + * FreeBSD stat functions * + * ---------------------------- */ + +#if defined(__FreeBSD__) + +#define EXIT_WITH(msg) (rich_error(msg, __FILE__, __LINE__)) +#define RICH_BUFLEN (213) /* left in error(char*) */ + +void rich_error(const char *reason, const char *file, const int line) { + char buf[RICH_BUFLEN]; + snprintf(buf, RICH_BUFLEN, "%s (%s:%i)", reason, file, line); + error(buf); +} +#undef RICH_BUFLEN + +static void util_measure(unsigned int **result_vec, int *result_sz) { + int no_of_cpus; + size_t size_cpu_times; + unsigned long *cpu_times; + unsigned int *rv = NULL; + int i; + + getsysctl("hw.ncpu", &no_of_cpus, sizeof(int)); + /* Header constant CPUSTATES = #long values per cpu. */ + size_cpu_times = sizeof(long) * CPUSTATES * no_of_cpus; + cpu_times = malloc(size_cpu_times); + if (!cpu_times) { + EXIT_WITH("badalloc"); + } + getsysctl("kern.cp_times", cpu_times, size_cpu_times); + + rv = *result_vec; + rv[0] = no_of_cpus; + rv[1] = CU_BSD_VALUES; + ++rv; /* first value is number of cpus */ + ++rv; /* second value is number of entries */ + + for (i = 0; i < no_of_cpus; ++i) { + int offset = i * CPUSTATES; + rv[ 0] = CU_CPU_ID; rv[ 1] = i; + rv[ 2] = CU_USER; rv[ 3] = cpu_times[CP_USER + offset]; + rv[ 4] = CU_NICE_USER; rv[ 5] = cpu_times[CP_NICE + offset]; + rv[ 6] = CU_KERNEL; rv[ 7] = cpu_times[CP_SYS + offset]; + rv[ 8] = CU_IDLE; rv[ 9] = cpu_times[CP_IDLE + offset]; + rv[10] = CU_HARD_IRQ; rv[11] = cpu_times[CP_INTR + offset]; + rv += CU_BSD_VALUES*2; + } + + *result_sz = 2 + 2*CU_BSD_VALUES * no_of_cpus; +} + +void getsysctl(const char *name, void *ptr, size_t len) +{ + size_t gotlen = len; + if (sysctlbyname(name, ptr, &gotlen, NULL, 0) != 0) { + EXIT_WITH("sysctlbyname failed"); + } + if (gotlen != len) { + EXIT_WITH("sysctlbyname: unexpected length"); + } +} +#endif + + +/* ---------------------------- * * Generic functions * * ---------------------------- */ -static void send(unsigned int data) { sendv(&data, 1); } +static void sendi(unsigned int data) { sendv(&data, 1); } static void sendv(unsigned int data[], int ints) { static unsigned char *buf = NULL; @@ -474,8 +659,7 @@ static void error(char* err_msg) { buffer[i++] = '\n'; /* try to use one write only */ - if(write(FD_ERR, buffer, i)); + if(write(FD_ERR, buffer, i)) + ; exit(-1); } - - diff --git a/lib/os_mon/doc/src/cpu_sup.xml b/lib/os_mon/doc/src/cpu_sup.xml index 59da876208..4a8f5bffa0 100644 --- a/lib/os_mon/doc/src/cpu_sup.xml +++ b/lib/os_mon/doc/src/cpu_sup.xml @@ -34,7 +34,7 @@ and CPU utilization. It is part of the OS_Mon application, see <seealso marker="os_mon_app">os_mon(6)</seealso>. Available for Unix, although CPU utilization values (<c>util/0,1</c>) are only - available for Solaris and Linux.</p> + available for Solaris, Linux and FreeBSD.</p> <p>The load values are proportional to how long time a runnable Unix process has to spend in the run queue before it is scheduled. Accordingly, higher values mean more system load. The returned diff --git a/lib/os_mon/src/cpu_sup.erl b/lib/os_mon/src/cpu_sup.erl index 66e7973e7e..d8cfd845bc 100644 --- a/lib/os_mon/src/cpu_sup.erl +++ b/lib/os_mon/src/cpu_sup.erl @@ -121,7 +121,7 @@ util(Args) when is_list (Args) -> util(_) -> erlang:error(badarg). --spec util() -> float(). +-spec util() -> float() | {'error', any()}. util() -> case util([]) of @@ -160,7 +160,8 @@ handle_call(?quit, _From, State) -> handle_call({?util, D, PC}, {Client, _Tag}, #state{os_type = {unix, Flavor}} = State) when Flavor == sunos; - Flavor == linux -> + Flavor == linux; + Flavor == freebsd -> case measurement_server_call(State#state.server, {?util, D, PC, Client}) of {error, Reason} -> { reply, @@ -217,8 +218,6 @@ code_change(_OldVsn, State, _Extra) -> %% internal functions %%---------------------------------------------------------------------- -get_uint32_measurement(Request, #internal{port = P, os_type = {unix, sunos}}) -> - port_server_call(P, Request); get_uint32_measurement(Request, #internal{os_type = {unix, linux}}) -> {ok,F} = file:open("/proc/loadavg",[read,raw]), {ok,D} = file:read_line(F), @@ -231,67 +230,13 @@ get_uint32_measurement(Request, #internal{os_type = {unix, linux}}) -> ?ping -> 4711; ?nprocs -> PTotal end; -get_uint32_measurement(Request, #internal{os_type = {unix, freebsd}}) -> - D = os:cmd("/sbin/sysctl -n vm.loadavg") -- "\n", - {ok,[Load1,Load5,Load15],_} = io_lib:fread("{ ~f ~f ~f }", D), - %% We could count the lines from the ps command as well - case Request of - ?avg1 -> sunify(Load1); - ?avg5 -> sunify(Load5); - ?avg15 -> sunify(Load15); - ?ping -> 4711; - ?nprocs -> - Ps = os:cmd("/bin/ps -ax | /usr/bin/wc -l"), - {ok, [N], _} = io_lib:fread("~d", Ps), - N-1 - end; -get_uint32_measurement(Request, #internal{os_type = {unix, dragonfly}}) -> - D = os:cmd("/sbin/sysctl -n vm.loadavg") -- "\n", - {ok,[Load1,Load5,Load15],_} = io_lib:fread("{ ~f ~f ~f }", D), - %% We could count the lines from the ps command as well - case Request of - ?avg1 -> sunify(Load1); - ?avg5 -> sunify(Load5); - ?avg15 -> sunify(Load15); - ?ping -> 4711; - ?nprocs -> - Ps = os:cmd("/bin/ps -ax | /usr/bin/wc -l"), - {ok, [N], _} = io_lib:fread("~d", Ps), - N-1 - end; -get_uint32_measurement(Request, #internal{os_type = {unix, openbsd}}) -> - D = os:cmd("/sbin/sysctl -n vm.loadavg") -- "\n", - {ok, [L1, L5, L15], _} = io_lib:fread("~f ~f ~f", D), - case Request of - ?avg1 -> sunify(L1); - ?avg5 -> sunify(L5); - ?avg15 -> sunify(L15); - ?ping -> 4711; - ?nprocs -> - Ps = os:cmd("/bin/ps -ax | /usr/bin/wc -l"), - {ok, [N], _} = io_lib:fread("~d", Ps), - N-1 - end; -get_uint32_measurement(Request, #internal{os_type = {unix, darwin}}) -> - %% Get the load average using uptime, overriding Locale setting. - D = os:cmd("LANG=C LC_ALL=C uptime") -- "\n", - %% Here is a sample uptime string from Mac OS 10.3.8 (C Locale): - %% "11:17 up 12 days, 20:39, 2 users, load averages: 1.07 0.95 0.66" - %% The safest way to extract the load averages seems to be grab everything - %% after the last colon and then do an fread on that. - Avg = lists:reverse(hd(string:tokens(lists:reverse(D), ":"))), - {ok,[L1,L5,L15],_} = io_lib:fread("~f ~f ~f", Avg), - - case Request of - ?avg1 -> sunify(L1); - ?avg5 -> sunify(L5); - ?avg15 -> sunify(L15); - ?ping -> 4711; - ?nprocs -> - Ps = os:cmd("/bin/ps -ax | /usr/bin/wc -l"), - {ok, [N], _} = io_lib:fread("~d", Ps), - N-1 - end; +get_uint32_measurement(Request, #internal{port = P, os_type = {unix, Sys}}) when + Sys == sunos; + Sys == dragonfly; + Sys == openbsd; + Sys == freebsd; + Sys == darwin -> + port_server_call(P, Request); get_uint32_measurement(Request, #internal{os_type = {unix, Sys}}) when Sys == irix64; Sys == irix -> %% Get the load average using uptime. @@ -541,14 +486,16 @@ measurement_server_init() -> process_flag(trap_exit, true), OS = os:type(), Server = case OS of - {unix, Flavor} when Flavor==sunos; - Flavor==linux -> - {ok, Pid} = port_server_start_link(), - Pid; - {unix, Flavor} when Flavor==darwin; + {unix, Flavor} when + Flavor==sunos; + Flavor==linux; + Flavor==darwin; Flavor==freebsd; Flavor==dragonfly; - Flavor==openbsd; + Flavor==openbsd -> + {ok, Pid} = port_server_start_link(), + Pid; + {unix, Flavor} when Flavor==irix64; Flavor==irix -> not_used; diff --git a/lib/os_mon/test/cpu_sup_SUITE.erl b/lib/os_mon/test/cpu_sup_SUITE.erl index 9f58e043db..7da819379c 100644 --- a/lib/os_mon/test/cpu_sup_SUITE.erl +++ b/lib/os_mon/test/cpu_sup_SUITE.erl @@ -64,6 +64,8 @@ all() -> [load_api, util_api, util_values, port, unavailable]; {unix, linux} -> [load_api, util_api, util_values, port, unavailable]; + {unix, freebsd} -> + [load_api, util_api, util_values, port, unavailable]; {unix, _OSname} -> [load_api]; _OS -> [unavailable] end. diff --git a/lib/os_mon/vsn.mk b/lib/os_mon/vsn.mk index 833e855e0e..7f2667e40a 100644 --- a/lib/os_mon/vsn.mk +++ b/lib/os_mon/vsn.mk @@ -1 +1 @@ -OS_MON_VSN = 2.3.1 +OS_MON_VSN = 2.4 diff --git a/lib/parsetools/src/parsetools.app.src b/lib/parsetools/src/parsetools.app.src index 9eeb8fcc05..a7b258820a 100644 --- a/lib/parsetools/src/parsetools.app.src +++ b/lib/parsetools/src/parsetools.app.src @@ -12,7 +12,7 @@ {env, [{file_util_search_methods,[{"", ""}, {"ebin", "esrc"}, {"ebin", "src"}]} ] }, - {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]} + {runtime_dependencies, ["stdlib-2.5","kernel-3.0","erts-6.0"]} ] }. diff --git a/lib/parsetools/vsn.mk b/lib/parsetools/vsn.mk index dd9cc2991c..b99b3bb713 100644 --- a/lib/parsetools/vsn.mk +++ b/lib/parsetools/vsn.mk @@ -1 +1 @@ -PARSETOOLS_VSN = 2.0.12 +PARSETOOLS_VSN = 2.1 diff --git a/lib/public_key/doc/src/Makefile b/lib/public_key/doc/src/Makefile index 17fb67e95c..d04819b5aa 100644 --- a/lib/public_key/doc/src/Makefile +++ b/lib/public_key/doc/src/Makefile @@ -42,8 +42,7 @@ XML_REF6_FILES = XML_PART_FILES = part.xml part_notes.xml XML_CHAPTER_FILES = \ introduction.xml \ - public_key_records.xml \ - cert_records.xml \ + public_key_records.xml \ using_public_key.xml \ notes.xml diff --git a/lib/public_key/doc/src/cert_records.xml b/lib/public_key/doc/src/cert_records.xml deleted file mode 100644 index 857a39bf40..0000000000 --- a/lib/public_key/doc/src/cert_records.xml +++ /dev/null @@ -1,690 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE chapter SYSTEM "chapter.dtd"> - -<chapter> - <header> - <copyright> - <year>2008</year> - <year>2014</year> - <holder>Ericsson AB, All Rights Reserved</holder> - </copyright> - <legalnotice> - The contents of this file are subject to the Erlang Public License, - Version 1.1, (the "License"); you may not use this file except in - compliance with the License. You should have received a copy of the - Erlang Public License along with this software. If not, it can be - retrieved online at http://www.erlang.org/. - - Software distributed under the License is distributed on an "AS IS" - basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - the License for the specific language governing rights and limitations - under the License. - - The Initial Developer of the Original Code is Ericsson AB. - </legalnotice> - - <title>Certificate records</title> - <prepared>Ingela Anderton Andin</prepared> - <responsible></responsible> - <docno></docno> - <approved></approved> - <checked></checked> - <date>2008-02-06</date> - <rev>A</rev> - <file>cert_records.xml</file> - </header> - - <p>This chapter briefly describes erlang records derived from ASN1 - specifications used to handle <c> X509 certificates</c> and <c>CertificationRequest</c>. - The intent is to describe the data types -and not to specify the semantics of each component. For information on the -semantics, please see <url - href="http://www.ietf.org/rfc/rfc5280.txt">RFC 5280</url> and - <url href="http://www.ietf.org/rfc/rfc5967.txt">PKCS-10</url>. - </p> - - <p>Use the following include directive to get access to the - records and constant macros (OIDs) described in the following sections.</p> - - <code> -include_lib("public_key/include/public_key.hrl"). </code> - - <p>The used ASN1 specifications are available <c>asn1</c> subdirectory - of the application <c>public_key</c>. - </p> - - <section> - <title>Common Data Types</title> - - <p>Common non standard erlang - data types used to described the record fields in the - below sections are defined in <seealso - marker="public_key">public key reference manual </seealso> or - follows here.</p> - - <p><c>time() = uct_time() | general_time()</c></p> - - <p><c>uct_time() = {utcTime, "YYMMDDHHMMSSZ"} </c></p> - - <p><c>general_time() = {generalTime, "YYYYMMDDHHMMSSZ"} </c></p> - - <p><c> - general_name() = {rfc822Name, string()} | {dNSName, string()} - | {x400Address, string()} | {directoryName, - {rdnSequence, [#AttributeTypeAndValue'{}]}} | - | {eidPartyName, special_string()} - | {eidPartyName, special_string(), special_string()} - | {uniformResourceIdentifier, string()} | {ipAddress, string()} | - {registeredId, oid()} | {otherName, term()} - </c></p> - - <p><c> - special_string() = - {teletexString, string()} | {printableString, string()} | - {universalString, string()} | {utf8String, binary()} | - {bmpString, string()} - </c></p> - - <p><c> - dist_reason() = unused | keyCompromise | cACompromise | - affiliationChanged | superseded | cessationOfOperation | - certificateHold | privilegeWithdrawn | - aACompromise - </c></p> - </section> - - <section> - <title> PKIX Certificates</title> -<code> -#'Certificate'{ - tbsCertificate, % #'TBSCertificate'{} - signatureAlgorithm, % #'AlgorithmIdentifier'{} - signature % bitstring() - }. - -#'TBSCertificate'{ - version, % v1 | v2 | v3 - serialNumber, % integer() - signature, % #'AlgorithmIdentifier'{} - issuer, % {rdnSequence, [#AttributeTypeAndValue'{}]} - validity, % #'Validity'{} - subject, % {rdnSequence, [#AttributeTypeAndValue'{}]} - subjectPublicKeyInfo, % #'SubjectPublicKeyInfo'{} - issuerUniqueID, % binary() | asn1_novalue - subjectUniqueID, % binary() | asn1_novalue - extensions % [#'Extension'{}] - }. - -#'AlgorithmIdentifier'{ - algorithm, % oid() - parameters % der_encoded() - }. -</code> - -<code> -#'OTPCertificate'{ - tbsCertificate, % #'OTPTBSCertificate'{} - signatureAlgorithm, % #'SignatureAlgorithm' - signature % bitstring() - }. - -#'OTPTBSCertificate'{ - version, % v1 | v2 | v3 - serialNumber, % integer() - signature, % #'SignatureAlgorithm' - issuer, % {rdnSequence, [#AttributeTypeAndValue'{}]} - validity, % #'Validity'{} - subject, % {rdnSequence, [#AttributeTypeAndValue'{}]} - subjectPublicKeyInfo, % #'OTPSubjectPublicKeyInfo'{} - issuerUniqueID, % binary() | asn1_novalue - subjectUniqueID, % binary() | asn1_novalue - extensions % [#'Extension'{}] - }. - -#'SignatureAlgorithm'{ - algorithm, % id_signature_algorithm() - parameters % asn1_novalue | #'Dss-Parms'{} - }. -</code> - -<p><c> id_signature_algorithm() = ?oid_name_as_erlang_atom</c> for available -oid names see table below. Ex: ?'id-dsa-with-sha1'</p> -<table> - <row> - <cell align="left" valign="middle">OID name</cell> - </row> - <row> - <cell align="left" valign="middle">id-dsa-with-sha1</cell> - </row> - <row> - <cell align="left" valign="middle">id-dsaWithSHA1 (ISO alt oid to above)</cell> - </row> - <row> - <cell align="left" valign="middle">md2WithRSAEncryption</cell> - </row> - <row> - <cell align="left" valign="middle">md5WithRSAEncryption</cell> - </row> - <row> - <cell align="left" valign="middle">sha1WithRSAEncryption</cell> - </row> - <row> - <cell align="left" valign="middle">sha-1WithRSAEncryption (ISO alt oid to above)</cell> - </row> - <row> - <cell align="left" valign="middle">sha224WithRSAEncryption</cell> - </row> - <row> - <cell align="left" valign="middle">sha256WithRSAEncryption</cell> - </row> - <row> - <cell align="left" valign="middle">sha512WithRSAEncryption</cell> - </row> - <row> - <cell align="left" valign="middle">ecdsa-with-SHA1</cell> - </row> - <tcaption>Signature algorithm oids </tcaption> -</table> - -<code> -#'AttributeTypeAndValue'{ - type, % id_attributes() - value % term() - }. -</code> - -<p><c>id_attributes() </c></p> -<table> - <row> - <cell align="left" valign="middle">OID name</cell> - <cell align="left" valign="middle">Value type</cell> - </row> - <row> - <cell align="left" valign="middle">id-at-name</cell> - <cell align="left" valign="middle">special_string()</cell> - </row> - <row> - <cell align="left" valign="middle">id-at-surname</cell> - <cell align="left" valign="middle">special_string()</cell> - </row> - <row> - <cell align="left" valign="middle">id-at-givenName</cell> - <cell align="left" valign="middle">special_string()</cell> - </row> - <row> - <cell align="left" valign="middle">id-at-initials </cell> - <cell align="left" valign="middle">special_string()</cell> - </row> - <row> - <cell align="left" valign="middle">id-at-generationQualifier</cell> - <cell align="left" valign="middle">special_string()</cell> - </row> - <row> - <cell align="left" valign="middle">id-at-commonName</cell> - <cell align="left" valign="middle">special_string()</cell> - </row> - <row> - <cell align="left" valign="middle">id-at-localityName</cell> - <cell align="left" valign="middle">special_string()</cell> - </row> - <row> - <cell align="left" valign="middle">id-at-stateOrProvinceName</cell> - <cell align="left" valign="middle">special_string()</cell> - </row> - <row> - <cell align="left" valign="middle">id-at-organizationName</cell> - <cell align="left" valign="middle">special_string()</cell> - </row> - <row> - <cell align="left" valign="middle">id-at-title</cell> - <cell align="left" valign="middle">special_string()</cell> - </row> - <row> - <cell align="left" valign="middle">id-at-dnQualifier</cell> - <cell align="left" valign="middle">{printableString, string()}</cell> - </row> - <row> - <cell align="left" valign="middle">id-at-countryName</cell> - <cell align="left" valign="middle">{printableString, string()}</cell> - </row> - <row> - <cell align="left" valign="middle">id-at-serialNumber</cell> - <cell align="left" valign="middle">{printableString, string()}</cell> - </row> - <row> - <cell align="left" valign="middle">id-at-pseudonym</cell> - <cell align="left" valign="middle">special_string()</cell> - </row> - <tcaption>Attribute oids </tcaption> -</table> - -<code> -#'Validity'{ - notBefore, % time() - notAfter % time() - }. - -#'SubjectPublicKeyInfo'{ - algorithm, % #AlgorithmIdentifier{} - subjectPublicKey % binary() - }. - -#'SubjectPublicKeyInfoAlgorithm'{ - algorithm, % id_public_key_algorithm() - parameters % public_key_params() - }. -</code> - -<p><c> id_public_key_algorithm() </c></p> -<table> - <row> - <cell align="left" valign="middle">OID name</cell> - </row> - <row> - <cell align="left" valign="middle">rsaEncryption</cell> - </row> - <row> - <cell align="left" valign="middle">id-dsa</cell> - </row> - <row> - <cell align="left" valign="middle">dhpublicnumber</cell> - </row> - <row> - <cell align="left" valign="middle">id-keyExchangeAlgorithm</cell> - </row> - <row> - <cell align="left" valign="middle">id-ecPublicKey</cell> - </row> - <tcaption>Public key algorithm oids </tcaption> -</table> - -<code> -#'Extension'{ - extnID, % id_extensions() | oid() - critical, % boolean() - extnValue % der_encoded() - }. -</code> - -<p><c>id_extensions()</c> - <seealso marker="#StdCertExt">Standard Certificate Extensions</seealso>, - <seealso marker="#PrivIntExt">Private Internet Extensions</seealso>, - <seealso marker="#CRLCertExt">CRL Extensions</seealso> and - <seealso marker="#CRLEntryExt">CRL Entry Extensions</seealso>. -</p> - -</section> - -<section> - <marker id="StdCertExt"></marker> - <title>Standard certificate extensions</title> - - <table> - <row> - <cell align="left" valign="middle">OID name</cell> - <cell align="left" valign="middle">Value type</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-authorityKeyIdentifier</cell> - <cell align="left" valign="middle">#'AuthorityKeyIdentifier'{}</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-subjectKeyIdentifier</cell> - <cell align="left" valign="middle">oid()</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-keyUsage</cell> - <cell align="left" valign="middle"> [key_usage()]</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-privateKeyUsagePeriod</cell> - <cell align="left" valign="middle">#'PrivateKeyUsagePeriod'{}</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-certificatePolicies</cell> - <cell align="left" valign="middle">#'PolicyInformation'{}</cell> - </row> - - <row> - <cell align="left" valign="middle">id-ce-policyMappings</cell> - <cell align="left" valign="middle">#'PolicyMappings_SEQOF'{}</cell> - </row> - - <row> - <cell align="left" valign="middle">id-ce-subjectAltName</cell> - <cell align="left" valign="middle">general_name()</cell> - </row> - - <row> - <cell align="left" valign="middle">id-ce-issuerAltName</cell> - <cell align="left" valign="middle">general_name()</cell> - </row> - - <row> - <cell align="left" valign="middle">id-ce-subjectDirectoryAttributes</cell> - <cell align="left" valign="middle"> [#'Attribute'{}]</cell> - </row> - - <row> - <cell align="left" valign="middle">id-ce-basicConstraints</cell> - <cell align="left" valign="middle">#'BasicConstraints'{}</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-nameConstraints</cell> - <cell align="left" valign="middle">#'NameConstraints'{}</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-policyConstraints</cell> - <cell align="left" valign="middle">#'PolicyConstraints'{}</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-extKeyUsage</cell> - <cell align="left" valign="middle">[id_key_purpose()]</cell> - </row> - - <row> - <cell align="left" valign="middle">id-ce-cRLDistributionPoints</cell> - <cell align="left" valign="middle">[#'DistributionPoint'{}]</cell> - </row> - - <row> - <cell align="left" valign="middle">id-ce-inhibitAnyPolicy</cell> - <cell align="left" valign="middle">integer()</cell> - </row> - - <row> - <cell align="left" valign="middle">id-ce-freshestCRL</cell> - <cell align="left" valign="middle">[#'DistributionPoint'{}]</cell> - </row> - - - <tcaption>Standard Certificate Extensions</tcaption> - </table> - - <p><c> - key_usage() = digitalSignature | nonRepudiation | keyEncipherment| - dataEncipherment | keyAgreement | keyCertSign | cRLSign | encipherOnly | - decipherOnly - </c></p> - - <p><c> id_key_purpose()</c></p> - -<table> - <row> - <cell align="left" valign="middle">OID name</cell> - </row> - <row> - <cell align="left" valign="middle">id-kp-serverAuth</cell> - </row> - <row> - <cell align="left" valign="middle">id-kp-clientAuth</cell> - </row> - <row> - <cell align="left" valign="middle">id-kp-codeSigning</cell> - </row> - <row> - <cell align="left" valign="middle">id-kp-emailProtection</cell> - </row> - <row> - <cell align="left" valign="middle">id-kp-timeStamping</cell> - </row> - <row> - <cell align="left" valign="middle">id-kp-OCSPSigning</cell> - </row> - <tcaption>Key purpose oids </tcaption> -</table> - - <code> -#'AuthorityKeyIdentifier'{ - keyIdentifier, % oid() - authorityCertIssuer, % general_name() - authorityCertSerialNumber % integer() - }. - -#'PrivateKeyUsagePeriod'{ - notBefore, % general_time() - notAfter % general_time() - }. - -#'PolicyInformation'{ - policyIdentifier, % oid() - policyQualifiers % [#PolicyQualifierInfo{}] - }. - -#'PolicyQualifierInfo'{ - policyQualifierId, % oid() - qualifier % string() | #'UserNotice'{} - }. - -#'UserNotice'{ - noticeRef, % #'NoticeReference'{} - explicitText % string() - }. - -#'NoticeReference'{ - organization, % string() - noticeNumbers % [integer()] - }. - -#'PolicyMappings_SEQOF'{ - issuerDomainPolicy, % oid() - subjectDomainPolicy % oid() - }. - -#'Attribute'{ - type, % oid() - values % [der_encoded()] - }). - -#'BasicConstraints'{ - cA, % boolean() - pathLenConstraint % integer() - }). - -#'NameConstraints'{ - permittedSubtrees, % [#'GeneralSubtree'{}] - excludedSubtrees % [#'GeneralSubtree'{}] - }). - -#'GeneralSubtree'{ - base, % general_name() - minimum, % integer() - maximum % integer() - }). - -#'PolicyConstraints'{ - requireExplicitPolicy, % integer() - inhibitPolicyMapping % integer() - }). - -#'DistributionPoint'{ - distributionPoint, % {fullName, [general_name()]} | {nameRelativeToCRLIssuer, - [#AttributeTypeAndValue{}]} - reasons, % [dist_reason()] - cRLIssuer % [general_name()] - }). -</code> - -</section> - - <section> - <marker id="PrivIntExt"></marker> - <title>Private Internet Extensions</title> - - <table> - <row> - <cell align="left" valign="middle">OID name</cell> - <cell align="left" valign="middle">Value type</cell> - </row> - <row> - <cell align="left" valign="middle">id-pe-authorityInfoAccess</cell> - <cell align="left" valign="middle">[#'AccessDescription'{}]</cell> - </row> - <row> - <cell align="left" valign="middle">id-pe-subjectInfoAccess</cell> - <cell align="left" valign="middle">[#'AccessDescription'{}]</cell> - </row> - <tcaption>Private Internet Extensions</tcaption> - </table> - -<code> -#'AccessDescription'{ - accessMethod, % oid() - accessLocation % general_name() - }). -</code> - - </section> - -<section> - <title> CRL and CRL Extensions Profile</title> - - <code> -#'CertificateList'{ - tbsCertList, % #'TBSCertList{} - signatureAlgorithm, % #'AlgorithmIdentifier'{} - signature % bitstring() - }). - -#'TBSCertList'{ - version, % v2 (if defined) - signature, % #AlgorithmIdentifier{} - issuer, % {rdnSequence, [#AttributeTypeAndValue'{}]} - thisUpdate, % time() - nextUpdate, % time() - revokedCertificates, % [#'TBSCertList_revokedCertificates_SEQOF'{}] - crlExtensions % [#'Extension'{}] - }). - -#'TBSCertList_revokedCertificates_SEQOF'{ - userCertificate, % integer() - revocationDate, % timer() - crlEntryExtensions % [#'Extension'{}] - }). - </code> - - <section> - <marker id="CRLCertExt"></marker> - <title>CRL Extensions </title> - - <table> - <row> - <cell align="left" valign="middle">OID name</cell> - <cell align="left" valign="middle">Value type</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-authorityKeyIdentifier</cell> - <cell align="left" valign="middle">#'AuthorityKeyIdentifier{}</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-issuerAltName</cell> - <cell align="left" valign="middle">{rdnSequence, [#AttributeTypeAndValue'{}]}</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-cRLNumber</cell> - <cell align="left" valign="middle">integer()</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-deltaCRLIndicator</cell> - <cell align="left" valign="middle">integer()</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-issuingDistributionPoint</cell> - <cell align="left" valign="middle">#'IssuingDistributionPoint'{}</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-freshestCRL</cell> - <cell align="left" valign="middle">[#'Distributionpoint'{}]</cell> - </row> - - <tcaption>CRL Extensions</tcaption> - </table> - - <code> -#'IssuingDistributionPoint'{ - distributionPoint, % {fullName, [general_name()]} | {nameRelativeToCRLIssuer, - [#AttributeTypeAndValue'{}]} - onlyContainsUserCerts, % boolean() - onlyContainsCACerts, % boolean() - onlySomeReasons, % [dist_reason()] - indirectCRL, % boolean() - onlyContainsAttributeCerts % boolean() - }). - </code> - </section> - - <section> - <marker id="CRLEntryExt"></marker> - <title> CRL Entry Extensions </title> - - <table> - <row> - <cell align="left" valign="middle">OID name</cell> - <cell align="left" valign="middle">Value type</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-cRLReason</cell> - <cell align="left" valign="middle">crl_reason()</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-holdInstructionCode</cell> - <cell align="left" valign="middle">oid()</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-invalidityDate</cell> - <cell align="left" valign="middle">general_time()</cell> - </row> - <row> - <cell align="left" valign="middle">id-ce-certificateIssuer</cell> - <cell align="left" valign="middle">general_name()</cell> - </row> - <tcaption>CRL Entry Extensions</tcaption> - </table> - <p><c> - crl_reason() = unspecified | keyCompromise | cACompromise | - affiliationChanged | superseded | cessationOfOperation | - certificateHold | removeFromCRL | privilegeWithdrawn | - aACompromise - </c></p> - </section> - - <section> - <marker id="PKCS10"></marker> - <title>PKCS#10 Certification Request</title> - <code> -#'CertificationRequest'{ - certificationRequestInfo #'CertificationRequestInfo'{}, - signatureAlgorithm #'CertificationRequest_signatureAlgorithm'{}}. - signature bitstring() - } - -#'CertificationRequestInfo'{ - version atom(), - subject {rdnSequence, [#AttributeTypeAndValue'{}]} , - subjectPKInfo #'CertificationRequestInfo_subjectPKInfo'{}, - attributes [#'AttributePKCS-10' {}] - } - -#'CertificationRequestInfo_subjectPKInfo'{ - algorithm #'CertificationRequestInfo_subjectPKInfo_algorithm'{} - subjectPublicKey bitstring() - } - -#'CertificationRequestInfo_subjectPKInfo_algorithm'{ - algorithm = oid(), - parameters = der_encoded() -} - -#'CertificationRequest_signatureAlgorithm'{ - algorithm = oid(), - parameters = der_encoded() - } - -#'AttributePKCS-10'{ - type = oid(), - values = [der_encoded()] -} - </code> - </section> - -</section> -</chapter> diff --git a/lib/public_key/doc/src/introduction.xml b/lib/public_key/doc/src/introduction.xml index bf11a092d8..6542c8c509 100644 --- a/lib/public_key/doc/src/introduction.xml +++ b/lib/public_key/doc/src/introduction.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2008</year> - <year>2013</year> + <year>2015</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> @@ -36,27 +36,28 @@ <section> <title>Purpose</title> - <p> public_key deals with public key related file formats, digital - signatures and <url href="http://www.ietf.org/rfc/rfc5280.txt"> + <p>The Public Key application deals with public-key related file + formats, digital signatures, and <url href="http://www.ietf.org/rfc/rfc5280.txt"> X-509 certificates</url>. It is a library application that - provides encode/decode, sign/verify, encrypt/decrypt and similar - functionality, it does not read or write files it expects or returns + provides encode/decode, sign/verify, encrypt/decrypt, and similar + functionality. It does not read or write files, it expects or returns file contents or partial file contents as binaries. </p> </section> <section> <title>Prerequisites</title> - <p>It is assumed that the reader has a basic understanding - of the concepts of using public keys and digital certificates.</p> + <p>It is assumed that the reader is familiar with the Erlang programming + language and has a basic understanding of the concepts of using public-keys + and digital certificates.</p> </section> <section> - <title>Performance tips</title> - <p>The public_key decode and encode functions will try to use the NIFs - which are in the ASN1 compilers runtime modules if they can be found. - So for the best performance you want to have the ASN1 application in the - path of your system. </p> + <title>Performance Tips</title> + <p>The Public Key decode- and encode-functions try to use the NIFs + in the ASN.1 compilers runtime modules, if they can be found. + Thus, to have the ASN1 application in the + path of your system gives the best performance.</p> </section> </chapter> diff --git a/lib/public_key/doc/src/part.xml b/lib/public_key/doc/src/part.xml index 73146c8e2a..465f311946 100644 --- a/lib/public_key/doc/src/part.xml +++ b/lib/public_key/doc/src/part.xml @@ -31,15 +31,14 @@ <file>part.xml</file> </header> <description> - <p> This application provides an API to public key infrastructure + <p>This application provides an API to public-key infrastructure from <url href="http://www.ietf.org/rfc/rfc5280.txt">RFC - 5280</url> (X.509 certificates) and public key formats defined by + 5280</url> (X.509 certificates) and public-key formats defined by the <url href="http://en.wikipedia.org/wiki/PKCS"> - PKCS-standard</url></p> + PKCS</url> standard.</p> </description> <xi:include href="introduction.xml"/> <xi:include href="public_key_records.xml"/> - <xi:include href="cert_records.xml"/> <xi:include href="using_public_key.xml"/> </part> diff --git a/lib/public_key/doc/src/public_key.xml b/lib/public_key/doc/src/public_key.xml index b86d0fe0ab..883c52393f 100644 --- a/lib/public_key/doc/src/public_key.xml +++ b/lib/public_key/doc/src/public_key.xml @@ -31,11 +31,11 @@ <rev></rev> </header> <module>public_key</module> - <modulesummary> API module for public key infrastructure.</modulesummary> + <modulesummary>API module for public-key infrastructure.</modulesummary> <description> - <p>This module provides functions to handle public key infrastructure. It can - encode/decode different file formats (PEM, openssh), sign and verify digital signatures and validate - certificate paths and certificate revocation lists. + <p>This module provides functions to handle public-key infrastructure. It can + encode/decode different file formats (PEM, OpenSSH), sign and verify digital signatures, + and validate certificate paths and certificate revocation lists. </p> </description> @@ -43,94 +43,156 @@ <title>public_key</title> <list type="bulleted"> - <item>public_key requires the crypto and asn1 applications, the latter since R16 (hopefully the runtime dependency on asn1 will + <item> Public Key requires the Crypto and ASN1 applications, + the latter as OTP R16 (hopefully the runtime dependency on ASN1 will be removed again in the future).</item> <item>Supports <url href="http://www.ietf.org/rfc/rfc5280.txt">RFC 5280 </url> - - Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile </item> - <item>Supports <url href="http://www.ietf.org/rfc/rfc3447.txt"> PKCS-1 </url> - RSA Cryptography Standard </item> - <item>Supports <url href="http://csrc.nist.gov/publications/fips/fips186-3/fips_186-3.pdf"> DSS</url>- Digital Signature Standard (DSA - Digital Signature Algorithm)</item> - <item>Supports <url href="http://www.emc.com/emc-plus/rsa-labs/standards-initiatives/pkcs-3-diffie-hellman-key-agreement-standar.htm"> PKCS-3 </url> - Diffie-Hellman Key Agreement Standard </item> - <item>Supports <url href="http://www.ietf.org/rfc/rfc2898.txt"> PKCS-5</url> - Password-Based Cryptography Standard </item> - <item>Supports <url href="http://www.ietf.org/rfc/rfc5208.txt"> PKCS-8</url> - Private-Key Information Syntax Standard</item> - <item>Supports <url href="http://www.ietf.org/rfc/rfc5967.txt"> PKCS-10</url> - Certification Request Syntax Standard</item> + Internet X.509 Public-Key Infrastructure Certificate and Certificate Revocation List + (CRL) Profile </item> + <item>Supports <url href="http://www.ietf.org/rfc/rfc3447.txt"> PKCS-1 </url> - + RSA Cryptography Standard </item> + <item>Supports <url href="http://csrc.nist.gov/publications/fips/fips186-3/fips_186-3.pdf"> DSS</url> - + Digital Signature Standard (DSA - Digital Signature Algorithm)</item> + <item>Supports + <url href="http://www.emc.com/emc-plus/rsa-labs/standards-initiatives/pkcs-3-diffie-hellman-key-agreement-standar.htm"> PKCS-3 </url> - + Diffie-Hellman Key Agreement Standard </item> + <item>Supports <url href="http://www.ietf.org/rfc/rfc2898.txt"> PKCS-5</url> - + Password-Based Cryptography Standard </item> + <item>Supports <url href="http://www.ietf.org/rfc/rfc5208.txt"> PKCS-8</url> - + Private-Key Information Syntax Standard</item> + <item>Supports <url href="http://www.ietf.org/rfc/rfc5967.txt"> PKCS-10</url> - + Certification Request Syntax Standard</item> </list> </section> <section> - <title>COMMON DATA TYPES </title> + <title>DATA TYPES</title> - <note><p>All records used in this manual + <note><p>All records used in this Reference Manual <!-- except #policy_tree_node{} --> are generated from ASN.1 specifications and are documented in the User's Guide. See <seealso - marker="public_key_records">Public key records</seealso> and <seealso - marker="cert_records">X.509 Certificate records</seealso>. + marker="public_key_records">Public-key Records</seealso>. </p></note> <p>Use the following include directive to get access to the - records and constant macros described here and in the User's Guide.</p> + records and constant macros described here and in the User's Guide:</p> <code> -include_lib("public_key/include/public_key.hrl").</code> - <p><em>Data Types </em></p> - - <p><code>oid() - Object Identifier, a tuple of integers as generated by the ASN1 compiler.</code></p> - - <p><code>boolean() = true | false</code></p> + <p>The following data types are used in the functions for <c>public_key</c>:</p> - <p><code>string() = [bytes()]</code></p> - - <p><code>der_encoded() = binary()</code></p> - - <p><code>pki_asn1_type() = 'Certificate' | 'RSAPrivateKey'| 'RSAPublicKey' | - 'DSAPrivateKey' | 'DSAPublicKey' | 'DHParameter' | - 'SubjectPublicKeyInfo' | 'PrivateKeyInfo' | - 'CertificationRequest' | 'ECPrivateKey' | 'EcpkParameters'</code></p> - - <p><code>pem_entry () = {pki_asn1_type(), binary(), %% DER or encrypted DER - not_encrypted | cipher_info()}</code></p> + <taglist> + <tag><c>oid()</c></tag> + <item><p>Object identifier, a tuple of integers as generated by the <c>ASN.1</c> compiler.</p></item> - <p><code>cipher_info() = {"RC2-CBC | "DES-CBC" | "DES-EDE3-CBC", - crypto:rand_bytes(8) | {#'PBEParameter{}, digest_type()} |#'PBES2-params'{}}</code></p> - - <p><code>public_key() = rsa_public_key() | dsa_public_key() | ec_public_key()</code></p> - <p><code>private_key() = rsa_private_key() | dsa_private_key() | ec_private_key()</code></p> - <p><code>rsa_public_key() = #'RSAPublicKey'{}</code></p> + <tag><c>boolean() =</c></tag> + <item><p><c>true | false</c></p></item> + + <tag><c>string() =</c></tag> + <item><p><c>[bytes()]</c></p></item> + + <tag><c>der_encoded() =</c></tag> + <item><p><c>binary()</c></p></item> + + <tag><c>pki_asn1_type() =</c></tag> + <item> + <p><c>'Certificate'</c></p> + <p><c>| 'RSAPrivateKey'</c></p> + <p><c>| 'RSAPublicKey'</c></p> + <p><c>| 'DSAPrivateKey'</c></p> + <p><c>| 'DSAPublicKey'</c></p> + <p><c>| 'DHParameter'</c></p> + <p><c>| 'SubjectPublicKeyInfo'</c></p> + <p><c>| 'PrivateKeyInfo'</c></p> + <p><c>| 'CertificationRequest'</c></p> + <p><c>| 'ECPrivateKey'</c></p> + <p><c>| 'EcpkParameters'</c></p> + </item> + + <tag><c>pem_entry () =</c></tag> + <item><p><c>{pki_asn1_type(), binary(), %% DER or encrypted DER not_encrypted</c></p> + <p><c>| cipher_info()}</c></p></item> + + <tag><c>cipher_info() = </c></tag> + <item><p><c>{"RC2-CBC" | "DES-CBC" | "DES-EDE3-CBC", crypto:rand_bytes(8)</c></p> + <p><c>| {#'PBEParameter{}, digest_type()} | #'PBES2-params'{}}</c></p> + </item> + + <tag><c>public_key() =</c></tag> + <item><p><c>rsa_public_key() | dsa_public_key() | ec_public_key()</c></p></item> + + <tag><c>private_key() =</c></tag> + <item><p><c>rsa_private_key() | dsa_private_key() | ec_private_key()</c></p></item> - <p><code>rsa_private_key() = #'RSAPrivateKey'{}</code></p> + <tag><c>rsa_public_key() =</c></tag> + <item><p><c>#'RSAPublicKey'{}</c></p></item> - <p><code>dsa_public_key() = {integer(), #'Dss-Parms'{}}</code></p> + <tag><c>rsa_private_key() =</c></tag> + <item><p><c>#'RSAPrivateKey'{}</c></p></item> - <p><code>dsa_private_key() = #'DSAPrivateKey'{}</code></p> + <tag><c>dsa_public_key() =</c></tag> + <item><p><c>{integer(), #'Dss-Parms'{}}</c></p></item> - <p><code>ec_public_key() = {#'ECPoint'{}, #'EcpkParameters'{} | - {namedCurve, oid()}}</code></p> - - <p><code>ec_private_key() = #'ECPrivateKey'{}</code></p> + <tag><c>dsa_private_key() =</c></tag> + <item><p><c>#'DSAPrivateKey'{}</c></p></item> - <p><code>public_crypt_options() = [{rsa_pad, rsa_padding()}].</code></p> + <tag><c>ec_public_key()</c></tag> + <item><p>= <c>{#'ECPoint'{}, #'EcpkParameters'{} | {namedCurve, oid()}}</c></p></item> - <p><code>rsa_padding() = 'rsa_pkcs1_padding' | 'rsa_pkcs1_oaep_padding' | - 'rsa_no_padding'</code></p> + <tag><c>ec_private_key() =</c></tag> + <item><p><c>#'ECPrivateKey'{}</c></p></item> - <p><code>digest_type() - Union of below digest types</code></p> - - <p><code>rsa_digest_type() = 'md5' | 'sha' | 'sha224' | 'sha256' | 'sha384' | - 'sha512'</code></p> + <tag><c>public_crypt_options() =</c></tag> + <item><p><c>[{rsa_pad, rsa_padding()}]</c></p></item> - <p><code>dss_digest_type() = 'sha'</code></p> + <tag><c>rsa_padding() =</c></tag> + <item> + <p><c>'rsa_pkcs1_padding'</c></p> + <p><c>| 'rsa_pkcs1_oaep_padding'</c></p> + <p><c>| 'rsa_no_padding'</c></p> + </item> - <p><code>ecdsa_digest_type() = 'sha'| 'sha224' | 'sha256' | 'sha384' | 'sha512'</code></p> + <tag><c>digest_type() = </c></tag> + <item><p>Union of <c>rsa_digest_type()</c>, <c>dss_digest_type()</c>, + and <c>ecdsa_digest_type()</c>.</p></item> - <p><code>crl_reason() = unspecified | keyCompromise | cACompromise | - affiliationChanged | superseded | cessationOfOperation | - certificateHold | privilegeWithdrawn | aACompromise</code></p> + <tag><c>rsa_digest_type() = </c></tag> + <item><p><c>'md5' | 'sha' | 'sha224' | 'sha256' | 'sha384' | 'sha512'</c></p></item> - <p><code>issuer_name() = {rdnSequence,[#'AttributeTypeAndValue'{}]} </code> </p> + <tag><c>dss_digest_type() = </c></tag> + <item><p><c>'sha'</c></p></item> - <p><code>ssh_file() = openssh_public_key | rfc4716_public_key | known_hosts | - auth_keys</code></p> + <tag><c>ecdsa_digest_type() = </c></tag> + <item><p><c>'sha'| 'sha224' | 'sha256' | 'sha384' | 'sha512'</c></p></item> + + <tag><c>crl_reason() = </c></tag> + <item> + <p><c>unspecified</c></p> + <p><c>| keyCompromise</c></p> + <p><c>| cACompromise</c></p> + <p><c>| affiliationChanged</c></p> + <p><c>| superseded</c></p> + <p><c>| cessationOfOperation</c></p> + <p><c>| certificateHold</c></p> + <p><c>| privilegeWithdrawn</c></p> + <p><c>| aACompromise</c></p> + </item> + + <tag><c>issuer_name() =</c></tag> + <item><p><c>{rdnSequence,[#'AttributeTypeAndValue'{}]}</c></p> + </item> + + <tag><c>ssh_file() =</c></tag> + <item> + <p><c>openssh_public_key</c></p> + <p><c>| rfc4716_public_key</c></p> + <p><c>| known_hosts</c></p> + <p><c>| auth_keys</c></p> + </item> + </taglist> + <!-- <p><code>policy_tree() = [Root, Children]</code></p> --> @@ -138,12 +200,12 @@ <!-- <p><code>Children = [] | policy_tree()</code></p> --> -<!-- <p> The policy_tree_node record has the following fields:</p> --> +<!-- <p>The <c>policy_tree_node</c> record has the following fields:</p> --> <!-- <taglist> --> <!-- <tag>valid_policy</tag> --> -<!-- <item> Is a single policy OID representing a --> +<!-- <item>A single policy OID representing a --> <!-- valid policy for the path of length x.</item> --> <!-- <tag>qualifier_set</tag> --> @@ -151,13 +213,13 @@ <!-- with the valid policy in certificate x.</item> --> <!-- <tag>critically_indicator</tag> --> -<!-- <item>The critically_indicator indicates whether the --> +<!-- <item>Indicates whether the --> <!-- certificate policy extension in certificate x was marked as --> -<!-- critical. </item> --> +<!-- critical.</item> --> <!-- <tag>expected_policy_set</tag> --> -<!-- <item>The expected_policy_set contains one or more policy OIDs --> -<!-- that would satisfy this policy in the certificate x+1. </item> --> +<!-- <item>Contains one or more policy OIDs --> +<!-- that would satisfy this policy in the certificate x+1.</item> --> <!-- </taglist> --> </section> @@ -166,27 +228,27 @@ <func> <name>compute_key(OthersKey, MyKey)-></name> <name>compute_key(OthersKey, MyKey, Params)-></name> - <fsummary> Compute shared secret</fsummary> + <fsummary>Computes shared secret.</fsummary> <type> <v>OthersKey = #'ECPoint'{} | binary(), MyKey = #'ECPrivateKey'{} | binary()</v> <v>Params = #'DHParameter'{}</v> </type> <desc> - <p> Compute shared secret </p> + <p>Computes shared secret.</p> </desc> </func> <func> <name>decrypt_private(CipherText, Key) -> binary()</name> <name>decrypt_private(CipherText, Key, Options) -> binary()</name> - <fsummary>Public key decryption.</fsummary> + <fsummary>Public-key decryption.</fsummary> <type> <v>CipherText = binary()</v> <v>Key = rsa_private_key()</v> <v>Options = public_crypt_options()</v> </type> <desc> - <p>Public key decryption using the private key. See also <seealso + <p>Public-key decryption using the private key. See also <seealso marker="crypto:crypto#private_decrypt/4">crypto:private_decrypt/4</seealso></p> </desc> </func> @@ -194,156 +256,156 @@ <func> <name>decrypt_public(CipherText, Key) - > binary()</name> <name>decrypt_public(CipherText, Key, Options) - > binary()</name> - <fsummary></fsummary> + <fsummary>Public-key decryption.</fsummary> <type> <v>CipherText = binary()</v> <v>Key = rsa_public_key()</v> <v>Options = public_crypt_options()</v> </type> <desc> - <p> Public key decryption using the public key. See also <seealso + <p>Public-key decryption using the public key. See also <seealso marker="crypto:crypto#public_decrypt/4">crypto:public_decrypt/4</seealso></p> </desc> </func> <func> <name>der_decode(Asn1type, Der) -> term()</name> - <fsummary> Decodes a public key ASN.1 DER encoded entity.</fsummary> + <fsummary>Decodes a public-key ASN.1 DER encoded entity.</fsummary> <type> <v>Asn1Type = atom()</v> - <d> ASN.1 type present in the public_key applications - asn1 specifications.</d> + <d>ASN.1 type present in the Public Key applications + ASN.1 specifications.</d> <v>Der = der_encoded()</v> </type> <desc> - <p> Decodes a public key ASN.1 DER encoded entity.</p> + <p>Decodes a public-key ASN.1 DER encoded entity.</p> </desc> </func> <func> <name>der_encode(Asn1Type, Entity) -> der_encoded()</name> - <fsummary> Encodes a public key entity with asn1 DER encoding.</fsummary> + <fsummary>Encodes a public-key entity with ASN.1 DER encoding.</fsummary> <type> <v>Asn1Type = atom()</v> - <d> Asn1 type present in the public_key applications + <d>ASN.1 type present in the Public Key applications ASN.1 specifications.</d> <v>Entity = term()</v> - <d>The erlang representation of <c>Asn1Type</c></d> + <d>Erlang representation of <c>Asn1Type</c></d> </type> <desc> - <p> Encodes a public key entity with ASN.1 DER encoding.</p> + <p>Encodes a public-key entity with ASN.1 DER encoding.</p> </desc> </func> + <func> + <name>encrypt_private(PlainText, Key) -> binary()</name> + <fsummary>Public-key encryption using the private key.</fsummary> + <type> + <v>PlainText = binary()</v> + <v>Key = rsa_private_key()</v> + </type> + <desc> + <p>Public-key encryption using the private key. + See also <seealso + marker="crypto:crypto#private_encrypt/4">crypto:private_encrypt/4</seealso>.</p> + </desc> + </func> + + <func> + <name>encrypt_public(PlainText, Key) -> binary()</name> + <fsummary>Public-key encryption using the public key.</fsummary> + <type> + <v>PlainText = binary()</v> + <v>Key = rsa_public_key()</v> + </type> + <desc> + <p>Public-key encryption using the public key. See also <seealso + marker="crypto:crypto#public_encrypt/4">crypto:public_encrypt/4</seealso>.</p> + </desc> + </func> + <func> <name>generate_key(Params) -> {Public::binary(), Private::binary()} | #'ECPrivateKey'{} </name> - <fsummary>Generates a new keypair</fsummary> + <fsummary>Generates a new keypair.</fsummary> <type> - <v> Params = #'DHParameter'{} | {namedCurve, oid()} | #'ECParameters'{} </v> + <v>Params = #'DHParameter'{} | {namedCurve, oid()} | #'ECParameters'{}</v> </type> <desc> - <p>Generates a new keypair</p> + <p>Generates a new keypair.</p> </desc> </func> <func> <name>pem_decode(PemBin) -> [pem_entry()]</name> - <fsummary>Decode PEM binary data and return - entries as ASN.1 DER encoded entities. </fsummary> + <fsummary>Decodes PEM binary data and returns + entries as ASN.1 DER encoded entities.</fsummary> <type> <v>PemBin = binary()</v> <d>Example {ok, PemBin} = file:read_file("cert.pem").</d> </type> <desc> - <p>Decode PEM binary data and return + <p>Decodes PEM binary data and returns entries as ASN.1 DER encoded entities.</p> </desc> </func> <func> <name>pem_encode(PemEntries) -> binary()</name> - <fsummary>Creates a PEM binary</fsummary> + <fsummary>Creates a PEM binary.</fsummary> <type> <v> PemEntries = [pem_entry()] </v> </type> <desc> - <p>Creates a PEM binary</p> + <p>Creates a PEM binary.</p> </desc> </func> <func> <name>pem_entry_decode(PemEntry) -> term()</name> <name>pem_entry_decode(PemEntry, Password) -> term()</name> - <fsummary>Decodes a pem entry.</fsummary> + <fsummary>Decodes a PEM entry.</fsummary> <type> - <v> PemEntry = pem_entry() </v> - <v> Password = string() </v> + <v>PemEntry = pem_entry()</v> + <v>Password = string()</v> </type> <desc> - <p>Decodes a PEM entry. pem_decode/1 returns a list of PEM - entries. Note that if the PEM entry is of type - 'SubjectPublickeyInfo' it will be further decoded to an - rsa_public_key() or dsa_public_key().</p> + <p>Decodes a PEM entry. <c>pem_decode/1</c> returns a list of PEM + entries. Notice that if the PEM entry is of type + 'SubjectPublickeyInfo', it is further decoded to an + <c>rsa_public_key()</c> or <c>dsa_public_key()</c>.</p> </desc> </func> <func> <name>pem_entry_encode(Asn1Type, Entity) -> pem_entry()</name> <name>pem_entry_encode(Asn1Type, Entity, {CipherInfo, Password}) -> pem_entry()</name> - <fsummary> Creates a PEM entry that can be fed to pem_encode/1.</fsummary> + <fsummary>Creates a PEM entry that can be fed to <c>pem_encode/1</c>.</fsummary> <type> <v>Asn1Type = pki_asn1_type()</v> <v>Entity = term()</v> - <d>The Erlang representation of - <c>Asn1Type</c>. If <c>Asn1Type</c> is 'SubjectPublicKeyInfo' - then <c>Entity</c> must be either an rsa_public_key() or a - dsa_public_key() and this function will create the appropriate + <d>Erlang representation of + <c>Asn1Type</c>. If <c>Asn1Type</c> is 'SubjectPublicKeyInfo', + <c>Entity</c> must be either an <c>rsa_public_key()</c> or a + <c>dsa_public_key()</c> and this function creates the appropriate 'SubjectPublicKeyInfo' entry. </d> <v>CipherInfo = cipher_info()</v> <v>Password = string()</v> </type> <desc> - <p> Creates a PEM entry that can be feed to pem_encode/1.</p> + <p>Creates a PEM entry that can be feed to <c>pem_encode/1</c>.</p> </desc> </func> - - <func> - <name>encrypt_private(PlainText, Key) -> binary()</name> - <fsummary> Public key encryption using the private key.</fsummary> - <type> - <v>PlainText = binary()</v> - <v>Key = rsa_private_key()</v> - </type> - <desc> - <p> Public key encryption using the private key. - See also <seealso - marker="crypto:crypto#private_encrypt/4">crypto:private_encrypt/4</seealso></p> - </desc> - </func> - - <func> - <name>encrypt_public(PlainText, Key) -> binary()</name> - <fsummary> Public key encryption using the public key.</fsummary> - <type> - <v>PlainText = binary()</v> - <v>Key = rsa_public_key()</v> - </type> - <desc> - <p> Public key encryption using the public key. See also <seealso - marker="crypto:crypto#public_encrypt/4">crypto:public_encrypt/4</seealso></p> - </desc> - </func> <func> <name>pkix_decode_cert(Cert, otp|plain) -> #'Certificate'{} | #'OTPCertificate'{}</name> - <fsummary> Decodes an ASN.1 DER encoded PKIX x509 certificate.</fsummary> + <fsummary>Decodes an ASN.1 DER-encoded PKIX x509 certificate.</fsummary> <type> <v>Cert = der_encoded()</v> </type> <desc> - <p>Decodes an ASN.1 DER encoded PKIX certificate. The otp option - will use the customized ASN.1 specification OTP-PKIX.asn1 for + <p>Decodes an ASN.1 DER-encoded PKIX certificate. Option <c>otp</c> + uses the customized ASN.1 specification OTP-PKIX.asn1 for decoding and also recursively decode most of the standard parts.</p> </desc> @@ -355,54 +417,54 @@ certificate.</fsummary> <type> <v>Asn1Type = atom()</v> - <d>The ASN.1 type can be 'Certificate', 'OTPCertificate' or a subtype of either .</d> + <d>The ASN.1 type can be 'Certificate', 'OTPCertificate' or a subtype of either.</d> <v>Entity = #'Certificate'{} | #'OTPCertificate'{} | a valid subtype</v> </type> <desc> <p>DER encodes a PKIX x509 certificate or part of such a certificate. This function must be used for encoding certificates or parts of certificates - that are decoded/created in the otp format, whereas for the plain format this - function will directly call der_encode/2. </p> + that are decoded/created in the <c>otp</c> format, whereas for the plain format this + function directly calls <c>der_encode/2</c>.</p> </desc> </func> <func> <name>pkix_is_issuer(Cert, IssuerCert) -> boolean()</name> - <fsummary> Checks if <c>IssuerCert</c> issued <c>Cert</c> </fsummary> + <fsummary>Checks if <c>IssuerCert</c> issued <c>Cert</c>.</fsummary> <type> <v>Cert = der_encoded() | #'OTPCertificate'{}</v> <v>IssuerCert = der_encoded() | #'OTPCertificate'{}</v> </type> <desc> - <p> Checks if <c>IssuerCert</c> issued <c>Cert</c> </p> + <p>Checks if <c>IssuerCert</c> issued <c>Cert</c>.</p> </desc> </func> <func> <name>pkix_is_fixed_dh_cert(Cert) -> boolean()</name> - <fsummary> Checks if a Certificate is a fixed Diffie-Hellman Cert.</fsummary> + <fsummary>Checks if a certificate is a fixed Diffie-Hellman certificate.</fsummary> <type> <v>Cert = der_encoded() | #'OTPCertificate'{}</v> </type> <desc> - <p> Checks if a Certificate is a fixed Diffie-Hellman Cert.</p> + <p>Checks if a certificate is a fixed Diffie-Hellman certificate.</p> </desc> </func> <func> <name>pkix_is_self_signed(Cert) -> boolean()</name> - <fsummary> Checks if a Certificate is self signed.</fsummary> + <fsummary>Checks if a certificate is self-signed.</fsummary> <type> <v>Cert = der_encoded() | #'OTPCertificate'{}</v> </type> <desc> - <p> Checks if a Certificate is self signed.</p> + <p>Checks if a certificate is self-signed.</p> </desc> </func> <func> <name>pkix_issuer_id(Cert, IssuedBy) -> {ok, IssuerID} | {error, Reason}</name> - <fsummary> Returns the issuer id.</fsummary> + <fsummary>Returns the issuer id.</fsummary> <type> <v>Cert = der_encoded() | #'OTPCertificate'{}</v> <v>IssuedBy = self | other</v> @@ -411,43 +473,43 @@ <v>Reason = term()</v> </type> <desc> - <p> Returns the issuer id.</p> + <p>Returns the issuer id.</p> </desc> </func> <func> <name>pkix_normalize_name(Issuer) -> Normalized</name> - <fsummary>Normalizes a issuer name so that it can be easily - compared to another issuer name. </fsummary> + <fsummary>Normalizes an issuer name so that it can be easily + compared to another issuer name.</fsummary> <type> <v>Issuer = issuer_name()</v> <v>Normalized = issuer_name()</v> </type> <desc> - <p>Normalizes a issuer name so that it can be easily + <p>Normalizes an issuer name so that it can be easily compared to another issuer name.</p> </desc> </func> <func> <name>pkix_path_validation(TrustedCert, CertChain, Options) -> {ok, {PublicKeyInfo, PolicyTree}} | {error, {bad_cert, Reason}} </name> - <fsummary> Performs a basic path validation according to RFC 5280.</fsummary> + <fsummary>Performs a basic path validation according to RFC 5280.</fsummary> <type> - <v> TrustedCert = #'OTPCertificate'{} | der_encoded() | atom() </v> - <d>Normally a trusted certificate but it can also be a path validation + <v>TrustedCert = #'OTPCertificate'{} | der_encode() | atom()</v> + <d>Normally a trusted certificate, but it can also be a path-validation error that can be discovered while - constructing the input to this function and that should be run through the <c>verify_fun</c>. - For example <c>unknown_ca </c> or <c>selfsigned_peer </c> + constructing the input to this function and that is to be run through the <c>verify_fun</c>. + Examples are <c>unknown_ca</c> and <c>selfsigned_peer.</c> </d> - <v> CertChain = [der_encoded()]</v> - <d>A list of DER encoded certificates in trust order ending with the peer certificate.</d> - <v> Options = proplists:proplist()</v> + <v>CertChain = [der_encode()]</v> + <d>A list of DER-encoded certificates in trust order ending with the peer certificate.</d> + <v>Options = proplists:proplist()</v> <v>PublicKeyInfo = {?'rsaEncryption' | ?'id-dsa', rsa_public_key() | integer(), 'NULL' | 'Dss-Parms'{}}</v> - <v> PolicyTree = term() </v> - <d>At the moment this will always be an empty list as Policies are not currently supported</d> - <v> Reason = cert_expired | invalid_issuer | invalid_signature | name_not_permitted | + <v>PolicyTree = term()</v> + <d>At the moment this is always an empty list as policies are not currently supported.</d> + <v>Reason = cert_expired | invalid_issuer | invalid_signature | name_not_permitted | missing_basic_constraint | invalid_key_usage | {revoked, crl_reason()} | atom() </v> </type> @@ -455,17 +517,17 @@ <p> Performs a basic path validation according to <url href="http://www.ietf.org/rfc/rfc5280.txt">RFC 5280.</url> - However CRL validation is done separately by <seealso - marker="#pkix_crls_validate-3">pkix_crls_validate/3 </seealso> and should be called - from the supplied <c>verify_fun</c> + However, CRL validation is done separately by <seealso + marker="#pkix_crls_validate-3">pkix_crls_validate/3 </seealso> and is to be called + from the supplied <c>verify_fun</c>. </p> - <taglist> - <p> Available options are: </p> + <p>Available options:</p> + <taglist> <tag>{verify_fun, fun()}</tag> <item> - <p>The fun should be defined as:</p> + <p>The fun must be defined as:</p> <code> fun(OtpCert :: #'OTPCertificate'{}, @@ -478,53 +540,53 @@ fun(OtpCert :: #'OTPCertificate'{}, {unknown, UserState :: term()}. </code> - <p>If the verify callback fun returns {fail, Reason}, the + <p>If the verify callback fun returns <c>{fail, Reason}</c>, the verification process is immediately stopped. If the verify - callback fun returns {valid, UserState}, the verification - process is continued, this can be used to accept specific path - validation errors such as <c>selfsigned_peer</c> as well as - verifying application specific extensions. If called with an - extension unknown to the user application the return value - {unknown, UserState} should be used.</p> + callback fun returns <c>{valid, UserState}</c>, the verification + process is continued. This can be used to accept specific path + validation errors, such as <c>selfsigned_peer</c>, as well as + verifying application-specific extensions. If called with an + extension unknown to the user application, the return value + <c>{unknown, UserState}</c> is to be used.</p> </item> <tag>{max_path_length, integer()}</tag> <item> The <c>max_path_length</c> is the maximum number of non-self-issued - intermediate certificates that may follow the peer certificate - in a valid certification path. So if <c>max_path_length</c> is 0 the PEER must - be signed by the trusted ROOT-CA directly, if 1 the path can - be PEER, CA, ROOT-CA, if it is 2 PEER, CA, CA, ROOT-CA and so - on. + intermediate certificates that can follow the peer certificate + in a valid certification path. So, if <c>max_path_length</c> is 0, the PEER must + be signed by the trusted ROOT-CA directly, if it is 1, the path can + be PEER, CA, ROOT-CA, if it is 2, the path can + be PEER, CA, CA, ROOT-CA, and so on. </item> </taglist> - <p> Possible reasons for a bad certificate are: </p> + <p>Possible reasons for a bad certificate: </p> <taglist> <tag>cert_expired</tag> - <item>The certificate is no longer valid as its expiration date has passed.</item> + <item><p>Certificate is no longer valid as its expiration date has passed.</p></item> <tag>invalid_issuer</tag> - <item>The certificate issuer name does not match the name of the issuer certificate in the chain.</item> + <item><p>Certificate issuer name does not match the name of the issuer certificate in the chain.</p></item> <tag>invalid_signature</tag> - <item>The certificate was not signed by its issuer certificate in the chain.</item> + <item><p>Certificate was not signed by its issuer certificate in the chain.</p></item> <tag>name_not_permitted</tag> - <item>Invalid Subject Alternative Name extension.</item> + <item><p>Invalid Subject Alternative Name extension.</p></item> <tag>missing_basic_constraint</tag> - <item>Certificate, required to have the basic constraints extension, does not have - a basic constraints extension.</item> + <item><p>Certificate, required to have the basic constraints extension, does not have + a basic constraints extension.</p></item> <tag>invalid_key_usage</tag> - <item>Certificate key is used in an invalid way according to the key usage extension.</item> + <item><p>Certificate key is used in an invalid way according to the key-usage extension.</p></item> <tag>{revoked, crl_reason()}</tag> - <item>Certificate has been revoked.</item> + <item><p>Certificate has been revoked.</p></item> <tag>atom()</tag> - <item>Application specific error reason that should be checked by the verify_fun</item> + <item><p>Application-specific error reason that is to be checked by the <c>verify_fun</c>.</p></item> </taglist> </desc> @@ -543,44 +605,47 @@ fun(OtpCert :: #'OTPCertificate'{}, <func> <name>pkix_crls_validate(OTPCertificate, DPAndCRLs, Options) -> CRLStatus()</name> - <fsummary> Performs CRL validation.</fsummary> + <fsummary>Performs CRL validation.</fsummary> <type> - <v> OTPCertificate = #'OTPCertificate'{}</v> - <v> DPAndCRLs = [{DP::#'DistributionPoint'{}, {DerCRL::der_encoded(), CRL::#'CertificateList'{}}}] </v> - <v> Options = proplists:proplist()</v> - <v> CRLStatus() = valid | {bad_cert, revocation_status_undetermined} | + <v>OTPCertificate = #'OTPCertificate'{}</v> + <v>DPAndCRLs = [{DP::#'DistributionPoint'{}, {DerCRL::der_encoded(), CRL::#'CertificateList'{}}}] </v> + <v>Options = proplists:proplist()</v> + <v>CRLStatus() = valid | {bad_cert, revocation_status_undetermined} | {bad_cert, {revoked, crl_reason()}}</v> </type> <desc> - <p> Performs CRL validation. It is intended to be called from + <p>Performs CRL validation. It is intended to be called from the verify fun of <seealso marker="#pkix_path_validation-3"> pkix_path_validation/3 - </seealso></p> + </seealso>.</p> + + <p>Available options:</p> + <taglist> - <p> Available options are: </p> + <tag>{update_crl, fun()}</tag> <item> - <p>The fun has the following type spec:</p> + <p>The fun has the following type specification:</p> <code> fun(#'DistributionPoint'{}, #'CertificateList'{}) -> #'CertificateList'{}</code> - <p>The fun should use the information in the distribution point to acesses - the lates possible version of the CRL. If this fun is not specified - public_key will use the default implementation: + <p>The fun uses the information in the distribution point to access + the latest possible version of the CRL. If this fun is not specified, + Public Key uses the default implementation: </p> <code> fun(_DP, CRL) -> CRL end</code> </item> <tag>{issuer_fun, fun()}</tag> <item> - <p>The fun has the following type spec:</p> + <p>The fun has the following type specification:</p> <code> fun(#'DistributionPoint'{}, #'CertificateList'{}, {rdnSequence,[#'AttributeTypeAndValue'{}]}, term()) -> {ok, #'OTPCertificate'{}, [der_encoded]}</code> - <p>The fun should return the root certificate and certificate chain + <p>The fun returns the root certificate and certificate chain that has signed the CRL. </p> <code> fun(DP, CRL, Issuer, UserState) -> {ok, RootCert, CertChain}</code> @@ -635,83 +700,83 @@ fun(#'DistributionPoint'{}, #'CertificateList'{}, <v>Key = rsa_public_key() | dsa_public_key()</v> </type> <desc> - <p>Signs a 'OTPTBSCertificate'. Returns the corresponding - der encoded certificate.</p> + <p>Signs an 'OTPTBSCertificate'. Returns the corresponding + DER-encoded certificate.</p> </desc> </func> <func> <name>pkix_sign_types(AlgorithmId) -> {DigestType, SignatureType}</name> - <fsummary>Translates signature algorithm oid to erlang digest and signature algorithm types.</fsummary> + <fsummary>Translates signature algorithm OID to Erlang digest and signature algorithm types.</fsummary> <type> <v>AlgorithmId = oid()</v> - <d>Signature oid from a certificate or a certificate revocation list</d> - <v>DigestType = rsa_digest_type() | dss_digest_type() </v> + <d>Signature OID from a certificate or a certificate revocation list.</d> + <v>DigestType = rsa_digest_type() | dss_digest_type()</v> <v>SignatureType = rsa | dsa</v> </type> <desc> - <p>Translates signature algorithm oid to erlang digest and signature types. + <p>Translates signature algorithm OID to Erlang digest and signature types. </p> </desc> </func> <func> <name>pkix_verify(Cert, Key) -> boolean()</name> - <fsummary> Verify pkix x.509 certificate signature.</fsummary> + <fsummary>Verifies PKIX x.509 certificate signature.</fsummary> <type> <v>Cert = der_encoded()</v> <v>Key = rsa_public_key() | dsa_public_key()</v> </type> <desc> - <p> Verify PKIX x.509 certificate signature.</p> + <p>Verifies PKIX x.509 certificate signature.</p> </desc> </func> <func> <name>sign(Msg, DigestType, Key) -> binary()</name> - <fsummary> Create digital signature.</fsummary> + <fsummary>Creates a digital signature.</fsummary> <type> <v>Msg = binary() | {digest,binary()}</v> - <d>The msg is either the binary "plain text" data to be - signed or it is the hashed value of "plain text" i.e. the + <d>The <c>Msg</c> is either the binary "plain text" data to be + signed or it is the hashed value of "plain text", that is, the digest.</d> <v>DigestType = rsa_digest_type() | dss_digest_type() | ecdsa_digest_type()</v> <v>Key = rsa_private_key() | dsa_private_key() | ec_private_key()</v> </type> <desc> - <p> Creates a digital signature.</p> + <p>Creates a digital signature.</p> </desc> </func> <func> <name>ssh_decode(SshBin, Type) -> [{public_key(), Attributes::list()}]</name> - <fsummary>Decodes a ssh file-binary. </fsummary> + <fsummary>Decodes an SSH file-binary.</fsummary> <type> <v>SshBin = binary()</v> <d>Example {ok, SshBin} = file:read_file("known_hosts").</d> - <v> Type = public_key | ssh_file()</v> - <d>If <c>Type</c> is <c>public_key</c> the binary may be either - a rfc4716 public key or a openssh public key.</d> + <v>Type = public_key | ssh_file()</v> + <d>If <c>Type</c> is <c>public_key</c> the binary can be either + an RFC4716 public key or an OpenSSH public key.</d> </type> <desc> - <p> Decodes a ssh file-binary. In the case of know_hosts or - auth_keys the binary may include one or more lines of the + <p>Decodes an SSH file-binary. In the case of <c>know_hosts</c> or + <c>auth_keys</c>, the binary can include one or more lines of the file. Returns a list of public keys and their attributes, possible attribute values depends on the file type represented by the binary. </p> <taglist> - <tag>rfc4716 attributes - see RFC 4716</tag> - <item>{headers, [{string(), utf8_string()}]}</item> - <tag>auth_key attributes - see man sshd </tag> + <tag>RFC4716 attributes - see RFC 4716.</tag> + <item><p>{headers, [{string(), utf8_string()}]}</p></item> + <tag>auth_key attributes - see manual page for sshd.</tag> <item>{comment, string()}</item> <item>{options, [string()]}</item> - <item>{bits, integer()} - In ssh version 1 files</item> - <tag>known_host attributes - see man sshd</tag> + <item><p>{bits, integer()} - In SSH version 1 files.</p></item> + <tag>known_host attributes - see manual page for sshd.</tag> <item>{hostnames, [string()]}</item> <item>{comment, string()}</item> - <item>{bits, integer()} - In ssh version 1 files</item> + <item><p>{bits, integer()} - In SSH version 1 files.</p></item> </taglist> </desc> @@ -719,16 +784,16 @@ fun(#'DistributionPoint'{}, #'CertificateList'{}, <func> <name>ssh_encode([{Key, Attributes}], Type) -> binary()</name> - <fsummary> Encodes a list of ssh file entries to a binary.</fsummary> + <fsummary>Encodes a list of SSH file entries to a binary.</fsummary> <type> <v>Key = public_key()</v> <v>Attributes = list()</v> <v>Type = ssh_file()</v> </type> <desc> - <p>Encodes a list of ssh file entries (public keys and attributes) to a binary. Possible - attributes depends on the file type, see <seealso - marker="#ssh_decode-2"> ssh_decode/2 </seealso></p> + <p>Encodes a list of SSH file entries (public keys and attributes) to a binary. Possible + attributes depend on the file type, see <seealso + marker="#ssh_decode-2"> ssh_decode/2 </seealso>.</p> </desc> </func> @@ -737,14 +802,14 @@ fun(#'DistributionPoint'{}, #'CertificateList'{}, <fsummary>Verifies a digital signature.</fsummary> <type> <v>Msg = binary() | {digest,binary()}</v> - <d>The msg is either the binary "plain text" data - or it is the hashed value of "plain text" i.e. the digest.</d> + <d>The <c>Msg</c> is either the binary "plain text" data + or it is the hashed value of "plain text", that is, the digest.</d> <v>DigestType = rsa_digest_type() | dss_digest_type() | ecdsa_digest_type()</v> <v>Signature = binary()</v> <v>Key = rsa_public_key() | dsa_public_key() | ec_public_key()</v> </type> <desc> - <p>Verifies a digital signature</p> + <p>Veryfies a digital signature.</p> </desc> </func> diff --git a/lib/public_key/doc/src/public_key_records.xml b/lib/public_key/doc/src/public_key_records.xml index a7dfc41449..fc2a74a353 100644 --- a/lib/public_key/doc/src/public_key_records.xml +++ b/lib/public_key/doc/src/public_key_records.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2008</year> - <year>2014</year> + <year>2015</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> @@ -23,7 +23,7 @@ The Initial Developer of the Original Code is Ericsson AB. </legalnotice> - <title>Public key records</title> + <title>Public-Key Records</title> <prepared>Ingela Anderton Andin</prepared> <responsible></responsible> <docno></docno> @@ -34,28 +34,85 @@ <file>public_key_records.xml</file> </header> - <p>This chapter briefly describes Erlang records derived from ASN1 - specifications used to handle public and private keys. - The intent is to describe the data types - and not to specify the semantics of each component. For information on the - semantics, please see the relevant standards and RFCs.</p> + <p>This chapter briefly describes Erlang records derived from ASN.1 + specifications used to handle public key infrastructure. + The scope is to describe the data types of each component, + not the semantics. For information on the + semantics, refer to the relevant standards and RFCs linked in the sections below.</p> <p>Use the following include directive to get access to the - records and constant macros described in the following sections.</p> + records and constant macros described in the following sections:</p> <code> -include_lib("public_key/include/public_key.hrl"). </code> - <section> - <title>Common Data Types</title> + <section> + <title>Data Types</title> <p>Common non-standard Erlang - data types used to described the record fields in the - below sections are defined in <seealso - marker="public_key">public key reference manual </seealso></p> - </section> + data types used to describe the record fields in the + following sections and which are not defined in the Public Key <seealso + marker="public_key">Reference Manual</seealso> + follows here:</p> + + <taglist> + <tag><c>time() =</c></tag> + <item><p><c>uct_time() | general_time()</c></p></item> + + <tag><c>uct_time() =</c></tag> + <item><p><c>{utcTime, "YYMMDDHHMMSSZ"}</c></p></item> + + <tag><c>general_time() =</c></tag> + <item><p><c>{generalTime, "YYYYMMDDHHMMSSZ"}</c></p></item> + + <tag><c>general_name() =</c></tag> + <item><p><c>{rfc822Name, string()}</c></p> + <p><c>| {dNSName, string()}</c></p> + <p><c>| {x400Address, string()}</c></p> + <p><c>| {directoryName, {rdnSequence, [#AttributeTypeAndValue'{}]}}</c></p> + <p><c>| {eidPartyName, special_string()}</c></p> + <p><c>| {eidPartyName, special_string(), special_string()}</c></p> + <p><c>| {uniformResourceIdentifier, string()}</c></p> + <p><c>| {ipAddress, string()}</c></p> + <p><c>| {registeredId, oid()}</c></p> + <p><c>| {otherName, term()}</c></p> + </item> + + <tag><c>special_string() =</c></tag> + <item><p><c>{teletexString, string()}</c></p> + <p><c>| {printableString, string()}</c></p> + <p><c>| {universalString, string()}</c></p> + <p><c>| {utf8String, binary()}</c></p> + <p><c>| {bmpString, string()}</c></p> + </item> + + <tag><c>dist_reason() =</c></tag> + <item><p><c>unused</c></p> + <p><c>| keyCompromise</c></p> + <p><c>| cACompromise</c></p> + <p><c>| affiliationChanged</c></p> + <p><c>| superseded</c></p> + <p><c>| cessationOfOperation</c></p> + <p><c>| certificateHold</c></p> + <p><c>| privilegeWithdrawn</c></p> + <p><c>| aACompromise</c></p> + </item> + <tag><c>OID_macro() =</c></tag> + <item><p><c>?OID_name()</c></p> + </item> + + <tag><c>OID_name() =</c></tag> + <item><p><c>atom()</c></p> + </item> + + </taglist> + + </section> + <section> - <title>RSA as defined by the PKCS-1 standard and <url href="http://www.ietf.org/rfc/rfc3447.txt"> RFC 3447 </url></title> + <title>RSA</title> + <p>Erlang representation of <url href="http://www.ietf.org/rfc/rfc3447.txt"> + Rivest-Shamir-Adleman cryptosystem (RSA)</url> keys follows:</p> <code> #'RSAPublicKey'{ @@ -80,16 +137,13 @@ prime, % integer() exponent, % integer() coefficient % integer() - }. - </code> + }. </code> </section> <section> - <title>DSA as defined by - <url href="http://csrc.nist.gov/publications/fips/fips186-3/fips_186-3.pdf"> Digital Signature Standard (NIST FIPS PUB 186-2) </url> - </title> - + <title>DSA</title> + <p>Erlang representation of <url href="http://www.ietf.org/rfc/rfc6979.txt">Digigital Signature Algorithm (DSA)</url> keys</p> <code> #'DSAPrivateKey',{ version, % integer() @@ -104,18 +158,18 @@ p, % integer() q, % integer() g % integer() - }. - </code> + }. </code> + </section> <section> - <title>ECC (Elliptic Curve) <url href="http://www.ietf.org/rfc/rfc3447.txt"> RFC 5480 </url> - </title> + <title>ECDSA </title> + <p>Erlang representation of <url href="http://www.ietf.org/rfc/rfc6979.txt">Elliptic Curve Digital Signature Algorithm (ECDSA)</url> keys follows:</p> <code> #'ECPrivateKey'{ version, % integer() - privateKey, % binary() + privateKey, % binary() parameters, % der_encoded() - {'EcpkParameters', #'ECParameters'{}} | {'EcpkParameters', {namedCurve, oid()}} | {'EcpkParameters', 'NULL'} % Inherited by CA @@ -126,14 +180,14 @@ version, % integer() fieldID, % #'FieldID'{} curve, % #'Curve'{} - base, % binary() + base, % binary() order, % integer() cofactor % integer() }. #'Curve'{ a, % binary() - b, % binary() + b, % binary() seed % bitstring() - optional }. @@ -144,10 +198,644 @@ }. #'ECPoint'{ - point % binary() - the public key - }. - - </code> + point % binary() - the public key + }.</code> </section> + <section> + <title>PKIX Certificates</title> + <p>Erlang representation of PKIX certificates derived from ASN.1 + specifications see also <url href="http://www.ietf.org/rfc/rfc5280.txt">X509 certificates (RFC 5280)</url>, also referred to as <c>plain</c> type, are as follows:</p> +<code> +#'Certificate'{ + tbsCertificate, % #'TBSCertificate'{} + signatureAlgorithm, % #'AlgorithmIdentifier'{} + signature % bitstring() + }. + +#'TBSCertificate'{ + version, % v1 | v2 | v3 + serialNumber, % integer() + signature, % #'AlgorithmIdentifier'{} + issuer, % {rdnSequence, [#AttributeTypeAndValue'{}]} + validity, % #'Validity'{} + subject, % {rdnSequence, [#AttributeTypeAndValue'{}]} + subjectPublicKeyInfo, % #'SubjectPublicKeyInfo'{} + issuerUniqueID, % binary() | asn1_novalue + subjectUniqueID, % binary() | asn1_novalue + extensions % [#'Extension'{}] + }. + +#'AlgorithmIdentifier'{ + algorithm, % oid() + parameters % der_encoded() + }.</code> + +<p>Erlang alternate representation of PKIX certificate, also referred to as <c>otp</c> type</p> + +<code> +#'OTPCertificate'{ + tbsCertificate, % #'OTPTBSCertificate'{} + signatureAlgorithm, % #'SignatureAlgorithm' + signature % bitstring() + }. + +#'OTPTBSCertificate'{ + version, % v1 | v2 | v3 + serialNumber, % integer() + signature, % #'SignatureAlgorithm' + issuer, % {rdnSequence, [#AttributeTypeAndValue'{}]} + validity, % #'Validity'{} + subject, % {rdnSequence, [#AttributeTypeAndValue'{}]} + subjectPublicKeyInfo, % #'OTPSubjectPublicKeyInfo'{} + issuerUniqueID, % binary() | asn1_novalue + subjectUniqueID, % binary() | asn1_novalue + extensions % [#'Extension'{}] + }. + +#'SignatureAlgorithm'{ + algorithm, % id_signature_algorithm() + parameters % asn1_novalue | #'Dss-Parms'{} + }.</code> + +<p><c>id_signature_algorithm() = OID_macro()</c></p> + +<p>The available OID names are as follows:</p> +<table> + <row> + <cell align="left" valign="middle"><em>OID Name</em></cell> + </row> + <row> + <cell align="left" valign="middle">id-dsa-with-sha1</cell> + </row> + <row> + <cell align="left" valign="middle">id-dsaWithSHA1 (ISO or OID to above)</cell> + </row> + <row> + <cell align="left" valign="middle">md2WithRSAEncryption</cell> + </row> + <row> + <cell align="left" valign="middle">md5WithRSAEncryption</cell> + </row> + <row> + <cell align="left" valign="middle">sha1WithRSAEncryption</cell> + </row> + <row> + <cell align="left" valign="middle">sha-1WithRSAEncryption (ISO or OID to above)</cell> + </row> + <row> + <cell align="left" valign="middle">sha224WithRSAEncryption</cell> + </row> + <row> + <cell align="left" valign="middle">sha256WithRSAEncryption</cell> + </row> + <row> + <cell align="left" valign="middle">sha512WithRSAEncryption</cell> + </row> + <row> + <cell align="left" valign="middle">ecdsa-with-SHA1</cell> + </row> + <tcaption>Signature Algorithm OIDs </tcaption> +</table> + +<p>The data type <c>'AttributeTypeAndValue'</c>, is represented as + the following erlang record:</p> + +<code> +#'AttributeTypeAndValue'{ + type, % id_attributes() + value % term() + }.</code> + +<p>The attribute OID name atoms and their corresponding value types +are as follows:</p> +<table> + <row> + <cell align="left" valign="middle"><em>OID Name</em></cell> + <cell align="left" valign="middle"><em>Value Type</em></cell> + </row> + <row> + <cell align="left" valign="middle">id-at-name</cell> + <cell align="left" valign="middle">special_string()</cell> + </row> + <row> + <cell align="left" valign="middle">id-at-surname</cell> + <cell align="left" valign="middle">special_string()</cell> + </row> + <row> + <cell align="left" valign="middle">id-at-givenName</cell> + <cell align="left" valign="middle">special_string()</cell> + </row> + <row> + <cell align="left" valign="middle">id-at-initials </cell> + <cell align="left" valign="middle">special_string()</cell> + </row> + <row> + <cell align="left" valign="middle">id-at-generationQualifier</cell> + <cell align="left" valign="middle">special_string()</cell> + </row> + <row> + <cell align="left" valign="middle">id-at-commonName</cell> + <cell align="left" valign="middle">special_string()</cell> + </row> + <row> + <cell align="left" valign="middle">id-at-localityName</cell> + <cell align="left" valign="middle">special_string()</cell> + </row> + <row> + <cell align="left" valign="middle">id-at-stateOrProvinceName</cell> + <cell align="left" valign="middle">special_string()</cell> + </row> + <row> + <cell align="left" valign="middle">id-at-organizationName</cell> + <cell align="left" valign="middle">special_string()</cell> + </row> + <row> + <cell align="left" valign="middle">id-at-title</cell> + <cell align="left" valign="middle">special_string()</cell> + </row> + <row> + <cell align="left" valign="middle">id-at-dnQualifier</cell> + <cell align="left" valign="middle">{printableString, string()}</cell> + </row> + <row> + <cell align="left" valign="middle">id-at-countryName</cell> + <cell align="left" valign="middle">{printableString, string()}</cell> + </row> + <row> + <cell align="left" valign="middle">id-at-serialNumber</cell> + <cell align="left" valign="middle">{printableString, string()}</cell> + </row> + <row> + <cell align="left" valign="middle">id-at-pseudonym</cell> + <cell align="left" valign="middle">special_string()</cell> + </row> + <tcaption>Attribute OIDs</tcaption> +</table> + +<p>The data types <c>'Validity'</c>, <c>'SubjectPublicKeyInfo'</c>, and +<c>'SubjectPublicKeyInfoAlgorithm'</c> are represented as the following Erlang records:</p> + +<code> +#'Validity'{ + notBefore, % time() + notAfter % time() + }. + +#'SubjectPublicKeyInfo'{ + algorithm, % #AlgorithmIdentifier{} + subjectPublicKey % binary() + }. + +#'SubjectPublicKeyInfoAlgorithm'{ + algorithm, % id_public_key_algorithm() + parameters % public_key_params() + }.</code> + +<p>The public-key algorithm OID name atoms are as follows:</p> +<table> + <row> + <cell align="left" valign="middle"><em>OID Name</em></cell> + </row> + <row> + <cell align="left" valign="middle">rsaEncryption</cell> + </row> + <row> + <cell align="left" valign="middle">id-dsa</cell> + </row> + <row> + <cell align="left" valign="middle">dhpublicnumber</cell> + </row> + <row> + <cell align="left" valign="middle">id-keyExchangeAlgorithm</cell> + </row> + <row> + <cell align="left" valign="middle">id-ecPublicKey</cell> + </row> + <tcaption>Public-Key Algorithm OIDs</tcaption> +</table> + +<code> +#'Extension'{ + extnID, % id_extensions() | oid() + critical, % boolean() + extnValue % der_encoded() + }.</code> + +<p><c>id_extensions()</c> + <seealso marker="#StdCertExt">Standard Certificate Extensions</seealso>, + <seealso marker="#PrivIntExt">Private Internet Extensions</seealso>, + <seealso marker="#CRLCertExt">CRL Extensions</seealso> and + <seealso marker="#CRLEntryExt">CRL Entry Extensions</seealso>. +</p> + +</section> + +<section> + <marker id="StdCertExt"></marker> + <title>Standard Certificate Extensions</title> + + <p>The standard certificate extensions OID name atoms and their + corresponding value types are as follows:</p> + + <table> + <row> + <cell align="left" valign="middle"><em>OID Name</em></cell> + <cell align="left" valign="middle"><em>Value Type</em></cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-authorityKeyIdentifier</cell> + <cell align="left" valign="middle">#'AuthorityKeyIdentifier'{}</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-subjectKeyIdentifier</cell> + <cell align="left" valign="middle">oid()</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-keyUsage</cell> + <cell align="left" valign="middle">[key_usage()]</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-privateKeyUsagePeriod</cell> + <cell align="left" valign="middle">#'PrivateKeyUsagePeriod'{}</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-certificatePolicies</cell> + <cell align="left" valign="middle">#'PolicyInformation'{}</cell> + </row> + + <row> + <cell align="left" valign="middle">id-ce-policyMappings</cell> + <cell align="left" valign="middle">#'PolicyMappings_SEQOF'{}</cell> + </row> + + <row> + <cell align="left" valign="middle">id-ce-subjectAltName</cell> + <cell align="left" valign="middle">general_name()</cell> + </row> + + <row> + <cell align="left" valign="middle">id-ce-issuerAltName</cell> + <cell align="left" valign="middle">general_name()</cell> + </row> + + <row> + <cell align="left" valign="middle">id-ce-subjectDirectoryAttributes</cell> + <cell align="left" valign="middle"> [#'Attribute'{}]</cell> + </row> + + <row> + <cell align="left" valign="middle">id-ce-basicConstraints</cell> + <cell align="left" valign="middle">#'BasicConstraints'{}</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-nameConstraints</cell> + <cell align="left" valign="middle">#'NameConstraints'{}</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-policyConstraints</cell> + <cell align="left" valign="middle">#'PolicyConstraints'{}</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-extKeyUsage</cell> + <cell align="left" valign="middle">[id_key_purpose()]</cell> + </row> + + <row> + <cell align="left" valign="middle">id-ce-cRLDistributionPoints</cell> + <cell align="left" valign="middle">[#'DistributionPoint'{}]</cell> + </row> + + <row> + <cell align="left" valign="middle">id-ce-inhibitAnyPolicy</cell> + <cell align="left" valign="middle">integer()</cell> + </row> + + <row> + <cell align="left" valign="middle">id-ce-freshestCRL</cell> + <cell align="left" valign="middle">[#'DistributionPoint'{}]</cell> + </row> + + + <tcaption>Standard Certificate Extensions</tcaption> + </table> + + <p>Here:</p> + <taglist> + <tag><c>key_usage()</c></tag> + <item>= <p><c>digitalSignature</c></p> + <p><c>| nonRepudiation</c></p> + <p><c>| keyEncipherment</c></p> + <p><c>| dataEncipherment</c></p> + <p><c>| keyAgreement</c></p> + <p><c>| keyCertSign</c></p> + <p><c>| cRLSign</c></p> + <p><c>| encipherOnly</c></p> + <p><c>| decipherOnly </c></p> + </item> + </taglist> + + <p>And for <c>id_key_purpose()</c>:</p> + +<table> + <row> + <cell align="left" valign="middle"><em>OID Name</em></cell> + </row> + <row> + <cell align="left" valign="middle">id-kp-serverAuth</cell> + </row> + <row> + <cell align="left" valign="middle">id-kp-clientAuth</cell> + </row> + <row> + <cell align="left" valign="middle">id-kp-codeSigning</cell> + </row> + <row> + <cell align="left" valign="middle">id-kp-emailProtection</cell> + </row> + <row> + <cell align="left" valign="middle">id-kp-timeStamping</cell> + </row> + <row> + <cell align="left" valign="middle">id-kp-OCSPSigning</cell> + </row> + <tcaption>Key Purpose OIDs</tcaption> +</table> + + <code> +#'AuthorityKeyIdentifier'{ + keyIdentifier, % oid() + authorityCertIssuer, % general_name() + authorityCertSerialNumber % integer() + }. + +#'PrivateKeyUsagePeriod'{ + notBefore, % general_time() + notAfter % general_time() + }. + +#'PolicyInformation'{ + policyIdentifier, % oid() + policyQualifiers % [#PolicyQualifierInfo{}] + }. + +#'PolicyQualifierInfo'{ + policyQualifierId, % oid() + qualifier % string() | #'UserNotice'{} + }. + +#'UserNotice'{ + noticeRef, % #'NoticeReference'{} + explicitText % string() + }. + +#'NoticeReference'{ + organization, % string() + noticeNumbers % [integer()] + }. + +#'PolicyMappings_SEQOF'{ + issuerDomainPolicy, % oid() + subjectDomainPolicy % oid() + }. + +#'Attribute'{ + type, % oid() + values % [der_encoded()] + }). + +#'BasicConstraints'{ + cA, % boolean() + pathLenConstraint % integer() + }). + +#'NameConstraints'{ + permittedSubtrees, % [#'GeneralSubtree'{}] + excludedSubtrees % [#'GeneralSubtree'{}] + }). + +#'GeneralSubtree'{ + base, % general_name() + minimum, % integer() + maximum % integer() + }). + +#'PolicyConstraints'{ + requireExplicitPolicy, % integer() + inhibitPolicyMapping % integer() + }). + +#'DistributionPoint'{ + distributionPoint, % {fullName, [general_name()]} | {nameRelativeToCRLIssuer, + [#AttributeTypeAndValue{}]} + reasons, % [dist_reason()] + cRLIssuer % [general_name()] + }).</code> + +</section> + + <section> + <marker id="PrivIntExt"></marker> + <title>Private Internet Extensions</title> + + <p>The private internet extensions OID name atoms and their corresponding value + types are as follows:</p> + + <table> + <row> + <cell align="left" valign="middle"><em>OID Name</em></cell> + <cell align="left" valign="middle"><em>Value Type</em></cell> + </row> + <row> + <cell align="left" valign="middle">id-pe-authorityInfoAccess</cell> + <cell align="left" valign="middle">[#'AccessDescription'{}]</cell> + </row> + <row> + <cell align="left" valign="middle">id-pe-subjectInfoAccess</cell> + <cell align="left" valign="middle">[#'AccessDescription'{}]</cell> + </row> + <tcaption>Private Internet Extensions</tcaption> + </table> + +<code> +#'AccessDescription'{ + accessMethod, % oid() + accessLocation % general_name() + }).</code> + + </section> + +<section> + <title>CRL and CRL Extensions Profile</title> + + <p>Erlang representation of CRL and CRL extensions profile + derived from ASN.1 specifications and RFC 5280 are as follows:</p> + + <code> +#'CertificateList'{ + tbsCertList, % #'TBSCertList{} + signatureAlgorithm, % #'AlgorithmIdentifier'{} + signature % bitstring() + }). + +#'TBSCertList'{ + version, % v2 (if defined) + signature, % #AlgorithmIdentifier{} + issuer, % {rdnSequence, [#AttributeTypeAndValue'{}]} + thisUpdate, % time() + nextUpdate, % time() + revokedCertificates, % [#'TBSCertList_revokedCertificates_SEQOF'{}] + crlExtensions % [#'Extension'{}] + }). + +#'TBSCertList_revokedCertificates_SEQOF'{ + userCertificate, % integer() + revocationDate, % timer() + crlEntryExtensions % [#'Extension'{}] + }).</code> + + <section> + <marker id="CRLCertExt"></marker> + <title>CRL Extensions</title> + + <p>The CRL extensions OID name atoms and their corresponding value types are as follows:</p> + + + <table> + <row> + <cell align="left" valign="middle"><em>OID Name</em></cell> + <cell align="left" valign="middle"><em>Value Type</em></cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-authorityKeyIdentifier</cell> + <cell align="left" valign="middle">#'AuthorityKeyIdentifier{}</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-issuerAltName</cell> + <cell align="left" valign="middle">{rdnSequence, [#AttributeTypeAndValue'{}]}</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-cRLNumber</cell> + <cell align="left" valign="middle">integer()</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-deltaCRLIndicator</cell> + <cell align="left" valign="middle">integer()</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-issuingDistributionPoint</cell> + <cell align="left" valign="middle">#'IssuingDistributionPoint'{}</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-freshestCRL</cell> + <cell align="left" valign="middle">[#'Distributionpoint'{}]</cell> + </row> + + <tcaption>CRL Extensions</tcaption> + </table> + + <p>Here, the data type <c>'IssuingDistributionPoint'</c> is represented as + the following Erlang record:</p> + + <code> +#'IssuingDistributionPoint'{ + distributionPoint, % {fullName, [general_name()]} | {nameRelativeToCRLIssuer, + [#AttributeTypeAndValue'{}]} + onlyContainsUserCerts, % boolean() + onlyContainsCACerts, % boolean() + onlySomeReasons, % [dist_reason()] + indirectCRL, % boolean() + onlyContainsAttributeCerts % boolean() + }).</code> + </section> + + <section> + <marker id="CRLEntryExt"></marker> + <title>CRL Entry Extensions</title> + + <p>The CRL entry extensions OID name atoms and their corresponding value types are as follows:</p> + + <table> + <row> + <cell align="left" valign="middle"><em>OID Name</em></cell> + <cell align="left" valign="middle"><em>Value Type</em></cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-cRLReason</cell> + <cell align="left" valign="middle">crl_reason()</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-holdInstructionCode</cell> + <cell align="left" valign="middle">oid()</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-invalidityDate</cell> + <cell align="left" valign="middle">general_time()</cell> + </row> + <row> + <cell align="left" valign="middle">id-ce-certificateIssuer</cell> + <cell align="left" valign="middle">general_name()</cell> + </row> + <tcaption>CRL Entry Extensions</tcaption> + </table> + + + <p>Here:</p> + <taglist> + <tag><c>crl_reason()</c></tag> + <item>= <p><c>unspecified</c></p> + <p><c>| keyCompromise</c></p> + <p><c>| cACompromise</c></p> + <p><c>| affiliationChanged</c></p> + <p><c>| superseded</c></p> + <p><c>| cessationOfOperation</c></p> + <p><c>| certificateHold</c></p> + <p><c>| removeFromCRL</c></p> + <p><c>| privilegeWithdrawn</c></p> + <p><c>| aACompromise</c></p> + </item> + </taglist> + + </section> + + <section> + <marker id="PKCS10"></marker> + <title>PKCS#10 Certification Request</title> + <p>Erlang representation of a PKCS#10 certification request + derived from ASN.1 specifications and RFC 5280 are as follows:</p> + <code> +#'CertificationRequest'{ + certificationRequestInfo #'CertificationRequestInfo'{}, + signatureAlgorithm #'CertificationRequest_signatureAlgorithm'{}}. + signature bitstring() + } + +#'CertificationRequestInfo'{ + version atom(), + subject {rdnSequence, [#AttributeTypeAndValue'{}]} , + subjectPKInfo #'CertificationRequestInfo_subjectPKInfo'{}, + attributes [#'AttributePKCS-10' {}] + } + +#'CertificationRequestInfo_subjectPKInfo'{ + algorithm #'CertificationRequestInfo_subjectPKInfo_algorithm'{} + subjectPublicKey bitstring() + } + +#'CertificationRequestInfo_subjectPKInfo_algorithm'{ + algorithm = oid(), + parameters = der_encoded() +} + +#'CertificationRequest_signatureAlgorithm'{ + algorithm = oid(), + parameters = der_encoded() + } + +#'AttributePKCS-10'{ + type = oid(), + values = [der_encoded()] +} </code> + </section> +</section> </chapter> diff --git a/lib/public_key/doc/src/ref_man.xml b/lib/public_key/doc/src/ref_man.xml index b7078891d4..9c80cf4b9f 100644 --- a/lib/public_key/doc/src/ref_man.xml +++ b/lib/public_key/doc/src/ref_man.xml @@ -31,8 +31,8 @@ <file>ref_man.xml</file> </header> <description> - <p> Provides functions to handle public key infrastructure - from RFC 3280 (X.509 certificates) and some parts of the PKCS-standard. + <p>The <c>public_key</c> application provides functions to handle public-key infrastructure + from RFC 3280 (X.509 certificates) and parts of the PKCS standard. </p> </description> <xi:include href="public_key.xml"/> diff --git a/lib/public_key/doc/src/using_public_key.xml b/lib/public_key/doc/src/using_public_key.xml index 450bd7e35f..03e4bedf3d 100644 --- a/lib/public_key/doc/src/using_public_key.xml +++ b/lib/public_key/doc/src/using_public_key.xml @@ -22,48 +22,50 @@ </legalnotice> <title>Getting Started</title> + <prepared></prepared> + <docno></docno> + <date></date> + <rev></rev> <file>using_public_key.xml</file> </header> - <section> - <title>General information</title> + <p>This section describes examples of how to use the + Public Key API. Keys and certificates used in the following + sections are generated only for testing the Public Key + application.</p> - <p> This chapter is dedicated to showing some - examples of how to use the public_key API. Keys and certificates - used in the following sections are generated only for the purpose - of testing the public key application.</p> + <p>Some shell printouts in the following examples + are abbreviated for increased readability.</p> - <p>Note that some shell printouts, in the following examples, - have been abbreviated for increased readability.</p> + + <section> + <title>PEM Files</title> + <p>Public-key data (keys, certificates, and so on) can be stored in + Privacy Enhanced Mail (PEM) format. + The PEM files have the following structure:</p> - </section> + <code> + <text> + -----BEGIN <SOMETHING>----- + <Attribute> : <Value> + <Base64 encoded DER data> + -----END <SOMETHING>----- + <text></code> - <section> - <title>PEM files</title> - <p> Public key data (keys, certificates etc) may be stored in PEM format. PEM files - comes from the Private Enhanced Mail Internet standard and has a - structure that looks like this:</p> - - <code><text> - -----BEGIN <SOMETHING>----- - <Attribute> : <Value> - <Base64 encoded DER data> - -----END <SOMETHING>----- - <text></code> - - <p>A file can contain several BEGIN/END blocks. Text lines between - blocks are ignored. Attributes, if present, are currently ignored except - for <c>Proc-Type</c> and <c>DEK-Info</c> that are used when the DER data is - encrypted.</p> + <p>A file can contain several <c>BEGIN/END</c> blocks. Text lines between + blocks are ignored. Attributes, if present, are ignored except + for <c>Proc-Type</c> and <c>DEK-Info</c>, which are used when <c>DER</c> + data is encrypted.</p> <section> - <title>DSA private key</title> + <title>DSA Private Key</title> + <p>A DSA private key can look as follows:</p> + <note><p>File handling is not done by the Public Key application.</p></note> - <p>Note file handling is not done by the public_key application. </p> <code>1> {ok, PemBin} = file:read_file("dsa.pem"). {ok,<<"-----BEGIN DSA PRIVATE KEY-----\nMIIBuw"...>>}</code> - <p>This PEM file only has one entry, a private DSA key.</p> + <p>The following PEM file has only one entry, a private DSA key:</p> <code>2> [DSAEntry] = public_key:pem_decode(PemBin). [{'DSAPrivateKey',<<48,130,1,187,2,1,0,2,129,129,0,183, 179,230,217,37,99,144,157,21,228,204, @@ -80,21 +82,20 @@ </section> <section> - <title>RSA private key encrypted with a password.</title> + <title>RSA Private Key with Password</title> + <p>An RSA private key encrypted with a password can look as follows:</p> <code>1> {ok, PemBin} = file:read_file("rsa.pem"). {ok,<<"Bag Attribut"...>>}</code> - <p>This PEM file only has one entry a private RSA key.</p> + <p>The following PEM file has only one entry, a private RSA key:</p> <code>2>[RSAEntry] = public_key:pem_decode(PemBin). [{'RSAPrivateKey',<<224,108,117,203,152,40,15,77,128,126, 221,195,154,249,85,208,202,251,109, 119,120,57,29,89,19,9,...>>, - {"DES-EDE3-CBC",<<"kÙeø¼pµL">>}}] + {"DES-EDE3-CBC",<<"kÙeø¼pµL">>}}]</code> - </code> - - <p>In this example the password is "abcd1234".</p> + <p>In this following example, the password is <c>"abcd1234"</c>:</p> <code>3> Key = public_key:pem_entry_decode(RSAEntry, "abcd1234"). #'RSAPrivateKey'{version = 'two-prime', modulus = 1112355156729921663373...2737107, @@ -110,11 +111,12 @@ <section> <title>X509 Certificates</title> + <p>The following is an example of X509 certificates:</p> <code>1> {ok, PemBin} = file:read_file("cacerts.pem"). {ok,<<"-----BEGIN CERTIFICATE-----\nMIIC7jCCAl"...>>}</code> - <p>This file includes two certificates</p> + <p>The following file includes two certificates:</p> <code>2> [CertEntry1, CertEntry2] = public_key:pem_decode(PemBin). [{'Certificate',<<48,130,2,238,48,130,2,87,160,3,2,1,2,2, 9,0,230,145,97,214,191,2,120,150,48,13, @@ -124,7 +126,7 @@ 1,48,13,6,9,42,134,72,134,247,...>>>, not_encrypted}]</code> - <p>Certificates may of course be decoded as usual ... </p> + <p>Certificates can be decoded as usual:</p> <code>2> Cert = public_key:pem_entry_decode(CertEntry1). #'Certificate'{ tbsCertificate = @@ -210,24 +212,24 @@ algorithm = {1,2,840,113549,1,1,5}, parameters = <<5,0>>}, signature = - {0, - <<163,186,7,163,216,152,63,47,154,234,139,73,154,96,120, - 165,2,52,196,195,109,167,192,...>>}} -</code> - - <p> Parts of certificates can be decoded with - public_key:der_decode/2 using that parts ASN.1 type. - Although application specific certificate - extension requires application specific ASN.1 decode/encode-functions. - Example, the first value of the rdnSequence above is of ASN.1 type - 'X520CommonName'. ({2,5,4,3} = ?id-at-commonName)</p> + <<163,186,7,163,216,152,63,47,154,234,139,73,154,96,120, + 165,2,52,196,195,109,167,192,...>>}</code> + + <p>Parts of certificates can be decoded with + <c>public_key:der_decode/2</c>, using the ASN.1 type of that part. + However, an application-specific certificate extension requires + application-specific ASN.1 decode/encode-functions. + In the recent example, the first value of <c>rdnSequence</c> is + of ASN.1 type <c>'X520CommonName'. ({2,5,4,3} = ?id-at-commonName)</c>:</p> <code>public_key:der_decode('X520CommonName', <<19,8,101,114,108,97,110,103,67,65>>). {printableString,"erlangCA"}</code> - <p>... but certificates can also be decode using the pkix_decode_cert/2 that - can customize and recursively decode standard parts of a certificate.</p> + <p>However, certificates can also be decoded using <c>pkix_decode_cert/2</c>, + which can customize and recursively decode standard parts of a certificate:</p> + <code>3>{_, DerCert, _} = CertEntry1.</code> + <code>4> public_key:pkix_decode_cert(DerCert, otp). #'OTPCertificate'{ tbsCertificate = @@ -314,30 +316,27 @@ algorithm = {1,2,840,113549,1,1,5}, parameters = 'NULL'}, signature = - {0, <<163,186,7,163,216,152,63,47,154,234,139,73,154,96,120, - 165,2,52,196,195,109,167,192,...>>}} -</code> + 165,2,52,196,195,109,167,192,...>>}</code> - <p>This call is equivalent to public_key:pem_entry_decode(CertEntry1)</p> + <p>This call is equivalent to <c>public_key:pem_entry_decode(CertEntry1)</c>:</p> <code>5> public_key:pkix_decode_cert(DerCert, plain). -#'Certificate'{ ...} -</code> +#'Certificate'{ ...}</code> </section> <section> - <title>Encoding public key data to PEM format</title> + <title>Encoding Public-Key Data to PEM Format</title> - <p>If you have public key data and and want to create a PEM file - you can do that by calling the functions - public_key:pem_entry_encode/2 and pem_encode/1 and then saving the - result to a file. For example assume you have PubKey = - 'RSAPublicKey'{} then you can create a PEM-"RSA PUBLIC KEY" file - (ASN.1 type 'RSAPublicKey') or a PEM-"PUBLIC KEY" file - ('SubjectPublicKeyInfo' ASN.1 type).</p> + <p>If you have public-key data and want to create a PEM file + this can be done by calling functions + <c>public_key:pem_entry_encode/2</c> and <c>pem_encode/1</c> and + saving the result to a file. For example, assume that you have + <c>PubKey = 'RSAPublicKey'{}</c>. Then you can create a PEM-"RSA PUBLIC KEY" + file (ASN.1 type <c>'RSAPublicKey'</c>) or a PEM-"PUBLIC KEY" file + (<c>'SubjectPublicKeyInfo'</c> ASN.1 type).</p> - <p> The second element of the PEM-entry will be the ASN.1 DER encoded - key data.</p> + <p>The second element of the PEM-entry is the ASN.1 <c>DER</c> encoded + key data:</p> <code>1> PemEntry = public_key:pem_entry_encode('RSAPublicKey', RSAPubKey). {'RSAPublicKey', <<48,72,...>>, not_encrypted} @@ -348,7 +347,7 @@ 3> file:write_file("rsa_pub_key.pem", PemBin). ok</code> - <p> or </p> + <p>or:</p> <code>1> PemEntry = public_key:pem_entry_encode('SubjectPublicKeyInfo', RSAPubKey). {'SubjectPublicKeyInfo', <<48,92...>>, not_encrypted} @@ -363,96 +362,108 @@ ok</code> </section> <section> - <title>RSA public key cryptography </title> - <p> Suppose you have PrivateKey = #'RSAPrivateKey{}' and the - plaintext Msg = binary() and the corresponding public key - PublicKey = #'RSAPublicKey'{} then you can do the following. - Note that you normally will only do one of the encrypt or - decrypt operations and the peer will do the other. - </p> - - <p>Encrypt with the private key </p> + <title>RSA Public-Key Cryptography</title> + <p>Suppose you have the following private key and a corresponding public key:</p> + <list type="bulleted"> + <item><c>PrivateKey = #'RSAPrivateKey{}'</c> and + the plaintext <c>Msg = binary()</c></item> + <item><c>PublicKey = #'RSAPublicKey'{}</c> + </item> + </list> + <p>Then you can proceed as follows:</p> + + <p>Encrypt with the private key:</p> <code>RsaEncrypted = public_key:encrypt_private(Msg, PrivateKey), Msg = public_key:decrypt_public(RsaEncrypted, PublicKey),</code> - <p>Encrypt with the public key </p> + <p>Encrypt with the public key:</p> <code>RsaEncrypted = public_key:encrypt_public(Msg, PublicKey), Msg = public_key:decrypt_private(RsaEncrypted, PrivateKey),</code> + + <note><p>You normally do only one of the encrypt or decrypt operations, + and the peer does the other. This normaly used in legacy applications + as a primitive digital signature. + </p></note> + </section> <section> - <title>Digital signatures</title> + <title>Digital Signatures</title> - <p> Suppose you have PrivateKey = #'RSAPrivateKey{}'or - #'DSAPrivateKey'{} and the plaintext Msg = binary() and the - corresponding public key PublicKey = #'RSAPublicKey'{} or - {integer(), #'DssParams'{}} then you can do the following. Note - that you normally will only do one of the sign or verify operations - and the peer will do the other. </p> + <p>Suppose you have the following private key and a corresponding public key:</p> + + <list type="bulleted"> + <item><c>PrivateKey = #'RSAPrivateKey{}'</c> or + <c>#'DSAPrivateKey'{}</c> and the plaintext <c>Msg = binary()</c></item> + <item><c>PublicKey = #'RSAPublicKey'{}</c> or + <c>{integer(), #'DssParams'{}}</c></item> + </list> + <p>Then you can proceed as follows:</p> <code>Signature = public_key:sign(Msg, sha, PrivateKey), true = public_key:verify(Msg, sha, Signature, PublicKey),</code> - <p>It might be appropriate to calculate the message digest before - calling sign or verify and then you can use the none as second - argument.</p> + <note><p>You normally do only one of the sign or verify operations, + and the peer does the other.</p></note> + + <p>It can be appropriate to calculate the message digest before + calling <c>sign</c> or <c>verify</c>, and then use <c>none</c> as + second argument:</p> <code>Digest = crypto:sha(Msg), Signature = public_key:sign(Digest, none, PrivateKey), -true = public_key:verify(Digest, none, Signature, PublicKey), - </code> +true = public_key:verify(Digest, none, Signature, PublicKey),</code> </section> <section> - <title>SSH files</title> + <title>SSH Files</title> <p>SSH typically uses PEM files for private keys but has its - own file format for storing public keys. The erlang public_key - application can be used to parse the content of SSH public key files.</p> + own file format for storing public keys. The <c>public_key</c> + application can be used to parse the content of SSH public-key files.</p> <section> - <title> RFC 4716 SSH public key files </title> + <title>RFC 4716 SSH Public-Key Files</title> <p>RFC 4716 SSH files looks confusingly like PEM files, - but there are some differences.</p> + but there are some differences:</p> <code>1> {ok, SshBin} = file:read_file("ssh2_rsa_pub"). {ok, <<"---- BEGIN SSH2 PUBLIC KEY ----\nAAAA"...>>}</code> - <p>This is equivalent to calling public_key:ssh_decode(SshBin, rfc4716_public_key). + <p>This is equivalent to calling <c>public_key:ssh_decode(SshBin, rfc4716_public_key)</c>: </p> <code>2> public_key:ssh_decode(SshBin, public_key). [{#'RSAPublicKey'{modulus = 794430685...91663, - publicExponent = 35}, []}] -</code> + publicExponent = 35}, []}]</code> </section> <section> - <title> Openssh public key format </title> + <title>OpenSSH Public-Key Format</title> + <p>OpenSSH public-key format looks as follows:</p> <code>1> {ok, SshBin} = file:read_file("openssh_dsa_pub"). {ok,<<"ssh-dss AAAAB3Nza"...>>}</code> - <p>This is equivalent to calling public_key:ssh_decode(SshBin, openssh_public_key). + <p>This is equivalent to calling <c>public_key:ssh_decode(SshBin, openssh_public_key)</c>: </p> <code>2> public_key:ssh_decode(SshBin, public_key). [{{15642692...694280725, #'Dss-Parms'{p = 17291273936...696123221, q = 1255626590179665817295475654204371833735706001853, g = 10454211196...480338645}}, - [{comment,"dhopson@VMUbuntu-DSH"}]}] -</code> + [{comment,"dhopson@VMUbuntu-DSH"}]}]</code> </section> <section> - <title> Known hosts - openssh format</title> - + <title>Known Hosts - OpenSSH Format</title> + <p>Known hosts - OpenSSH format looks as follows:</p> <code>1> {ok, SshBin} = file:read_file("known_hosts"). {ok,<<"hostname.domain.com,192.168.0.1 ssh-rsa AAAAB...>>}</code> - <p>Returns a list of public keys and their related attributes - each pair of key and attributes corresponds to one entry in - the known hosts file.</p> + <p>Returns a list of public keys and their related attributes. + Each pair of key and attribute corresponds to one entry in + the known hosts file:</p> <code>2> public_key:ssh_decode(SshBin, known_hosts). [{#'RSAPublicKey'{modulus = 1498979460408...72721699, @@ -461,19 +472,19 @@ true = public_key:verify(Digest, none, Signature, PublicKey), {#'RSAPublicKey'{modulus = 14989794604088...2721699, publicExponent = 35}, [{comment,"[email protected]"}, - {hostnames,["|1|BWO5qDxk/cFH0wa05JLdHn+j6xQ=|rXQvIxh5cDD3C43k5DPDamawVNA="]}]}] -</code> + {hostnames,["|1|BWO5qDxk/cFH0wa05JLdHn+j6xQ=|rXQvIxh5cDD3C43k5DPDamawVNA="]}]}]</code> </section> <section> - <title> Authorized keys - openssh format</title> + <title>Authorized Keys - OpenSSH Format</title> + <p>Authorized keys - OpenSSH format looks as follows:</p> <code>1> {ok, SshBin} = file:read_file("auth_keys"). {ok, <<"command=\"dump /home\",no-pty,no-port-forwarding ssh-rsa AAA...>>}</code> - <p>Returns a list of public keys and their related attributes - each pair of key and attributes corresponds to one entry in - the authorized key file.</p> + <p>Returns a list of public keys and their related attributes. + Each pair of key and attribute corresponds to one entry in + the authorized key file:</p> <code>2> public_key:ssh_decode(SshBin, auth_keys). [{#'RSAPublicKey'{modulus = 794430685...691663, @@ -485,16 +496,15 @@ true = public_key:verify(Digest, none, Signature, PublicKey), #'Dss-Parms'{p = 17291273936185...763696123221, q = 1255626590179665817295475654204371833735706001853, g = 10454211195705...60511039590076780999046480338645}}, - [{comment,"dhopson@VMUbuntu-DSH"}]}] -</code> + [{comment,"dhopson@VMUbuntu-DSH"}]}]</code> </section> <section> - <title> Creating an SSH file from public key data </title> + <title>Creating an SSH File from Public-Key Data</title> <p>If you got a public key <c>PubKey</c> and a related list of attributes <c>Attributes</c> as returned - by ssh_decode/2 you can create a new ssh file for example</p> + by <c>ssh_decode/2</c>, you can create a new SSH file, for example:</p> <code>N> SshBin = public_key:ssh_encode([{PubKey, Attributes}], openssh_public_key), <<"ssh-rsa "...>> N+1> file:write_file("id_rsa.pub", SshBin). diff --git a/lib/sasl/doc/src/sasl_app.xml b/lib/sasl/doc/src/sasl_app.xml index 9c3c80bd13..572e550061 100644 --- a/lib/sasl/doc/src/sasl_app.xml +++ b/lib/sasl/doc/src/sasl_app.xml @@ -92,6 +92,13 @@ <item>Installs <c>sasl_report_file_h</c> in the error logger. This makes all reports go to the file <c>FileName</c>. <c>FileName</c> is a string.</item> + <tag><c>{file,FileName,Modes}</c></tag> + <item>Same as <c>{file,FileName}</c> except that the <c>Modes</c> + allows to specify the modes used for opening the <c>FileName</c> + given to the <seealso marker="kernel:file#open/2">file:open/2</seealso> + call. When not specified, the <c>Modes</c> defaults to <c>[write]</c>. + Use <c>[append]</c> for having the <c>FileName</c> open in append mode. + <c>FileName</c> is a string.</item> <tag><c>false</c></tag> <item> <p>No SASL error logger handler is installed.</p> diff --git a/lib/sasl/src/sasl.erl b/lib/sasl/src/sasl.erl index fdea6da13e..4a220f0511 100644 --- a/lib/sasl/src/sasl.erl +++ b/lib/sasl/src/sasl.erl @@ -55,7 +55,9 @@ get_sasl_error_logger() -> case application:get_env(sasl, sasl_error_logger) of {ok, false} -> undefined; {ok, tty} -> tty; - {ok, {file, File}} when is_list(File) -> {file, File}; + {ok, {file, File}} when is_list(File) -> {file, File, [write]}; + {ok, {file, File, Modes}} when is_list(File), is_list(Modes) -> + {file, File, Modes}; {ok, Bad} -> exit({bad_config, {sasl, {sasl_error_logger, Bad}}}); _ -> undefined end. @@ -125,9 +127,9 @@ delete_sasl_error_logger(Type) -> error_logger:delete_report_handler(mod(Type)). mod(tty) -> sasl_report_tty_h; -mod({file, _File}) -> sasl_report_file_h. +mod({file, _File, _Modes}) -> sasl_report_file_h. -args({file, File}, Type) -> {File, type(Type)}; +args({file, File, Modes}, Type) -> {File, Modes, type(Type)}; args(_, Type) -> type(Type). type(error) -> error; diff --git a/lib/sasl/src/sasl_report_file_h.erl b/lib/sasl/src/sasl_report_file_h.erl index f42b4b5ff2..a5bd0ac055 100644 --- a/lib/sasl/src/sasl_report_file_h.erl +++ b/lib/sasl/src/sasl_report_file_h.erl @@ -28,9 +28,9 @@ handle_event/2, handle_call/2, handle_info/2, terminate/2]). -init({File, Type}) -> +init({File, Modes, Type}) when is_list(Modes) -> process_flag(trap_exit, true), - case file:open(File, [write]) of + case file:open(File, Modes) of {ok,Fd} -> {ok, {Fd, File, Type}}; What -> diff --git a/lib/sasl/test/sasl_SUITE.erl b/lib/sasl/test/sasl_SUITE.erl index d7b99d506e..d9ab9e551c 100644 --- a/lib/sasl/test/sasl_SUITE.erl +++ b/lib/sasl/test/sasl_SUITE.erl @@ -26,10 +26,11 @@ %% Test cases must be exported. -export([app_test/1, appup_test/1, - log_mf_h_env/1]). + log_mf_h_env/1, + log_file/1]). all() -> - [log_mf_h_env, app_test, appup_test]. + [log_mf_h_env, log_file, app_test, appup_test]. groups() -> []. @@ -151,10 +152,9 @@ check_appup([],_,_) -> log_mf_h_env(Config) -> PrivDir = ?config(priv_dir,Config), LogDir = filename:join(PrivDir,sasl_SUITE_log_dir), - ok = file:make_dir(LogDir), + ok = filelib:ensure_dir(LogDir), application:stop(sasl), - SaslEnv = application:get_all_env(sasl), - lists:foreach(fun({E,_V}) -> application:unset_env(sasl,E) end, SaslEnv), + clear_env(sasl), ok = application:set_env(sasl,error_logger_mf_dir,LogDir), match_error(missing_config,application:start(sasl)), @@ -178,6 +178,23 @@ log_mf_h_env(Config) -> ok = application:set_env(sasl,error_logger_mf_dir,LogDir), ok = application:start(sasl). +log_file(Config) -> + PrivDir = ?config(priv_dir,Config), + LogDir = filename:join(PrivDir,sasl_SUITE_log_dir), + ok = filelib:ensure_dir(LogDir), + File = filename:join(LogDir, "file.log"), + application:stop(sasl), + clear_env(sasl), + + ok = application:set_env(sasl,sasl_error_logger,{file, File}, [{persistent, true}]), + ok = application:start(sasl), + application:stop(sasl), + ok = application:set_env(sasl,sasl_error_logger,{file, File, [append]}, [{persistent, true}]), + ok = application:start(sasl), + application:stop(sasl), + ok = application:set_env(sasl,sasl_error_logger, tty, [{persistent, false}]), + ok = application:start(sasl). + %%----------------------------------------------------------------- %% Internal @@ -185,3 +202,7 @@ match_error(Expected,{error,{bad_return,{_,{'EXIT',{Expected,{sasl,_}}}}}}) -> ok; match_error(Expected,Actual) -> ?t:fail({unexpected_return,Expected,Actual}). + +clear_env(App) -> + [application:unset_env(App,Opt) || {Opt,_} <- application:get_all_env(App)], + ok. diff --git a/lib/sasl/vsn.mk b/lib/sasl/vsn.mk index 4259a2d76c..8d1a043410 100644 --- a/lib/sasl/vsn.mk +++ b/lib/sasl/vsn.mk @@ -1 +1 @@ -SASL_VSN = 2.4.1 +SASL_VSN = 2.4.2 diff --git a/lib/snmp/doc/src/snmp_app.xml b/lib/snmp/doc/src/snmp_app.xml index 86f0981988..e36908a5b9 100644 --- a/lib/snmp/doc/src/snmp_app.xml +++ b/lib/snmp/doc/src/snmp_app.xml @@ -587,7 +587,7 @@ <marker id="manager_server_timeout"></marker> <tag><c><![CDATA[server_timeout() = integer() <optional>]]></c></tag> <item> - <p>Asynchroneous request cleanup time. For every requests, + <p>Asynchronous request cleanup time. For every requests, some info is stored internally, in order to be able to deliver the reply (when it arrives) to the proper destination. If the reply arrives, this info will be deleted. But if diff --git a/lib/snmp/doc/src/snmp_config.xml b/lib/snmp/doc/src/snmp_config.xml index 0ec8bb91cf..d1ee6545dd 100644 --- a/lib/snmp/doc/src/snmp_config.xml +++ b/lib/snmp/doc/src/snmp_config.xml @@ -616,7 +616,7 @@ in so far as it will be converted to the new format if found. <marker id="manager_server_timeout"></marker> <tag><c><![CDATA[server_timeout() = integer() <optional>]]></c></tag> <item> - <p>Asynchroneous request cleanup time. For every requests, + <p>Asynchronous request cleanup time. For every requests, some info is stored internally, in order to be able to deliver the reply (when it arrives) to the proper destination. If the reply arrives, this info will be deleted. But if diff --git a/lib/snmp/src/app/snmp.app.src b/lib/snmp/src/app/snmp.app.src index cbd292e4c3..a55bb389ba 100644 --- a/lib/snmp/src/app/snmp.app.src +++ b/lib/snmp/src/app/snmp.app.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -137,5 +137,5 @@ %% before snmp. {applications, [kernel, stdlib]}, {mod, {snmp_app, []}}, - {runtime_dependencies, ["stdlib-2.0","runtime_tools-1.8.14","mnesia-4.12", + {runtime_dependencies, ["stdlib-2.5","runtime_tools-1.8.14","mnesia-4.12", "kernel-3.0","erts-6.0","crypto-3.3"]}]}. diff --git a/lib/snmp/src/app/snmp.appup.src b/lib/snmp/src/app/snmp.appup.src index 081163b368..a21ff863be 100644 --- a/lib/snmp/src/app/snmp.appup.src +++ b/lib/snmp/src/app/snmp.appup.src @@ -28,6 +28,8 @@ %% {update, snmpa_local_db, soft, soft_purge, soft_purge, []} %% {add_module, snmpm_net_if_mt} [ + {"5.1.2", [ % Only runtime dependencies change + ]}, {"5.1.1", [{restart_application, snmp}]}, {"5.1", [ % Only compiler changes ]}, @@ -47,6 +49,8 @@ %% {remove, {snmpm_net_if_mt, soft_purge, soft_purge}} [ + {"5.1.2", [ % Only runtime dependencies change + ]}, {"5.1.1", [{restart_application, snmp}]}, {"5.1", [ % Only compiler changes ]}, diff --git a/lib/snmp/src/manager/snmpm.erl b/lib/snmp/src/manager/snmpm.erl index 8976322c4e..96e3d55b46 100644 --- a/lib/snmp/src/manager/snmpm.erl +++ b/lib/snmp/src/manager/snmpm.erl @@ -520,7 +520,7 @@ sync_get(UserId, TargetName, Context, Oids, Timeout, ExtraInfo) -> -%% --- asynchroneous get-request --- +%% --- asynchronous get-request --- %% %% The reply will be delivered to the user %% through a call to handle_pdu/5 @@ -588,7 +588,7 @@ sync_get_next(UserId, TargetName, Context, Oids, Timeout, ExtraInfo) -> %% </BACKWARD-COMPAT> -%% --- asynchroneous get_next-request --- +%% --- asynchronous get_next-request --- %% async_get_next2(UserId, TargetName, Oids) -> @@ -654,7 +654,7 @@ sync_set(UserId, TargetName, Context, VarsAndVals, Timeout, ExtraInfo) -> %% </BACKWARD-COMPAT> -%% --- asynchroneous set-request --- +%% --- asynchronous set-request --- %% async_set2(UserId, TargetName, VarsAndVals) -> @@ -746,7 +746,7 @@ sync_get_bulk(UserId, TargetName, NonRep, MaxRep, Context, Oids, Timeout, %% </BACKWARD-COMPAT> -%% --- asynchroneous get-bulk --- +%% --- asynchronous get-bulk --- %% async_get_bulk2(UserId, TargetName, NonRep, MaxRep, Oids) -> diff --git a/lib/snmp/vsn.mk b/lib/snmp/vsn.mk index 67adf0a34f..14da37a225 100644 --- a/lib/snmp/vsn.mk +++ b/lib/snmp/vsn.mk @@ -18,6 +18,6 @@ # %CopyrightEnd% APPLICATION = snmp -SNMP_VSN = 5.1.2 +SNMP_VSN = 5.2 PRE_VSN = APP_VSN = "$(APPLICATION)-$(SNMP_VSN)$(PRE_VSN)" diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 41885c684c..c77ee1e77a 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -29,6 +29,52 @@ <file>notes.xml</file> </header> +<section><title>Ssh 3.2.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Gracefully terminate if sockets is unexpectedly closed.</p> + <p> + Own Id: OTP-12782</p> + </item> + <item> + <p> + Made Codenomicon Defensics test suite pass: <list> + <item>limit number of algorithms in kexinit + message</item> <item>check 'e' and 'f' parameters in + kexdh</item> <item>implement 'keyboard-interactive' user + authentication on server side</item> <item> return plain + text message to bad version exchange message</item> + </list></p> + <p> + Own Id: OTP-12784</p> + </item> + </list> + </section> + +</section> + +<section><title>Ssh 3.2.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + A new option for handling the SSH_MSG_DEBUG message's + printouts. A fun could be given in the options that will + be called whenever the SSH_MSG_DEBUG message arrives. + This enables the user to format the printout or just + discard it.</p> + <p> + Own Id: OTP-12738 Aux Id: seq12860 </p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 3.2.2</title> <section><title>Improvements and New Features</title> diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index d49d3ac2a7..cf58806aa8 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -35,13 +35,15 @@ <section> <title>SSH</title> - + <marker id="supported"/> <list type="bulleted"> <item>For application dependencies see <seealso marker="SSH_app"> ssh(6)</seealso> </item> <item>Supported SSH version is 2.0.</item> + <item>Supported public key algorithms: ssh-rsa and ssh-dss.</item> <item>Supported MAC algorithms: hmac-sha2-256 and hmac-sha1.</item> <item>Supported encryption algorithms: aes128-ctr, aes128-cb and 3des-cbc.</item> <item>Supported key exchange algorithms: diffie-hellman-group1-sha1.</item> + <item>Supported compression algorithms: none, zlib, [email protected],</item> <item>Supports unicode filenames if the emulator and the underlaying OS support it. See section DESCRIPTION in the <seealso marker="kernel:file">file</seealso> manual page in <c>kernel</c> @@ -57,29 +59,40 @@ this module, or abstractions to indicate the intended use of the data type, or both:</p> <taglist> - <tag><c>boolean()</c></tag> - <item><p>= <c>true | false</c></p></item> - <tag><c>string()</c></tag> - <item><p>= <c>[byte()]</c></p></item> - <tag><c>ssh_daemon_ref()</c></tag> - <item><p>Opaque to the user, - returned by <c>ssh:daemon/[1,2,3]</c></p></item> - <tag><c>ssh_connection_ref()</c></tag> - <item><p>Opaque to the user, - returned by <c>ssh:connect/3</c></p></item> - <tag><c>ip_address()</c></tag> + <tag><c>boolean() =</c></tag> + <item><p><c>true | false</c></p></item> + <tag><c>string() =</c></tag> + <item><p><c>[byte()]</c></p></item> + <tag><c>ssh_daemon_ref() =</c></tag> + <item><p>opaque() - + as returned by <c>ssh:daemon/[1,2,3]</c></p></item> + <tag><c>ssh_connection_ref() =</c></tag> + <item><p>opaque() - as returned by <c>ssh:connect/3</c></p></item> + <tag><c>ip_address() =</c></tag> <item><p><c>inet::ip_address</c></p></item> - <tag><c>subsystem_spec()</c></tag> - <item><p>= <c>{subsystem_name(), - {channel_callback(), channel_init_args()}}</c></p></item> - <tag><c>subsystem_name()</c></tag> - <item><p>= <c>string()</c></p></item> - <tag><c>channel_callback()</c></tag> - <item><p>= <c>atom()</c> - Name of the Erlang module - implementing the subsystem using the <c>ssh_channel</c> behavior, see - <seealso marker="ssh_channel">ssh_channel(3)</seealso></p></item> - <tag><c>channel_init_args()</c></tag> - <item><p>= <c>list()</c></p></item> + <tag><c>subsystem_spec() =</c></tag> + <item><p><c>{subsystem_name(), + {channel_callback(), channel_init_args()}}</c></p></item> + <tag><c>subsystem_name() =</c></tag> + <item><p><c>string()</c></p></item> + <tag><c>channel_callback() =</c></tag> + <item><p><c>atom()</c> - Name of the Erlang module + implementing the subsystem using the <c>ssh_channel</c> behavior, see + <seealso marker="ssh_channel">ssh_channel(3)</seealso></p></item> + <tag><c>channel_init_args() =</c></tag> + <item><p><c>list()</c></p></item> + + <tag><c>algs_list() =</c></tag> + <item><p><c>list( alg_entry() )</c></p></item> + + <tag><c>alg_entry() =</c></tag> + <item><p><c>{kex, simple_algs()} | {public_key, simple_algs()} | {cipher, double_algs()} | {mac, double_algs()} | {compression, double_algs()}</c></p></item> + + <tag><c>simple_algs() =</c></tag> + <item><p><c>list( atom() )</c></p></item> + + <tag><c>double_algs() =</c></tag> + <item><p><c>[{client2serverlist,simple_algs()},{server2client,simple_algs()}] | simple_algs()</c></p></item> </taglist> </section> @@ -161,19 +174,58 @@ and <c>password</c>. However, those optins are not always desirable to use from a security point of view.</p> </item> + <tag><c><![CDATA[{public_key_alg, 'ssh-rsa' | 'ssh-dss'}]]></c></tag> <item> + <note> + <p>This option is kept for compatibility. It is ignored if the <c>preferred_algorithms</c> + option is used. The equivalence of <c>{public_key_alg,'ssh-dss'}</c> is + <c>{preferred_algorithms, [{public_key,['ssh-dss','ssh-rsa']}]}</c>.</p> + </note> <p>Sets the preferred public key algorithm to use for user authentication. If the preferred algorithm fails, the other algorithm is tried. The default is to try <c><![CDATA['ssh-rsa']]></c> first.</p> </item> + <tag><c><![CDATA[{pref_public_key_algs, list()}]]></c></tag> <item> + <note> + <p>This option is kept for compatibility. It is ignored if the <c>preferred_algorithms</c> + option is used. The equivalence of <c>{pref_public_key_algs,['ssh-dss']}</c> is + <c>{preferred_algorithms, [{public_key,['ssh-dss']}]}</c>.</p> + </note> <p>List of public key algorithms to try to use. <c>'ssh-rsa'</c> and <c>'ssh-dss'</c> are available. Overrides <c><![CDATA[{public_key_alg, 'ssh-rsa' | 'ssh-dss'}]]></c></p> </item> + + <tag><c><![CDATA[{preferred_algorithms, algs_list()}]]></c></tag> + <item> + <p>List of algorithms to use in the algorithm negotiation. The default <c>algs_list()</c> can + be obtained from <seealso marker="#default_algorithms/0">default_algorithms/0</seealso>. + </p> + <p>Here is an example of this option:</p> + <code> +{preferred_algorithms, + [{public_key,['ssh-rsa','ssh-dss']}, + {cipher,[{client2server,['aes128-ctr']}, + {server2client,['aes128-cbc','3des-cbc']}]}, + {mac,['hmac-sha2-256','hmac-sha1']}, + {compression,[none,zlib]} +} +</code> + <p>The example specifies different algorithms in the two directions (client2server and server2client), for cipher but specifies the same +algorithms for mac and compression in both directions. The kex (key exchange) and public key algorithms are set to their default values, +kex is implicit but public_key is set explicitly.</p> + + <warning> + <p>Changing the values can make a connection less secure. Do not change unless you + know exactly what you are doing. If you do not understand the values then you + are not supposed to change them.</p> + </warning> + </item> + <tag><c><![CDATA[{connect_timeout, timeout()}]]></c></tag> <item> <p>Sets a time-out on the transport layer @@ -227,6 +279,13 @@ <item> <p>Sets a time-out on a connection when no channels are active. Defaults to <c>infinity</c>.</p></item> + <tag><c><![CDATA[{ssh_msg_debug_fun, fun(ConnectionRef::ssh_connection_ref(), AlwaysDisplay::boolean(), Msg::binary(), LanguageTag::binary()) -> _}]]></c></tag> + <item> + <p>Provide a fun to implement your own logging of the SSH message SSH_MSG_DEBUG. The last three parameters are from the message, see RFC4253, section 11.3. The <c>ConnectionRef</c> is the reference to the connection on which the message arrived. The return value from the fun is not checked.</p> + <p>The default behaviour is ignore the message. + To get a printout for each message with <c>AlwaysDisplay = true</c>, use for example <c>{ssh_msg_debug_fun, fun(_,true,M,_)-> io:format("DEBUG: ~p~n", [M]) end}</c></p> + </item> + </taglist> </desc> </func> @@ -335,6 +394,33 @@ user. From a security perspective this option makes the server very vulnerable.</p> </item> + + <tag><c><![CDATA[{preferred_algorithms, algs_list()}]]></c></tag> + <item> + <p>List of algorithms to use in the algorithm negotiation. The default <c>algs_list()</c> can + be obtained from <seealso marker="#default_algorithms/0">default_algorithms/0</seealso>. + </p> + <p>Here is an example of this option:</p> + <code> +{preferred_algorithms, + [{public_key,['ssh-rsa','ssh-dss']}, + {cipher,[{client2server,['aes128-ctr']}, + {server2client,['aes128-cbc','3des-cbc']}]}, + {mac,['hmac-sha2-256','hmac-sha1']}, + {compression,[none,zlib]} +} +</code> + <p>The example specifies different algorithms in the two directions (client2server and server2client), for cipher but specifies the same +algorithms for mac and compression in both directions. The kex (key exchange) and public key algorithms are set to their default values, +kex is implicit but public_key is set explicitly.</p> + + <warning> + <p>Changing the values can make a connection less secure. Do not change unless you + know exactly what you are doing. If you do not understand the values then you + are not supposed to change them.</p> + </warning> + </item> + <tag><c><![CDATA[{pwdfun, fun(User::string(), password::string()) -> boolean()}]]></c></tag> <item> <p>Provides a function for password validation. This function is called @@ -427,10 +513,38 @@ <item> <p>Provides a fun to implement your own logging when a user disconnects from the server.</p> </item> - </taglist> - </desc> + + <tag><c><![CDATA[{ssh_msg_debug_fun, fun(ConnectionRef::ssh_connection_ref(), AlwaysDisplay::boolean(), Msg::binary(), LanguageTag::binary()) -> _}]]></c></tag> + <item> + <p>Provide a fun to implement your own logging of the SSH message SSH_MSG_DEBUG. The last three parameters are from the message, see RFC4253, section 11.3. The <c>ConnectionRef</c> is the reference to the connection on which the message arrived. The return value from the fun is not checked.</p> + <p>The default behaviour is ignore the message. + To get a printout for each message with <c>AlwaysDisplay = true</c>, use for example <c>{ssh_msg_debug_fun, fun(_,true,M,_)-> io:format("DEBUG: ~p~n", [M]) end}</c></p> + </item> + + </taglist> + </desc> </func> + <func> + <name>default_algorithms() -> algs_list()</name> + <fsummary>Get a list declaring the supported algorithms</fsummary> + <desc> + <p>Returns a key-value list, where the keys are the different types of algorithms and the values are the + algorithms themselves. An example:</p> + <code> +20> ssh:default_algorithms(). +[{kex,['diffie-hellman-group1-sha1']}, + {public_key,['ssh-rsa','ssh-dss']}, + {cipher,[{client2server,['aes128-ctr','aes128-cbc','3des-cbc']}, + {server2client,['aes128-ctr','aes128-cbc','3des-cbc']}]}, + {mac,[{client2server,['hmac-sha2-256','hmac-sha1']}, + {server2client,['hmac-sha2-256','hmac-sha1']}]}, + {compression,[{client2server,[none,zlib]}, + {server2client,[none,zlib]}]}] +21> +</code> + </desc> + </func> <func> <name>shell(Host) -> </name> diff --git a/lib/ssh/doc/src/ssh_channel.xml b/lib/ssh/doc/src/ssh_channel.xml index b8a03c350a..2fdecf9072 100644 --- a/lib/ssh/doc/src/ssh_channel.xml +++ b/lib/ssh/doc/src/ssh_channel.xml @@ -62,22 +62,22 @@ type, or both:</p> <taglist> - <tag><c>boolean()</c></tag> - <item><p>= <c>true | false</c></p></item> - <tag><c>string()</c></tag> - <item><p>= list of ASCII characters</p></item> - <tag><c>timeout()</c></tag> - <item><p>= <c>infinity | integer()</c> in milliseconds</p></item> - <tag><c>ssh_connection_ref()</c></tag> - <item><p>Opaque to the user, returned by - <c>ssh:connect/3</c> or sent to an SSH channel process</p></item> - <tag><c>ssh_channel_id()</c></tag> - <item><p>= <c>integer()</c></p></item> - <tag><c>ssh_data_type_code()</c></tag> - <item><p>= <c>1</c> ("stderr") | <c>0</c> ("normal") are - the valid values, - see <url href="http://www.ietf.org/rfc/rfc4254.txt">RFC 4254</url> - Section 5.2</p></item> + <tag><c>boolean() =</c></tag> + <item><p><c>true | false</c></p></item> + <tag><c>string() =</c></tag> + <item><p>list of ASCII characters</p></item> + <tag><c>timeout() =</c></tag> + <item><p><c>infinity | integer()</c> in milliseconds</p></item> + <tag><c>ssh_connection_ref() =</c></tag> + <item><p>opaque() -as returned by + <c>ssh:connect/3</c> or sent to an SSH channel process</p></item> + <tag><c>ssh_channel_id() =</c></tag> + <item><p><c>integer()</c></p></item> + <tag><c>ssh_data_type_code() =</c></tag> + <item><p><c>1</c> ("stderr") | <c>0</c> ("normal") are + the valid values, + see <url href="http://www.ietf.org/rfc/rfc4254.txt">RFC 4254</url> + Section 5.2</p></item> </taglist> </section> diff --git a/lib/ssh/doc/src/ssh_client_key_api.xml b/lib/ssh/doc/src/ssh_client_key_api.xml index a8dda042c9..9a892d71fd 100644 --- a/lib/ssh/doc/src/ssh_client_key_api.xml +++ b/lib/ssh/doc/src/ssh_client_key_api.xml @@ -50,16 +50,16 @@ <seealso marker="public_key:public_key_records"> public_key user's guide:</seealso> </p> <taglist> - <tag><c>boolean()</c></tag> - <item><p>= <c>true | false</c></p></item> - <tag><c>string()</c></tag> - <item><p>= <c>[byte()]</c></p></item> - <tag><c>public_key()</c></tag> - <item><p>= <c>#'RSAPublicKey'{}| {integer(), #'Dss-Parms'{}}| term()</c></p></item> - <tag><c>private_key()</c></tag> - <item><p>= <c>#'RSAPrivateKey'{} | #'DSAPrivateKey'{} | term()</c></p></item> - <tag><c>public_key_algorithm()</c></tag> - <item><p>= <c>'ssh-rsa'| 'ssh-dss' | atom()</c></p></item> + <tag><c>boolean() =</c></tag> + <item><p><c>true | false</c></p></item> + <tag><c>string() =</c></tag> + <item><p><c>[byte()]</c></p></item> + <tag><c>public_key() =</c></tag> + <item><p><c>#'RSAPublicKey'{}| {integer(), #'Dss-Parms'{}}| term()</c></p></item> + <tag><c>private_key() =</c></tag> + <item><p><c>#'RSAPrivateKey'{} | #'DSAPrivateKey'{} | term()</c></p></item> + <tag><c>public_key_algorithm() =</c></tag> + <item><p><c>'ssh-rsa'| 'ssh-dss' | atom()</c></p></item> </taglist> </section> diff --git a/lib/ssh/doc/src/ssh_connection.xml b/lib/ssh/doc/src/ssh_connection.xml index 669a361db9..5422633dc3 100644 --- a/lib/ssh/doc/src/ssh_connection.xml +++ b/lib/ssh/doc/src/ssh_connection.xml @@ -56,29 +56,29 @@ type, or both:</p> <taglist> - <tag><c>boolean()</c></tag> - <item><p>= <c>true | false </c></p></item> - <tag><c>string()</c></tag> - <item><p>= list of ASCII characters</p></item> - <tag><c>timeout()</c></tag> - <item><p>= <c>infinity | integer()</c> in milliseconds</p></item> - <tag><c>ssh_connection_ref()</c></tag> - <item><p>Opaque to the user, returned by - <c>ssh:connect/3</c> or sent to an SSH channel processes</p></item> - <tag><c>ssh_channel_id()</c></tag> - <item><p>= <c>integer()</c></p></item> - <tag><c>ssh_data_type_code()</c></tag> - <item><p>= <c>1</c> ("stderr") | <c>0</c> ("normal") are + <tag><c>boolean() =</c></tag> + <item><p><c>true | false </c></p></item> + <tag><c>string() =</c></tag> + <item><p>list of ASCII characters</p></item> + <tag><c>timeout() =</c></tag> + <item><p><c>infinity | integer()</c> in milliseconds</p></item> + <tag><c>ssh_connection_ref() =</c></tag> + <item><p>opaque() -as returned by + <c>ssh:connect/3</c> or sent to an SSH channel processes</p></item> + <tag><c>ssh_channel_id() =</c></tag> + <item><p><c>integer()</c></p></item> + <tag><c>ssh_data_type_code() =</c></tag> + <item><p><c>1</c> ("stderr") | <c>0</c> ("normal") are valid values, see <url href="http://www.ietf.org/rfc/rfc4254.txt">RFC 4254</url> Section 5.2.</p></item> - <tag><c>ssh_request_status() ssh_request_status()</c></tag> - <item><p>= <c>success | failure</c></p></item> - <tag><c>event()</c></tag> - <item><p>= <c>{ssh_cm, ssh_connection_ref(), ssh_event_msg()}</c></p></item> - <tag><c>ssh_event_msg()</c></tag> - <item><p>= <c>data_events() | status_events() | terminal_events()</c></p></item> - <tag><c>reason()</c></tag> - <item><p>= <c>timeout | closed</c></p></item> + <tag><c>ssh_request_status() =</c></tag> + <item><p> <c>success | failure</c></p></item> + <tag><c>event() =</c></tag> + <item><p><c>{ssh_cm, ssh_connection_ref(), ssh_event_msg()}</c></p></item> + <tag><c>ssh_event_msg() =</c></tag> + <item><p><c>data_events() | status_events() | terminal_events()</c></p></item> + <tag><c>reason() =</c></tag> + <item><p><c>timeout | closed</c></p></item> </taglist> <taglist> diff --git a/lib/ssh/doc/src/ssh_server_key_api.xml b/lib/ssh/doc/src/ssh_server_key_api.xml index 34ce7f7660..73dd90c962 100644 --- a/lib/ssh/doc/src/ssh_server_key_api.xml +++ b/lib/ssh/doc/src/ssh_server_key_api.xml @@ -50,20 +50,20 @@ <seealso marker="public_key:public_key_records"> public_key user's guide</seealso>. </p> -<taglist> - <tag><c>boolean()</c></tag> - <item><p>= <c>true | false</c></p></item> - <tag><c>string()</c></tag> - <item><p>= <c>[byte()]</c></p></item> - <tag><c>public_key()</c></tag> - <item><p>= <c>#'RSAPublicKey'{}| {integer(), #'Dss-Parms'{}}| term()</c></p></item> - <tag><c>private_key()</c></tag> - <item><p>= <c>#'RSAPrivateKey'{} | #'DSAPrivateKey'{} | term()</c></p></item> - <tag><c>public_key_algorithm()</c></tag> - <item><p>= <c>'ssh-rsa'| 'ssh-dss' | atom()</c></p></item> + <taglist> + <tag><c>boolean() =</c></tag> + <item><p><c>true | false</c></p></item> + <tag><c>string() =</c></tag> + <item><p><c>[byte()]</c></p></item> + <tag><c>public_key() =</c></tag> + <item><p><c>#'RSAPublicKey'{}| {integer(), #'Dss-Parms'{}}| term()</c></p></item> + <tag><c>private_key() =</c></tag> + <item><p><c>#'RSAPrivateKey'{} | #'DSAPrivateKey'{} | term()</c></p></item> + <tag><c>public_key_algorithm() =</c></tag> + <item><p><c>'ssh-rsa'| 'ssh-dss' | atom()</c></p></item> </taglist> </section> - + <funcs> <func> <name>Module:host_key(Algorithm, DaemonOptions) -> diff --git a/lib/ssh/doc/src/ssh_sftp.xml b/lib/ssh/doc/src/ssh_sftp.xml index 643130fe6b..fc418bc934 100644 --- a/lib/ssh/doc/src/ssh_sftp.xml +++ b/lib/ssh/doc/src/ssh_sftp.xml @@ -43,8 +43,8 @@ </p> <taglist> - <tag><c>ssh_connection_ref()</c></tag> - <item><p>Opaque to the user, returned by <c>ssh:connect/3</c></p></item> + <tag><c>ssh_connection_ref() =</c></tag> + <item><p>opaque() - as returned by <c>ssh:connect/3</c></p></item> <tag><c>timeout()</c></tag> <item><p>= <c>infinity | integer() in milliseconds. Default infinity.</c></p></item> </taglist> diff --git a/lib/ssh/doc/src/ssh_sftpd.xml b/lib/ssh/doc/src/ssh_sftpd.xml index bc2660f595..8b2497e6a3 100644 --- a/lib/ssh/doc/src/ssh_sftpd.xml +++ b/lib/ssh/doc/src/ssh_sftpd.xml @@ -37,16 +37,16 @@ <section> <title>DATA TYPES</title> <taglist> - <tag><c>subsystem_spec()</c></tag> - <item><p>= <c>{subsystem_name(), {channel_callback(), channel_init_args()}}</c></p></item> - <tag><c>subsystem_name()</c></tag> - <item><p>= <c>"sftp"</c></p></item> - <tag><c>channel_callback()</c></tag> - <item><p>= <c>atom()</c> - Name of the Erlang module implementing the subsystem using the + <tag><c>subsystem_spec() =</c></tag> + <item><p><c>{subsystem_name(), {channel_callback(), channel_init_args()}}</c></p></item> + <tag><c>subsystem_name() =</c></tag> + <item><p><c>"sftp"</c></p></item> + <tag><c>channel_callback() =</c></tag> + <item><p><c>atom()</c> - Name of the Erlang module implementing the subsystem using the <c>ssh_channel</c> behavior, see the <seealso marker="ssh_channel">ssh_channel(3)</seealso> manual page.</p></item> - <tag><c>channel_init_args()</c></tag> - <item><p>= <c>list()</c> - The one given as argument to function <c>subsystem_spec/1</c>.</p></item> + <tag><c>channel_init_args() =</c></tag> + <item><p><c>list()</c> - The one given as argument to function <c>subsystem_spec/1</c>.</p></item> </taglist> </section> <funcs> diff --git a/lib/ssh/src/ssh.erl b/lib/ssh/src/ssh.erl index d4b02a024e..4a07473f74 100644 --- a/lib/ssh/src/ssh.erl +++ b/lib/ssh/src/ssh.erl @@ -28,6 +28,7 @@ -export([start/0, start/1, stop/0, connect/3, connect/4, close/1, connection_info/2, channel_info/3, daemon/1, daemon/2, daemon/3, + default_algorithms/0, stop_listener/1, stop_listener/2, stop_daemon/1, stop_daemon/2, shell/1, shell/2, shell/3]). @@ -208,6 +209,11 @@ shell(Host, Port, Options) -> end. %%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +default_algorithms() -> + ssh_transport:default_algorithms(). + +%%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- fix_idle_time(SshOptions) -> @@ -259,7 +265,7 @@ do_start_daemon(Host, Port, Options, SocketOptions) -> end. handle_options(Opts) -> - try handle_option(proplists:unfold(Opts), [], []) of + try handle_option(algs_compatibility(proplists:unfold(Opts)), [], []) of {Inet, Ssh} -> {handle_ip(Inet), Ssh} catch @@ -267,6 +273,35 @@ handle_options(Opts) -> Error end. + +algs_compatibility(Os) -> + %% Take care of old options 'public_key_alg' and 'pref_public_key_algs' + comp_pk(proplists:get_value(preferred_algorithms,Os), + proplists:get_value(pref_public_key_algs,Os), + proplists:get_value(public_key_alg, Os), + [{K,V} || {K,V} <- Os, + K =/= public_key_alg, + K =/= pref_public_key_algs] + ). + +comp_pk(undefined, undefined, undefined, Os) -> Os; +comp_pk( PrefAlgs, _, _, Os) when PrefAlgs =/= undefined -> Os; + +comp_pk(undefined, undefined, ssh_dsa, Os) -> comp_pk(undefined, undefined, 'ssh-dss', Os); +comp_pk(undefined, undefined, ssh_rsa, Os) -> comp_pk(undefined, undefined, 'ssh-rsa', Os); +comp_pk(undefined, undefined, PK, Os) -> + PKs = [PK | ssh_transport:supported_algorithms(public_key)--[PK]], + [{preferred_algorithms, [{public_key,PKs}] } | Os]; + +comp_pk(undefined, PrefPKs, _, Os) when PrefPKs =/= undefined -> + PKs = [case PK of + ssh_dsa -> 'ssh-dss'; + ssh_rsa -> 'ssh-rsa'; + _ -> PK + end || PK <- PrefPKs], + [{preferred_algorithms, [{public_key,PKs}]} | Os]. + + handle_option([], SocketOptions, SshOptions) -> {SocketOptions, SshOptions}; handle_option([{system_dir, _} = Opt | Rest], SocketOptions, SshOptions) -> @@ -279,8 +314,6 @@ handle_option([{silently_accept_hosts, _} = Opt | Rest], SocketOptions, SshOptio handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{user_interaction, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{public_key_alg, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{connect_timeout, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{user, _} = Opt | Rest], SocketOptions, SshOptions) -> @@ -297,10 +330,6 @@ handle_option([{pwdfun, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{key_cb, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{role, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{compression, _} = Opt | Rest], SocketOptions, SshOptions) -> - handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); %%Backwards compatibility handle_option([{allow_user_interaction, Value} | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option({user_interaction, Value}) | SshOptions]); @@ -312,6 +341,8 @@ handle_option([{disconnectfun, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{failfun, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); +handle_option([{ssh_msg_debug_fun, _} = Opt | Rest], SocketOptions, SshOptions) -> + handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); %%Backwards compatibility should not be underscore between ip and v6 in API handle_option([{ip_v6_disabled, Value} | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option({ipv6_disabled, Value}) | SshOptions]); @@ -329,7 +360,9 @@ handle_option([{exec, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{auth_methods, _} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); -handle_option([{pref_public_key_algs, _} = Opt | Rest], SocketOptions, SshOptions) -> +handle_option([{auth_method_kb_interactive_data, _} = Opt | Rest], SocketOptions, SshOptions) -> + handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); +handle_option([{preferred_algorithms,_} = Opt | Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); handle_option([{quiet_mode, _} = Opt|Rest], SocketOptions, SshOptions) -> handle_option(Rest, SocketOptions, [handle_ssh_option(Opt) | SshOptions]); @@ -365,19 +398,8 @@ handle_ssh_option({silently_accept_hosts, Value} = Opt) when is_boolean(Value) - Opt; handle_ssh_option({user_interaction, Value} = Opt) when is_boolean(Value) -> Opt; -handle_ssh_option({public_key_alg, ssh_dsa}) -> - {public_key_alg, 'ssh-dss'}; -handle_ssh_option({public_key_alg, ssh_rsa}) -> - {public_key_alg, 'ssh-rsa'}; -handle_ssh_option({public_key_alg, Value} = Opt) when Value == 'ssh-rsa'; Value == 'ssh-dss' -> - Opt; -handle_ssh_option({pref_public_key_algs, Value} = Opt) when is_list(Value), length(Value) >= 1 -> - case handle_pref_algs(Value, []) of - {true, NewOpts} -> - NewOpts; - _ -> - throw({error, {eoptions, Opt}}) - end; +handle_ssh_option({preferred_algorithms,[_|_]} = Opt) -> + handle_pref_algs(Opt); handle_ssh_option({connect_timeout, Value} = Opt) when is_integer(Value); Value == infinity -> Opt; handle_ssh_option({max_sessions, Value} = Opt) when is_integer(Value), Value>0 -> @@ -409,6 +431,13 @@ handle_ssh_option({exec, Function} = Opt) when is_function(Function) -> Opt; handle_ssh_option({auth_methods, Value} = Opt) when is_list(Value) -> Opt; +handle_ssh_option({auth_method_kb_interactive_data, {Name,Instruction,Prompt,Echo}} = Opt) when is_list(Name), + is_list(Instruction), + is_list(Prompt), + is_boolean(Echo) -> + Opt; +handle_ssh_option({auth_method_kb_interactive_data, F} = Opt) when is_function(F,3) -> + Opt; handle_ssh_option({infofun, Value} = Opt) when is_function(Value) -> Opt; handle_ssh_option({connectfun, Value} = Opt) when is_function(Value) -> @@ -417,6 +446,8 @@ handle_ssh_option({disconnectfun , Value} = Opt) when is_function(Value) -> Opt; handle_ssh_option({failfun, Value} = Opt) when is_function(Value) -> Opt; +handle_ssh_option({ssh_msg_debug_fun, Value} = Opt) when is_function(Value,4) -> + Opt; handle_ssh_option({ipv6_disabled, Value} = Opt) when is_boolean(Value) -> throw({error, {{ipv6_disabled, Opt}, option_no_longer_valid_use_inet_option_instead}}); @@ -461,23 +492,83 @@ handle_inet_option({reuseaddr, _} = Opt) -> %% Option verified by inet handle_inet_option(Opt) -> Opt. + + %% Check preferred algs -handle_pref_algs([], Acc) -> - {true, lists:reverse(Acc)}; -handle_pref_algs([H|T], Acc) -> - case H of - ssh_dsa -> - handle_pref_algs(T, ['ssh-dss'| Acc]); - ssh_rsa -> - handle_pref_algs(T, ['ssh-rsa'| Acc]); - 'ssh-dss' -> - handle_pref_algs(T, ['ssh-dss'| Acc]); - 'ssh-rsa' -> - handle_pref_algs(T, ['ssh-rsa'| Acc]); - _ -> - false + +handle_pref_algs({preferred_algorithms,Algs}) -> + try alg_duplicates(Algs, [], []) of + [] -> + {preferred_algorithms, + [try ssh_transport:supported_algorithms(Key) + of + DefAlgs -> handle_pref_alg(Key,Vals,DefAlgs) + catch + _:_ -> throw({error, {{eoptions, {preferred_algorithms,Key}}, + "Bad preferred_algorithms key"}}) + end || {Key,Vals} <- Algs] + }; + + Dups -> + throw({error, {{eoptions, {preferred_algorithms,Dups}}, "Duplicates found"}}) + catch + _:_ -> + throw({error, {{eoptions, preferred_algorithms}, "Malformed"}}) end. +alg_duplicates([{K,V}|KVs], Ks, Dups0) -> + Dups = + case lists:member(K,Ks) of + true -> + [K|Dups0]; + false -> + Dups0 + end, + case V--lists:usort(V) of + [] -> + alg_duplicates(KVs, [K|Ks], Dups); + Ds -> + alg_duplicates(KVs, [K|Ks], Dups++Ds) + end; +alg_duplicates([], _Ks, Dups) -> + Dups. + +handle_pref_alg(Key, + Vs=[{client2server,C2Ss=[_|_]},{server2client,S2Cs=[_|_]}], + [{client2server,Sup_C2Ss},{server2client,Sup_S2Cs}] + ) -> + chk_alg_vs(Key, C2Ss, Sup_C2Ss), + chk_alg_vs(Key, S2Cs, Sup_S2Cs), + {Key, Vs}; + +handle_pref_alg(Key, + Vs=[{server2client,[_|_]},{client2server,[_|_]}], + Sup=[{client2server,_},{server2client,_}] + ) -> + handle_pref_alg(Key, lists:reverse(Vs), Sup); + +handle_pref_alg(Key, + Vs=[V|_], + Sup=[{client2server,_},{server2client,_}] + ) when is_atom(V) -> + handle_pref_alg(Key, [{client2server,Vs},{server2client,Vs}], Sup); + +handle_pref_alg(Key, + Vs=[V|_], + Sup=[S|_] + ) when is_atom(V), is_atom(S) -> + chk_alg_vs(Key, Vs, Sup), + {Key, Vs}; + +handle_pref_alg(Key, Vs, _) -> + throw({error, {{eoptions, {preferred_algorithms,[{Key,Vs}]}}, "Badly formed list"}}). + +chk_alg_vs(OptKey, Values, SupportedValues) -> + case (Values -- SupportedValues) of + [] -> Values; + Bad -> throw({error, {{eoptions, {OptKey,Bad}}, "Unsupported value(s) found"}}) + end. + handle_ip(Inet) -> %% Default to ipv4 case lists:member(inet, Inet) of true -> diff --git a/lib/ssh/src/ssh_auth.erl b/lib/ssh/src/ssh_auth.erl index 45c4d52d7e..df9a97c8f8 100644 --- a/lib/ssh/src/ssh_auth.erl +++ b/lib/ssh/src/ssh_auth.erl @@ -30,7 +30,8 @@ -export([publickey_msg/1, password_msg/1, keyboard_interactive_msg/1, service_request_msg/1, init_userauth_request_msg/1, userauth_request_msg/1, handle_userauth_request/3, - handle_userauth_info_request/3, handle_userauth_info_response/2 + handle_userauth_info_request/3, handle_userauth_info_response/2, + default_public_key_algorithms/0 ]). %%-------------------------------------------------------------------- @@ -115,33 +116,16 @@ init_userauth_request_msg(#ssh{opts = Opts} = Ssh) -> service = "ssh-connection", method = "none", data = <<>>}, - case proplists:get_value(pref_public_key_algs, Opts, false) of - false -> - FirstAlg = proplists:get_value(public_key_alg, Opts, ?PREFERRED_PK_ALG), - SecondAlg = other_alg(FirstAlg), - Prefs = method_preference(FirstAlg, SecondAlg), - ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, - userauth_preference = Prefs, - userauth_methods = none, - service = "ssh-connection"}); - Algs -> - FirstAlg = lists:nth(1, Algs), - case length(Algs) =:= 2 of - true -> - SecondAlg = other_alg(FirstAlg), - Prefs = method_preference(FirstAlg, SecondAlg), - ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, - userauth_preference = Prefs, - userauth_methods = none, - service = "ssh-connection"}); - _ -> - Prefs = method_preference(FirstAlg), - ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, - userauth_preference = Prefs, - userauth_methods = none, - service = "ssh-connection"}) - end - end; + + + Algs = proplists:get_value(public_key, + proplists:get_value(preferred_algorithms, Opts, []), + default_public_key_algorithms()), + Prefs = method_preference(Algs), + ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, + userauth_preference = Prefs, + userauth_methods = none, + service = "ssh-connection"}); {error, no_user} -> ErrStr = "Could not determine the users name", throw(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_ILLEGAL_USER_NAME, @@ -259,6 +243,54 @@ handle_userauth_request(#ssh_msg_userauth_request{user = User, handle_userauth_request(#ssh_msg_userauth_request{user = User, service = "ssh-connection", + method = "keyboard-interactive", + data = _}, + _, #ssh{opts = Opts} = Ssh) -> + %% RFC4256 + %% The data field contains: + %% - language tag (deprecated). If =/=[] SHOULD use it however. We skip + %% it for simplicity. + %% - submethods. "... the user can give a hint of which actual methods + %% he wants to use. ...". It's a "MAY use" so we skip + %% it. It also needs an understanding between the client + %% and the server. + %% + %% "The server MUST reply with an SSH_MSG_USERAUTH_SUCCESS, + %% SSH_MSG_USERAUTH_FAILURE, or SSH_MSG_USERAUTH_INFO_REQUEST message." + Default = {"SSH server", + "Enter password for \""++User++"\"", + "pwd: ", + false}, + + {Name, Instruction, Prompt, Echo} = + case proplists:get_value(auth_method_kb_interactive_data, Opts) of + undefined -> + Default; + {_,_,_,_}=V -> + V; + F when is_function(F) -> + {_,PeerName} = Ssh#ssh.peer, + F(PeerName, User, "ssh-connection") + end, + EchoEnc = case Echo of + true -> <<?TRUE>>; + false -> <<?FALSE>> + end, + Msg = #ssh_msg_userauth_info_request{name = unicode:characters_to_list(Name), + instruction = unicode:characters_to_list(Instruction), + language_tag = "", + num_prompts = 1, + data = <<?STRING(unicode:characters_to_binary(Prompt)), + EchoEnc/binary + >> + }, + {not_authorized, {User, undefined}, + ssh_transport:ssh_packet(Msg, Ssh#ssh{user = User, + opts = [{max_kb_tries,3},{kb_userauth_info_msg,Msg}|Opts] + })}; + +handle_userauth_request(#ssh_msg_userauth_request{user = User, + service = "ssh-connection", method = Other}, _, #ssh{userauth_supported_methods = Methods} = Ssh) -> {not_authorized, {User, {authmethod, Other}}, @@ -280,6 +312,38 @@ handle_userauth_info_request( #ssh_msg_userauth_info_response{num_responses = NumPrompts, data = Responses}, Ssh)}. +handle_userauth_info_response(#ssh_msg_userauth_info_response{num_responses = 1, + data = <<?UINT32(Sz), Password:Sz/binary>>}, + #ssh{opts = Opts0, + user = User} = Ssh) -> + NumTriesLeft = proplists:get_value(max_kb_tries, Opts0, 0) - 1, + Opts = lists:keydelete(max_kb_tries,1,Opts0), + case check_password(User, unicode:characters_to_list(Password), Opts) of + true -> + {authorized, User, + ssh_transport:ssh_packet(#ssh_msg_userauth_success{}, Ssh)}; + false when NumTriesLeft > 0 -> + UserAuthInfoMsg = + (proplists:get_value(kb_userauth_info_msg,Opts)) + #ssh_msg_userauth_info_request{name = "", + instruction = + lists:concat( + ["Bad user or password, try again. ", + integer_to_list(NumTriesLeft), + " tries left."])}, + {not_authorized, {User, undefined}, + ssh_transport:ssh_packet(UserAuthInfoMsg, + Ssh#ssh{opts = [{max_kb_tries,NumTriesLeft}|Opts]})}; + + false -> + {not_authorized, {User, {error,"Bad user or password"}}, + ssh_transport:ssh_packet(#ssh_msg_userauth_failure{ + authentications = "", + partial_success = false}, + Ssh#ssh{opts = lists:keydelete(kb_userauth_info_msg,1,Opts)} + )} + end; + handle_userauth_info_response(#ssh_msg_userauth_info_response{}, _Auth) -> throw(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_SERVICE_NOT_AVAILABLE, @@ -287,20 +351,20 @@ handle_userauth_info_response(#ssh_msg_userauth_info_response{}, "keyboard-interactive", language = "en"}). + +default_public_key_algorithms() -> ?PREFERRED_PK_ALGS. + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- -method_preference(Alg1, Alg2) -> - [{"publickey", ?MODULE, publickey_msg, [Alg1]}, - {"publickey", ?MODULE, publickey_msg,[Alg2]}, - {"password", ?MODULE, password_msg, []}, - {"keyboard-interactive", ?MODULE, keyboard_interactive_msg, []} - ]. -method_preference(Alg1) -> - [{"publickey", ?MODULE, publickey_msg, [Alg1]}, - {"password", ?MODULE, password_msg, []}, - {"keyboard-interactive", ?MODULE, keyboard_interactive_msg, []} - ]. +method_preference(Algs) -> + lists:foldr(fun(A, Acc) -> + [{"publickey", ?MODULE, publickey_msg, [A]} | Acc] + end, + [{"password", ?MODULE, password_msg, []}, + {"keyboard-interactive", ?MODULE, keyboard_interactive_msg, []} + ], + Algs). user_name(Opts) -> Env = case os:type() of @@ -418,10 +482,6 @@ keyboard_interact_fun(KbdInteractFun, Name, Instr, PromptInfos, NumPrompts) -> language = "en"}}) end. -other_alg('ssh-rsa') -> - 'ssh-dss'; -other_alg('ssh-dss') -> - 'ssh-rsa'. decode_public_key_v2(<<?UINT32(Len0), _:Len0/binary, ?UINT32(Len1), BinE:Len1/binary, ?UINT32(Len2), BinN:Len2/binary>> diff --git a/lib/ssh/src/ssh_auth.hrl b/lib/ssh/src/ssh_auth.hrl index 6cd8e6bf14..764c9f4246 100644 --- a/lib/ssh/src/ssh_auth.hrl +++ b/lib/ssh/src/ssh_auth.hrl @@ -23,7 +23,7 @@ -define(SUPPORTED_AUTH_METHODS, "publickey,keyboard-interactive,password"). --define(PREFERRED_PK_ALG, 'ssh-rsa'). +-define(PREFERRED_PK_ALGS, ['ssh-rsa','ssh-dss']). -define(SSH_MSG_USERAUTH_REQUEST, 50). -define(SSH_MSG_USERAUTH_FAILURE, 51). diff --git a/lib/ssh/src/ssh_connection.erl b/lib/ssh/src/ssh_connection.erl index 388c080d99..d532d41009 100644 --- a/lib/ssh/src/ssh_connection.erl +++ b/lib/ssh/src/ssh_connection.erl @@ -196,15 +196,16 @@ reply_request(_,false, _, _) -> %%-------------------------------------------------------------------- ptty_alloc(ConnectionHandler, Channel, Options) -> ptty_alloc(ConnectionHandler, Channel, Options, infinity). -ptty_alloc(ConnectionHandler, Channel, Options, TimeOut) -> +ptty_alloc(ConnectionHandler, Channel, Options0, TimeOut) -> + Options = backwards_compatible(Options0, []), {Width, PixWidth} = pty_default_dimensions(width, Options), - {Hight, PixHight} = pty_default_dimensions(hight, Options), + {Height, PixHeight} = pty_default_dimensions(height, Options), pty_req(ConnectionHandler, Channel, proplists:get_value(term, Options, os:getenv("TERM", ?DEFAULT_TERMINAL)), proplists:get_value(width, Options, Width), - proplists:get_value(hight, Options, Hight), + proplists:get_value(height, Options, Height), proplists:get_value(pixel_widh, Options, PixWidth), - proplists:get_value(pixel_hight, Options, PixHight), + proplists:get_value(pixel_height, Options, PixHeight), proplists:get_value(pty_opts, Options, []), TimeOut ). %%-------------------------------------------------------------------- @@ -1339,3 +1340,12 @@ decode_ip(Addr) when is_binary(Addr) -> {error,_} -> Addr; {ok,A} -> A end. + +backwards_compatible([], Acc) -> + Acc; +backwards_compatible([{hight, Value} | Rest], Acc) -> + backwards_compatible(Rest, [{height, Value} | Acc]); +backwards_compatible([{pixel_hight, Value} | Rest], Acc) -> + backwards_compatible(Rest, [{height, Value} | Acc]); +backwards_compatible([Value| Rest], Acc) -> + backwards_compatible(Rest, [ Value | Acc]). diff --git a/lib/ssh/src/ssh_connection_handler.erl b/lib/ssh/src/ssh_connection_handler.erl index 4dea284071..3bdca4ba94 100644 --- a/lib/ssh/src/ssh_connection_handler.erl +++ b/lib/ssh/src/ssh_connection_handler.erl @@ -33,7 +33,7 @@ -include("ssh_transport.hrl"). -include("ssh_auth.hrl"). -include("ssh_connect.hrl"). - +-compile(export_all). -export([start_link/3]). %% Internal application API @@ -71,6 +71,7 @@ key_exchange_init_msg, % #ssh_msg_kexinit{} renegotiate = false, % boolean() last_size_rekey = 0, + event_queue = [], connection_queue, address, port, @@ -83,6 +84,11 @@ {next_state, state_name(), term(), timeout()} | {stop, term(), term()}. +-type gen_fsm_sync_return() :: {next_state, state_name(), term()} | + {next_state, state_name(), term(), timeout()} | + {reply, term(), state_name(), term()} | + {stop, term(), term(), term()}. + %%==================================================================== %% Internal application API %%==================================================================== @@ -327,22 +333,25 @@ info(ConnectionHandler, ChannelProcess) -> hello(socket_control, #state{socket = Socket, ssh_params = Ssh} = State) -> VsnMsg = ssh_transport:hello_version_msg(string_version(Ssh)), send_msg(VsnMsg, State), - {ok, [{recbuf, Size}]} = inet:getopts(Socket, [recbuf]), - inet:setopts(Socket, [{packet, line}, {active, once}, {recbuf, ?MAX_PROTO_VERSION}]), - {next_state, hello, State#state{recbuf = Size}}; + case getopt(recbuf, Socket) of + {ok, Size} -> + inet:setopts(Socket, [{packet, line}, {active, once}, {recbuf, ?MAX_PROTO_VERSION}]), + {next_state, hello, State#state{recbuf = Size}}; + {error, Reason} -> + {stop, {shutdown, Reason}, State} + end; hello({info_line, _Line},#state{role = client, socket = Socket} = State) -> %% The server may send info lines before the version_exchange inet:setopts(Socket, [{active, once}]), {next_state, hello, State}; -hello({info_line, _Line},#state{role = server} = State) -> - DisconnectMsg = - #ssh_msg_disconnect{code = - ?SSH_DISCONNECT_PROTOCOL_ERROR, - description = "Did not receive expected protocol version exchange", - language = "en"}, - handle_disconnect(DisconnectMsg, State); +hello({info_line, _Line},#state{role = server, + socket = Socket, + transport_cb = Transport } = State) -> + %% as openssh + Transport:send(Socket, "Protocol mismatch."), + {stop, {shutdown,"Protocol mismatch in version exchange."}, State}; hello({version_exchange, Version}, #state{ssh_params = Ssh0, socket = Socket, @@ -433,9 +442,7 @@ key_exchange(#ssh_msg_kex_dh_gex_reply{} = Msg, new_keys(#ssh_msg_newkeys{} = Msg, #state{ssh_params = Ssh0} = State0) -> {ok, Ssh} = ssh_transport:handle_new_keys(Msg, Ssh0), - {NextStateName, State} = - after_new_keys(State0#state{ssh_params = Ssh}), - {next_state, NextStateName, next_packet(State)}. + after_new_keys(next_packet(State0#state{ssh_params = Ssh})). %%-------------------------------------------------------------------- -spec userauth(#ssh_msg_service_request{} | #ssh_msg_service_accept{} | @@ -497,10 +504,21 @@ userauth(#ssh_msg_userauth_info_request{} = Msg, {next_state, userauth, next_packet(State#state{ssh_params = Ssh})}; userauth(#ssh_msg_userauth_info_response{} = Msg, - #state{ssh_params = #ssh{role = server} = Ssh0} = State) -> - {ok, {Reply, Ssh}} = ssh_auth:handle_userauth_info_response(Msg, Ssh0), - send_msg(Reply, State), - {next_state, userauth, next_packet(State#state{ssh_params = Ssh})}; + #state{ssh_params = #ssh{role = server, + peer = {_, Address}} = Ssh0, + opts = Opts, starter = Pid} = State) -> + case ssh_auth:handle_userauth_info_response(Msg, Ssh0) of + {authorized, User, {Reply, Ssh}} -> + send_msg(Reply, State), + Pid ! ssh_connected, + connected_fun(User, Address, "keyboard-interactive", Opts), + {next_state, connected, + next_packet(State#state{auth_user = User, ssh_params = Ssh})}; + {not_authorized, {User, Reason}, {Reply, Ssh}} -> + retry_fun(User, Address, Reason, Opts), + send_msg(Reply, State), + {next_state, userauth, next_packet(State#state{ssh_params = Ssh})} + end; userauth(#ssh_msg_userauth_success{}, #state{ssh_params = #ssh{role = client} = Ssh, starter = Pid} = State) -> @@ -559,11 +577,13 @@ userauth(#ssh_msg_userauth_banner{message = Msg}, -spec connected({#ssh_msg_kexinit{}, binary()}, %%| %% #ssh_msg_kexdh_init{}, #state{}) -> gen_fsm_state_return(). %%-------------------------------------------------------------------- -connected({#ssh_msg_kexinit{}, _Payload} = Event, State) -> - kexinit(Event, State#state{renegotiate = true}). -%% ; -%% connected(#ssh_msg_kexdh_init{} = Event, State) -> -%% key_exchange(Event, State#state{renegotiate = true}). +connected({#ssh_msg_kexinit{}, _Payload} = Event, #state{ssh_params = Ssh0} = State0) -> + {KeyInitMsg, SshPacket, Ssh} = ssh_transport:key_exchange_init_msg(Ssh0), + State = State0#state{ssh_params = Ssh, + key_exchange_init_msg = KeyInitMsg, + renegotiate = true}, + send_msg(SshPacket, State), + kexinit(Event, State). %%-------------------------------------------------------------------- -spec handle_event(#ssh_msg_disconnect{} | #ssh_msg_ignore{} | #ssh_msg_debug{} | @@ -581,44 +601,17 @@ handle_event(#ssh_msg_disconnect{description = Desc} = DisconnectMsg, _StateName handle_event(#ssh_msg_ignore{}, StateName, State) -> {next_state, StateName, next_packet(State)}; -handle_event(#ssh_msg_debug{always_display = true, message = DbgMsg}, - StateName, State) -> - io:format("DEBUG: ~p\n", [DbgMsg]), - {next_state, StateName, next_packet(State)}; - -handle_event(#ssh_msg_debug{}, StateName, State) -> +handle_event(#ssh_msg_debug{always_display = Display, message = DbgMsg, language=Lang}, + StateName, #state{opts = Opts} = State) -> + F = proplists:get_value(ssh_msg_debug_fun, Opts, + fun(_ConnRef, _AlwaysDisplay, _Msg, _Language) -> ok end + ), + catch F(self(), Display, DbgMsg, Lang), {next_state, StateName, next_packet(State)}; handle_event(#ssh_msg_unimplemented{}, StateName, State) -> {next_state, StateName, next_packet(State)}; -handle_event({adjust_window, ChannelId, Bytes}, StateName, - #state{connection_state = - #connection{channel_cache = Cache}} = State0) -> - State = - case ssh_channel:cache_lookup(Cache, ChannelId) of - #channel{recv_window_size = WinSize, remote_id = Id} = Channel -> - ssh_channel:cache_update(Cache, Channel#channel{recv_window_size = - WinSize + Bytes}), - Msg = ssh_connection:channel_adjust_window_msg(Id, Bytes), - send_replies([{connection_reply, Msg}], State0); - undefined -> - State0 - end, - {next_state, StateName, next_packet(State)}; - -handle_event({reply_request, success, ChannelId}, StateName, - #state{connection_state = - #connection{channel_cache = Cache}} = State0) -> - State = case ssh_channel:cache_lookup(Cache, ChannelId) of - #channel{remote_id = RemoteId} -> - Msg = ssh_connection:channel_success_msg(RemoteId), - send_replies([{connection_reply, Msg}], State0); - undefined -> - State0 - end, - {next_state, StateName, State}; - handle_event(renegotiate, connected, #state{ssh_params = Ssh0} = State) -> {KeyInitMsg, SshPacket, Ssh} = ssh_transport:key_exchange_init_msg(Ssh0), @@ -630,8 +623,7 @@ handle_event(renegotiate, connected, #state{ssh_params = Ssh0} renegotiate = true})}; handle_event(renegotiate, StateName, State) -> - timer:apply_after(?REKEY_TIMOUT, gen_fsm, send_all_state_event, [self(), renegotiate]), - %% Allready in keyexcahange so ignore + %% Already in key-exchange so safe to ignore {next_state, StateName, State}; %% Rekey due to sent data limit reached? @@ -653,6 +645,38 @@ handle_event(data_size, connected, #state{ssh_params = Ssh0} = State) -> {next_state, connected, next_packet(State)} end; handle_event(data_size, StateName, State) -> + %% Already in key-exchange so safe to ignore + {next_state, StateName, State}; + +handle_event(Event, StateName, State) when StateName /= connected -> + Events = [{event, Event} | State#state.event_queue], + {next_state, StateName, State#state{event_queue = Events}}; + +handle_event({adjust_window, ChannelId, Bytes}, StateName, + #state{connection_state = + #connection{channel_cache = Cache}} = State0) -> + State = + case ssh_channel:cache_lookup(Cache, ChannelId) of + #channel{recv_window_size = WinSize, remote_id = Id} = Channel -> + ssh_channel:cache_update(Cache, Channel#channel{recv_window_size = + WinSize + Bytes}), + Msg = ssh_connection:channel_adjust_window_msg(Id, Bytes), + send_replies([{connection_reply, Msg}], State0); + undefined -> + State0 + end, + {next_state, StateName, next_packet(State)}; + +handle_event({reply_request, success, ChannelId}, StateName, + #state{connection_state = + #connection{channel_cache = Cache}} = State0) -> + State = case ssh_channel:cache_lookup(Cache, ChannelId) of + #channel{remote_id = RemoteId} -> + Msg = ssh_connection:channel_success_msg(RemoteId), + send_replies([{connection_reply, Msg}], State0); + undefined -> + State0 + end, {next_state, StateName, State}; handle_event({request, ChannelPid, ChannelId, Type, Data}, StateName, State0) -> @@ -683,8 +707,65 @@ handle_event({unknown, Data}, StateName, State) -> sockname]} | {channel_info, channel_id(), [recv_window | send_window]} | {close, channel_id()} | stop, term(), state_name(), #state{}) - -> gen_fsm_state_return(). + -> gen_fsm_sync_return(). %%-------------------------------------------------------------------- +handle_sync_event(get_print_info, _From, StateName, State) -> + Reply = + try + {inet:sockname(State#state.socket), + inet:peername(State#state.socket) + } + of + {{ok,Local}, {ok,Remote}} -> {{Local,Remote},io_lib:format("statename=~p",[StateName])}; + _ -> {{"-",0},"-"} + catch + _:_ -> {{"?",0},"?"} + end, + {reply, Reply, StateName, State}; + +handle_sync_event({connection_info, Options}, _From, StateName, State) -> + Info = ssh_info(Options, State, []), + {reply, Info, StateName, State}; + +handle_sync_event({channel_info, ChannelId, Options}, _From, StateName, + #state{connection_state = #connection{channel_cache = Cache}} = State) -> + case ssh_channel:cache_lookup(Cache, ChannelId) of + #channel{} = Channel -> + Info = ssh_channel_info(Options, Channel, []), + {reply, Info, StateName, State}; + undefined -> + {reply, [], StateName, State} + end; + +handle_sync_event({info, ChannelPid}, _From, StateName, + #state{connection_state = + #connection{channel_cache = Cache}} = State) -> + Result = ssh_channel:cache_foldl( + fun(Channel, Acc) when ChannelPid == all; + Channel#channel.user == ChannelPid -> + [Channel | Acc]; + (_, Acc) -> + Acc + end, [], Cache), + {reply, {ok, Result}, StateName, State}; + +handle_sync_event(stop, _, _StateName, #state{connection_state = Connection0, + role = Role, + opts = Opts} = State0) -> + {disconnect, Reason, {{replies, Replies}, Connection}} = + ssh_connection:handle_msg(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_BY_APPLICATION, + description = "User closed down connection", + language = "en"}, Connection0, Role), + State = send_replies(Replies, State0), + SSHOpts = proplists:get_value(ssh_opts, Opts), + disconnect_fun(Reason, SSHOpts), + {stop, normal, ok, State#state{connection_state = Connection}}; + + +handle_sync_event(Event, From, StateName, State) when StateName /= connected -> + Events = [{sync, Event, From} | State#state.event_queue], + {next_state, StateName, State#state{event_queue = Events}}; + handle_sync_event({request, ChannelPid, ChannelId, Type, Data, Timeout}, From, StateName, State0) -> {{replies, Replies}, State1} = handle_request(ChannelPid, ChannelId, Type, Data, @@ -787,46 +868,6 @@ handle_sync_event({recv_window, ChannelId}, _From, StateName, end, {reply, Reply, StateName, next_packet(State)}; -handle_sync_event(get_print_info, _From, StateName, State) -> - Reply = - try - {inet:sockname(State#state.socket), - inet:peername(State#state.socket) - } - of - {{ok,Local}, {ok,Remote}} -> {{Local,Remote},io_lib:format("statename=~p",[StateName])}; - _ -> {{"-",0},"-"} - catch - _:_ -> {{"?",0},"?"} - end, - {reply, Reply, StateName, State}; - -handle_sync_event({connection_info, Options}, _From, StateName, State) -> - Info = ssh_info(Options, State, []), - {reply, Info, StateName, State}; - -handle_sync_event({channel_info, ChannelId, Options}, _From, StateName, - #state{connection_state = #connection{channel_cache = Cache}} = State) -> - case ssh_channel:cache_lookup(Cache, ChannelId) of - #channel{} = Channel -> - Info = ssh_channel_info(Options, Channel, []), - {reply, Info, StateName, State}; - undefined -> - {reply, [], StateName, State} - end; - -handle_sync_event({info, ChannelPid}, _From, StateName, - #state{connection_state = - #connection{channel_cache = Cache}} = State) -> - Result = ssh_channel:cache_foldl( - fun(Channel, Acc) when ChannelPid == all; - Channel#channel.user == ChannelPid -> - [Channel | Acc]; - (_, Acc) -> - Acc - end, [], Cache), - {reply, {ok, Result}, StateName, State}; - handle_sync_event({close, ChannelId}, _, StateName, #state{connection_state = #connection{channel_cache = Cache}} = State0) -> @@ -841,19 +882,7 @@ handle_sync_event({close, ChannelId}, _, StateName, undefined -> State0 end, - {reply, ok, StateName, next_packet(State)}; - -handle_sync_event(stop, _, _StateName, #state{connection_state = Connection0, - role = Role, - opts = Opts} = State0) -> - {disconnect, Reason, {{replies, Replies}, Connection}} = - ssh_connection:handle_msg(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_BY_APPLICATION, - description = "User closed down connection", - language = "en"}, Connection0, Role), - State = send_replies(Replies, State0), - SSHOpts = proplists:get_value(ssh_opts, Opts), - disconnect_fun(Reason, SSHOpts), - {stop, normal, ok, State#state{connection_state = Connection}}. + {reply, ok, StateName, next_packet(State)}. %%-------------------------------------------------------------------- -spec handle_info({atom(), port(), binary()} | {atom(), port()} | @@ -1141,54 +1170,38 @@ init_ssh(server = Role, Vsn, Version, Options, Socket) -> supported_host_keys(client, _, Options) -> try - case extract_algs(proplists:get_value(pref_public_key_algs, Options, false), []) of - false -> - ["ssh-rsa", "ssh-dss"]; - Algs -> - Algs + case proplists:get_value(public_key, + proplists:get_value(preferred_algorithms,Options,[]) + ) of + undefined -> + ssh_auth:default_public_key_algorithms(); + L -> + L -- (L--ssh_auth:default_public_key_algorithms()) end + of + [] -> + {stop, {shutdown, "No public key algs"}}; + Algs -> + [atom_to_list(A) || A<-Algs] catch exit:Reason -> {stop, {shutdown, Reason}} end; supported_host_keys(server, KeyCb, Options) -> - lists:foldl(fun(Type, Acc) -> - case available_host_key(KeyCb, Type, Options) of - {error, _} -> - Acc; - Alg -> - [Alg | Acc] - end - end, [], - %% Prefered alg last so no need to reverse - ["ssh-dss", "ssh-rsa"]). -extract_algs(false, _) -> - false; -extract_algs([],[]) -> - false; -extract_algs([], NewList) -> - lists:reverse(NewList); -extract_algs([H|T], NewList) -> - case H of - 'ssh-dss' -> - extract_algs(T, ["ssh-dss"|NewList]); - 'ssh-rsa' -> - extract_algs(T, ["ssh-rsa"|NewList]) - end. -available_host_key(KeyCb, "ssh-dss"= Alg, Opts) -> - case KeyCb:host_key('ssh-dss', Opts) of - {ok, _} -> - Alg; - Other -> - Other - end; -available_host_key(KeyCb, "ssh-rsa" = Alg, Opts) -> - case KeyCb:host_key('ssh-rsa', Opts) of - {ok, _} -> - Alg; - Other -> - Other - end. + Algs= + [atom_to_list(A) || A <- proplists:get_value(public_key, + proplists:get_value(preferred_algorithms,Options,[]), + ssh_auth:default_public_key_algorithms() + ), + available_host_key(KeyCb, A, Options) + ], + Algs. + + +%% Alg :: atom() +available_host_key(KeyCb, Alg, Opts) -> + element(1, catch KeyCb:host_key(Alg, Opts)) == ok. + send_msg(Msg, #state{socket = Socket, transport_cb = Transport}) -> Transport:send(Socket, Msg). @@ -1282,8 +1295,17 @@ generate_event(<<?BYTE(Byte), _/binary>> = Msg, StateName, ConnectionMsg = ssh_message:decode(Msg), State1 = generate_event_new_state(State0, EncData), try ssh_connection:handle_msg(ConnectionMsg, Connection0, Role) of - {{replies, Replies}, Connection} -> - State = send_replies(Replies, State1#state{connection_state = Connection}), + {{replies, Replies0}, Connection} -> + if StateName == connected -> + Replies = Replies0, + State2 = State1; + true -> + {ConnReplies, Replies} = + lists:splitwith(fun not_connected_filter/1, Replies0), + Q = State1#state.event_queue ++ ConnReplies, + State2 = State1#state{ event_queue = Q } + end, + State = send_replies(Replies, State2#state{connection_state = Connection}), {next_state, StateName, next_packet(State)}; {noreply, Connection} -> {next_state, StateName, next_packet(State1#state{connection_state = Connection})}; @@ -1456,15 +1478,43 @@ next_packet(#state{socket = Socket} = State) -> State. after_new_keys(#state{renegotiate = true} = State) -> - {connected, State#state{renegotiate = false}}; + State1 = State#state{renegotiate = false, event_queue = []}, + lists:foldr(fun after_new_keys_events/2, {next_state, connected, State1}, State#state.event_queue); after_new_keys(#state{renegotiate = false, ssh_params = #ssh{role = client} = Ssh0} = State) -> {Msg, Ssh} = ssh_auth:service_request_msg(Ssh0), send_msg(Msg, State), - {userauth, State#state{ssh_params = Ssh}}; + {next_state, userauth, State#state{ssh_params = Ssh}}; after_new_keys(#state{renegotiate = false, ssh_params = #ssh{role = server}} = State) -> - {userauth, State}. + {next_state, userauth, State}. + +after_new_keys_events({sync, _Event, From}, {stop, _Reason, _StateData}=Terminator) -> + gen_fsm:reply(From, {error, closed}), + Terminator; +after_new_keys_events(_, {stop, _Reason, _StateData}=Terminator) -> + Terminator; +after_new_keys_events({sync, Event, From}, {next_state, StateName, StateData}) -> + case handle_sync_event(Event, From, StateName, StateData) of + {reply, Reply, NextStateName, NewStateData} -> + gen_fsm:reply(From, Reply), + {next_state, NextStateName, NewStateData}; + {next_state, NextStateName, NewStateData}-> + {next_state, NextStateName, NewStateData}; + {stop, Reason, Reply, NewStateData} -> + gen_fsm:reply(From, Reply), + {stop, Reason, NewStateData} + end; +after_new_keys_events({event, Event}, {next_state, StateName, StateData}) -> + case handle_event(Event, StateName, StateData) of + {next_state, NextStateName, NewStateData}-> + {next_state, NextStateName, NewStateData}; + {stop, Reason, NewStateData} -> + {stop, Reason, NewStateData} + end; +after_new_keys_events({connection_reply, _Data} = Reply, {StateName, State}) -> + NewState = send_replies([Reply], State), + {next_state, StateName, NewState}. handle_ssh_packet_data(RemainingSshPacketLen, DecData, EncData, StateName, State) -> @@ -1625,6 +1675,11 @@ log_error(Reason) -> error_logger:error_report(Report), "Internal error". +not_connected_filter({connection_reply, _Data}) -> + true; +not_connected_filter(_) -> + false. + send_replies([], State) -> State; send_replies([{connection_reply, Data} | Rest], #state{ssh_params = Ssh0} = State) -> @@ -1722,3 +1777,12 @@ start_timeout(_,_, infinity) -> ok; start_timeout(Channel, From, Time) -> erlang:send_after(Time, self(), {timeout, {Channel, From}}). + +getopt(Opt, Socket) -> + case inet:getopts(Socket, [Opt]) of + {ok, [{Opt, Value}]} -> + {ok, Value}; + Other -> + {error, {unexpected_getopts_return, Other}} + end. + diff --git a/lib/ssh/src/ssh_transport.erl b/lib/ssh/src/ssh_transport.erl index 8669be570e..ea9bca2390 100644 --- a/lib/ssh/src/ssh_transport.erl +++ b/lib/ssh/src/ssh_transport.erl @@ -31,6 +31,8 @@ -export([versions/2, hello_version_msg/1]). -export([next_seqnum/1, decrypt_first_block/2, decrypt_blocks/3, + supported_algorithms/0, supported_algorithms/1, + default_algorithms/0, default_algorithms/1, is_valid_mac/3, handle_hello_version/1, key_exchange_init_msg/1, @@ -42,6 +44,68 @@ unpack/3, decompress/2, ssh_packet/2, pack/2, msg_data/1, sign/3, verify/4]). +%%%---------------------------------------------------------------------------- +%%% +%%% There is a difference between supported and default algorithms. The +%%% SUPPORTED algorithms can be handled (maybe untested...). The DEFAULT ones +%%% are announced in ssh_msg_kexinit and in ssh:default_algorithms/0 to the +%%% user. +%%% +%%% A supported algorithm can be requested in the option 'preferred_algorithms', +%%% but may give unexpected results because of being promoted to default. +%%% +%%% This makes it possible to add experimental algorithms (in supported_algorithms) +%%% and test them without letting the default users know about them. +%%% + +default_algorithms() -> [{K,default_algorithms(K)} || K <- algo_classes()]. + +algo_classes() -> [kex, public_key, cipher, mac, compression]. + +default_algorithms(compression) -> + %% Do not announce '[email protected]' because there seem to be problems + supported_algorithms(compression, same(['[email protected]'])); +default_algorithms(Alg) -> + supported_algorithms(Alg). + + +supported_algorithms() -> [{K,supported_algorithms(K)} || K <- algo_classes()]. + +supported_algorithms(kex) -> + ['diffie-hellman-group1-sha1']; +supported_algorithms(public_key) -> + ssh_auth:default_public_key_algorithms(); +supported_algorithms(cipher) -> + Supports = crypto:supports(), + CipherAlgos = [{aes_ctr, 'aes128-ctr'}, {aes_cbc128, 'aes128-cbc'}, {des3_cbc, '3des-cbc'}], + Algs = [SshAlgo || + {CryptoAlgo, SshAlgo} <- CipherAlgos, + lists:member(CryptoAlgo, proplists:get_value(ciphers, Supports, []))], + same(Algs); +supported_algorithms(mac) -> + Supports = crypto:supports(), + HashAlgos = [{sha256, 'hmac-sha2-256'}, {sha, 'hmac-sha1'}], + Algs = [SshAlgo || + {CryptoAlgo, SshAlgo} <- HashAlgos, + lists:member(CryptoAlgo, proplists:get_value(hashs, Supports, []))], + same(Algs); +supported_algorithms(compression) -> + same(['none','zlib','[email protected]']). + + +supported_algorithms(Key, [{client2server,BL1},{server2client,BL2}]) -> + [{client2server,As1},{server2client,As2}] = supported_algorithms(Key), + [{client2server,As1--BL1},{server2client,As2--BL2}]; +supported_algorithms(Key, BlackList) -> + supported_algorithms(Key) -- BlackList. + + + + +same(Algs) -> [{client2server,Algs}, {server2client,Algs}]. + + +%%%---------------------------------------------------------------------------- versions(client, Options)-> Vsn = proplists:get_value(vsn, Options, ?DEFAULT_CLIENT_VERSION), {Vsn, format_version(Vsn, software_version(Options))}; @@ -128,62 +192,45 @@ key_exchange_init_msg(Ssh0) -> kex_init(#ssh{role = Role, opts = Opts, available_host_keys = HostKeyAlgs}) -> Random = ssh_bits:random(16), - Compression = case proplists:get_value(compression, Opts, none) of - openssh_zlib -> ["[email protected]", "none"]; - zlib -> ["zlib", "none"]; - none -> ["none", "zlib"] - end, - kexinit_messsage(Role, Random, Compression, HostKeyAlgs). + PrefAlgs = + case proplists:get_value(preferred_algorithms,Opts) of + undefined -> + default_algorithms(); + Algs0 -> + Algs0 + end, + kexinit_message(Role, Random, PrefAlgs, HostKeyAlgs). key_init(client, Ssh, Value) -> Ssh#ssh{c_keyinit = Value}; key_init(server, Ssh, Value) -> Ssh#ssh{s_keyinit = Value}. -available_ssh_algos() -> - Supports = crypto:supports(), - CipherAlgos = [{aes_ctr, "aes128-ctr"}, {aes_cbc128, "aes128-cbc"}, {des3_cbc, "3des-cbc"}], - Ciphers = [SshAlgo || - {CryptoAlgo, SshAlgo} <- CipherAlgos, - lists:member(CryptoAlgo, proplists:get_value(ciphers, Supports, []))], - HashAlgos = [{sha256, "hmac-sha2-256"}, {sha, "hmac-sha1"}], - Hashs = [SshAlgo || - {CryptoAlgo, SshAlgo} <- HashAlgos, - lists:member(CryptoAlgo, proplists:get_value(hashs, Supports, []))], - {Ciphers, Hashs}. - -kexinit_messsage(client, Random, Compression, HostKeyAlgs) -> - {CipherAlgs, HashAlgs} = available_ssh_algos(), - #ssh_msg_kexinit{ - cookie = Random, - kex_algorithms = ["diffie-hellman-group1-sha1"], - server_host_key_algorithms = HostKeyAlgs, - encryption_algorithms_client_to_server = CipherAlgs, - encryption_algorithms_server_to_client = CipherAlgs, - mac_algorithms_client_to_server = HashAlgs, - mac_algorithms_server_to_client = HashAlgs, - compression_algorithms_client_to_server = Compression, - compression_algorithms_server_to_client = Compression, - languages_client_to_server = [], - languages_server_to_client = [] - }; -kexinit_messsage(server, Random, Compression, HostKeyAlgs) -> - {CipherAlgs, HashAlgs} = available_ssh_algos(), +kexinit_message(_Role, Random, Algs, HostKeyAlgs) -> #ssh_msg_kexinit{ cookie = Random, - kex_algorithms = ["diffie-hellman-group1-sha1"], + kex_algorithms = to_strings( get_algs(kex,Algs) ), server_host_key_algorithms = HostKeyAlgs, - encryption_algorithms_client_to_server = CipherAlgs, - encryption_algorithms_server_to_client = CipherAlgs, - mac_algorithms_client_to_server = HashAlgs, - mac_algorithms_server_to_client = HashAlgs, - compression_algorithms_client_to_server = Compression, - compression_algorithms_server_to_client = Compression, + encryption_algorithms_client_to_server = c2s(cipher,Algs), + encryption_algorithms_server_to_client = s2c(cipher,Algs), + mac_algorithms_client_to_server = c2s(mac,Algs), + mac_algorithms_server_to_client = s2c(mac,Algs), + compression_algorithms_client_to_server = c2s(compression,Algs), + compression_algorithms_server_to_client = s2c(compression,Algs), languages_client_to_server = [], languages_server_to_client = [] }. +c2s(Key, Algs) -> x2y(client2server, Key, Algs). +s2c(Key, Algs) -> x2y(server2client, Key, Algs). + +x2y(DirectionKey, Key, Algs) -> to_strings(proplists:get_value(DirectionKey, get_algs(Key,Algs))). + +get_algs(Key, Algs) -> proplists:get_value(Key, Algs, default_algorithms(Key)). + +to_strings(L) -> lists:map(fun erlang:atom_to_list/1, L). + new_keys_message(Ssh0) -> {SshPacket, Ssh} = ssh_packet(#ssh_msg_newkeys{}, Ssh0), @@ -240,20 +287,30 @@ key_exchange_first_msg('diffie-hellman-group-exchange-sha1', Ssh0) -> handle_kexdh_init(#ssh_msg_kexdh_init{e = E}, Ssh0) -> {G, P} = dh_group1(), - {Private, Public} = dh_gen_key(G, P, 1024), - K = ssh_math:ipow(E, Private, P), - Key = get_host_key(Ssh0), - H = kex_h(Ssh0, Key, E, Public, K), - H_SIG = sign_host_key(Ssh0, Key, H), - {SshPacket, Ssh1} = ssh_packet(#ssh_msg_kexdh_reply{public_host_key = Key, - f = Public, - h_sig = H_SIG - }, Ssh0), - - {ok, SshPacket, Ssh1#ssh{keyex_key = {{Private, Public}, {G, P}}, - shared_secret = K, - exchanged_hash = H, - session_id = sid(Ssh1, H)}}. + if + 1=<E, E=<(P-1) -> + {Private, Public} = dh_gen_key(G, P, 1024), + K = ssh_math:ipow(E, Private, P), + Key = get_host_key(Ssh0), + H = kex_h(Ssh0, Key, E, Public, K), + H_SIG = sign_host_key(Ssh0, Key, H), + {SshPacket, Ssh1} = ssh_packet(#ssh_msg_kexdh_reply{public_host_key = Key, + f = Public, + h_sig = H_SIG + }, Ssh0), + + {ok, SshPacket, Ssh1#ssh{keyex_key = {{Private, Public}, {G, P}}, + shared_secret = K, + exchanged_hash = H, + session_id = sid(Ssh1, H)}}; + true -> + Error = {error,bad_e_from_peer}, + Disconnect = #ssh_msg_disconnect{ + code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + description = "Key exchange failed, 'f' out of bounds", + language = "en"}, + throw({Error, Disconnect}) + end. handle_kex_dh_gex_group(#ssh_msg_kex_dh_gex_group{p = P, g = G}, Ssh0) -> {Private, Public} = dh_gen_key(G,P,1024), @@ -277,7 +334,7 @@ handle_new_keys(#ssh_msg_newkeys{}, Ssh0) -> %% %% Select algorithms handle_kexdh_reply(#ssh_msg_kexdh_reply{public_host_key = HostKey, f = F, h_sig = H_SIG}, - #ssh{keyex_key = {{Private, Public}, {_G, P}}} = Ssh0) -> + #ssh{keyex_key = {{Private, Public}, {_G, P}}} = Ssh0) when 1=<F, F=<(P-1)-> K = ssh_math:ipow(F, Private, P), H = kex_h(Ssh0, HostKey, Public, F, K), @@ -293,7 +350,15 @@ handle_kexdh_reply(#ssh_msg_kexdh_reply{public_host_key = HostKey, f = F, description = "Key exchange failed", language = "en"}, throw({Error, Disconnect}) - end. + end; +handle_kexdh_reply(#ssh_msg_kexdh_reply{}, _SSH) -> + Error = {error,bad_f_from_peer}, + Disconnect = #ssh_msg_disconnect{ + code = ?SSH_DISCONNECT_KEY_EXCHANGE_FAILED, + description = "Key exchange failed, 'f' out of bounds", + language = "en"}, + throw({Error, Disconnect}). + handle_kex_dh_gex_request(#ssh_msg_kex_dh_gex_request{min = _Min, n = _NBits, @@ -430,6 +495,7 @@ select_algorithm(Role, Client, Server) -> decompress = Decompression, c_lng = C_Lng, s_lng = S_Lng}, +%%ct:pal("~p~n Client=~p~n Server=~p~n Alg=~p~n",[Role,Client,Server,Alg]), {ok, Alg}. select_encrypt_decrypt(client, Client, Server) -> @@ -519,10 +585,15 @@ alg_final(SSH0) -> {ok,SSH6} = decompress_final(SSH5), SSH6. -select_all(CL, SL) -> +select_all(CL, SL) when length(CL) + length(SL) < 50 -> A = CL -- SL, %% algortihms only used by client %% algorithms used by client and server (client pref) - lists:map(fun(ALG) -> list_to_atom(ALG) end, (CL -- A)). + lists:map(fun(ALG) -> list_to_atom(ALG) end, (CL -- A)); +select_all(_CL, _SL) -> + throw(#ssh_msg_disconnect{code = ?SSH_DISCONNECT_PROTOCOL_ERROR, + description = "Too many algorithms", + language = "en"}). + select([], []) -> none; diff --git a/lib/ssh/test/Makefile b/lib/ssh/test/Makefile index 740dbd0235..39b2f57d26 100644 --- a/lib/ssh/test/Makefile +++ b/lib/ssh/test/Makefile @@ -40,7 +40,8 @@ MODULES= \ ssh_connection_SUITE \ ssh_echo_server \ ssh_peername_sockname_server \ - ssh_test_cli + ssh_test_cli \ + ssh_relay HRL_FILES_NEEDED_IN_TEST= \ $(ERL_TOP)/lib/ssh/src/ssh.hrl \ diff --git a/lib/ssh/test/ssh_basic_SUITE.erl b/lib/ssh/test/ssh_basic_SUITE.erl index bd029ad420..cff695681e 100644 --- a/lib/ssh/test/ssh_basic_SUITE.erl +++ b/lib/ssh/test/ssh_basic_SUITE.erl @@ -29,6 +29,7 @@ -define(NEWLINE, <<"\r\n">>). +-define(REKEY_DATA_TMO, 65000). %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- @@ -44,6 +45,7 @@ all() -> {group, dsa_pass_key}, {group, rsa_pass_key}, {group, internal_error}, + {group, renegotiate}, daemon_already_started, server_password_option, server_userpassword_option, @@ -52,6 +54,9 @@ all() -> ssh_connect_arg4_timeout, packet_size_zero, ssh_daemon_minimal_remote_max_packet_size_option, + ssh_msg_debug_fun_option_client, + ssh_msg_debug_fun_option_server, + preferred_algorithms, id_string_no_opt_client, id_string_own_string_client, id_string_random_client, @@ -67,6 +72,7 @@ groups() -> {dsa_pass_key, [], [pass_phrase]}, {rsa_pass_key, [], [pass_phrase]}, {internal_error, [], [internal_error]}, + {renegotiate, [], [rekey, rekey_limit, renegotiate1, renegotiate2]}, {hardening_tests, [], [ssh_connect_nonegtimeout_connected_parallel, ssh_connect_nonegtimeout_connected_sequential, ssh_connect_negtimeout_parallel, @@ -82,12 +88,12 @@ groups() -> basic_tests() -> [send, close, peername_sockname, exec, exec_compressed, shell, cli, known_hosts, - idle_time, rekey, openssh_zlib_basic_test, - misc_ssh_options, inet_option]. + idle_time, openssh_zlib_basic_test, misc_ssh_options, inet_option]. %%-------------------------------------------------------------------- init_per_suite(Config) -> + catch crypto:stop(), case catch crypto:start() of ok -> Config; @@ -285,7 +291,7 @@ exec_compressed(Config) when is_list(Config) -> UserDir = ?config(priv_dir, Config), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir},{user_dir, UserDir}, - {compression, zlib}, + {preferred_algorithms,[{compression, [zlib]}]}, {failfun, fun ssh_test_lib:failfun/2}]), ConnectionRef = @@ -331,25 +337,175 @@ idle_time(Config) -> rekey() -> [{doc, "Idle timeout test"}]. rekey(Config) -> - SystemDir = filename:join(?config(priv_dir, Config), system), + SystemDir = ?config(data_dir, Config), UserDir = ?config(priv_dir, Config), {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, - {user_dir, UserDir}, + {user_dir, UserDir}, {failfun, fun ssh_test_lib:failfun/2}, + {user_passwords, + [{"simon", "says"}]}, {rekey_limit, 0}]), + ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, {user_dir, UserDir}, + {user, "simon"}, + {password, "says"}, {user_interaction, false}, {rekey_limit, 0}]), receive - after 200000 -> + after ?REKEY_DATA_TMO -> %%By this time rekeying would have been done ssh:close(ConnectionRef), ssh:stop_daemon(Pid) end. %%-------------------------------------------------------------------- +rekey_limit() -> + [{doc, "Test rekeying by data volume"}]. +rekey_limit(Config) -> + SystemDir = ?config(data_dir, Config), + UserDir = ?config(priv_dir, Config), + DataFile = filename:join(UserDir, "rekey.data"), + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {user_passwords, + [{"simon", "says"}]}]), + {ok, SftpPid, ConnectionRef} = + ssh_sftp:start_channel(Host, Port, [{system_dir, SystemDir}, + {user_dir, UserDir}, + {user, "simon"}, + {password, "says"}, + {rekey_limit, 2500}, + {user_interaction, false}, + {silently_accept_hosts, true}]), + + Kex1 = get_kex_init(ConnectionRef), + + ct:sleep(?REKEY_DATA_TMO), + Kex1 = get_kex_init(ConnectionRef), + + Data = lists:duplicate(9000,1), + ok = ssh_sftp:write_file(SftpPid, DataFile, Data), + + ct:sleep(?REKEY_DATA_TMO), + Kex2 = get_kex_init(ConnectionRef), + + false = (Kex2 == Kex1), + + ct:sleep(?REKEY_DATA_TMO), + Kex2 = get_kex_init(ConnectionRef), + + ok = ssh_sftp:write_file(SftpPid, DataFile, "hi\n"), + + ct:sleep(?REKEY_DATA_TMO), + Kex2 = get_kex_init(ConnectionRef), + + false = (Kex2 == Kex1), + + ct:sleep(?REKEY_DATA_TMO), + Kex2 = get_kex_init(ConnectionRef), + + + ssh_sftp:stop_channel(SftpPid), + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid). + +%%-------------------------------------------------------------------- +renegotiate1() -> + [{doc, "Test rekeying with simulataneous send request"}]. +renegotiate1(Config) -> + SystemDir = ?config(data_dir, Config), + UserDir = ?config(priv_dir, Config), + DataFile = filename:join(UserDir, "renegotiate1.data"), + + {Pid, Host, DPort} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {user_passwords, + [{"simon", "says"}]}]), + RPort = ssh_test_lib:inet_port(), + + {ok,RelayPid} = ssh_relay:start_link({0,0,0,0}, RPort, Host, DPort), + + {ok, SftpPid, ConnectionRef} = + ssh_sftp:start_channel(Host, RPort, [{system_dir, SystemDir}, + {user_dir, UserDir}, + {user, "simon"}, + {password, "says"}, + {user_interaction, false}, + {silently_accept_hosts, true}]), + + Kex1 = get_kex_init(ConnectionRef), + + {ok, Handle} = ssh_sftp:open(SftpPid, DataFile, [write]), + + ok = ssh_sftp:write(SftpPid, Handle, "hi\n"), + + ssh_relay:hold(RelayPid, rx, 20, 1000), + ssh_connection_handler:renegotiate(ConnectionRef), + spawn(fun() -> ok=ssh_sftp:write(SftpPid, Handle, "another hi\n") end), + + ct:sleep(2000), + + Kex2 = get_kex_init(ConnectionRef), + + false = (Kex2 == Kex1), + + ssh_relay:stop(RelayPid), + ssh_sftp:stop_channel(SftpPid), + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid). + +%%-------------------------------------------------------------------- +renegotiate2() -> + [{doc, "Test rekeying with inflight messages from peer"}]. +renegotiate2(Config) -> + SystemDir = ?config(data_dir, Config), + UserDir = ?config(priv_dir, Config), + DataFile = filename:join(UserDir, "renegotiate1.data"), + + {Pid, Host, DPort} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {user_passwords, + [{"simon", "says"}]}]), + RPort = ssh_test_lib:inet_port(), + + {ok,RelayPid} = ssh_relay:start_link({0,0,0,0}, RPort, Host, DPort), + + {ok, SftpPid, ConnectionRef} = + ssh_sftp:start_channel(Host, RPort, [{system_dir, SystemDir}, + {user_dir, UserDir}, + {user, "simon"}, + {password, "says"}, + {user_interaction, false}, + {silently_accept_hosts, true}]), + + Kex1 = get_kex_init(ConnectionRef), + + {ok, Handle} = ssh_sftp:open(SftpPid, DataFile, [write]), + + ok = ssh_sftp:write(SftpPid, Handle, "hi\n"), + + ssh_relay:hold(RelayPid, rx, 20, infinity), + spawn(fun() -> ok=ssh_sftp:write(SftpPid, Handle, "another hi\n") end), + %% need a small pause here to ensure ssh_sftp:write is executed + ct:sleep(10), + ssh_connection_handler:renegotiate(ConnectionRef), + ssh_relay:release(RelayPid, rx), + + ct:sleep(2000), + + Kex2 = get_kex_init(ConnectionRef), + + false = (Kex2 == Kex1), + + ssh_relay:stop(RelayPid), + ssh_sftp:stop_channel(SftpPid), + ssh:close(ConnectionRef), + ssh:stop_daemon(Pid). + +%%-------------------------------------------------------------------- shell() -> [{doc, "Test that ssh:shell/2 works"}]. shell(Config) when is_list(Config) -> @@ -494,6 +650,94 @@ server_userpassword_option(Config) when is_list(Config) -> ssh:stop_daemon(Pid). %%-------------------------------------------------------------------- +ssh_msg_debug_fun_option_client() -> + [{doc, "validate client that uses the 'ssh_msg_debug_fun' option"}]. +ssh_msg_debug_fun_option_client(Config) -> + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {failfun, fun ssh_test_lib:failfun/2}]), + Parent = self(), + DbgFun = fun(ConnRef,Displ,Msg,Lang) -> Parent ! {msg_dbg,{ConnRef,Displ,Msg,Lang}} end, + + ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {user_interaction, false}, + {ssh_msg_debug_fun,DbgFun}]), + %% Beware, implementation knowledge: + gen_fsm:send_all_state_event(ConnectionRef,{ssh_msg_debug,false,<<"Hello">>,<<>>}), + receive + {msg_dbg,X={ConnectionRef,false,<<"Hello">>,<<>>}} -> + ct:log("Got expected dbg msg ~p",[X]), + ssh:stop_daemon(Pid); + {msg_dbg,X={_,false,<<"Hello">>,<<>>}} -> + ct:log("Got dbg msg but bad ConnectionRef (~p expected) ~p",[ConnectionRef,X]), + ssh:stop_daemon(Pid), + {fail, "Bad ConnectionRef received"}; + {msg_dbg,X} -> + ct:log("Got bad dbg msg ~p",[X]), + ssh:stop_daemon(Pid), + {fail,"Bad msg received"} + after 1000 -> + ssh:stop_daemon(Pid), + {fail,timeout} + end. + +%%-------------------------------------------------------------------- +ssh_msg_debug_fun_option_server() -> + [{doc, "validate client that uses the 'ssh_msg_debug_fun' option"}]. +ssh_msg_debug_fun_option_server(Config) -> + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + SysDir = ?config(data_dir, Config), + + Parent = self(), + DbgFun = fun(ConnRef,Displ,Msg,Lang) -> Parent ! {msg_dbg,{ConnRef,Displ,Msg,Lang}} end, + ConnFun = fun(_,_,_) -> Parent ! {connection_pid,self()} end, + + {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SysDir}, + {user_dir, UserDir}, + {password, "morot"}, + {failfun, fun ssh_test_lib:failfun/2}, + {connectfun, ConnFun}, + {ssh_msg_debug_fun, DbgFun}]), + _ConnectionRef = + ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, + {user, "foo"}, + {password, "morot"}, + {user_dir, UserDir}, + {user_interaction, false}]), + receive + {connection_pid,Server} -> + %% Beware, implementation knowledge: + gen_fsm:send_all_state_event(Server,{ssh_msg_debug,false,<<"Hello">>,<<>>}), + receive + {msg_dbg,X={_,false,<<"Hello">>,<<>>}} -> + ct:log("Got expected dbg msg ~p",[X]), + ssh:stop_daemon(Pid); + {msg_dbg,X} -> + ct:log("Got bad dbg msg ~p",[X]), + ssh:stop_daemon(Pid), + {fail,"Bad msg received"} + after 3000 -> + ssh:stop_daemon(Pid), + {fail,timeout2} + end + after 3000 -> + ssh:stop_daemon(Pid), + {fail,timeout1} + end. + +%%-------------------------------------------------------------------- known_hosts() -> [{doc, "check that known_hosts is updated correctly"}]. known_hosts(Config) when is_list(Config) -> @@ -822,57 +1066,114 @@ ssh_daemon_minimal_remote_max_packet_size_option(Config) -> ssh:stop_daemon(Server). %%-------------------------------------------------------------------- +%% This test try every algorithm by connecting to an Erlang server +preferred_algorithms(Config) -> + SystemDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + UserDir = filename:join(PrivDir, nopubkey), % to make sure we don't use public-key-auth + file:make_dir(UserDir), + + {Server, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, + {user_dir, UserDir}, + {user_passwords, [{"vego", "morot"}]}, + {failfun, fun ssh_test_lib:failfun/2}]), + Available = ssh:default_algorithms(), + Tests = [[{Tag,[Alg]}] || {Tag, SubAlgs} <- Available, + is_atom(hd(SubAlgs)), + Alg <- SubAlgs] + ++ [[{Tag,[{T1,[A1]},{T2,[A2]}]}] || {Tag, [{T1,As1},{T2,As2}]} <- Available, + A1 <- As1, + A2 <- As2], + ct:log("TESTS: ~p",[Tests]), + [connect_exec_channel(Host,Port,PrefAlgs) || PrefAlgs <- Tests], + ssh:stop_daemon(Server). + + +connect_exec_channel(_Host, Port, Algs) -> + ct:log("Try ~p",[Algs]), + ConnectionRef = ssh_test_lib:connect(Port, [{silently_accept_hosts, true}, + {user_interaction, false}, + {user, "vego"}, + {password, "morot"}, + {preferred_algorithms,Algs} + ]), + chan_exec(ConnectionRef, "2*21.", <<"42\n">>), + ssh:close(ConnectionRef). + +chan_exec(ConnectionRef, Cmnd, Expected) -> + {ok, ChannelId0} = ssh_connection:session_channel(ConnectionRef, infinity), + success = ssh_connection:exec(ConnectionRef, ChannelId0,Cmnd, infinity), + Data0 = {ssh_cm, ConnectionRef, {data, ChannelId0, 0, Expected}}, + case ssh_test_lib:receive_exec_result(Data0) of + expected -> + ssh_test_lib:receive_exec_end(ConnectionRef, ChannelId0); + {unexpected_msg,{ssh_cm, ConnectionRef, {exit_status, ChannelId0, 0}} + = ExitStatus0} -> + ct:pal("0: Collected data ~p", [ExitStatus0]), + ssh_test_lib:receive_exec_result(Data0, + ConnectionRef, ChannelId0); + Other0 -> + ct:fail(Other0) + end. + +%%-------------------------------------------------------------------- id_string_no_opt_client(Config) -> - {Server, Host, Port} = fake_daemon(Config), - {error,_} = ssh:connect(Host, Port, []), + {Server, _Host, Port} = fake_daemon(Config), + {error,_} = ssh:connect("localhost", Port, [], 1000), receive {id,Server,"SSH-2.0-Erlang/"++Vsn} -> true = expected_ssh_vsn(Vsn); {id,Server,Other} -> ct:fail("Unexpected id: ~s.",[Other]) + after 5000 -> + {fail,timeout} end. %%-------------------------------------------------------------------- id_string_own_string_client(Config) -> - {Server, Host, Port} = fake_daemon(Config), - {error,_} = ssh:connect(Host, Port, [{id_string,"Pelle"}]), + {Server, _Host, Port} = fake_daemon(Config), + {error,_} = ssh:connect("localhost", Port, [{id_string,"Pelle"}], 1000), receive {id,Server,"SSH-2.0-Pelle\r\n"} -> ok; {id,Server,Other} -> ct:fail("Unexpected id: ~s.",[Other]) + after 5000 -> + {fail,timeout} end. %%-------------------------------------------------------------------- id_string_random_client(Config) -> - {Server, Host, Port} = fake_daemon(Config), - {error,_} = ssh:connect(Host, Port, [{id_string,random}]), + {Server, _Host, Port} = fake_daemon(Config), + {error,_} = ssh:connect("localhost", Port, [{id_string,random}], 1000), receive {id,Server,Id="SSH-2.0-Erlang"++_} -> ct:fail("Unexpected id: ~s.",[Id]); {id,Server,Rnd="SSH-2.0-"++_} -> - ct:log("Got ~s.",[Rnd]); + ct:log("Got correct ~s",[Rnd]); {id,Server,Id} -> ct:fail("Unexpected id: ~s.",[Id]) + after 5000 -> + {fail,timeout} end. %%-------------------------------------------------------------------- id_string_no_opt_server(Config) -> {_Server, Host, Port} = std_daemon(Config, []), - {ok,S1}=gen_tcp:connect(Host,Port,[{active,false}]), + {ok,S1}=gen_tcp:connect(Host,Port,[{active,false},{packet,line}]), {ok,"SSH-2.0-Erlang/"++Vsn} = gen_tcp:recv(S1, 0, 2000), true = expected_ssh_vsn(Vsn). %%-------------------------------------------------------------------- id_string_own_string_server(Config) -> {_Server, Host, Port} = std_daemon(Config, [{id_string,"Olle"}]), - {ok,S1}=gen_tcp:connect(Host,Port,[{active,false}]), + {ok,S1}=gen_tcp:connect(Host,Port,[{active,false},{packet,line}]), {ok,"SSH-2.0-Olle\r\n"} = gen_tcp:recv(S1, 0, 2000). %%-------------------------------------------------------------------- id_string_random_server(Config) -> {_Server, Host, Port} = std_daemon(Config, [{id_string,random}]), - {ok,S1}=gen_tcp:connect(Host,Port,[{active,false}]), + {ok,S1}=gen_tcp:connect(Host,Port,[{active,false},{packet,line}]), {ok,"SSH-2.0-"++Rnd} = gen_tcp:recv(S1, 0, 2000), case Rnd of "Erlang"++_ -> ct:log("Id=~p",[Rnd]), @@ -985,12 +1286,15 @@ openssh_zlib_basic_test(Config) -> {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, {user_dir, UserDir}, + {preferred_algorithms,[{compression, ['[email protected]']}]}, {failfun, fun ssh_test_lib:failfun/2}]), ConnectionRef = ssh_test_lib:connect(Host, Port, [{silently_accept_hosts, true}, {user_dir, UserDir}, {user_interaction, false}, - {compression, openssh_zlib}]), + {preferred_algorithms,[{compression, ['[email protected]', + none]}]} + ]), ok = ssh:close(ConnectionRef), ssh:stop_daemon(Pid). @@ -1183,13 +1487,14 @@ expected_ssh_vsn(Str) -> _:_ -> true %% ssh not started so we dont't know end. - + fake_daemon(_Config) -> Parent = self(), %% start the server Server = spawn(fun() -> - {ok,Sl} = gen_tcp:listen(0,[]), + {ok,Sl} = gen_tcp:listen(0,[{packet,line}]), {ok,{Host,Port}} = inet:sockname(Sl), + ct:log("fake_daemon listening on ~p:~p~n",[Host,Port]), Parent ! {sockname,self(),Host,Port}, Rsa = gen_tcp:accept(Sl), ct:log("Server gen_tcp:accept got ~p",[Rsa]), @@ -1203,3 +1508,18 @@ fake_daemon(_Config) -> {sockname,Server,ServerHost,ServerPort} -> {Server, ServerHost, ServerPort} end. +%% get_kex_init - helper function to get key_exchange_init_msg +get_kex_init(Conn) -> + %% First, validate the key exchange is complete (StateName == connected) + {connected,S} = sys:get_state(Conn), + %% Next, walk through the elements of the #state record looking + %% for the #ssh_msg_kexinit record. This method is robust against + %% changes to either record. The KEXINIT message contains a cookie + %% unique to each invocation of the key exchange procedure (RFC4253) + SL = tuple_to_list(S), + case lists:keyfind(ssh_msg_kexinit, 1, SL) of + false -> + throw(not_found); + KexInit -> + KexInit + end. diff --git a/lib/ssh/test/ssh_connection_SUITE.erl b/lib/ssh/test/ssh_connection_SUITE.erl index 6fc09876ad..f0c337cf2f 100644 --- a/lib/ssh/test/ssh_connection_SUITE.erl +++ b/lib/ssh/test/ssh_connection_SUITE.erl @@ -65,6 +65,7 @@ ptty() -> %%-------------------------------------------------------------------- init_per_suite(Config) -> + catch crypto:stop(), case catch crypto:start() of ok -> Config; @@ -271,7 +272,7 @@ ptty_alloc(Config) when is_list(Config) -> {user_interaction, false}]), {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), success = ssh_connection:ptty_alloc(ConnectionRef, ChannelId, - [{term, os:getenv("TERM", ?DEFAULT_TERMINAL)}, {width, 70}, {high, 20}]), + [{term, os:getenv("TERM", ?DEFAULT_TERMINAL)}, {width, 70}, {height, 20}]), ssh:close(ConnectionRef). diff --git a/lib/ssh/test/ssh_relay.erl b/lib/ssh/test/ssh_relay.erl new file mode 100644 index 0000000000..a4f2bad2e2 --- /dev/null +++ b/lib/ssh/test/ssh_relay.erl @@ -0,0 +1,407 @@ +%%%------------------------------------------------------------------- +%%% @author Simon Cornish <[email protected]> +%%% @copyright (C) 2015, Simon Cornish +%%% @doc +%%% Provide manipulatable TCP-level relaying for testing SSH +%%% @end +%%% Created : 7 May 2015 by Simon Cornish <[email protected]> +%%%------------------------------------------------------------------- +-module(ssh_relay). + +-behaviour(gen_server). + +%% API +-export([start_link/4]). +-export([stop/1]). +-export([hold/4, release/2, release_next/3]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-record(hold, { + port, + n, + tmo, + tref, + q = [] + }). + +-record(state, { + local_addr, + local_port, + peer_addr, + peer_port, + lpid, + local, + peer, + tx_hold, + rx_hold + }). + +-define(ACCEPT_TMO, 200). +%%%=================================================================== +%%% API +%%%=================================================================== +%%-------------------------------------------------------------------- +%% @doc +%% Hold N (or 'all') messages in given direction. +%% Messages will be released after the N+1th message or +%% Tmo ms or 'infinity' +%% +%% Dir is 'tx' for direction local -> peer +%% and 'rx' for direction peer -> local +%% +%% An Error, ealready, is returned if there is already a hold +%% in the given direction +%% +%% @spec hold(Srv, Dir, N, Tmo) -> ok | {error, Error} +%% @end +%%-------------------------------------------------------------------- +hold(Srv, Dir, N, Tmo) -> + gen_server:call(Srv, {hold, Dir, N, Tmo}). + +%%-------------------------------------------------------------------- +%% @doc +%% Release all held messages in given direction. +%% +%% An Error, enoent, is returned if there is no hold +%% in the given direction +%% +%% @spec release(Srv, Dir) -> ok | {error, Error} +%% @end +%%-------------------------------------------------------------------- +release(Srv, Dir) -> + gen_server:call(Srv, {release, Dir}). + +%%-------------------------------------------------------------------- +%% @doc +%% Release all held messages in given direction after the +%% next message in the trigger direction +%% +%% An Error, enoent, is returned if there is no hold +%% in the given direction +%% +%% @spec release_next(Srv, Dir, TriggerDir) -> ok | {error, Error} +%% @end +%%-------------------------------------------------------------------- +release_next(Srv, Dir, TriggerDir) -> + gen_server:call(Srv, {release_next, Dir, TriggerDir}). + +%%-------------------------------------------------------------------- +%% @doc +%% Starts the server +%% +%% @spec start_link() -> {ok, Pid} | ignore | {error, Error} +%% @end +%%-------------------------------------------------------------------- +start_link(ListenAddr, ListenPort, PeerAddr, PeerPort) -> + gen_server:start_link(?MODULE, [ListenAddr, ListenPort, PeerAddr, PeerPort], []). + +stop(Srv) -> + unlink(Srv), + Srv ! stop. + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Initializes the server +%% +%% @spec init(Args) -> {ok, State} | +%% {ok, State, Timeout} | +%% ignore | +%% {stop, Reason} +%% @end +%%-------------------------------------------------------------------- +init([ListenAddr, ListenPort, PeerAddr, PeerPort | Options]) -> + IfAddr = case ListenAddr of + {0,0,0,0} -> + []; + _ -> + [{ifaddr, ListenAddr}] + end, + case gen_tcp:listen(ListenPort, [{reuseaddr, true}, {backlog, 1}, {active, false}, binary | IfAddr]) of + {ok, LSock} -> + Parent = self(), + {LPid, _LMod} = spawn_monitor(fun() -> listen(Parent, LSock) end), + S = #state{local_addr = ListenAddr, + local_port = ListenPort, + lpid = LPid, + peer_addr = PeerAddr, + peer_port = PeerPort + }, + {ok, S}; + Error -> + {stop, Error} + end. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling call messages +%% +%% @spec handle_call(Request, From, State) -> +%% {reply, Reply, State} | +%% {reply, Reply, State, Timeout} | +%% {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, Reply, State} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_call({hold, Dir, N, Tmo}, _From, State) -> + case Dir of + tx -> + do_hold(#state.tx_hold, State#state.peer, N, Tmo, State); + rx -> + do_hold(#state.rx_hold, State#state.local, N, Tmo, State); + _ -> + {reply, {error, einval}, State} + end; +handle_call({release, Dir}, _From, State) -> + case Dir of + tx -> + do_release(#state.tx_hold, State); + rx -> + do_release(#state.rx_hold, State); + _ -> + {reply, {error, einval}, State} + end; +handle_call({release_next, _Dir, _TriggerDir}, _From, State) -> + {reply, {error, nyi}, State}; + +handle_call(Request, _From, State) -> + Reply = {unhandled, Request}, + {reply, Reply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling cast messages +%% +%% @spec handle_cast(Msg, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_cast(_Msg, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling all non call/cast messages +%% +%% @spec handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} +%% @end +%%-------------------------------------------------------------------- +handle_info({tcp, Local, Data}, S) when S#state.local == Local -> + S1 = do_local(Data, S), + {noreply, S1}; + +handle_info({tcp_error, Local, Error}, S) when S#state.local == Local -> + S1 = do_local({error, Error}, S), + {noreply, S1}; + +handle_info({tcp_closed, Local}, S) when S#state.local == Local -> + S1 = do_local(closed, S), + {noreply, S1}; + +handle_info({tcp, Peer, Data}, S) when S#state.peer == Peer -> + S1 = do_peer(Data, S), + {noreply, S1}; + +handle_info({tcp_error, Peer, Error}, S) when S#state.peer == Peer -> + S1 = do_peer({error, Error}, S), + {noreply, S1}; + +handle_info({tcp_closed, Peer}, S) when S#state.peer == Peer -> + S1 = do_peer(closed, S), + {noreply, S1}; + +handle_info({accept, Local}, S) -> + S1 = do_accept(Local, S), + {noreply, S1}; + +handle_info({activate, Local}, State) -> + inet:setopts(Local, [{active, true}]), + {noreply, State}; + +handle_info({release, Pos}, S) -> + {reply, _, S1} = do_release(Pos,S), + {noreply, S1}; + +handle_info(stop, State) -> + {stop, normal, State}; + +handle_info({'DOWN', _Ref, _process, LPid, Reason}, S) when S#state.lpid == LPid -> + io:format("Acceptor has finished: ~p~n", [Reason]), + {noreply, S}; + +handle_info(_Info, State) -> + io:format("Unhandled info: ~p~n", [_Info]), + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_server terminates +%% with Reason. The return value is ignored. +%% +%% @spec terminate(Reason, State) -> void() +%% @end +%%-------------------------------------------------------------------- +terminate(_Reason, _State) -> + ok. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%% +%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} +%% @end +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== +do_hold(Pos, _Port, _N, _Tmo, S) when element(Pos, S) /= undefined -> + {reply, {error, ealready}, S}; +do_hold(Pos, Port, N, Tmo, S) -> + TRef = if is_integer(Tmo) andalso Tmo > 0 -> + erlang:send_after(Tmo, self(), {release, Pos}); + true -> + undefined + end, + Hold = #hold{port = Port, n = N, tmo = Tmo, tref = TRef}, + {reply, ok, setelement(Pos, S, Hold)}. + +do_release(HPos, S) when element(HPos, S) == undefined -> + {reply, {error, enoent}, S}; +do_release(HPos, S) -> + #hold{port = Port, tref = TRef, q = Q} = element(HPos, S), + lists:foreach(fun(M) -> gen_tcp:send(Port, M), erlang:yield() end, Q), + catch erlang:cancel_timer(TRef), + receive + {release, HPos} -> ok + after 0 -> + ok + end, + {reply, ok, setelement(HPos, S, undefined)}. + +listen(Parent, LSock) -> + monitor(process, Parent), + do_listen(Parent, LSock). + +do_listen(Parent, LSock) -> + %% So annoying there is no select-like sematic for this + case gen_tcp:accept(LSock, ?ACCEPT_TMO) of + {ok, Sock} -> + Parent ! {accept, Sock}, + gen_tcp:controlling_process(Sock, Parent), + Parent ! {activate, Sock}, + do_flush(Parent, Sock), + gen_tcp:close(LSock); + {error, timeout} -> + receive + DOWN when element(1, DOWN) == 'DOWN' -> + ok; + stop -> + ok + after 1 -> + do_listen(Parent, LSock) + end; + Error -> + gen_tcp:close(LSock), + exit({accept,Error}) + end. + +do_flush(Parent, Sock) -> + receive + {Tcp, Sock, _} = Msg when Tcp == tcp; Tcp == tcp_error -> + Parent ! Msg, + do_flush(Parent, Sock); + {tcp_closed, Sock} = Msg -> + Parent ! Msg, + do_flush(Parent, Sock) + after 1 -> + ok + end. + +do_accept(Local, S) -> + case gen_tcp:connect(S#state.peer_addr, S#state.peer_port, [{active, true}, binary]) of + {ok, Peer} -> + S#state{local = Local, peer = Peer}; + Error -> + exit({connect, Error}) + end. + +do_local(Data, S) when is_binary(Data) -> + TxH = S#state.tx_hold, + if TxH == undefined -> + gen_tcp:send(S#state.peer, Data), + S; + TxH#hold.n == 0 -> + lists:foreach(fun(M) -> gen_tcp:send(S#state.peer, M) end, TxH#hold.q), + gen_tcp:send(S#state.peer, Data), + catch erlang:cancel_timer(TxH#hold.tref), + TxP = #state.tx_hold, + receive + {release, TxP} -> + ok + after 0 -> + ok + end, + S#state{tx_hold = undefined}; + true -> + Q = TxH#hold.q ++ [Data], + N = if is_integer(TxH#hold.n) -> + TxH#hold.n -1; + true -> + TxH#hold.n + end, + S#state{tx_hold = TxH#hold{q = Q, n = N}} + end; +do_local(Error, _S) -> + exit({local, Error}). + +do_peer(Data, S) when is_binary(Data) -> + RxH = S#state.rx_hold, + if RxH == undefined -> + gen_tcp:send(S#state.local, Data), + S; + RxH#hold.n == 0 -> + lists:foreach(fun(M) -> gen_tcp:send(S#state.local, M) end, RxH#hold.q), + gen_tcp:send(S#state.local, Data), + catch erlang:cancel_timer(RxH#hold.tref), + RxP = #state.rx_hold, + receive + {release, RxP} -> + ok + after 0 -> + ok + end, + S#state{rx_hold = undefined}; + true -> + Q = RxH#hold.q ++ [Data], + N = if is_integer(RxH#hold.n) -> + RxH#hold.n -1; + true -> + RxH#hold.n + end, + S#state{rx_hold = RxH#hold{q = Q, n = N}} + end; +do_peer(Error, _S) -> + exit({peer, Error}). + diff --git a/lib/ssh/test/ssh_sftp_SUITE.erl b/lib/ssh/test/ssh_sftp_SUITE.erl index cb74a27638..850b1cbf6b 100644 --- a/lib/ssh/test/ssh_sftp_SUITE.erl +++ b/lib/ssh/test/ssh_sftp_SUITE.erl @@ -49,6 +49,7 @@ all() -> init_per_suite(Config) -> + catch crypto:stop(), case (catch crypto:start()) of ok -> ssh:start(), diff --git a/lib/ssh/test/ssh_sftpd_SUITE.erl b/lib/ssh/test/ssh_sftpd_SUITE.erl index 0ce8eec906..925b02a437 100644 --- a/lib/ssh/test/ssh_sftpd_SUITE.erl +++ b/lib/ssh/test/ssh_sftpd_SUITE.erl @@ -68,6 +68,7 @@ groups() -> %%-------------------------------------------------------------------- init_per_suite(Config) -> + catch crypto:stop(), case (catch crypto:start()) of ok -> DataDir = ?config(data_dir, Config), diff --git a/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl b/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl index cc34cc0793..eac7575486 100644 --- a/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl +++ b/lib/ssh/test/ssh_sftpd_erlclient_SUITE.erl @@ -52,6 +52,7 @@ groups() -> init_per_suite(Config) -> catch ssh:stop(), + catch crypto:stop(), case catch crypto:start() of ok -> DataDir = ?config(data_dir, Config), diff --git a/lib/ssh/test/ssh_to_openssh_SUITE.erl b/lib/ssh/test/ssh_to_openssh_SUITE.erl index a61fd2dd41..277e3a1b08 100644 --- a/lib/ssh/test/ssh_to_openssh_SUITE.erl +++ b/lib/ssh/test/ssh_to_openssh_SUITE.erl @@ -60,6 +60,7 @@ groups() -> ]. init_per_suite(Config) -> + catch crypto:stop(), case catch crypto:start() of ok -> case gen_tcp:connect("localhost", 22, []) of @@ -166,9 +167,11 @@ erlang_client_openssh_server_exec_compressed() -> [{doc, "Test that compression option works"}]. erlang_client_openssh_server_exec_compressed(Config) when is_list(Config) -> + CompressAlgs = [zlib, '[email protected]',none], ConnectionRef = ssh_test_lib:connect(?SSH_DEFAULT_PORT, [{silently_accept_hosts, true}, {user_interaction, false}, - {compression, zlib}]), + {preferred_algorithms, + [{compression,CompressAlgs}]}]), {ok, ChannelId} = ssh_connection:session_channel(ConnectionRef, infinity), success = ssh_connection:exec(ConnectionRef, ChannelId, "echo testing", infinity), @@ -326,8 +329,11 @@ erlang_server_openssh_client_exec_compressed(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), KnownHosts = filename:join(PrivDir, "known_hosts"), +%% CompressAlgs = [zlib, '[email protected]'], % Does not work + CompressAlgs = [zlib], {Pid, Host, Port} = ssh_test_lib:daemon([{system_dir, SystemDir}, - {compression, zlib}, + {preferred_algorithms, + [{compression, CompressAlgs}]}, {failfun, fun ssh_test_lib:failfun/2}]), ct:sleep(500), diff --git a/lib/ssh/test/ssh_unicode_SUITE.erl b/lib/ssh/test/ssh_unicode_SUITE.erl index cc916673b3..07d51335c6 100644 --- a/lib/ssh/test/ssh_unicode_SUITE.erl +++ b/lib/ssh/test/ssh_unicode_SUITE.erl @@ -55,6 +55,7 @@ all() -> init_per_suite(Config) -> + catch crypto:stop(), case {file:native_name_encoding(), (catch crypto:start())} of {utf8, ok} -> ssh:start(), diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 352563700b..fe0606b1a3 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -25,7 +25,23 @@ <file>notes.xml</file> </header> <p>This document describes the changes made to the SSL application.</p> - <section><title>SSL 6.0</title> + <section><title>SSL 6.0.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Terminate gracefully when receving bad input to premaster + secret calculation</p> + <p> + Own Id: OTP-12783</p> + </item> + </list> + </section> + +</section> + +<section><title>SSL 6.0</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index cdf6870c25..18d98e5efb 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -67,15 +67,15 @@ <taglist> - <tag><c>boolean()</c></tag> - <item><p><c>= true | false</c></p></item> + <tag><c>boolean() =</c></tag> + <item><p><c>true | false</c></p></item> - <tag><c>option()</c></tag> - <item><p><c>= socketoption() | ssloption() | transportoption()</c></p> + <tag><c>option() =</c></tag> + <item><p><c>socketoption() | ssloption() | transportoption()</c></p> </item> - <tag><c>socketoption()</c></tag> - <item><p><c>= proplists:property()</c></p> + <tag><c>socketoption() =</c></tag> + <item><p><c>proplists:property()</c></p> <p>The default socket options are <c>[{mode,list},{packet, 0},{header, 0},{active, true}]</c>.</p> <p>For valid options, see the @@ -83,32 +83,37 @@ <seealso marker="kernel:gen_tcp">gen_tcp(3)</seealso> manual pages in Kernel.</p></item> - <tag><marker id="type-ssloption"></marker><c>ssloption()</c></tag> - <item><p><c>= {verify, verify_type()}</c></p> - <p><c>| {verify_fun, {fun(), term()}}</c></p> - <p><c>| {fail_if_no_peer_cert, boolean()} {depth, integer()}</c></p> - <p><c>| {cert, public_key:der_encoded()}</c></p> - <p><c>| {certfile, path()}</c></p> - <p><c>| {key, {'RSAPrivateKey'| 'DSAPrivateKey' | 'ECPrivateKey' - | 'PrivateKeyInfo', public_key:der_encoded()}}</c></p> - <p><c>| {keyfile, path()}</c></p> - <p><c>| {password, string()}</c></p> - <p><c>| {cacerts, [public_key:der_encoded()]}</c></p> - <p><c>| {cacertfile, path()}</c></p> - <p><c>| {dh, public_key:der_encoded()}</c></p> - <p><c>| {dhfile, path()}</c></p> - <p><c>| {ciphers, ciphers()}</c></p> - <p><c>| {user_lookup_fun, {fun(), term()}}, {psk_identity, string()}, - {srp_identity, {string(), string()}}</c></p> - <p><c>| {reuse_sessions, boolean()}</c></p> - <p><c>| {reuse_session, fun()} {next_protocols_advertised, [binary()]}</c></p> - <p><c>| {client_preferred_next_protocols, {client | server, - [binary()]} | {client | server, [binary()], binary()}}</c></p> - <p><c>| {log_alert, boolean()}</c></p> - <p><c>| {server_name_indication, hostname() | disable}</c></p></item> - - <tag><c>transportoption()</c></tag> - <item><p><c>= {cb_info, {CallbackModule::atom(), DataTag::atom(), + <tag><marker id="type-ssloption"></marker><c>ssloption() =</c></tag> + <item> + <p><c>{verify, verify_type()}</c></p> + <p><c>| {verify_fun, {fun(), term()}}</c></p> + <p><c>| {fail_if_no_peer_cert, boolean()} {depth, integer()}</c></p> + <p><c>| {cert, public_key:der_encoded()}</c></p> + <p><c>| {certfile, path()}</c></p> + <p><c>| {key, {'RSAPrivateKey'| 'DSAPrivateKey' | 'ECPrivateKey' + | 'PrivateKeyInfo', public_key:der_encoded()}}</c></p> + <p><c>| {keyfile, path()}</c></p> + <p><c>| {password, string()}</c></p> + <p><c>| {cacerts, [public_key:der_encoded()]}</c></p> + <p><c>| {cacertfile, path()}</c></p> + <p><c>| {dh, public_key:der_encoded()}</c></p> + <p><c>| {dhfile, path()}</c></p> + <p><c>| {ciphers, ciphers()}</c></p> + <p><c>| {user_lookup_fun, {fun(), term()}}, {psk_identity, string()}, + {srp_identity, {string(), string()}}</c></p> + <p><c>| {reuse_sessions, boolean()}</c></p> + <p><c>| {reuse_session, fun()} {next_protocols_advertised, [binary()]}</c></p> + <p><c>| {client_preferred_next_protocols, {client | server, + [binary()]} | {client | server, [binary()], binary()}}</c></p> + <p><c>| {log_alert, boolean()}</c></p> + <p><c>| {server_name_indication, hostname() | disable}</c></p> + <p><c>| {sni_hosts, [{hostname(), ssloptions()}]}</c></p> + <p><c>| {sni_fun, SNIfun::fun()}</c></p> + </item> + + <tag><c>transportoption() =</c></tag> + <item><p><c>{cb_info, {CallbackModule::atom(), DataTag::atom(), + ClosedTag::atom(), ErrTag:atom()}}</c></p> <p>Defaults to <c>{gen_tcp, tcp, tcp_closed, tcp_error}</c>. Can be used to customize the transport layer. The callback module must implement a @@ -118,70 +123,73 @@ The callback <c>gen_tcp</c> is treated specially and calls <c>inet</c> directly.</p> <taglist> - <tag><c>CallbackModule</c></tag> - <item><p><c>= atom()</c></p></item> - <tag><c>DataTag</c></tag> - <item><p><c>= atom()</c></p> + <tag><c>CallbackModule =</c></tag> + <item><p><c>atom()</c></p></item> + <tag><c>DataTag =</c></tag> + <item><p><c>atom()</c></p> <p>Used in socket data message.</p></item> - <tag><c>ClosedTag</c></tag> - <item><p><c>= atom()</c></p> + <tag><c>ClosedTag =</c></tag> + <item><p><c>atom()</c></p> <p>Used in socket close message.</p></item> </taglist> </item> - <tag><c>verify_type()</c></tag> - <item><p><c>= verify_none | verify_peer</c></p></item> + <tag><c>verify_type() =</c></tag> + <item><p><c>verify_none | verify_peer</c></p></item> - <tag><c>path()</c></tag> - <item><p><c>= string()</c></p> + <tag><c>path() =</c></tag> + <item><p><c>string()</c></p> <p>Represents a file path.</p></item> - <tag><c>public_key:der_encoded()</c></tag> - <item><p><c>= binary()</c></p> + <tag><c>public_key:der_encoded() =</c></tag> + <item><p><c>binary()</c></p> <p>ASN.1 DER-encoded entity as an Erlang binary.</p></item> - <tag><c>host()</c></tag> - <item><p><c>= hostname() | ipaddress()</c></p></item> + <tag><c>host() =</c></tag> + <item><p><c>hostname() | ipaddress()</c></p></item> - <tag><c>hostname()</c></tag> - <item><p><c>= string()</c></p></item> + <tag><c>hostname() =</c></tag> + <item><p><c>string()</c></p></item> - <tag><c>ip_address()</c></tag> - <item><p><c>= {N1,N2,N3,N4} % IPv4 | {K1,K2,K3,K4,K5,K6,K7,K8} % IPv6 + <tag><c>ip_address() =</c></tag> + <item><p><c>{N1,N2,N3,N4} % IPv4 | {K1,K2,K3,K4,K5,K6,K7,K8} % IPv6 </c></p></item> - <tag><c>sslsocket()</c></tag> - <item><p>Opaque to the user.</p></item> + <tag><c>sslsocket() =</c></tag> + <item><p>opaque()</p></item> - <tag><c>protocol()</c></tag> - <item><p><c>= sslv3 | tlsv1 | 'tlsv1.1' | 'tlsv1.2'</c></p></item> + <tag><c>protocol() =</c></tag> + <item><p><c>sslv3 | tlsv1 | 'tlsv1.1' | 'tlsv1.2'</c></p></item> - <tag><c>ciphers()</c></tag> + <tag><c>ciphers() =</c></tag> <item><p><c>= [ciphersuite()] | string()</c></p> <p>According to old API.</p></item> - <tag><c>ciphersuite()</c></tag> - <item><p><c>= {key_exchange(), cipher(), hash()}</c></p></item> + <tag><c>ciphersuite() =</c></tag> + <item><p><c>{key_exchange(), cipher(), hash()}</c></p></item> - <tag><c>key_exchange()</c></tag> - <item><p><c>= rsa | dhe_dss | dhe_rsa | dh_anon | psk | dhe_psk + <tag><c>key_exchange()=</c></tag> + <item><p><c>rsa | dhe_dss | dhe_rsa | dh_anon | psk | dhe_psk | rsa_psk | srp_anon | srp_dss | srp_rsa | ecdh_anon | ecdh_ecdsa | ecdhe_ecdsa | ecdh_rsa | ecdhe_rsa</c></p></item> - <tag><c>cipher()</c></tag> - <item><p><c>= rc4_128 | des_cbc | '3des_ede_cbc' + <tag><c>cipher() =</c></tag> + <item><p><c>rc4_128 | des_cbc | '3des_ede_cbc' | aes_128_cbc | aes_256_cbc | aes_128_gcm | aes_256_gcm</c></p></item> - <tag><c>hash()</c></tag> - <item><p><c>= md5 | sha</c></p></item> + <tag><c>hash() =</c></tag> + <item><p><c>md5 | sha</c></p></item> - <tag><c>prf_random()</c></tag> - <item><p><c>= client_random | server_random</c></p></item> + <tag><c>prf_random() =</c></tag> + <item><p><c>client_random | server_random</c></p></item> - <tag><c>srp_param_type()</c></tag> - <item><p><c>= srp_1024 | srp_1536 | srp_2048 | srp_3072 + <tag><c>srp_param_type() =</c></tag> + <item><p><c>srp_1024 | srp_1536 | srp_2048 | srp_3072 | srp_4096 | srp_6144 | srp_8192</c></p></item> + <tag><c>SNIfun::fun()</c></tag> + <item><p><c>= fun(ServerName :: string()) -> ssloptions()</c></p></item> + </taglist> </section> @@ -268,7 +276,7 @@ atom()}} | application. It differentiates between the peer certificate and the CA certificates by using <c>valid_peer</c> or <c>valid</c> as second argument to the verification fun. See the - <seealso marker="public_key:cert_records">public_key User's + <seealso marker="public_key:public_key_records">public_key User's Guide</seealso> for definition of <c>#'OTPCertificate'{}</c> and <c>#'Extension'{}</c>.</p> @@ -364,10 +372,10 @@ marker="public_key:public_key#pkix_path_validation-3">public_key:pkix_path_valid empty argument list. The following arguments may be specified for the internal cache.</p> <taglist> <tag><c>{http, timeout()}</c></tag> - <item> + <item><p> Enables fetching of CRLs specified as http URIs in<seealso - marker="public_key:cert_records"> X509 cerificate extensions.</seealso> - Requires the OTP inets application. + marker="public_key:public_key_records"> X509 cerificate extensions.</seealso> + Requires the OTP inets application.</p> </item> </taglist> </item> @@ -624,7 +632,24 @@ fun(srp, Username :: string(), UserState :: term()) -> selection. If set to <c>false</c> (the default), use the client preference.</p></item> - + <tag><c>{sni_hosts, [{hostname(), ssloptions()}]}</c></tag> + <item><p>If the server receives a SNI (Server Name Indication) from the client + matching a host listed in the <c>sni_hosts</c> option, the speicific options for + that host will override previously specified options. + + The option <c>sni_fun</c>, and <c>sni_hosts</c> are mutually exclusive.</p></item> + + <tag><c>{sni_fun, SNIfun::fun()}</c></tag> + <item><p>If the server receives a SNI (Server Name Indication) from the client, + the given function will be called to retrive <c>ssloptions()</c> for indicated server. + These options will be merged into predefined <c>ssloptions()</c>. + + The function should be defined as: + <c>fun(ServerName :: string()) -> ssloptions()</c> + and can be specified as a fun or as named <c>fun module:function/1</c> + + The option <c>sni_fun</c>, and <c>sni_hosts</c> are mutually exclusive.</p></item> + </taglist> </section> @@ -752,6 +777,45 @@ fun(srp, Username :: string(), UserState :: term()) -> </func> <func> + <name>connection_information(SslSocket) -> + {ok, Info} | {error, Reason} </name> + <fsummary>Returns all the connection information. + </fsummary> + <type> + <v>Info = [InfoTuple]</v> + <v>InfoTuple = {protocol, Protocol} | {cipher_suite, CipherSuite} | {sni_hostname, SNIHostname}</v> + <v>CipherSuite = ciphersuite()</v> + <v>ProtocolVersion = protocol()</v> + <v>SNIHostname = string()</v> + <v>Reason = term()</v> + </type> + <desc><p>Return all the connection information containing negotiated protocol version, cipher suite, and the hostname of SNI extension. + Info will be a proplists containing all the connection information on success, otherwise <c>{error, Reason}</c> will be returned.</p> + </desc> + </func> + + <func> + <name>connection_information(SslSocket, Items) -> + {ok, Info} | {error, Reason} </name> + <fsummary>Returns the requested connection information. + </fsummary> + <type> + <v>Items = [Item]</v> + <v>Item = protocol | cipher_suite | sni_hostname</v> + <v>Info = [InfoTuple]</v> + <v>InfoTuple = {protocol, Protocol} | {cipher_suite, CipherSuite} | {sni_hostname, SNIHostname}</v> + <v>CipherSuite = ciphersuite()</v> + <v>ProtocolVersion = protocol()</v> + <v>SNIHostname = string()</v> + <v>Reason = term()</v> + </type> + <desc><p>Returns the connection information you requested. The connection information you can request contains protocol, cipher_suite, and sni_hostname. + <c>{ok, Info}</c> will be returned if it executes sucessfully. The Info is a proplists containing the information you requested. + Otherwise, <c>{error, Reason}</c> will be returned.</p> + </desc> + </func> + + <func> <name>format_error(Reason) -> string()</name> <fsummary>Returns an error string.</fsummary> <type> diff --git a/lib/ssl/doc/src/ssl_crl_cache_api.xml b/lib/ssl/doc/src/ssl_crl_cache_api.xml index 1d9353a2cc..9230442ae0 100644 --- a/lib/ssl/doc/src/ssl_crl_cache_api.xml +++ b/lib/ssl/doc/src/ssl_crl_cache_api.xml @@ -47,11 +47,11 @@ <taglist> - <tag><c>cache_ref()</c></tag> - <item> = opaque()</item> - <tag><c>dist_point()</c></tag> - <item> = #'DistributionPoint'{} see <seealso - marker="public_key:cert_records"> X509 certificates records</seealso></item> + <tag><c>cache_ref() =</c></tag> + <item>opaque()</item> + <tag><c>dist_point() =</c></tag> + <item><p>#'DistributionPoint'{} see <seealso + marker="public_key:public_key_records"> X509 certificates records</seealso></p></item> </taglist> diff --git a/lib/ssl/doc/src/ssl_session_cache_api.xml b/lib/ssl/doc/src/ssl_session_cache_api.xml index c89d3874a1..28b5f4ce23 100644 --- a/lib/ssl/doc/src/ssl_session_cache_api.xml +++ b/lib/ssl/doc/src/ssl_session_cache_api.xml @@ -40,20 +40,20 @@ <c>ssl_session_cache_api</c>:</p> <taglist> - <tag><c>cache_ref()</c></tag> - <item><p>= <c>opaque()</c></p></item> + <tag><c>cache_ref() =</c></tag> + <item><p><c>opaque()</c></p></item> - <tag><c>key()</c></tag> - <item><p>= <c>{partialkey(), session_id()}</c></p></item> + <tag><c>key() =</c></tag> + <item><p><c>{partialkey(), session_id()}</c></p></item> - <tag><c>partialkey()</c></tag> - <item><p>= <c>opaque()</c></p></item> + <tag><c>partialkey() =</c></tag> + <item><p><c>opaque()</c></p></item> - <tag><c>session_id()</c></tag> - <item><p>= <c>binary()</c></p></item> + <tag><c>session_id() =</c></tag> + <item><p><c>binary()</c></p></item> - <tag><c>session()</c></tag> - <item><p>= <c>opaque()</c></p></item> + <tag><c>session()</c> =</tag> + <item><p><c>opaque()</c></p></item> </taglist> </section> diff --git a/lib/ssl/src/ssl.appup.src b/lib/ssl/src/ssl.appup.src index 1476336039..d100e41930 100644 --- a/lib/ssl/src/ssl.appup.src +++ b/lib/ssl/src/ssl.appup.src @@ -1,14 +1,16 @@ %% -*- erlang -*- {"%VSN%", [ - {<<"6\\..*">>, [{restart_application, ssl}]}, - {<<"5\\..*">>, [{restart_application, ssl}]}, + {<<"6.0">>, [{load_module, ssl_handshake, soft_purge, soft_purge, []}]}, + {<<"5\\.3\\.[1-7]($|\\..*)">>, [{restart_application, ssl}]}, + {<<"5\\.[0-2]($|\\..*)">>, [{restart_application, ssl}]}, {<<"4\\..*">>, [{restart_application, ssl}]}, {<<"3\\..*">>, [{restart_application, ssl}]} ], [ - {<<"6\\..*">>, [{restart_application, ssl}]}, - {<<"5\\..*">>, [{restart_application, ssl}]}, + {<<"6.0">>, [{load_module, ssl_handshake, soft_purge, soft_purge, []}]}, + {<<"5\\.3\\.[1-7]($|\\..*)">>, [{restart_application, ssl}]}, + {<<"5\\.[0-2]($|\\..*)">>, [{restart_application, ssl}]}, {<<"4\\..*">>, [{restart_application, ssl}]}, {<<"3\\..*">>, [{restart_application, ssl}]} ] diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 6461f64c1c..225a9be66f 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -38,11 +38,13 @@ %% SSL/TLS protocol handling -export([cipher_suites/0, cipher_suites/1, suite_definition/1, connection_info/1, versions/0, session_info/1, format_error/1, - renegotiate/1, prf/5, negotiated_protocol/1, negotiated_next_protocol/1]). + renegotiate/1, prf/5, negotiated_protocol/1, negotiated_next_protocol/1, + connection_information/1, connection_information/2]). %% Misc --export([random_bytes/1]). +-export([random_bytes/1, handle_options/2]). -deprecated({negotiated_next_protocol, 1, next_major_release}). +-deprecated({connection_info, 1, next_major_release}). -include("ssl_api.hrl"). -include("ssl_internal.hrl"). @@ -286,16 +288,42 @@ controlling_process(#sslsocket{pid = {Listen, is_pid(NewOwner) -> Transport:controlling_process(Listen, NewOwner). + +%%-------------------------------------------------------------------- +-spec connection_information(#sslsocket{}) -> {ok, list()} | {error, reason()}. +%% +%% Description: Return SSL information for the connection +%%-------------------------------------------------------------------- +connection_information(#sslsocket{pid = Pid}) when is_pid(Pid) -> ssl_connection:connection_information(Pid); +connection_information(#sslsocket{pid = {Listen, _}}) when is_port(Listen) -> {error, enotconn}. + + +%%-------------------------------------------------------------------- +-spec connection_information(#sslsocket{}, [atom]) -> {ok, list()} | {error, reason()}. +%% +%% Description: Return SSL information for the connection +%%-------------------------------------------------------------------- +connection_information(#sslsocket{} = SSLSocket, Items) -> + case connection_information(SSLSocket) of + {ok, I} -> + {ok, lists:filter(fun({K, _}) -> lists:foldl(fun(K1, Acc) when K1 =:= K -> Acc + 1; (_, Acc) -> Acc end, 0, Items) > 0 end, I)}; + E -> + E + end. + %%-------------------------------------------------------------------- -spec connection_info(#sslsocket{}) -> {ok, {tls_record:tls_atom_version(), ssl_cipher:erl_cipher_suite()}} | {error, reason()}. %% %% Description: Returns ssl protocol and cipher used for the connection %%-------------------------------------------------------------------- -connection_info(#sslsocket{pid = Pid}) when is_pid(Pid) -> - ssl_connection:info(Pid); -connection_info(#sslsocket{pid = {Listen, _}}) when is_port(Listen) -> - {error, enotconn}. +connection_info(#sslsocket{} = SSLSocket) -> + case connection_information(SSLSocket) of + {ok, Result} -> + {ok, {proplists:get_value(protocol, Result), proplists:get_value(cipher_suite, Result)}}; + Error -> + Error + end. %%-------------------------------------------------------------------- -spec peername(#sslsocket{}) -> {ok, {inet:ip_address(), inet:port_number()}} | {error, reason()}. @@ -671,6 +699,8 @@ handle_options(Opts0) -> handle_option(client_preferred_next_protocols, Opts, undefined)), log_alert = handle_option(log_alert, Opts, true), server_name_indication = handle_option(server_name_indication, Opts, undefined), + sni_hosts = handle_option(sni_hosts, Opts, []), + sni_fun = handle_option(sni_fun, Opts, undefined), honor_cipher_order = handle_option(honor_cipher_order, Opts, false), protocol = proplists:get_value(protocol, Opts, tls), padding_check = proplists:get_value(padding_check, Opts, true), @@ -687,7 +717,7 @@ handle_options(Opts0) -> user_lookup_fun, psk_identity, srp_identity, ciphers, reuse_session, reuse_sessions, ssl_imp, cb_info, renegotiate_at, secure_renegotiate, hibernate_after, - erl_dist, alpn_advertised_protocols, + erl_dist, alpn_advertised_protocols, sni_hosts, sni_fun, alpn_preferred_protocols, next_protocols_advertised, client_preferred_next_protocols, log_alert, server_name_indication, honor_cipher_order, padding_check, crl_check, crl_cache, @@ -704,6 +734,18 @@ handle_options(Opts0) -> inet_user = SockOpts, transport_info = CbInfo, connection_cb = ConnetionCb }}. +handle_option(sni_fun, Opts, Default) -> + OptFun = validate_option(sni_fun, + proplists:get_value(sni_fun, Opts, Default)), + OptHosts = proplists:get_value(sni_hosts, Opts, undefined), + case {OptFun, OptHosts} of + {Default, _} -> + Default; + {_, undefined} -> + OptFun; + _ -> + throw({error, {conflict_options, [sni_fun, sni_hosts]}}) + end; handle_option(OptionName, Opts, Default) -> validate_option(OptionName, proplists:get_value(OptionName, Opts, Default)). @@ -881,6 +923,20 @@ validate_option(server_name_indication, disable) -> disable; validate_option(server_name_indication, undefined) -> undefined; +validate_option(sni_hosts, []) -> + []; +validate_option(sni_hosts, [{Hostname, SSLOptions} | Tail]) when is_list(Hostname) -> + RecursiveSNIOptions = proplists:get_value(sni_hosts, SSLOptions, undefined), + case RecursiveSNIOptions of + undefined -> + [{Hostname, validate_options(SSLOptions)} | validate_option(sni_hosts, Tail)]; + _ -> + throw({error, {options, {sni_hosts, RecursiveSNIOptions}}}) + end; +validate_option(sni_fun, undefined) -> + undefined; +validate_option(sni_fun, Fun) when is_function(Fun) -> + Fun; validate_option(honor_cipher_order, Value) when is_boolean(Value) -> Value; validate_option(padding_check, Value) when is_boolean(Value) -> @@ -896,6 +952,12 @@ validate_option(crl_cache, {Cb, {_Handle, Options}} = Value) when is_atom(Cb) an validate_option(Opt, Value) -> throw({error, {options, {Opt, Value}}}). + +validate_options([]) -> + []; +validate_options([{Opt, Value} | Tail]) -> + [{Opt, validate_option(Opt, Value)} | validate_options(Tail)]. + validate_npn_ordering(client) -> ok; validate_npn_ordering(server) -> diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index 4a839872a6..64fa7bab0d 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -41,8 +41,9 @@ %% User Events -export([send/2, recv/3, close/1, shutdown/2, - new_user/2, get_opts/2, set_opts/2, info/1, session_info/1, - peer_certificate/1, renegotiation/1, negotiated_protocol/1, prf/5 + new_user/2, get_opts/2, set_opts/2, session_info/1, + peer_certificate/1, renegotiation/1, negotiated_protocol/1, prf/5, + connection_information/1 ]). -export([handle_session/7]). @@ -161,6 +162,14 @@ recv(Pid, Length, Timeout) -> sync_send_all_state_event(Pid, {recv, Length, Timeout}). %%-------------------------------------------------------------------- +-spec connection_information(pid()) -> {ok, list()} | {error, reason()}. +%% +%% Description: Get the SNI hostname +%%-------------------------------------------------------------------- +connection_information(Pid) when is_pid(Pid) -> + sync_send_all_state_event(Pid, connection_information). + +%%-------------------------------------------------------------------- -spec close(pid()) -> ok | {error, reason()}. %% %% Description: Close an ssl connection @@ -214,14 +223,6 @@ set_opts(ConnectionPid, Options) -> sync_send_all_state_event(ConnectionPid, {set_opts, Options}). %%-------------------------------------------------------------------- --spec info(pid()) -> {ok, {atom(), tuple()}} | {error, reason()}. -%% -%% Description: Returns ssl protocol and cipher used for the connection -%%-------------------------------------------------------------------- -info(ConnectionPid) -> - sync_send_all_state_event(ConnectionPid, info). - -%%-------------------------------------------------------------------- -spec session_info(pid()) -> {ok, list()} | {error, reason()}. %% %% Description: Returns info about the ssl session @@ -829,13 +830,6 @@ handle_sync_event({prf, Secret, Label, Seed, WantedLength}, _, StateName, error:Reason -> {error, Reason} end, {reply, Reply, StateName, State, get_timeout(State)}; -handle_sync_event(info, _, StateName, - #state{negotiated_version = Version, - session = #session{cipher_suite = Suite}} = State) -> - - AtomVersion = tls_record:protocol_version(Version), - {reply, {ok, {AtomVersion, ssl:suite_definition(Suite)}}, - StateName, State, get_timeout(State)}; handle_sync_event(session_info, _, StateName, #state{session = #session{session_id = Id, cipher_suite = Suite}} = State) -> @@ -845,7 +839,10 @@ handle_sync_event(session_info, _, StateName, handle_sync_event(peer_certificate, _, StateName, #state{session = #session{peer_certificate = Cert}} = State) -> - {reply, {ok, Cert}, StateName, State, get_timeout(State)}. + {reply, {ok, Cert}, StateName, State, get_timeout(State)}; +handle_sync_event(connection_information, _, StateName, #state{sni_hostname = SNIHostname, session = #session{cipher_suite = CipherSuite}, negotiated_version = Version} = State) -> + {reply, {ok, [{protocol, tls_record:protocol_version(Version)}, {cipher_suite, ssl:suite_definition(CipherSuite)}, {sni_hostname, SNIHostname}]}, StateName, State, get_timeout(State)}. + handle_info({ErrorTag, Socket, econnaborted}, StateName, #state{socket = Socket, transport_cb = Transport, diff --git a/lib/ssl/src/ssl_connection.hrl b/lib/ssl/src/ssl_connection.hrl index e569d706af..d95b51132a 100644 --- a/lib/ssl/src/ssl_connection.hrl +++ b/lib/ssl/src/ssl_connection.hrl @@ -80,7 +80,8 @@ expecting_finished = false ::boolean(), negotiated_protocol = undefined :: undefined | binary(), client_ecc, % {Curves, PointFmt} - tracker :: pid() %% Tracker process for listen socket + tracker :: pid(), %% Tracker process for listen socket + sni_hostname = undefined }). -define(DEFAULT_DIFFIE_HELLMAN_PARAMS, diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index b538fefe53..12a17cb6ac 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -476,19 +476,27 @@ update_handshake_history({Handshake0, _Prev}, Data) -> %% end. premaster_secret(OtherPublicDhKey, MyPrivateKey, #'DHParameter'{} = Params) -> - public_key:compute_key(OtherPublicDhKey, MyPrivateKey, Params); - + try + public_key:compute_key(OtherPublicDhKey, MyPrivateKey, Params) + catch + error:computation_failed -> + throw(?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)) + end; premaster_secret(PublicDhKey, PrivateDhKey, #server_dh_params{dh_p = Prime, dh_g = Base}) -> - crypto:compute_key(dh, PublicDhKey, PrivateDhKey, [Prime, Base]); + try + crypto:compute_key(dh, PublicDhKey, PrivateDhKey, [Prime, Base]) + catch + error:computation_failed -> + throw(?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)) + end; premaster_secret(#client_srp_public{srp_a = ClientPublicKey}, ServerKey, #srp_user{prime = Prime, verifier = Verifier}) -> case crypto:compute_key(srp, ClientPublicKey, ServerKey, {host, [Verifier, Prime, '6a']}) of error -> - ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER); + throw(?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)); PremasterSecret -> PremasterSecret end; - premaster_secret(#server_srp_params{srp_n = Prime, srp_g = Generator, srp_s = Salt, srp_b = Public}, ClientKeys, {Username, Password}) -> case ssl_srp_primes:check_srp_params(Generator, Prime) of @@ -496,21 +504,19 @@ premaster_secret(#server_srp_params{srp_n = Prime, srp_g = Generator, srp_s = Sa DerivedKey = crypto:hash(sha, [Salt, crypto:hash(sha, [Username, <<$:>>, Password])]), case crypto:compute_key(srp, Public, ClientKeys, {user, [DerivedKey, Prime, Generator, '6a']}) of error -> - ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER); + throw(?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)); PremasterSecret -> PremasterSecret end; _ -> - ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER) + throw(?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)) end; - premaster_secret(#client_rsa_psk_identity{ identity = PSKIdentity, exchange_keys = #encrypted_premaster_secret{premaster_secret = EncPMS} }, #'RSAPrivateKey'{} = Key, PSKLookup) -> PremasterSecret = premaster_secret(EncPMS, Key), psk_secret(PSKIdentity, PSKLookup, PremasterSecret); - premaster_secret(#server_dhe_psk_params{ hint = IdentityHint, dh_params = #server_dh_params{dh_y = PublicDhKey} = Params}, @@ -518,7 +524,6 @@ premaster_secret(#server_dhe_psk_params{ LookupFun) -> PremasterSecret = premaster_secret(PublicDhKey, PrivateDhKey, Params), psk_secret(IdentityHint, LookupFun, PremasterSecret); - premaster_secret({rsa_psk, PSKIdentity}, PSKLookup, RSAPremasterSecret) -> psk_secret(PSKIdentity, PSKLookup, RSAPremasterSecret). @@ -527,13 +532,10 @@ premaster_secret(#client_dhe_psk_identity{ dh_public = PublicDhKey}, PrivateKey, #'DHParameter'{} = Params, PSKLookup) -> PremasterSecret = premaster_secret(PublicDhKey, PrivateKey, Params), psk_secret(PSKIdentity, PSKLookup, PremasterSecret). - premaster_secret(#client_psk_identity{identity = PSKIdentity}, PSKLookup) -> psk_secret(PSKIdentity, PSKLookup); - premaster_secret({psk, PSKIdentity}, PSKLookup) -> psk_secret(PSKIdentity, PSKLookup); - premaster_secret(#'ECPoint'{} = ECPoint, #'ECPrivateKey'{} = ECDHKeys) -> public_key:compute_key(ECPoint, ECDHKeys); premaster_secret(EncSecret, #'RSAPrivateKey'{} = RSAPrivateKey) -> @@ -2036,7 +2038,7 @@ psk_secret(PSKIdentity, PSKLookup) -> #alert{} = Alert -> Alert; _ -> - ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER) + throw(?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)) end. psk_secret(PSKIdentity, PSKLookup, PremasterSecret) -> @@ -2048,7 +2050,7 @@ psk_secret(PSKIdentity, PSKLookup, PremasterSecret) -> #alert{} = Alert -> Alert; _ -> - ?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER) + throw(?ALERT_REC(?FATAL, ?ILLEGAL_PARAMETER)) end. handle_psk_identity(_PSKIdentity, LookupFun) diff --git a/lib/ssl/src/ssl_internal.hrl b/lib/ssl/src/ssl_internal.hrl index 90f8b8a412..baeae68bc4 100644 --- a/lib/ssl/src/ssl_internal.hrl +++ b/lib/ssl/src/ssl_internal.hrl @@ -122,6 +122,8 @@ next_protocol_selector = undefined, %% fun([binary()]) -> binary()) log_alert :: boolean(), server_name_indication = undefined, + sni_hosts :: [{inet:hostname(), [tuple()]}], + sni_fun :: function() | undefined, %% Should the server prefer its own cipher order over the one provided by %% the client? honor_cipher_order = false :: boolean(), diff --git a/lib/ssl/src/ssl_tls_dist_proxy.erl b/lib/ssl/src/ssl_tls_dist_proxy.erl index a22af6b960..d23b42ace5 100644 --- a/lib/ssl/src/ssl_tls_dist_proxy.erl +++ b/lib/ssl/src/ssl_tls_dist_proxy.erl @@ -227,7 +227,10 @@ loop_conn_setup(World, Erts) -> {tcp_closed, Erts} -> ssl:close(World); {ssl_closed, World} -> - gen_tcp:close(Erts) + gen_tcp:close(Erts); + {ssl_error, World, _} -> + + ssl:close(World) end. loop_conn(World, Erts) -> @@ -241,7 +244,9 @@ loop_conn(World, Erts) -> {tcp_closed, Erts} -> ssl:close(World); {ssl_closed, World} -> - gen_tcp:close(Erts) + gen_tcp:close(Erts); + {ssl_error, World, _} -> + ssl:close(World) end. get_ssl_options(Type) -> diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 0577222980..3304ffcddb 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -398,6 +398,23 @@ initial_state(Role, Host, Port, Socket, {SSLOptions, SocketOptions, Tracker}, Us tracker = Tracker }. + +update_ssl_options_from_sni(OrigSSLOptions, SNIHostname) -> + SSLOption = + case OrigSSLOptions#ssl_options.sni_fun of + undefined -> + proplists:get_value(SNIHostname, + OrigSSLOptions#ssl_options.sni_hosts); + SNIFun -> + SNIFun(SNIHostname) + end, + case SSLOption of + undefined -> + undefined; + _ -> + ssl:handle_options(SSLOption, OrigSSLOptions) + end. + next_state(Current,_, #alert{} = Alert, #state{negotiated_version = Version} = State) -> handle_own_alert(Alert, Version, Current, State); @@ -426,15 +443,17 @@ next_state(Current, Next, #ssl_tls{type = ?HANDSHAKE, fragment = Data}, %% This message should not be included in handshake %% message hashes. Already in negotiation so it will be ignored! ?MODULE:SName(Packet, State); - ({#client_hello{} = Packet, Raw}, {next_state, connection = SName, State}) -> + ({#client_hello{} = Packet, Raw}, {next_state, connection = SName, HState0}) -> + HState = handle_sni_extension(Packet, HState0), Version = Packet#client_hello.client_version, Hs0 = ssl_handshake:init_handshake_history(), Hs1 = ssl_handshake:update_handshake_history(Hs0, Raw), - ?MODULE:SName(Packet, State#state{tls_handshake_history=Hs1, - renegotiation = {true, peer}}); - ({Packet, Raw}, {next_state, SName, State = #state{tls_handshake_history=Hs0}}) -> + ?MODULE:SName(Packet, HState#state{tls_handshake_history=Hs1, + renegotiation = {true, peer}}); + ({Packet, Raw}, {next_state, SName, HState0 = #state{tls_handshake_history=Hs0}}) -> + HState = handle_sni_extension(Packet, HState0), Hs1 = ssl_handshake:update_handshake_history(Hs0, Raw), - ?MODULE:SName(Packet, State#state{tls_handshake_history=Hs1}); + ?MODULE:SName(Packet, HState#state{tls_handshake_history=Hs1}); (_, StopState) -> StopState end, try @@ -981,3 +1000,32 @@ convert_options_partial_chain(Options, up) -> list_to_tuple(Head ++ [{partial_chain, fun(_) -> unknown_ca end}] ++ Tail); convert_options_partial_chain(Options, down) -> list_to_tuple(proplists:delete(partial_chain, tuple_to_list(Options))). + +handle_sni_extension(#client_hello{extensions = HelloExtensions}, State0) -> + case HelloExtensions#hello_extensions.sni of + undefined -> + State0; + #sni{hostname = Hostname} -> + NewOptions = update_ssl_options_from_sni(State0#state.ssl_options, Hostname), + case NewOptions of + undefined -> + State0; + _ -> + {ok, Ref, CertDbHandle, FileRefHandle, CacheHandle, CRLDbHandle, OwnCert, Key, DHParams} = + ssl_config:init(NewOptions, State0#state.role), + State0#state{ + session = State0#state.session#session{own_certificate = OwnCert}, + file_ref_db = FileRefHandle, + cert_db_ref = Ref, + cert_db = CertDbHandle, + crl_db = CRLDbHandle, + session_cache = CacheHandle, + private_key = Key, + diffie_hellman_params = DHParams, + ssl_options = NewOptions, + sni_hostname = Hostname + } + end + end; +handle_sni_extension(_, State0) -> + State0. diff --git a/lib/ssl/test/Makefile b/lib/ssl/test/Makefile index 8c45a788a4..886cc7726b 100644 --- a/lib/ssl/test/Makefile +++ b/lib/ssl/test/Makefile @@ -53,6 +53,7 @@ MODULES = \ ssl_to_openssl_SUITE \ ssl_ECC_SUITE \ ssl_upgrade_SUITE\ + ssl_sni_SUITE \ make_certs\ erl_make_certs diff --git a/lib/ssl/test/make_certs.erl b/lib/ssl/test/make_certs.erl index 77631f62d3..4a193d48fe 100644 --- a/lib/ssl/test/make_certs.erl +++ b/lib/ssl/test/make_certs.erl @@ -81,7 +81,7 @@ all(DataDir, PrivDir, C = #config{}) -> create_rnd(DataDir, PrivDir), % For all requests rootCA(PrivDir, "erlangCA", C), intermediateCA(PrivDir, "otpCA", "erlangCA", C), - endusers(PrivDir, "otpCA", ["client", "server", "revoked"], C), + endusers(PrivDir, "otpCA", ["client", "server", "revoked", "a.server", "b.server"], C), endusers(PrivDir, "erlangCA", ["localhost"], C), %% Create keycert files SDir = filename:join([PrivDir, "server"]), diff --git a/lib/ssl/test/ssl_alpn_handshake_SUITE.erl b/lib/ssl/test/ssl_alpn_handshake_SUITE.erl index ccd70fa605..ae76f5849e 100644 --- a/lib/ssl/test/ssl_alpn_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_alpn_handshake_SUITE.erl @@ -384,7 +384,7 @@ ssl_receive_and_assert_alpn(Socket, Protocol, Data) -> ssl_send(Socket, Data) -> ct:log("Connection info: ~p~n", - [ssl:connection_info(Socket)]), + [ssl:connection_information(Socket)]), ssl:send(Socket, Data). ssl_receive(Socket, Data) -> @@ -392,7 +392,7 @@ ssl_receive(Socket, Data) -> ssl_receive(Socket, Data, Buffer) -> ct:log("Connection info: ~p~n", - [ssl:connection_info(Socket)]), + [ssl:connection_information(Socket)]), receive {ssl, Socket, MoreData} -> ct:log("Received ~p~n",[MoreData]), @@ -411,4 +411,4 @@ ssl_receive(Socket, Data, Buffer) -> end. connection_info_result(Socket) -> - ssl:connection_info(Socket). + ssl:connection_information(Socket). diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 50d5fb411f..e1a36dbbd4 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -384,7 +384,7 @@ new_options_in_accept(Config) when is_list(Config) -> %%-------------------------------------------------------------------- connection_info() -> - [{doc,"Test the API function ssl:connection_info/1"}]. + [{doc,"Test the API function ssl:connection_information/1"}]. connection_info(Config) when is_list(Config) -> ClientOpts = ?config(client_opts, Config), ServerOpts = ?config(server_opts, Config), @@ -2831,7 +2831,7 @@ listen_socket(Config) -> {error, enotconn} = ssl:send(ListenSocket, <<"data">>), {error, enotconn} = ssl:recv(ListenSocket, 0), - {error, enotconn} = ssl:connection_info(ListenSocket), + {error, enotconn} = ssl:connection_information(ListenSocket), {error, enotconn} = ssl:peername(ListenSocket), {error, enotconn} = ssl:peercert(ListenSocket), {error, enotconn} = ssl:session_info(ListenSocket), @@ -3445,7 +3445,7 @@ renegotiate_immediately(Socket) -> end, ok = ssl:renegotiate(Socket), {error, renegotiation_rejected} = ssl:renegotiate(Socket), - ct:sleep(?RENEGOTIATION_DISABLE_TIME +1), + ct:sleep(?RENEGOTIATION_DISABLE_TIME + ?SLEEP), ok = ssl:renegotiate(Socket), ct:log("Renegotiated again"), ssl:send(Socket, "Hello world"), @@ -3836,10 +3836,10 @@ cipher(CipherSuite, Version, Config, ClientOpts, ServerOpts) -> end. connection_info_result(Socket) -> - ssl:connection_info(Socket). - + {ok, Info} = ssl:connection_information(Socket, [protocol, cipher_suite]), + {ok, {proplists:get_value(protocol, Info), proplists:get_value(cipher_suite, Info)}}. version_info_result(Socket) -> - {ok, {Version, _}} = ssl:connection_info(Socket), + {ok, [{version, Version}]} = ssl:connection_information(Socket, [version]), {ok, Version}. connect_dist_s(S) -> diff --git a/lib/ssl/test/ssl_npn_handshake_SUITE.erl b/lib/ssl/test/ssl_npn_handshake_SUITE.erl index 326f907e66..8e95679306 100644 --- a/lib/ssl/test/ssl_npn_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_npn_handshake_SUITE.erl @@ -332,7 +332,7 @@ ssl_receive_and_assert_npn(Socket, Protocol, Data) -> ssl_send(Socket, Data) -> ct:log("Connection info: ~p~n", - [ssl:connection_info(Socket)]), + [ssl:connection_information(Socket)]), ssl:send(Socket, Data). ssl_receive(Socket, Data) -> @@ -340,7 +340,7 @@ ssl_receive(Socket, Data) -> ssl_receive(Socket, Data, Buffer) -> ct:log("Connection info: ~p~n", - [ssl:connection_info(Socket)]), + [ssl:connection_information(Socket)]), receive {ssl, Socket, MoreData} -> ct:log("Received ~p~n",[MoreData]), @@ -360,4 +360,4 @@ ssl_receive(Socket, Data, Buffer) -> connection_info_result(Socket) -> - ssl:connection_info(Socket). + ssl:connection_information(Socket). diff --git a/lib/ssl/test/ssl_sni_SUITE.erl b/lib/ssl/test/ssl_sni_SUITE.erl new file mode 100644 index 0000000000..b059ff991b --- /dev/null +++ b/lib/ssl/test/ssl_sni_SUITE.erl @@ -0,0 +1,179 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2015-2015. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% + +-module(ssl_sni_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("public_key/include/public_key.hrl"). + +%%-------------------------------------------------------------------- +%% Common Test interface functions ----------------------------------- +%%-------------------------------------------------------------------- +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> [no_sni_header, + sni_match, + sni_no_match, + no_sni_header_fun, + sni_match_fun, + sni_no_match_fun]. + +init_per_suite(Config0) -> + catch crypto:stop(), + try crypto:start() of + ok -> + ssl:start(), + Result = + (catch make_certs:all(?config(data_dir, Config0), + ?config(priv_dir, Config0))), + ct:log("Make certs ~p~n", [Result]), + ssl_test_lib:cert_options(Config0) + catch _:_ -> + {skip, "Crypto did not start"} + end. + +end_per_suite(_) -> + ssl:stop(), + application:stop(crypto). + +%%-------------------------------------------------------------------- +%% Test Cases -------------------------------------------------------- +%%-------------------------------------------------------------------- +no_sni_header(Config) -> + run_handshake(Config, undefined, undefined, "server"). + +no_sni_header_fun(Config) -> + run_sni_fun_handshake(Config, undefined, undefined, "server"). + +sni_match(Config) -> + run_handshake(Config, "a.server", "a.server", "a.server"). + +sni_match_fun(Config) -> + run_sni_fun_handshake(Config, "a.server", "a.server", "a.server"). + +sni_no_match(Config) -> + run_handshake(Config, "c.server", undefined, "server"). + +sni_no_match_fun(Config) -> + run_sni_fun_handshake(Config, "c.server", undefined, "server"). + + +%%-------------------------------------------------------------------- +%% Internal Functions ------------------------------------------------ +%%-------------------------------------------------------------------- +ssl_recv(SSLSocket, Expect) -> + ssl_recv(SSLSocket, "", Expect). + +ssl_recv(SSLSocket, CurrentData, ExpectedData) -> + receive + {ssl, SSLSocket, Data} -> + NeweData = CurrentData ++ Data, + case NeweData of + ExpectedData -> + ok; + _ -> + ssl_recv(SSLSocket, NeweData, ExpectedData) + end; + Other -> + ct:fail({unexpected_message, Other}) + after 4000 -> + ct:fail({timeout, CurrentData, ExpectedData}) + end. + +send_and_hostname(SSLSocket) -> + ssl:send(SSLSocket, "OK"), + {ok, [{sni_hostname, Hostname}]} = ssl:connection_information(SSLSocket, [sni_hostname]), + Hostname. + +rdnPart([[#'AttributeTypeAndValue'{type=Type, value=Value} | _] | _], Type) -> + Value; +rdnPart([_ | Tail], Type) -> + rdnPart(Tail, Type); +rdnPart([], _) -> + unknown. + +rdn_to_string({utf8String, Binary}) -> + erlang:binary_to_list(Binary); +rdn_to_string({printableString, String}) -> + String. + +recv_and_certificate(SSLSocket) -> + ssl_recv(SSLSocket, "OK"), + {ok, PeerCert} = ssl:peercert(SSLSocket), + #'OTPCertificate'{tbsCertificate = #'OTPTBSCertificate'{subject = {rdnSequence, Subject}}} + = public_key:pkix_decode_cert(PeerCert, otp), + ct:log("Subject of certificate received from server: ~p", [Subject]), + rdn_to_string(rdnPart(Subject, ?'id-at-commonName')). + +run_sni_fun_handshake(Config, SNIHostname, ExpectedSNIHostname, ExpectedCN) -> + ct:log("Start running handshake for sni_fun, Config: ~p, SNIHostname: ~p, " + "ExpectedSNIHostname: ~p, ExpectedCN: ~p", + [Config, SNIHostname, ExpectedSNIHostname, ExpectedCN]), + [{sni_hosts, ServerSNIConf}] = ?config(sni_server_opts, Config), + SNIFun = fun(Domain) -> proplists:get_value(Domain, ServerSNIConf, undefined) end, + ServerOptions = ?config(server_opts, Config) ++ [{sni_fun, SNIFun}], + ClientOptions = + case SNIHostname of + undefined -> + ?config(client_opts, Config); + _ -> + [{server_name_indication, SNIHostname}] ++ ?config(client_opts, Config) + end, + ct:log("Options: ~p", [[ServerOptions, ClientOptions]]), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, {mfa, {?MODULE, send_and_hostname, []}}, + {options, ServerOptions}]), + Port = ssl_test_lib:inet_port(Server), + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, {from, self()}, + {mfa, {?MODULE, recv_and_certificate, []}}, + {options, ClientOptions}]), + ssl_test_lib:check_result(Server, ExpectedSNIHostname, Client, ExpectedCN), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + +run_handshake(Config, SNIHostname, ExpectedSNIHostname, ExpectedCN) -> + ct:log("Start running handshake, Config: ~p, SNIHostname: ~p, " + "ExpectedSNIHostname: ~p, ExpectedCN: ~p", + [Config, SNIHostname, ExpectedSNIHostname, ExpectedCN]), + ServerOptions = ?config(sni_server_opts, Config) ++ ?config(server_opts, Config), + ClientOptions = + case SNIHostname of + undefined -> + ?config(client_opts, Config); + _ -> + [{server_name_indication, SNIHostname}] ++ ?config(client_opts, Config) + end, + ct:log("Options: ~p", [[ServerOptions, ClientOptions]]), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, {mfa, {?MODULE, send_and_hostname, []}}, + {options, ServerOptions}]), + Port = ssl_test_lib:inet_port(Server), + Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, {from, self()}, + {mfa, {?MODULE, recv_and_certificate, []}}, + {options, ClientOptions}]), + ssl_test_lib:check_result(Server, ExpectedSNIHostname, Client, ExpectedCN), + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl index d19e3b7fdb..a3bfdf8893 100644 --- a/lib/ssl/test/ssl_test_lib.erl +++ b/lib/ssl/test/ssl_test_lib.erl @@ -354,6 +354,11 @@ cert_options(Config) -> BadKeyFile = filename:join([?config(priv_dir, Config), "badkey.pem"]), PskSharedSecret = <<1,2,3,4,5,6,7,8,9,10,11,12,13,14,15>>, + + SNIServerACertFile = filename:join([?config(priv_dir, Config), "a.server", "cert.pem"]), + SNIServerAKeyFile = filename:join([?config(priv_dir, Config), "a.server", "key.pem"]), + SNIServerBCertFile = filename:join([?config(priv_dir, Config), "b.server", "cert.pem"]), + SNIServerBKeyFile = filename:join([?config(priv_dir, Config), "b.server", "key.pem"]), [{client_opts, [{ssl_imp, new},{reuseaddr, true}]}, {client_verification_opts, [{cacertfile, ClientCaCertFile}, {certfile, ClientCertFile}, @@ -414,7 +419,17 @@ cert_options(Config) -> {server_bad_cert, [{ssl_imp, new},{cacertfile, ServerCaCertFile}, {certfile, BadCertFile}, {keyfile, ServerKeyFile}]}, {server_bad_key, [{ssl_imp, new},{cacertfile, ServerCaCertFile}, - {certfile, ServerCertFile}, {keyfile, BadKeyFile}]} + {certfile, ServerCertFile}, {keyfile, BadKeyFile}]}, + {sni_server_opts, [{sni_hosts, [ + {"a.server", [ + {certfile, SNIServerACertFile}, + {keyfile, SNIServerAKeyFile} + ]}, + {"b.server", [ + {certfile, SNIServerBCertFile}, + {keyfile, SNIServerBKeyFile} + ]} + ]}]} | Config]. @@ -934,7 +949,8 @@ der_to_pem(File, Entries) -> file:write_file(File, PemBin). cipher_result(Socket, Result) -> - Result = ssl:connection_info(Socket), + {ok, Info} = ssl:connection_information(Socket), + Result = {ok, {proplists:get_value(protocol, Info), proplists:get_value(cipher_suite, Info)}}, ct:log("~p:~p~nSuccessfull connect: ~p~n", [?MODULE,?LINE, Result]), %% Importante to send two packets here %% to properly test "cipher state" handling diff --git a/lib/ssl/test/ssl_to_openssl_SUITE.erl b/lib/ssl/test/ssl_to_openssl_SUITE.erl index 94426a3061..aca34cb6e9 100644 --- a/lib/ssl/test/ssl_to_openssl_SUITE.erl +++ b/lib/ssl/test/ssl_to_openssl_SUITE.erl @@ -50,9 +50,9 @@ all() -> groups() -> [{basic, [], basic_tests()}, - {'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests()}, - {'tlsv1.1', [], all_versions_tests() ++ alpn_tests() ++ npn_tests()}, - {'tlsv1', [], all_versions_tests()++ alpn_tests() ++ npn_tests()}, + {'tlsv1.2', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, + {'tlsv1.1', [], all_versions_tests() ++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, + {'tlsv1', [], all_versions_tests()++ alpn_tests() ++ npn_tests() ++ sni_server_tests()}, {'sslv3', [], all_versions_tests()}]. basic_tests() -> @@ -101,6 +101,14 @@ npn_tests() -> erlang_client_openssl_server_npn_only_client, erlang_client_openssl_server_npn_only_server]. +sni_server_tests() -> + [erlang_server_openssl_client_sni_match, + erlang_server_openssl_client_sni_match_fun, + erlang_server_openssl_client_sni_no_match, + erlang_server_openssl_client_sni_no_match_fun, + erlang_server_openssl_client_sni_no_header, + erlang_server_openssl_client_sni_no_header_fun]. + init_per_suite(Config0) -> Dog = ct:timetrap(?LONG_TIMEOUT *2), @@ -222,6 +230,15 @@ special_init(TestCase, Config) check_openssl_npn_support(Config) end; +special_init(TestCase, Config) + when TestCase == erlang_server_openssl_client_sni_match; + TestCase == erlang_server_openssl_client_sni_no_match; + TestCase == erlang_server_openssl_client_sni_no_header; + TestCase == erlang_server_openssl_client_sni_match_fun; + TestCase == erlang_server_openssl_client_sni_no_match_fun; + TestCase == erlang_server_openssl_client_sni_no_header_fun -> + check_openssl_sni_support(Config); + special_init(_, Config) -> Config. @@ -1181,6 +1198,25 @@ erlang_server_openssl_client_npn_only_client(Config) when is_list(Config) -> ssl_test_lib:check_result(Server, ok) end), ok. +%-------------------------------------------------------------------------- +erlang_server_openssl_client_sni_no_header(Config) when is_list(Config) -> + erlang_server_openssl_client_sni_test(Config, undefined, undefined, "server"). + +erlang_server_openssl_client_sni_no_header_fun(Config) when is_list(Config) -> + erlang_server_openssl_client_sni_test_sni_fun(Config, undefined, undefined, "server"). + +erlang_server_openssl_client_sni_match(Config) when is_list(Config) -> + erlang_server_openssl_client_sni_test(Config, "a.server", "a.server", "a.server"). + +erlang_server_openssl_client_sni_match_fun(Config) when is_list(Config) -> + erlang_server_openssl_client_sni_test_sni_fun(Config, "a.server", "a.server", "a.server"). + +erlang_server_openssl_client_sni_no_match(Config) when is_list(Config) -> + erlang_server_openssl_client_sni_test(Config, "c.server", undefined, "server"). + +erlang_server_openssl_client_sni_no_match_fun(Config) when is_list(Config) -> + erlang_server_openssl_client_sni_test_sni_fun(Config, "c.server", undefined, "server"). + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ @@ -1207,6 +1243,94 @@ run_suites(Ciphers, Version, Config, Type) -> ct:fail(cipher_suite_failed_see_test_case_log) end. +client_read_check([], _Data) -> + ok; +client_read_check([Hd | T], Data) -> + case binary:match(Data, list_to_binary(Hd)) of + nomatch -> + nomatch; + _ -> + client_read_check(T, Data) + end. +client_check_result(Port, DataExpected, DataReceived) -> + receive + {Port, {data, TheData}} -> + Data = list_to_binary(TheData), + NewData = <<DataReceived/binary, Data/binary>>, + ct:log("New Data: ~p", [NewData]), + case client_read_check(DataExpected, NewData) of + ok -> + ok; + _ -> + client_check_result(Port, DataExpected, NewData) + end + after 3000 -> + ct:fail({"Time out on opensssl Client", {expected, DataExpected}, + {got, DataReceived}}) + end. +client_check_result(Port, DataExpected) -> + client_check_result(Port, DataExpected, <<"">>). + +send_and_hostname(SSLSocket) -> + ssl:send(SSLSocket, "OK"), + {ok, [{sni_hostname, Hostname}]} = ssl:connection_information(SSLSocket, [sni_hostname]), + Hostname. + +erlang_server_openssl_client_sni_test(Config, SNIHostname, ExpectedSNIHostname, ExpectedCN) -> + ct:log("Start running handshake, Config: ~p, SNIHostname: ~p, ExpectedSNIHostname: ~p, ExpectedCN: ~p", [Config, SNIHostname, ExpectedSNIHostname, ExpectedCN]), + ServerOptions = ?config(sni_server_opts, Config) ++ ?config(server_opts, Config), + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, {mfa, {?MODULE, send_and_hostname, []}}, + {options, ServerOptions}]), + Port = ssl_test_lib:inet_port(Server), + ClientCommand = case SNIHostname of + undefined -> + "openssl s_client -connect " ++ Hostname ++ ":" ++ integer_to_list(Port); + _ -> + "openssl s_client -connect " ++ Hostname ++ ":" ++ integer_to_list(Port) ++ " -servername " ++ SNIHostname + end, + ct:log("Options: ~p", [[ServerOptions, ClientCommand]]), + ClientPort = open_port({spawn, ClientCommand}, [stderr_to_stdout]), + + %% Client check needs to be done befor server check, + %% or server check might consume client messages + ExpectedClientOutput = ["OK", "/CN=" ++ ExpectedCN ++ "/"], + client_check_result(ClientPort, ExpectedClientOutput), + ssl_test_lib:check_result(Server, ExpectedSNIHostname), + ssl_test_lib:close_port(ClientPort), + ssl_test_lib:close(Server), + ok. + + +erlang_server_openssl_client_sni_test_sni_fun(Config, SNIHostname, ExpectedSNIHostname, ExpectedCN) -> + ct:log("Start running handshake for sni_fun, Config: ~p, SNIHostname: ~p, ExpectedSNIHostname: ~p, ExpectedCN: ~p", [Config, SNIHostname, ExpectedSNIHostname, ExpectedCN]), + [{sni_hosts, ServerSNIConf}] = ?config(sni_server_opts, Config), + SNIFun = fun(Domain) -> proplists:get_value(Domain, ServerSNIConf, undefined) end, + ServerOptions = ?config(server_opts, Config) ++ [{sni_fun, SNIFun}], + {_, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, {mfa, {?MODULE, send_and_hostname, []}}, + {options, ServerOptions}]), + Port = ssl_test_lib:inet_port(Server), + ClientCommand = case SNIHostname of + undefined -> + "openssl s_client -connect " ++ Hostname ++ ":" ++ integer_to_list(Port); + _ -> + "openssl s_client -connect " ++ Hostname ++ ":" ++ integer_to_list(Port) ++ " -servername " ++ SNIHostname + end, + ct:log("Options: ~p", [[ServerOptions, ClientCommand]]), + ClientPort = open_port({spawn, ClientCommand}, [stderr_to_stdout]), + + %% Client check needs to be done befor server check, + %% or server check might consume client messages + ExpectedClientOutput = ["OK", "/CN=" ++ ExpectedCN ++ "/"], + client_check_result(ClientPort, ExpectedClientOutput), + ssl_test_lib:check_result(Server, ExpectedSNIHostname), + ssl_test_lib:close_port(ClientPort), + ssl_test_lib:close(Server). + + cipher(CipherSuite, Version, Config, ClientOpts, ServerOpts) -> process_flag(trap_exit, true), ct:log("Testing CipherSuite ~p~n", [CipherSuite]), @@ -1545,7 +1669,7 @@ erlang_ssl_receive_and_assert_negotiated_protocol(Socket, Protocol, Data) -> erlang_ssl_receive(Socket, Data) -> ct:log("Connection info: ~p~n", - [ssl:connection_info(Socket)]), + [ssl:connection_information(Socket)]), receive {ssl, Socket, Data} -> io:format("Received ~p~n",[Data]), @@ -1564,16 +1688,16 @@ erlang_ssl_receive(Socket, Data) -> end. connection_info(Socket, Version) -> - case ssl:connection_info(Socket) of - {ok, {Version, _} = Info} -> + case ssl:connection_information(Socket, [version]) of + {ok, [{version, Version}] = Info} -> ct:log("Connection info: ~p~n", [Info]), ok; - {ok, {OtherVersion, _}} -> + {ok, [{version, OtherVersion}]} -> {wrong_version, OtherVersion} end. connection_info_result(Socket) -> - ssl:connection_info(Socket). + ssl:connection_information(Socket). delayed_send(Socket, [ErlData, OpenSslData]) -> @@ -1588,6 +1712,14 @@ server_sent_garbage(Socket) -> end. +check_openssl_sni_support(Config) -> + HelpText = os:cmd("openssl s_client --help"), + case string:str(HelpText, "-servername") of + 0 -> + {skip, "Current openssl doesn't support SNI"}; + _ -> + Config + end. check_openssl_npn_support(Config) -> HelpText = os:cmd("openssl s_client --help"), diff --git a/lib/stdlib/doc/src/c.xml b/lib/stdlib/doc/src/c.xml index b49fa6ad67..b43d4786ae 100644 --- a/lib/stdlib/doc/src/c.xml +++ b/lib/stdlib/doc/src/c.xml @@ -232,6 +232,14 @@ compile:file(<anno>File</anno>, <anno>Options</anno> ++ [report_errors, report_w </desc> </func> <func> + <name name="uptime" arity="0"/> + <fsummary>Print node uptime</fsummary> + <desc> + <p>Prints the node uptime (as given by + <c>erlang:statistics(wall_clock)</c>), in human-readable form.</p> + </desc> + </func> + <func> <name>xm(ModSpec) -> void()</name> <fsummary>Cross reference check a module</fsummary> <type> diff --git a/lib/stdlib/doc/src/ets.xml b/lib/stdlib/doc/src/ets.xml index 6b9524ef63..2bfe074c3e 100644 --- a/lib/stdlib/doc/src/ets.xml +++ b/lib/stdlib/doc/src/ets.xml @@ -1435,7 +1435,9 @@ is_integer(X), is_integer(Y), X + Y < 4711]]></code> <p>Whenever the <c>extended_info</c> option is used, it results in a file not readable by versions of ets prior to that in stdlib-1.15.1</p> - + <p>The <c>sync</c> option, if set to <c>true</c>, ensures that + the content of the file is actually written to the disk before + <c>tab2file</c> returns. Default is <c>{sync, false}</c>.</p> </desc> </func> <func> diff --git a/lib/stdlib/doc/src/gb_sets.xml b/lib/stdlib/doc/src/gb_sets.xml index ea96c14472..405bae5698 100644 --- a/lib/stdlib/doc/src/gb_sets.xml +++ b/lib/stdlib/doc/src/gb_sets.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2001</year><year>2014</year> + <year>2001</year><year>2015</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -306,6 +306,17 @@ </desc> </func> <func> + <name name="iterator_from" arity="2"/> + <fsummary>Return an iterator for a set starting from a specified element</fsummary> + <desc> + <p>Returns an iterator that can be used for traversing the + entries of <c><anno>Set</anno></c>; see <c>next/1</c>. + The difference as compared to the iterator returned by + <c>iterator/1</c> is that the first element greater than + or equal to <c><anno>Element</anno></c> is returned.</p> + </desc> + </func> + <func> <name name="largest" arity="1"/> <fsummary>Return largest element</fsummary> <desc> diff --git a/lib/stdlib/doc/src/gb_trees.xml b/lib/stdlib/doc/src/gb_trees.xml index b2f237e1d7..82167e1083 100644 --- a/lib/stdlib/doc/src/gb_trees.xml +++ b/lib/stdlib/doc/src/gb_trees.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2001</year><year>2014</year> + <year>2001</year><year>2015</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -183,6 +183,17 @@ </desc> </func> <func> + <name name="iterator_from" arity="2"/> + <fsummary>Return an iterator for a tree starting from specified key</fsummary> + <desc> + <p>Returns an iterator that can be used for traversing the + entries of <c><anno>Tree</anno></c>; see <c>next/1</c>. + The difference as compared to the iterator returned by + <c>iterator/1</c> is that the first key greater than + or equal to <c><anno>Key</anno></c> is returned.</p> + </desc> + </func> + <func> <name name="keys" arity="1"/> <fsummary>Return a list of the keys in a tree</fsummary> <desc> diff --git a/lib/stdlib/doc/src/maps.xml b/lib/stdlib/doc/src/maps.xml index e46068230a..7345a9357a 100644 --- a/lib/stdlib/doc/src/maps.xml +++ b/lib/stdlib/doc/src/maps.xml @@ -33,6 +33,28 @@ <funcs> <func> + <name name="filter" arity="2"/> + <fsummary>Choose pairs which satisfy a predicate</fsummary> + <desc> + <p> + Returns a map <c><anno>Map2</anno></c> for which predicate + <c><anno>Pred</anno></c> holds true in <c><anno>Map1</anno></c>. + </p> + <p> + The call will fail with a <c>{badmap,Map}</c> exception if + <c><anno>Map1</anno></c> is not a map or with <c>badarg</c> if + <c><anno>Pred</anno></c> is not a function of arity 2. + </p> + <p>Example:</p> + <code type="none"> +> M = #{a => 2, b => 3, c=> 4, "a" => 1, "b" => 2, "c" => 4}, + Pred = fun(K,V) -> is_atom(K) andalso (V rem 2) =:= 0 end, + maps:filter(Pred,M). +#{a => 2,c => 4} </code> + </desc> + </func> + + <func> <name name="find" arity="2"/> <fsummary></fsummary> <desc> diff --git a/lib/stdlib/doc/src/orddict.xml b/lib/stdlib/doc/src/orddict.xml index ec1e43f29c..c853b402d4 100644 --- a/lib/stdlib/doc/src/orddict.xml +++ b/lib/stdlib/doc/src/orddict.xml @@ -48,8 +48,11 @@ <datatypes> <datatype> - <name name="orddict"/> - <desc><p>As returned by new/0.</p></desc> + <name name="orddict" n_vars="2"/> + <desc><p>Dictionary as returned by <c>new/0</c>.</p></desc> + </datatype> + <datatype> + <name name="orddict" n_vars="0"/> </datatype> </datatypes> diff --git a/lib/stdlib/doc/src/supervisor.xml b/lib/stdlib/doc/src/supervisor.xml index ffac1c0bd7..6ff477a42d 100644 --- a/lib/stdlib/doc/src/supervisor.xml +++ b/lib/stdlib/doc/src/supervisor.xml @@ -386,9 +386,15 @@ added to the supervisor and the function returns the same value.</p> <p>If the child process start function returns <c>ignore</c>, - the child specification is added to the supervisor, the pid - is set to <c>undefined</c>, and the function returns - <c>{ok,undefined}</c>.</p> + the child specification is added to the supervisor (unless the + supervisor is a <c>simple_one_for_one</c> supervisor, see below), + the pid is set to <c>undefined</c> and the function returns + <c>{ok,undefined}</c>. + </p> + <p>In the case of a <c>simple_one_for_one</c> supervisor, when a child + process start function returns <c>ignore</c> the functions returns + <c>{ok,undefined}</c> and no child is added to the supervisor. + </p> <p>If the child process start function returns an error tuple or an erroneous value, or if it fails, the child specification is discarded, and the function returns <c>{error,Error}</c> where diff --git a/lib/stdlib/src/c.erl b/lib/stdlib/src/c.erl index 9860adf04d..d5b24d3c32 100644 --- a/lib/stdlib/src/c.erl +++ b/lib/stdlib/src/c.erl @@ -27,7 +27,7 @@ lc_batch/0, lc_batch/1, i/3,pid/3,m/0,m/1, bt/1, q/0, - erlangrc/0,erlangrc/1,bi/1, flush/0, regs/0, + erlangrc/0,erlangrc/1,bi/1, flush/0, regs/0, uptime/0, nregs/0,pwd/0,ls/0,ls/1,cd/1,memory/1,memory/0, xm/1]). -export([display_info/1]). @@ -65,6 +65,7 @@ help() -> "q() -- quit - shorthand for init:stop()\n" "regs() -- information about registered processes\n" "nregs() -- information about all registered processes\n" + "uptime() -- print node uptime\n" "xm(M) -- cross reference check a module\n" "y(File) -- generate a Yecc parser\n">>). @@ -774,6 +775,26 @@ memory() -> erlang:memory(). memory(TypeSpec) -> erlang:memory(TypeSpec). %% +%% uptime/0 +%% + +-spec uptime() -> 'ok'. + +uptime() -> + io:format("~s~n", [uptime(get_uptime())]). + +uptime({D, {H, M, S}}) -> + lists:flatten( + [[ io_lib:format("~p days, ", [D]) || D > 0 ], + [ io_lib:format("~p hours, ", [H]) || D+H > 0 ], + [ io_lib:format("~p minutes and ", [M]) || D+H+M > 0 ], + io_lib:format("~p seconds", [S])]). + +get_uptime() -> + {UpTime, _} = erlang:statistics(wall_clock), + calendar:seconds_to_daystime(UpTime div 1000). + +%% %% Cross Reference Check %% %%-spec xm(module() | file:filename()) -> xref:m/1 return diff --git a/lib/stdlib/src/digraph.erl b/lib/stdlib/src/digraph.erl index 0c21271529..1f8caa88a4 100644 --- a/lib/stdlib/src/digraph.erl +++ b/lib/stdlib/src/digraph.erl @@ -36,7 +36,7 @@ -export([get_short_path/3, get_short_cycle/2]). --export_type([graph/0, d_type/0, vertex/0, edge/0]). +-export_type([graph/0, d_type/0, vertex/0, edge/0, label/0]). -record(digraph, {vtab = notable :: ets:tab(), etab = notable :: ets:tab(), diff --git a/lib/stdlib/src/erl_anno.erl b/lib/stdlib/src/erl_anno.erl index 963b7278a6..fa83375c34 100644 --- a/lib/stdlib/src/erl_anno.erl +++ b/lib/stdlib/src/erl_anno.erl @@ -147,12 +147,10 @@ is_anno2(_, _) -> false. is_filename(T) -> - is_string(T) orelse is_binary(T). + is_list(T) orelse is_binary(T). is_string(T) -> - try lists:all(fun(C) when is_integer(C), C >= 0 -> true end, T) - catch _:_ -> false - end. + is_list(T). -spec column(Anno) -> column() | 'undefined' when Anno :: anno(). diff --git a/lib/stdlib/src/erl_lint.erl b/lib/stdlib/src/erl_lint.erl index 821d81a6b4..b13848c501 100644 --- a/lib/stdlib/src/erl_lint.erl +++ b/lib/stdlib/src/erl_lint.erl @@ -120,13 +120,13 @@ value_option(Flag, Default, On, OnVal, Off, OffVal, Opts) -> func=[], %Current function warn_format=0, %Warn format calls enabled_warnings=[], %All enabled warnings (ordset). + nowarn_bif_clash=[], %All no warn bif clashes (ordset). errors=[], %Current errors warnings=[], %Current warnings file = "" :: string(), %From last file attribute recdef_top=false :: boolean(), %true in record initialisation %outside any fun or lc xqlc= false :: boolean(), %true if qlc.hrl included - new = false :: boolean(), %Has user-defined 'new/N' called= [] :: [{fa(),line()}], %Called functions usage = #usage{} :: #usage{}, specs = dict:new() %Type specifications @@ -569,6 +569,7 @@ start(File, Opts) -> warn_format = value_option(warn_format, 1, warn_format, 1, nowarn_format, 0, Opts), enabled_warnings = Enabled, + nowarn_bif_clash = nowarn_function(nowarn_bif_clash, Opts), file = File }. @@ -608,22 +609,30 @@ pack_warnings(Ws) -> %% add_warning(ErrorDescriptor, State) -> State' %% add_warning(Line, Error, State) -> State' -add_error(E, St) -> St#lint{errors=[{St#lint.file,E}|St#lint.errors]}. +add_error(E, St) -> add_lint_error(E, St#lint.file, St). add_error(Anno, E, St) -> - {File,Location} = loc(Anno), - add_error({Location,erl_lint,E}, St#lint{file = File}). + {File,Location} = loc(Anno, St), + add_lint_error({Location,erl_lint,E}, File, St). -add_warning(W, St) -> St#lint{warnings=[{St#lint.file,W}|St#lint.warnings]}. +add_lint_error(E, File, St) -> + St#lint{errors=[{File,E}|St#lint.errors]}. + +add_warning(W, St) -> add_lint_warning(W, St#lint.file, St). add_warning(FileLine, W, St) -> - {File,Location} = loc(FileLine), - add_warning({Location,erl_lint,W}, St#lint{file = File}). + {File,Location} = loc(FileLine, St), + add_lint_warning({Location,erl_lint,W}, File, St). + +add_lint_warning(W, File, St) -> + St#lint{warnings=[{File,W}|St#lint.warnings]}. -loc(Anno) -> - File = erl_anno:file(Anno), +loc(Anno, St) -> Location = erl_anno:location(Anno), - {File,Location}. + case erl_anno:file(Anno) of + undefined -> {St#lint.file,Location}; + File -> {File,Location} + end. %% forms([Form], State) -> State' @@ -640,8 +649,6 @@ forms(Forms0, St0) -> St4 = foldl(fun form/2, pre_scan(Forms, St3), Forms), post_traversal_check(Forms, St4). -pre_scan([{function,_L,new,_A,_Cs} | Fs], St) -> - pre_scan(Fs, St#lint{new=true}); pre_scan([{attribute,L,compile,C} | Fs], St) -> case is_warn_enabled(export_all, St) andalso member(export_all, lists:flatten([C])) of @@ -668,11 +675,21 @@ eval_file_attribute(Forms, St) -> eval_file_attr([{attribute,_L,file,{File,_Line}}=Form | Forms], _File) -> [Form | eval_file_attr(Forms, File)]; eval_file_attr([Form0 | Forms], File) -> - Form = set_file(Form0, File), + Form = set_form_file(Form0, File), [Form | eval_file_attr(Forms, File)]; eval_file_attr([], _File) -> []. +%% Sets the file only on the form. This is used on post-traversal. +%% For the remaining of the AST we rely on #lint.file. + +set_form_file({attribute,L,K,V}, File) -> + {attribute,erl_anno:set_file(File, L),K,V}; +set_form_file({function,L,N,A,C}, File) -> + {function,erl_anno:set_file(File, L),N,A,C}; +set_form_file(Form, _File) -> + Form. + set_file(T, File) -> F = fun(Anno) -> erl_anno:set_file(File, Anno) end, erl_parse:map_anno(F, T). @@ -772,8 +789,7 @@ eof(_Line, St0) -> %% bif_clashes(Forms, State0) -> State. -bif_clashes(Forms, St) -> - Nowarn = nowarn_function(nowarn_bif_clash, St#lint.compile), +bif_clashes(Forms, #lint{nowarn_bif_clash=Nowarn} = St) -> Clashes0 = [{Name,Arity} || {function,_L,Name,Arity,_Cs} <- Forms, erl_internal:bif(Name, Arity)], Clashes = ordsets:subtract(ordsets:from_list(Clashes0), Nowarn), @@ -798,10 +814,10 @@ disallowed_compile_flags(Forms, St0) -> %% There are (still) no line numbers in St0#lint.compile. Errors0 = [ {St0#lint.file,{L,erl_lint,disallowed_nowarn_bif_clash}} || {attribute,A,compile,nowarn_bif_clash} <- Forms, - {_,L} <- [loc(A)] ], + {_,L} <- [loc(A, St0)] ], Errors1 = [ {St0#lint.file,{L,erl_lint,disallowed_nowarn_bif_clash}} || {attribute,A,compile,{nowarn_bif_clash, {_,_}}} <- Forms, - {_,L} <- [loc(A)] ], + {_,L} <- [loc(A, St0)] ], Disabled = (not is_warn_enabled(bif_clash, St0)), Errors = if Disabled andalso Errors0 =:= [] -> @@ -926,7 +942,7 @@ behaviour_conflicting(AllBfs, St) -> behaviour_add_conflicts(R, St). behaviour_add_conflicts([{Cb,[{FirstLoc,FirstB}|Cs]}|T], St0) -> - FirstL = element(2, loc(FirstLoc)), + FirstL = element(2, loc(FirstLoc, St0)), St = behaviour_add_conflict(Cs, Cb, FirstL, FirstB, St0), behaviour_add_conflicts(T, St); behaviour_add_conflicts([], St) -> St. @@ -1144,7 +1160,7 @@ check_unused_records(Forms, St0) -> end, St0#lint.records, UsedRecords), Unused = [{Name,FileLine} || {Name,{FileLine,_Fields}} <- dict:to_list(URecs), - element(1, loc(FileLine)) =:= FirstFile], + element(1, loc(FileLine, St0)) =:= FirstFile], foldl(fun ({N,L}, St) -> add_warning(L, {unused_record, N}, St) end, St0, Unused); @@ -1337,14 +1353,15 @@ check_on_load(St) -> St. -spec call_function(line(), atom(), arity(), lint_state()) -> lint_state(). %% Add to both called and calls. -call_function(Line, F, A, #lint{usage=Usage0,called=Cd,func=Func}=St) -> +call_function(Line, F, A, #lint{usage=Usage0,called=Cd,func=Func,file=File}=St) -> #usage{calls = Cs} = Usage0, NA = {F,A}, Usage = case Cs of undefined -> Usage0; _ -> Usage0#usage{calls=dict:append(Func, NA, Cs)} end, - St#lint{called=[{NA,Line}|Cd], usage=Usage}. + Anno = erl_anno:set_file(File, Line), + St#lint{called=[{NA,Anno}|Cd], usage=Usage}. %% function(Line, Name, Arity, Clauses, State) -> State. @@ -2123,7 +2140,7 @@ expr({'receive',Line,Cs,To,ToEs}, Vt, St0) -> {Cvt,St3} = icrt_clauses(Cs, Vt, St2), %% Csvts = [vtnew(Tevt, Vt)|Cvt], %This is just NEW variables! Csvts = [Tevt|Cvt], - Rvt = icrt_export(Csvts, Vt, {'receive',Line}), + Rvt = icrt_export(Csvts, Vt, {'receive',Line}, St3), {vtmerge([Tvt,Tevt,Rvt]),St3}; expr({'fun',Line,Body}, Vt, St) -> %%No one can think funs export! @@ -2826,10 +2843,9 @@ check_record_types([{type, _, field_type, [{atom, AL, FName}, Type]}|Left], check_record_types([], _Name, _DefFields, SeenVars, St, _SeenFields) -> {SeenVars, St}. -used_type(TypePair, L, St) -> - Usage = St#lint.usage, +used_type(TypePair, L, #lint{usage = Usage, file = File} = St) -> OldUsed = Usage#usage.used_types, - UsedTypes = dict:store(TypePair, L, OldUsed), + UsedTypes = dict:store(TypePair, erl_anno:set_file(File, L), OldUsed), St#lint{usage=Usage#usage{used_types=UsedTypes}}. is_default_type({Name, NumberOfTypeVariables}) -> @@ -2984,7 +3000,7 @@ check_unused_types(Forms, #lint{usage=Usage, types=Ts, exp_types=ExpTs}=St) -> UsedTypes = gb_sets:from_list(L), FoldFun = fun(Type, #typeinfo{line = FileLine}, AccSt) -> - case loc(FileLine) of + case loc(FileLine, AccSt) of {FirstFile, _} -> case gb_sets:is_member(Type, UsedTypes) of true -> AccSt; @@ -3022,7 +3038,7 @@ check_local_opaque_types(St) -> icrt_clauses(Cs, In, Vt, St0) -> {Csvt,St1} = icrt_clauses(Cs, Vt, St0), - UpdVt = icrt_export(Csvt, Vt, In), + UpdVt = icrt_export(Csvt, Vt, In, St1), {UpdVt,St1}. %% icrt_clauses(Clauses, ImportVarTable, State) -> @@ -3039,8 +3055,8 @@ icrt_clause({clause,_Line,H,G,B}, Vt0, St0) -> {Bvt,St3} = exprs(B, vtupdate(Vt2, Vt0), St2), {vtupdate(Bvt, Vt2),St3}. -icrt_export(Vts, Vt, {Tag,Attrs}) -> - {_File,Loc} = loc(Attrs), +icrt_export(Vts, Vt, {Tag,Attrs}, St) -> + {_File,Loc} = loc(Attrs, St), icrt_export(lists:merge(Vts), Vt, {Tag,Loc}, length(Vts), []). icrt_export([{V,{{export,_},_,_}}|Vs0], [{V,{{export,_}=S0,_,Ls}}|Vt], @@ -3397,7 +3413,7 @@ vtupdate(Uvt, Vt0) -> %% Return all new variables in UpdVarTable as unsafe. vtunsafe({Tag,FileLine}, Uvt, Vt) -> - {_File,Line} = loc(FileLine), + Line = erl_anno:location(FileLine), [{V,{{unsafe,{Tag,Line}},U,Ls}} || {V,{_,U,Ls}} <- vtnew(Uvt, Vt)]. %% vtmerge(VarTable, VarTable) -> VarTable. @@ -3781,8 +3797,7 @@ is_autoimport_suppressed(NoAutoSet,{Func,Arity}) -> gb_sets:is_element({Func,Arity},NoAutoSet). %% Predicate to find out if a function specific bif-clash suppression (old deprecated) is present bif_clash_specifically_disabled(St,{F,A}) -> - Nowarn = nowarn_function(nowarn_bif_clash, St#lint.compile), - lists:member({F,A},Nowarn). + lists:member({F,A},St#lint.nowarn_bif_clash). %% Predicate to find out if an autoimported guard_bif is not overriden in some way %% Guard Bif without module name is disallowed if diff --git a/lib/stdlib/src/ets.erl b/lib/stdlib/src/ets.erl index 1df069755d..0e2d59d0c3 100644 --- a/lib/stdlib/src/ets.erl +++ b/lib/stdlib/src/ets.erl @@ -739,7 +739,8 @@ do_filter(Tab, Key, F, A, Ack) -> -record(filetab_options, { object_count = false :: boolean(), - md5sum = false :: boolean() + md5sum = false :: boolean(), + sync = false :: boolean() }). -spec tab2file(Tab, Filename) -> 'ok' | {'error', Reason} when @@ -754,7 +755,7 @@ tab2file(Tab, File) -> Tab :: tab(), Filename :: file:name(), Options :: [Option], - Option :: {'extended_info', [ExtInfo]}, + Option :: {'extended_info', [ExtInfo]} | {'sync', boolean()}, ExtInfo :: 'md5sum' | 'object_count', Reason :: term(). @@ -835,6 +836,15 @@ tab2file(Tab, File, Options) -> List -> LogFun(NewState1,[['$end_of_table',List]]) end, + case FtOptions#filetab_options.sync of + true -> + case disk_log:sync(Name) of + ok -> ok; + {error, Reason2} -> throw(Reason2) + end; + false -> + ok + end, disk_log:close(Name) catch throw:TReason -> @@ -887,23 +897,24 @@ md5terms(State, [H|T]) -> {FinState, [B|TL]}. parse_ft_options(Options) when is_list(Options) -> - {Opt,Rest} = case (catch lists:keytake(extended_info,1,Options)) of - false -> - {[],Options}; - {value,{extended_info,L},R} when is_list(L) -> - {L,R} - end, - case Rest of - [] -> - parse_ft_info_options(#filetab_options{}, Opt); - Other -> - throw({unknown_option, Other}) - end; -parse_ft_options(Malformed) -> + {ok, parse_ft_options(Options, #filetab_options{}, false)}. + +parse_ft_options([], FtOpt, _) -> + FtOpt; +parse_ft_options([{sync,true} | Rest], FtOpt, EI) -> + parse_ft_options(Rest, FtOpt#filetab_options{sync = true}, EI); +parse_ft_options([{sync,false} | Rest], FtOpt, EI) -> + parse_ft_options(Rest, FtOpt, EI); +parse_ft_options([{extended_info,L} | Rest], FtOpt0, false) -> + FtOpt1 = parse_ft_info_options(FtOpt0, L), + parse_ft_options(Rest, FtOpt1, true); +parse_ft_options([Other | _], _, _) -> + throw({unknown_option, Other}); +parse_ft_options(Malformed, _, _) -> throw({malformed_option, Malformed}). parse_ft_info_options(FtOpt,[]) -> - {ok,FtOpt}; + FtOpt; parse_ft_info_options(FtOpt,[object_count | T]) -> parse_ft_info_options(FtOpt#filetab_options{object_count = true}, T); parse_ft_info_options(FtOpt,[md5sum | T]) -> diff --git a/lib/stdlib/src/gb_sets.erl b/lib/stdlib/src/gb_sets.erl index 393fb07229..d3fbd542f7 100644 --- a/lib/stdlib/src/gb_sets.erl +++ b/lib/stdlib/src/gb_sets.erl @@ -137,6 +137,10 @@ %% approach is that it does not require the complete list of all %% elements to be built in memory at one time. %% +%% - iterator_from(X, S): returns an iterator that can be used for +%% traversing the elements of set S greater than or equal to X; +%% see `next'. +%% %% - next(T): returns {X, T1} where X is the smallest element referred %% to by the iterator T, and T1 is the new iterator to be used for %% traversing the remaining elements, or the atom `none' if no @@ -157,8 +161,8 @@ insert/2, add/2, delete/2, delete_any/2, balance/1, union/2, union/1, intersection/2, intersection/1, is_disjoint/2, difference/2, is_subset/2, to_list/1, from_list/1, from_ordset/1, smallest/1, - largest/1, take_smallest/1, take_largest/1, iterator/1, next/1, - filter/2, fold/3, is_set/1]). + largest/1, take_smallest/1, take_largest/1, iterator/1, + iterator_from/2, next/1, filter/2, fold/3, is_set/1]). %% `sets' compatibility aliases: @@ -500,6 +504,22 @@ iterator({_, L, _} = T, As) -> iterator(nil, As) -> As. +-spec iterator_from(Element, Set) -> Iter when + Set :: set(Element), + Iter :: iter(Element). + +iterator_from(S, {_, T}) -> + iterator_from(S, T, []). + +iterator_from(S, {K, _, T}, As) when K < S -> + iterator_from(S, T, As); +iterator_from(_, {_, nil, _} = T, As) -> + [T | As]; +iterator_from(S, {_, L, _} = T, As) -> + iterator_from(S, L, [T | As]); +iterator_from(_, nil, As) -> + As. + -spec next(Iter1) -> {Element, Iter2} | 'none' when Iter1 :: iter(Element), Iter2 :: iter(Element). diff --git a/lib/stdlib/src/gb_trees.erl b/lib/stdlib/src/gb_trees.erl index 7069b61873..259e8f718b 100644 --- a/lib/stdlib/src/gb_trees.erl +++ b/lib/stdlib/src/gb_trees.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2014. All Rights Reserved. +%% Copyright Ericsson AB 2001-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -102,6 +102,10 @@ %% approach is that it does not require the complete list of all %% elements to be built in memory at one time. %% +%% - iterator_from(K, T): returns an iterator that can be used for +%% traversing the entries of tree T with key greater than or +%% equal to K; see `next'. +%% %% - next(S): returns {X, V, S1} where X is the smallest key referred to %% by the iterator S, and S1 is the new iterator to be used for %% traversing the remaining entries, or the atom `none' if no entries @@ -117,7 +121,7 @@ update/3, enter/3, delete/2, delete_any/2, balance/1, is_defined/2, keys/1, values/1, to_list/1, from_orddict/1, smallest/1, largest/1, take_smallest/1, take_largest/1, - iterator/1, next/1, map/2]). + iterator/1, iterator_from/2, next/1, map/2]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -529,6 +533,29 @@ iterator({_, _, L, _} = T, As) -> iterator(nil, As) -> As. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +-spec iterator_from(Key, Tree) -> Iter when + Tree :: tree(Key, Value), + Iter :: iter(Key, Value). + +iterator_from(S, {_, T}) -> + iterator_1_from(S, T). + +iterator_1_from(S, T) -> + iterator_from(S, T, []). + +iterator_from(S, {K, _, _, T}, As) when K < S -> + iterator_from(S, T, As); +iterator_from(_, {_, _, nil, _} = T, As) -> + [T | As]; +iterator_from(S, {_, _, L, _} = T, As) -> + iterator_from(S, L, [T | As]); +iterator_from(_, nil, As) -> + As. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + -spec next(Iter1) -> 'none' | {Key, Value, Iter2} when Iter1 :: iter(Key, Value), Iter2 :: iter(Key, Value). diff --git a/lib/stdlib/src/maps.erl b/lib/stdlib/src/maps.erl index 3877c150ec..533ff08726 100644 --- a/lib/stdlib/src/maps.erl +++ b/lib/stdlib/src/maps.erl @@ -19,7 +19,8 @@ -module(maps). --export([get/3,fold/3, map/2, size/1, +-export([get/3,filter/2,fold/3, map/2, + size/1, without/2, with/2]). @@ -145,6 +146,19 @@ get(Key,Map,Default) -> erlang:error({badmap,Map},[Key,Map,Default]). +-spec filter(Pred,Map1) -> Map2 when + Pred :: fun((Key, Value) -> boolean()), + Key :: term(), + Value :: term(), + Map1 :: map(), + Map2 :: map(). + +filter(Pred,Map) when is_function(Pred,2), is_map(Map) -> + maps:from_list([{K,V}||{K,V}<-maps:to_list(Map),Pred(K,V)]); +filter(Pred,Map) -> + erlang:error(error_type(Map),[Pred,Map]). + + -spec fold(Fun,Init,Map) -> Acc when Fun :: fun((K, V, AccIn) -> AccOut), Init :: term(), @@ -169,10 +183,7 @@ fold(Fun,Init,Map) -> V2 :: term(). map(Fun,Map) when is_function(Fun, 2), is_map(Map) -> - maps:from_list(lists:map(fun - ({K,V}) -> - {K,Fun(K,V)} - end,maps:to_list(Map))); + maps:from_list([{K,Fun(K,V)}||{K,V}<-maps:to_list(Map)]); map(Fun,Map) -> erlang:error(error_type(Map),[Fun,Map]). diff --git a/lib/stdlib/src/orddict.erl b/lib/stdlib/src/orddict.erl index af5d917840..cbdf25d757 100644 --- a/lib/stdlib/src/orddict.erl +++ b/lib/stdlib/src/orddict.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -25,11 +25,13 @@ -export([store/3,append/3,append_list/3,update/3,update/4,update_counter/3]). -export([fold/3,map/2,filter/2,merge/3]). --export_type([orddict/0]). +-export_type([orddict/0, orddict/2]). %%--------------------------------------------------------------------------- --type orddict() :: [{Key :: term(), Value :: term()}]. +-type orddict() :: orddict(_, _). + +-type orddict(Key, Value) :: [{Key, Value}]. %%--------------------------------------------------------------------------- @@ -38,8 +40,7 @@ new() -> []. -spec is_key(Key, Orddict) -> boolean() when - Key :: term(), - Orddict :: orddict(). + Orddict :: orddict(Key, Value :: term()). is_key(Key, [{K,_}|_]) when Key < K -> false; is_key(Key, [{K,_}|Dict]) when Key > K -> is_key(Key, Dict); @@ -47,14 +48,14 @@ is_key(_Key, [{_K,_Val}|_]) -> true; %Key == K is_key(_, []) -> false. -spec to_list(Orddict) -> List when - Orddict :: orddict(), - List :: [{Key :: term(), Value :: term()}]. + Orddict :: orddict(Key, Value), + List :: [{Key, Value}]. to_list(Dict) -> Dict. -spec from_list(List) -> Orddict when - List :: [{Key :: term(), Value :: term()}], - Orddict :: orddict(). + List :: [{Key, Value}], + Orddict :: orddict(Key, Value). from_list([]) -> []; from_list([{_,_}]=Pair) -> Pair; @@ -73,17 +74,13 @@ is_empty([]) -> true; is_empty([_|_]) -> false. -spec fetch(Key, Orddict) -> Value when - Key :: term(), - Value :: term(), - Orddict :: orddict(). + Orddict :: orddict(Key, Value). fetch(Key, [{K,_}|D]) when Key > K -> fetch(Key, D); fetch(Key, [{K,Value}|_]) when Key == K -> Value. -spec find(Key, Orddict) -> {'ok', Value} | 'error' when - Key :: term(), - Orddict :: orddict(), - Value :: term(). + Orddict :: orddict(Key, Value). find(Key, [{K,_}|_]) when Key < K -> error; find(Key, [{K,_}|D]) when Key > K -> find(Key, D); @@ -91,17 +88,16 @@ find(_Key, [{_K,Value}|_]) -> {ok,Value}; %Key == K find(_, []) -> error. -spec fetch_keys(Orddict) -> Keys when - Orddict :: orddict(), - Keys :: [term()]. + Orddict :: orddict(Key, Value :: term()), + Keys :: [Key]. fetch_keys([{Key,_}|Dict]) -> [Key|fetch_keys(Dict)]; fetch_keys([]) -> []. -spec erase(Key, Orddict1) -> Orddict2 when - Key :: term(), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value). erase(Key, [{K,_}=E|Dict]) when Key < K -> [E|Dict]; erase(Key, [{K,_}=E|Dict]) when Key > K -> @@ -110,10 +106,8 @@ erase(_Key, [{_K,_Val}|Dict]) -> Dict; %Key == K erase(_, []) -> []. -spec store(Key, Value, Orddict1) -> Orddict2 when - Key :: term(), - Value :: term(), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value). store(Key, New, [{K,_}|_]=Dict) when Key < K -> [{Key,New}|Dict]; @@ -124,10 +118,8 @@ store(Key, New, [{_K,_Old}|Dict]) -> %Key == K store(Key, New, []) -> [{Key,New}]. -spec append(Key, Value, Orddict1) -> Orddict2 when - Key :: term(), - Value :: term(), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value). append(Key, New, [{K,_}|_]=Dict) when Key < K -> [{Key,[New]}|Dict]; @@ -138,10 +130,9 @@ append(Key, New, [{_K,Old}|Dict]) -> %Key == K append(Key, New, []) -> [{Key,[New]}]. -spec append_list(Key, ValList, Orddict1) -> Orddict2 when - Key :: term(), - ValList :: [Value :: term()], - Orddict1 :: orddict(), - Orddict2 :: orddict(). + ValList :: [Value], + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value). append_list(Key, NewList, [{K,_}|_]=Dict) when Key < K -> [{Key,NewList}|Dict]; @@ -153,10 +144,9 @@ append_list(Key, NewList, []) -> [{Key,NewList}]. -spec update(Key, Fun, Orddict1) -> Orddict2 when - Key :: term(), - Fun :: fun((Value1 :: term()) -> Value2 :: term()), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Fun :: fun((Value1 :: Value) -> Value2 :: Value), + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value). update(Key, Fun, [{K,_}=E|Dict]) when Key > K -> [E|update(Key, Fun, Dict)]; @@ -164,11 +154,10 @@ update(Key, Fun, [{K,Val}|Dict]) when Key == K -> [{Key,Fun(Val)}|Dict]. -spec update(Key, Fun, Initial, Orddict1) -> Orddict2 when - Key :: term(), - Initial :: term(), - Fun :: fun((Value1 :: term()) -> Value2 :: term()), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Initial :: Value, + Fun :: fun((Value1 :: Value) -> Value2 :: Value), + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value). update(Key, _, Init, [{K,_}|_]=Dict) when Key < K -> [{Key,Init}|Dict]; @@ -179,10 +168,9 @@ update(Key, Fun, _Init, [{_K,Val}|Dict]) -> %Key == K update(Key, _, Init, []) -> [{Key,Init}]. -spec update_counter(Key, Increment, Orddict1) -> Orddict2 when - Key :: term(), - Increment :: number(), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value), + Increment :: number(). update_counter(Key, Incr, [{K,_}|_]=Dict) when Key < K -> [{Key,Incr}|Dict]; @@ -193,28 +181,30 @@ update_counter(Key, Incr, [{_K,Val}|Dict]) -> %Key == K update_counter(Key, Incr, []) -> [{Key,Incr}]. -spec fold(Fun, Acc0, Orddict) -> Acc1 when - Fun :: fun((Key :: term(), Value :: term(), AccIn :: term()) -> AccOut :: term()), - Acc0 :: term(), - Acc1 :: term(), - Orddict :: orddict(). + Fun :: fun((Key, Value, AccIn) -> AccOut), + Orddict :: orddict(Key, Value), + Acc0 :: Acc, + Acc1 :: Acc, + AccIn :: Acc, + AccOut :: Acc. fold(F, Acc, [{Key,Val}|D]) -> fold(F, F(Key, Val, Acc), D); fold(F, Acc, []) when is_function(F, 3) -> Acc. -spec map(Fun, Orddict1) -> Orddict2 when - Fun :: fun((Key :: term(), Value1 :: term()) -> Value2 :: term()), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Fun :: fun((Key, Value1) -> Value2), + Orddict1 :: orddict(Key, Value1), + Orddict2 :: orddict(Key, Value2). map(F, [{Key,Val}|D]) -> [{Key,F(Key, Val)}|map(F, D)]; map(F, []) when is_function(F, 2) -> []. -spec filter(Pred, Orddict1) -> Orddict2 when - Pred :: fun((Key :: term(), Value :: term()) -> boolean()), - Orddict1 :: orddict(), - Orddict2 :: orddict(). + Pred :: fun((Key, Value) -> boolean()), + Orddict1 :: orddict(Key, Value), + Orddict2 :: orddict(Key, Value). filter(F, [{Key,Val}=E|D]) -> case F(Key, Val) of @@ -224,10 +214,10 @@ filter(F, [{Key,Val}=E|D]) -> filter(F, []) when is_function(F, 2) -> []. -spec merge(Fun, Orddict1, Orddict2) -> Orddict3 when - Fun :: fun((Key :: term(), Value1 :: term(), Value2 :: term()) -> Value :: term()), - Orddict1 :: orddict(), - Orddict2 :: orddict(), - Orddict3 :: orddict(). + Fun :: fun((Key, Value1, Value2) -> Value), + Orddict1 :: orddict(Key, Value1), + Orddict2 :: orddict(Key, Value2), + Orddict3 :: orddict(Key, Value). merge(F, [{K1,_}=E1|D1], [{K2,_}=E2|D2]) when K1 < K2 -> [E1|merge(F, D1, [E2|D2])]; diff --git a/lib/stdlib/src/otp_internal.erl b/lib/stdlib/src/otp_internal.erl index 24721da187..0340015c35 100644 --- a/lib/stdlib/src/otp_internal.erl +++ b/lib/stdlib/src/otp_internal.erl @@ -631,6 +631,9 @@ obsolete_1(erl_lint, modify_line, 2) -> obsolete_1(ssl, negotiated_next_protocol, 1) -> {deprecated,{ssl,negotiated_protocol,1}}; +obsolete_1(ssl, connection_info, 1) -> + {deprecated, "deprecated; use connection_information/[1,2] instead"}; + obsolete_1(_, _, _) -> no. diff --git a/lib/stdlib/src/shell_default.erl b/lib/stdlib/src/shell_default.erl index 3fe359af0e..0fca7ff8c7 100644 --- a/lib/stdlib/src/shell_default.erl +++ b/lib/stdlib/src/shell_default.erl @@ -23,7 +23,7 @@ -module(shell_default). -export([help/0,lc/1,c/1,c/2,nc/1,nl/1,l/1,i/0,pid/3,i/3,m/0,m/1, - memory/0,memory/1, + memory/0,memory/1,uptime/0, erlangrc/1,bi/1, regs/0, flush/0,pwd/0,ls/0,ls/1,cd/1, y/1, y/2, xm/1, bt/1, q/0, @@ -92,6 +92,7 @@ pid(X,Y,Z) -> c:pid(X,Y,Z). pwd() -> c:pwd(). q() -> c:q(). regs() -> c:regs(). +uptime() -> c:uptime(). xm(Mod) -> c:xm(Mod). y(File) -> c:y(File). y(File, Opts) -> c:y(File, Opts). diff --git a/lib/stdlib/src/stdlib.app.src b/lib/stdlib/src/stdlib.app.src index a27a35dca2..c33130cf8c 100644 --- a/lib/stdlib/src/stdlib.app.src +++ b/lib/stdlib/src/stdlib.app.src @@ -104,7 +104,7 @@ dets]}, {applications, [kernel]}, {env, []}, - {runtime_dependencies, ["sasl-2.4","kernel-3.0.2","erts-7.0","crypto-3.3", + {runtime_dependencies, ["sasl-2.4","kernel-4.0","erts-7.0","crypto-3.3", "compiler-5.0"]} ]}. diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src index ee87a8ddb2..b3569c2848 100644 --- a/lib/stdlib/src/stdlib.appup.src +++ b/lib/stdlib/src/stdlib.appup.src @@ -1,7 +1,7 @@ %% -*- erlang -*- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2014. All Rights Reserved. +%% Copyright Ericsson AB 1999-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -17,9 +17,7 @@ %% %CopyrightEnd% {"%VSN%", %% Up from - max one major revision back - [{<<"2\\.[1-3](\\.[0-9]+)*">>,[restart_new_emulator]}, %% 17.1-17.3 - {<<"2\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}], %% 17.0 + [{<<"2\\.[0-4](\\.[0-9]+)*">>,[restart_new_emulator]}], %% 17.0-17.5 %% Down to - max one major revision back - [{<<"2\\.[1-3](\\.[0-9]+)*">>,[restart_new_emulator]}, %% 17.1-17.3 - {<<"2\\.0(\\.[0-9]+)*">>,[restart_new_emulator]}] %% 17.0 + [{<<"2\\.[0-4](\\.[0-9]+)*">>,[restart_new_emulator]}] %% 17.0-17.5 }. diff --git a/lib/stdlib/src/supervisor.erl b/lib/stdlib/src/supervisor.erl index 67655b1145..1d7396adee 100644 --- a/lib/stdlib/src/supervisor.erl +++ b/lib/stdlib/src/supervisor.erl @@ -381,7 +381,7 @@ handle_call({start_child, EArgs}, _From, State) when ?is_simple(State) -> #child{mfargs = {M, F, A}} = Child, Args = A ++ EArgs, case do_start_child_i(M, F, Args) of - {ok, undefined} when Child#child.restart_type =:= temporary -> + {ok, undefined} -> {reply, {ok, undefined}, State}; {ok, Pid} -> NState = save_dynamic_child(Child#child.restart_type, Pid, Args, State), diff --git a/lib/stdlib/src/zip.erl b/lib/stdlib/src/zip.erl index 3c67bd67c6..f986c0081d 100644 --- a/lib/stdlib/src/zip.erl +++ b/lib/stdlib/src/zip.erl @@ -24,7 +24,7 @@ list_dir/1, list_dir/2, table/1, table/2, t/1, tt/1]). -%% unzipping peicemeal +%% unzipping piecemeal -export([openzip_open/1, openzip_open/2, openzip_get/1, openzip_get/2, openzip_t/1, openzip_tt/1, diff --git a/lib/stdlib/test/binary_module_SUITE.erl b/lib/stdlib/test/binary_module_SUITE.erl index 5248870744..8d26c77c9b 100644 --- a/lib/stdlib/test/binary_module_SUITE.erl +++ b/lib/stdlib/test/binary_module_SUITE.erl @@ -993,43 +993,51 @@ random_parts(X,N) -> random_ref_comp(doc) -> ["Test pseudorandomly generated cases against reference imlementation"]; random_ref_comp(Config) when is_list(Config) -> - ?line put(success_counter,0), - ?line random:seed({1271,769940,559934}), - ?line do_random_match_comp(5000,{1,40},{30,1000}), + put(success_counter,0), + random:seed({1271,769940,559934}), + Nr = {1,40}, + Hr = {30,1000}, + I1 = 1500, + I2 = 5, + do_random_match_comp(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_match_comp2(5000,{1,40},{30,1000}), + do_random_match_comp2(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_match_comp3(5000,{1,40},{30,1000}), + do_random_match_comp3(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_match_comp4(5000,{1,40},{30,1000}), + do_random_match_comp4(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_matches_comp(5000,{1,40},{30,1000}), + do_random_matches_comp(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_matches_comp2(5000,{1,40},{30,1000}), + do_random_matches_comp2(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_matches_comp3(5,{1,40},{30,1000}), - ?line erts_debug:set_internal_state(available_internal_state,true), - ?line io:format("oldlimit: ~p~n",[ erts_debug:set_internal_state(binary_loop_limit,100)]), - ?line do_random_match_comp(5000,{1,40},{30,1000}), - ?line do_random_matches_comp3(5,{1,40},{30,1000}), - ?line io:format("limit was: ~p~n",[ erts_debug:set_internal_state(binary_loop_limit,default)]), - ?line erts_debug:set_internal_state(available_internal_state,false), + do_random_matches_comp3(I2,Nr,Hr), + erts_debug:set_internal_state(available_internal_state,true), + io:format("oldlimit: ~p~n",[ erts_debug:set_internal_state(binary_loop_limit,100)]), + do_random_match_comp(I1,Nr,Hr), + do_random_matches_comp3(I2,Nr,Hr), + io:format("limit was: ~p~n",[ erts_debug:set_internal_state(binary_loop_limit,default)]), + erts_debug:set_internal_state(available_internal_state,false), ok. random_ref_sr_comp(doc) -> ["Test pseudorandomly generated cases against reference imlementation of split and replace"]; random_ref_sr_comp(Config) when is_list(Config) -> - ?line put(success_counter,0), - ?line random:seed({1271,769940,559934}), - ?line do_random_split_comp(5000,{1,40},{30,1000}), + put(success_counter,0), + random:seed({1271,769940,559934}), + Nr = {1,40}, + Hr = {30,1000}, + I1 = 1500, + do_random_split_comp(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_replace_comp(5000,{1,40},{30,1000}), + do_random_replace_comp(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_split_comp2(5000,{1,40},{30,1000}), + do_random_split_comp2(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), - ?line do_random_replace_comp2(5000,{1,40},{30,1000}), + do_random_replace_comp2(I1,Nr,Hr), io:format("Number of successes: ~p~n",[get(success_counter)]), ok. + random_ref_fla_comp(doc) -> ["Test pseudorandomly generated cases against reference imlementation of split and replace"]; random_ref_fla_comp(Config) when is_list(Config) -> diff --git a/lib/stdlib/test/dict_SUITE.erl b/lib/stdlib/test/dict_SUITE.erl index 69814e12ce..ab624e8dd2 100644 --- a/lib/stdlib/test/dict_SUITE.erl +++ b/lib/stdlib/test/dict_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2013. All Rights Reserved. +%% Copyright Ericsson AB 2008-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -25,16 +25,16 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2,end_per_testcase/2, - create/1,store/1]). + create/1,store/1,iterate/1]). -include_lib("test_server/include/test_server.hrl"). --import(lists, [foldl/3,reverse/1]). +-import(lists, [foldl/3]). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [create, store]. + [create, store, iterate]. groups() -> []. @@ -93,6 +93,48 @@ store_1(List, M) -> D0. %%% +%%% Test specifics for gb_trees. +%%% + +iterate(Config) when is_list(Config) -> + test_all(fun iterate_1/1). + +iterate_1(M) -> + case M(module, []) of + gb_trees -> iterate_2(M); + _ -> ok + end, + M(empty, []). + +iterate_2(M) -> + random:seed(1, 2, 42), + iter_tree(M, 1000). + +iter_tree(_M, 0) -> + ok; +iter_tree(M, N) -> + L = [{I, I} || I <- lists:seq(1, N)], + T = M(from_list, L), + L = lists:reverse(iterate_tree(M, T)), + R = random:uniform(N), + KV = lists:reverse(iterate_tree_from(M, R, T)), + KV = [P || P={K,_} <- L, K >= R], + iter_tree(M, N-1). + +iterate_tree(M, Tree) -> + I = M(iterator, Tree), + iterate_tree_1(M, M(next, I), []). + +iterate_tree_from(M, Start, Tree) -> + I = M(iterator_from, {Start, Tree}), + iterate_tree_1(M, M(next, I), []). + +iterate_tree_1(_, none, R) -> + R; +iterate_tree_1(M, {K, V, I}, R) -> + iterate_tree_1(M, M(next, I), [{K, V} | R]). + +%%% %%% Helper functions. %%% diff --git a/lib/stdlib/test/dict_test_lib.erl b/lib/stdlib/test/dict_test_lib.erl index 4fdb4fa0bd..81d26ce5f8 100644 --- a/lib/stdlib/test/dict_test_lib.erl +++ b/lib/stdlib/test/dict_test_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2013. All Rights Reserved. +%% Copyright Ericsson AB 2008-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,6 +29,9 @@ new(Mod, Eq) -> (module, []) -> Mod; (size, D) -> Mod:size(D); (is_empty, D) -> Mod:is_empty(D); + (iterator, S) -> Mod:iterator(S); + (iterator_from, {Start, S}) -> Mod:iterator_from(Start, S); + (next, I) -> Mod:next(I); (to_list, D) -> to_list(Mod, D) end. diff --git a/lib/stdlib/test/erl_anno_SUITE.erl b/lib/stdlib/test/erl_anno_SUITE.erl index 7632fbd324..d024f6907d 100644 --- a/lib/stdlib/test/erl_anno_SUITE.erl +++ b/lib/stdlib/test/erl_anno_SUITE.erl @@ -89,7 +89,6 @@ is_anno(_Config) -> false = erl_anno:is_anno([{generated,true}]), false = erl_anno:is_anno([{location,1},{file,nofile}]), false = erl_anno:is_anno([{location,1},{text,notext}]), - false = erl_anno:is_anno([{location,1},{text,[a,b,c]}]), true = erl_anno:is_anno(erl_anno:new(1)), A0 = erl_anno:new({1, 17}), diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index 5774d774b5..fff6b11a38 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -3061,13 +3061,13 @@ time_lookup(Config) when is_list(Config) -> "~p ets lookups/s",[Values]))}. time_lookup_do(Opts) -> - ?line Tab = ets_new(foo,Opts), - ?line fill_tab(Tab,foo), - ?line ets:insert(Tab,{{a,key},foo}), - ?line {Time,_} = ?t:timecall(test_server,do_times, - [10000,ets,lookup,[Tab,{a,key}]]), - ?line true = ets:delete(Tab), - round(10000 / Time). % lookups/s + Tab = ets_new(foo,Opts), + fill_tab(Tab,foo), + ets:insert(Tab,{{a,key},foo}), + {Time,_} = ?t:timecall(test_server,do_times, + [100000,ets,lookup,[Tab,{a,key}]]), + true = ets:delete(Tab), + round(100000 / Time). % lookups/s badlookup(doc) -> ["Check proper return values from bad lookups in existing/non existing " @@ -3821,46 +3821,99 @@ match_object(Config) when is_list(Config) -> repeat_for_opts(match_object_do). match_object_do(Opts) -> - ?line EtsMem = etsmem(), - ?line Tab = ets_new(foobar, Opts), - ?line fill_tab(Tab, foo), - ?line ets:insert(Tab, {{one, 4}, 4}), - ?line ets:insert(Tab,{{one,5},5}), - ?line ets:insert(Tab,{{two,4},4}), - ?line ets:insert(Tab,{{two,5},6}), - ?line ets:insert(Tab, {#{camembert=>cabécou},7}), - ?line case ets:match_object(Tab, {{one, '_'}, '$0'}) of + EtsMem = etsmem(), + Tab = ets_new(foobar, Opts), + fill_tab(Tab, foo), + ets:insert(Tab,{{one,4},4}), + ets:insert(Tab,{{one,5},5}), + ets:insert(Tab,{{two,4},4}), + ets:insert(Tab,{{two,5},6}), + ets:insert(Tab, {#{camembert=>cabécou},7}), + ets:insert(Tab, {#{"hi"=>"hello","wazzup"=>"awesome","1337"=>"42"},8}), + ets:insert(Tab, {#{"hi"=>"hello",#{"wazzup"=>3}=>"awesome","1337"=>"42"},9}), + ets:insert(Tab, {#{"hi"=>"hello","wazzup"=>#{"awesome"=>3},"1337"=>"42"},10}), + Is = lists:seq(1,100), + M1 = maps:from_list([{I,I}||I <- Is]), + M2 = maps:from_list([{I,"hi"}||I <- Is]), + ets:insert(Tab, {M1,11}), + ets:insert(Tab, {M2,12}), + + case ets:match_object(Tab, {{one, '_'}, '$0'}) of [{{one,5},5},{{one,4},4}] -> ok; [{{one,4},4},{{one,5},5}] -> ok; _ -> ?t:fail("ets:match_object() returned something funny.") end, - ?line case ets:match_object(Tab, {{two, '$1'}, '$0'}) of + case ets:match_object(Tab, {{two, '$1'}, '$0'}) of [{{two,5},6},{{two,4},4}] -> ok; [{{two,4},4},{{two,5},6}] -> ok; _ -> ?t:fail("ets:match_object() returned something funny.") end, - ?line case ets:match_object(Tab, {{two, '$9'}, '$4'}) of + case ets:match_object(Tab, {{two, '$9'}, '$4'}) of [{{two,5},6},{{two,4},4}] -> ok; [{{two,4},4},{{two,5},6}] -> ok; _ -> ?t:fail("ets:match_object() returned something funny.") end, - ?line case ets:match_object(Tab, {{two, '$9'}, '$22'}) of + case ets:match_object(Tab, {{two, '$9'}, '$22'}) of [{{two,5},6},{{two,4},4}] -> ok; [{{two,4},4},{{two,5},6}] -> ok; _ -> ?t:fail("ets:match_object() returned something funny.") end, + % Check that maps are inspected for variables. - [{#{camembert:=cabécou},7}] = - ets:match_object(Tab, {#{camembert=>'_'},7}), + [{#{camembert:=cabécou},7}] = ets:match_object(Tab, {#{camembert=>'_'},7}), + + [{#{"hi":="hello",#{"wazzup"=>3}:="awesome","1337":="42"},9}] = + ets:match_object(Tab, {#{#{"wazzup"=>3}=>"awesome","hi"=>"hello","1337"=>"42"},9}), + [{#{"hi":="hello",#{"wazzup"=>3}:="awesome","1337":="42"},9}] = + ets:match_object(Tab, {#{#{"wazzup"=>3}=>"awesome","hi"=>"hello","1337"=>'_'},'_'}), + [{#{"hi":="hello","wazzup":=#{"awesome":=3},"1337":="42"},10}] = + ets:match_object(Tab, {#{"wazzup"=>'_',"hi"=>'_',"1337"=>'_'},10}), + + %% multiple patterns + Pat = {{#{#{"wazzup"=>3}=>"awesome","hi"=>"hello","1337"=>'_'},'$1'},[{is_integer,'$1'}],['$_']}, + [{#{"hi":="hello",#{"wazzup"=>3}:="awesome","1337":="42"},9}] = + ets:select(Tab, [Pat,Pat,Pat,Pat]), + case ets:match_object(Tab, {#{"hi"=>"hello","wazzup"=>'_',"1337"=>"42"},'_'}) of + [{#{"1337" := "42","hi" := "hello","wazzup" := "awesome"},8}, + {#{"1337" := "42","hi" := "hello","wazzup" := #{"awesome" := 3}},10}] -> ok; + [{#{"1337" := "42","hi" := "hello","wazzup" := #{"awesome" := 3}},10}, + {#{"1337" := "42","hi" := "hello","wazzup" := "awesome"},8}] -> ok; + _ -> ?t:fail("ets:match_object() returned something funny.") + end, + case ets:match_object(Tab, {#{"hi"=>'_'},'_'}) of + [{#{"1337":="42", "hi":="hello"},_}, + {#{"1337":="42", "hi":="hello"},_}, + {#{"1337":="42", "hi":="hello"},_}] -> ok; + _ -> ?t:fail("ets:match_object() returned something funny.") + end, + + %% match large maps + [{#{1:=1,2:=2,99:=99,100:=100},11}] = ets:match_object(Tab, {M1,11}), + [{#{1:="hi",2:="hi",99:="hi",100:="hi"},12}] = ets:match_object(Tab, {M2,12}), + case ets:match_object(Tab, {#{1=>'_',2=>'_'},'_'}) of + %% only match a part of the map + [{#{1:=1,5:=5,99:=99,100:=100},11},{#{1:="hi",6:="hi",99:="hi"},12}] -> ok; + [{#{1:="hi",2:="hi",59:="hi"},12},{#{1:=1,2:=2,39:=39,100:=100},11}] -> ok; + _ -> ?t:fail("ets:match_object() returned something funny.") + end, + case ets:match_object(Tab, {maps:from_list([{I,'_'}||I<-Is]),'_'}) of + %% only match a part of the map + [{#{1:=1,5:=5,99:=99,100:=100},11},{#{1:="hi",6:="hi",99:="hi"},12}] -> ok; + [{#{1:="hi",2:="hi",59:="hi"},12},{#{1:=1,2:=2,39:=39,100:=100},11}] -> ok; + _ -> ?t:fail("ets:match_object() returned something funny.") + end, {'EXIT',{badarg,_}} = (catch ets:match_object(Tab, {#{'$1'=>'_'},7})), - % Check that unsucessful match returns an empty list. - ?line [] = ets:match_object(Tab, {{three,'$0'}, '$92'}), + Mve = maps:from_list([{list_to_atom([$$|integer_to_list(I)]),'_'}||I<-Is]), + {'EXIT',{badarg,_}} = (catch ets:match_object(Tab, {Mve,11})), + + % Check that unsuccessful match returns an empty list. + [] = ets:match_object(Tab, {{three,'$0'}, '$92'}), % Check that '$0' equals '_'. Len = length(ets:match_object(Tab, '$0')), Len = length(ets:match_object(Tab, '_')), - ?line if Len > 4 -> ok end, - ?line true = ets:delete(Tab), - ?line verify_etsmem(EtsMem). + if Len > 4 -> ok end, + true = ets:delete(Tab), + verify_etsmem(EtsMem). match_object2(suite) -> []; match_object2(doc) -> ["Tests that db_match_object does not generate " @@ -4025,12 +4078,22 @@ tab2file(doc) -> ["Check the ets:tab2file function on an empty " "ets table."]; tab2file(suite) -> []; tab2file(Config) when is_list(Config) -> + ?line FName = filename:join([?config(priv_dir, Config),"tab2file_case"]), + tab2file_do(FName, []), + tab2file_do(FName, [{sync,true}]), + tab2file_do(FName, [{sync,false}]), + {'EXIT',{{badmatch,{error,_}},_}} = (catch tab2file_do(FName, [{sync,yes}])), + {'EXIT',{{badmatch,{error,_}},_}} = (catch tab2file_do(FName, [sync])), + ok. + +tab2file_do(FName, Opts) -> %% Write an empty ets table to a file, read back and check properties. ?line Tab = ets_new(ets_SUITE_foo_tab, [named_table, set, private, {keypos, 2}]), - ?line FName = filename:join([?config(priv_dir, Config),"tab2file_case"]), - ?line ok = ets:tab2file(Tab, FName), - ?line true = ets:delete(Tab), + catch file:delete(FName), + Res = ets:tab2file(Tab, FName, Opts), + true = ets:delete(Tab), + ok = Res, % ?line EtsMem = etsmem(), ?line {ok, Tab2} = ets:file2tab(FName), @@ -4040,6 +4103,7 @@ tab2file(Config) when is_list(Config) -> ?line set = ets:info(Tab2, type), ?line true = ets:delete(Tab2), ?line verify_etsmem(EtsMem). + tab2file2(doc) -> ["Check the ets:tab2file function on a ", "filled set/bag type ets table."]; diff --git a/lib/stdlib/test/io_proto_SUITE.erl b/lib/stdlib/test/io_proto_SUITE.erl index 858a78b1d2..78432789cd 100644 --- a/lib/stdlib/test/io_proto_SUITE.erl +++ b/lib/stdlib/test/io_proto_SUITE.erl @@ -482,7 +482,7 @@ unicode_options_gen(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), AllModes = [utf8,utf16,{utf16,big},{utf16,little}, utf32,{utf32,big},{utf32,little}], - FSize = 17*1024, + FSize = 9*1024, NumItersRead = 2, NumItersWrite = 2, Dir = filename:join(PrivDir, "GENDATA1"), diff --git a/lib/stdlib/test/maps_SUITE.erl b/lib/stdlib/test/maps_SUITE.erl index 1d9c041a74..f8f241d834 100644 --- a/lib/stdlib/test/maps_SUITE.erl +++ b/lib/stdlib/test/maps_SUITE.erl @@ -34,18 +34,21 @@ -export([init_per_testcase/2]). -export([end_per_testcase/2]). --export([t_get_3/1, +-export([t_get_3/1, t_filter_2/1, t_fold_3/1,t_map_2/1,t_size_1/1, t_with_2/1,t_without_2/1]). --define(badmap(V,F,Args), {'EXIT', {{badmap,V}, [{maps,F,Args,_}|_]}}). --define(badarg(F,Args), {'EXIT', {badarg, [{maps,F,Args,_}|_]}}). +%-define(badmap(V,F,Args), {'EXIT', {{badmap,V}, [{maps,F,Args,_}|_]}}). +%-define(badarg(F,Args), {'EXIT', {badarg, [{maps,F,Args,_}|_]}}). +% silly broken hipe +-define(badmap(V,F,_Args), {'EXIT', {{badmap,V}, [{maps,F,_,_}|_]}}). +-define(badarg(F,_Args), {'EXIT', {badarg, [{maps,F,_,_}|_]}}). suite() -> [{ct_hooks, [ts_install_cth]}]. all() -> - [t_get_3, + [t_get_3,t_filter_2, t_fold_3,t_map_2,t_size_1, t_with_2,t_without_2]. @@ -99,6 +102,16 @@ t_with_2(_Config) -> ?badarg(with,[a,#{}]) = (catch maps:with(a,#{})), ok. +t_filter_2(Config) when is_list(Config) -> + M = #{a => 2, b => 3, c=> 4, "a" => 1, "b" => 2, "c" => 4}, + Pred1 = fun(K,V) -> is_atom(K) andalso (V rem 2) =:= 0 end, + Pred2 = fun(K,V) -> is_list(K) andalso (V rem 2) =:= 0 end, + #{a := 2,c := 4} = maps:filter(Pred1,M), + #{"b" := 2,"c" := 4} = maps:filter(Pred2,M), + %% error case + ?badmap(a,filter,[_,a]) = (catch maps:filter(fun(_,_) -> ok end,id(a))), + ?badarg(filter,[<<>>,#{}]) = (catch maps:filter(id(<<>>),#{})), + ok. t_fold_3(Config) when is_list(Config) -> Vs = lists:seq(1,200), diff --git a/lib/stdlib/test/qlc_SUITE.erl b/lib/stdlib/test/qlc_SUITE.erl index 348c308f5d..56829fac5c 100644 --- a/lib/stdlib/test/qlc_SUITE.erl +++ b/lib/stdlib/test/qlc_SUITE.erl @@ -6120,7 +6120,7 @@ otp_6964(Config) when is_list(Config) -> lists:flatten(qlc:format_error(ErrReply)), qlc_SUITE:install_error_logger(), 20000 = length(F(warning_msg)), - {error, joining} = qlc_SUITE:read_error_logger(), + {warning, joining} = qlc_SUITE:read_error_logger(), 20000 = length(F(info_msg)), {info, joining} = qlc_SUITE:read_error_logger(), 20000 = length(F(error_msg)), @@ -6155,8 +6155,8 @@ otp_6964(Config) when is_list(Config) -> {error, caching} = qlc_SUITE:read_error_logger(), {error, caching} = qlc_SUITE:read_error_logger(), 1 = length(F(warning_msg)), - {error, caching} = qlc_SUITE:read_error_logger(), - {error, caching} = qlc_SUITE:read_error_logger(), + {warning, caching} = qlc_SUITE:read_error_logger(), + {warning, caching} = qlc_SUITE:read_error_logger(), 1 = length(F(info_msg)), {info, caching} = qlc_SUITE:read_error_logger(), {info, caching} = qlc_SUITE:read_error_logger(), @@ -6188,7 +6188,7 @@ otp_6964(Config) when is_list(Config) -> L = F(info_msg), {info, sorting} = qlc_SUITE:read_error_logger(), L = F(warning_msg), - {error, sorting} = qlc_SUITE:read_error_logger(), + {warning, sorting} = qlc_SUITE:read_error_logger(), qlc_SUITE:uninstall_error_logger(), ets:delete(E1), ets:delete(E2)">>], @@ -6215,7 +6215,7 @@ otp_6964(Config) when is_list(Config) -> R = lists:sort(F(error_msg)), {error, caching} = qlc_SUITE:read_error_logger(), R = lists:sort(F(warning_msg)), - {error, caching} = qlc_SUITE:read_error_logger(), + {warning, caching} = qlc_SUITE:read_error_logger(), qlc_SUITE:uninstall_error_logger(), ErrReply = F(not_allowed), {error,qlc,{tmpdir_usage,caching}} = ErrReply, @@ -8178,6 +8178,8 @@ read_error_logger() -> {error, Why}; {info, Why} -> {info, Why}; + {warning, Why} -> + {warning, Why}; {error, Pid, Tuple} -> {error, Pid, Tuple} after 1000 -> @@ -8192,8 +8194,7 @@ read_error_logger() -> init(Tester) -> {ok, Tester}. -handle_event({error, _GL, {_Pid, _Msg, [Why, _]}}, Tester) - when is_atom(Why) -> +handle_event({error, _GL, {_Pid, _Msg, [Why, _]}}, Tester) when is_atom(Why) -> Tester ! {error, Why}, {ok, Tester}; handle_event({error, _GL, {_Pid, _Msg, [P, T]}}, Tester) when is_pid(P) -> @@ -8202,6 +8203,9 @@ handle_event({error, _GL, {_Pid, _Msg, [P, T]}}, Tester) when is_pid(P) -> handle_event({info_msg, _GL, {_Pid, _Msg, [Why, _]}}, Tester) -> Tester ! {info, Why}, {ok, Tester}; +handle_event({warning_msg, _GL, {_Pid, _Msg, [Why, _]}}, Tester) when is_atom(Why) -> + Tester ! {warning, Why}, + {ok, Tester}; handle_event(_Event, State) -> {ok, State}. diff --git a/lib/stdlib/test/rand_SUITE.erl b/lib/stdlib/test/rand_SUITE.erl index 9a1f37aa75..39ce1bd89a 100644 --- a/lib/stdlib/test/rand_SUITE.erl +++ b/lib/stdlib/test/rand_SUITE.erl @@ -33,7 +33,7 @@ -include_lib("test_server/include/test_server.hrl"). % Default timetrap timeout (set in init_per_testcase). --define(default_timeout, ?t:minutes(1)). +-define(default_timeout, ?t:minutes(3)). -define(LOOP, 1000000). init_per_testcase(_Case, Config) -> diff --git a/lib/stdlib/test/sets_SUITE.erl b/lib/stdlib/test/sets_SUITE.erl index c0cf1fc7e8..24f5d65f82 100644 --- a/lib/stdlib/test/sets_SUITE.erl +++ b/lib/stdlib/test/sets_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2013. All Rights Reserved. +%% Copyright Ericsson AB 2004-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -28,7 +28,7 @@ create/1,add_element/1,del_element/1, subtract/1,intersection/1,union/1,is_subset/1, is_set/1,fold/1,filter/1, - take_smallest/1,take_largest/1]). + take_smallest/1,take_largest/1, iterate/1]). -include_lib("test_server/include/test_server.hrl"). @@ -48,7 +48,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [create, add_element, del_element, subtract, intersection, union, is_subset, is_set, fold, filter, - take_smallest, take_largest]. + take_smallest, take_largest, iterate]. groups() -> []. @@ -426,6 +426,44 @@ take_largest_3(S0, List0, M) -> take_largest_3(S, List, M) end. +iterate(Config) when is_list(Config) -> + test_all(fun iterate_1/1). + +iterate_1(M) -> + case M(module, []) of + gb_sets -> iterate_2(M); + _ -> ok + end, + M(empty, []). + +iterate_2(M) -> + random:seed(1, 2, 42), + iter_set(M, 1000). + +iter_set(_M, 0) -> + ok; +iter_set(M, N) -> + L = [I || I <- lists:seq(1, N)], + T = M(from_list, L), + L = lists:reverse(iterate_set(M, T)), + R = random:uniform(N), + S = lists:reverse(iterate_set(M, R, T)), + S = [E || E <- L, E >= R], + iter_set(M, N-1). + +iterate_set(M, Set) -> + I = M(iterator, Set), + iterate_set_1(M, M(next, I), []). + +iterate_set(M, Start, Set) -> + I = M(iterator_from, {Start, Set}), + iterate_set_1(M, M(next, I), []). + +iterate_set_1(_, none, R) -> + R; +iterate_set_1(M, {E, I}, R) -> + iterate_set_1(M, M(next, I), [E | R]). + %%% %%% Helper functions. %%% diff --git a/lib/stdlib/test/sets_test_lib.erl b/lib/stdlib/test/sets_test_lib.erl index 86f009a8f9..772139406d 100644 --- a/lib/stdlib/test/sets_test_lib.erl +++ b/lib/stdlib/test/sets_test_lib.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2013. All Rights Reserved. +%% Copyright Ericsson AB 2004-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -34,7 +34,10 @@ new(Mod, Eq) -> (is_empty, S) -> is_empty(Mod, S); (is_set, S) -> Mod:is_set(S); (is_subset, {S,Set}) -> is_subset(Mod, Eq, S, Set); + (iterator, S) -> Mod:iterator(S); + (iterator_from, {Start, S}) -> Mod:iterator_from(Start, S); (module, []) -> Mod; + (next, I) -> Mod:next(I); (singleton, E) -> singleton(Mod, E); (size, S) -> Mod:size(S); (subtract, {S1,S2}) -> subtract(Mod, S1, S2); diff --git a/lib/stdlib/test/supervisor_SUITE.erl b/lib/stdlib/test/supervisor_SUITE.erl index 9dcf19707c..015b09f35e 100644 --- a/lib/stdlib/test/supervisor_SUITE.erl +++ b/lib/stdlib/test/supervisor_SUITE.erl @@ -37,6 +37,7 @@ sup_start_ignore_child/1, sup_start_ignore_temporary_child/1, sup_start_ignore_temporary_child_start_child/1, sup_start_ignore_temporary_child_start_child_simple/1, + sup_start_ignore_permanent_child_start_child_simple/1, sup_start_error_return/1, sup_start_fail/1, sup_start_map/1, sup_start_map_faulty_specs/1, sup_stop_infinity/1, sup_stop_timeout/1, sup_stop_brutal_kill/1, @@ -99,6 +100,7 @@ groups() -> sup_start_ignore_child, sup_start_ignore_temporary_child, sup_start_ignore_temporary_child_start_child, sup_start_ignore_temporary_child_start_child_simple, + sup_start_ignore_permanent_child_start_child_simple, sup_start_error_return, sup_start_fail]}, {sup_start_map, [], [sup_start_map, sup_start_map_faulty_specs]}, @@ -250,6 +252,27 @@ sup_start_ignore_temporary_child_start_child_simple(Config) [1,1,0,1] = get_child_counts(sup_test). %%------------------------------------------------------------------------- +%% Tests what happens if child's init-callback returns ignore for a +%% permanent child when child is started with start_child/2, and the +%% supervisor is simple_one_for_one. +%% Child spec shall NOT be saved!!! +sup_start_ignore_permanent_child_start_child_simple(Config) + when is_list(Config) -> + process_flag(trap_exit, true), + Child1 = {child1, {supervisor_1, start_child, [ignore]}, + permanent, 1000, worker, []}, + {ok, Pid} = start_link({ok, {{simple_one_for_one, 2, 3600}, [Child1]}}), + + {ok, undefined} = supervisor:start_child(sup_test, []), + {ok, CPid2} = supervisor:start_child(sup_test, []), + + [{undefined, CPid2, worker, []}] = supervisor:which_children(sup_test), + [1,1,0,1] = get_child_counts(sup_test), + + %% Regression test: check that the supervisor terminates without error. + exit(Pid, shutdown), + check_exit_reason(Pid, shutdown). +%%------------------------------------------------------------------------- %% Tests what happens if init-callback returns a invalid value. sup_start_error_return(Config) when is_list(Config) -> process_flag(trap_exit, true), diff --git a/lib/stdlib/test/unicode_SUITE.erl b/lib/stdlib/test/unicode_SUITE.erl index 613be99ccd..9f5d485df6 100644 --- a/lib/stdlib/test/unicode_SUITE.erl +++ b/lib/stdlib/test/unicode_SUITE.erl @@ -87,8 +87,9 @@ ex_binaries_errors_utf8(Config) when is_list(Config) -> %% Now, try with longer binary (trapping) BrokenPart = list_to_binary(lists:seq(128,255)), BrokenSz = byte_size(BrokenPart), + Seq255 = lists:seq(1,255), [ begin - OKList = lists:flatten(lists:duplicate(N,lists:seq(1,255))), + OKList = lists:flatten(lists:duplicate(N,Seq255)), OKBin = unicode:characters_to_binary(OKList), OKLen = length(OKList), %% Copy to avoid that the binary get's writable diff --git a/lib/stdlib/vsn.mk b/lib/stdlib/vsn.mk index f57f31c8de..a1f2a946b1 100644 --- a/lib/stdlib/vsn.mk +++ b/lib/stdlib/vsn.mk @@ -1 +1 @@ -STDLIB_VSN = 2.4 +STDLIB_VSN = 2.5 diff --git a/lib/syntax_tools/doc/overview.edoc b/lib/syntax_tools/doc/overview.edoc index df02ad0b3a..3111633a99 100644 --- a/lib/syntax_tools/doc/overview.edoc +++ b/lib/syntax_tools/doc/overview.edoc @@ -2,79 +2,34 @@ Syntax Tools overview page - @author Richard Carlsson <[email protected]> -@copyright 1997-2004 Richard Carlsson +@copyright 1997-2014 Richard Carlsson @version {@version} -@title Erlang Syntax Tools +@title Erlang Syntax and Metaprogramming tools -@doc This package contains modules for handling abstract Erlang syntax -trees, in a way that is compatible with the "parse trees" of the -standard library module `erl_parse', together with utilities for reading -source files in unusual ways and pretty-printing syntax trees. Also -included is an amazing module merger and renamer called Igor, as well as -an automatic code-cleaner. +@doc This package contains modules for handling abstract syntax trees (ASTs) +in Erlang, in a way that is compatible with the "abstract format" parse +trees of the stdlib module `erl_parse', together with utilities for reading +source files, {@link erl_prettypr. pretty-printing syntax trees}, {@link +igor. merging and renaming modules}, {@link erl_tidy. cleaning up obsolete +constructs}, and doing {@link merl. metaprogramming} in Erlang. -<p>The abstract layer (defined in {@link erl_syntax}) is nicely +The abstract layer (defined in {@link erl_syntax}) is nicely structured and the node types are context-independent. The layer makes it possible to transparently attach source-code comments and user annotations to nodes of the tree. Using the abstract layer makes applications less sensitive to changes in the {@link //stdlib/erl_parse} -data structures, only requiring the {@link erl_syntax} module to be -up-to-date.</p> +data structures, only requiring the `erl_syntax' module to be up-to-date. -<p>The pretty printer {@link erl_prettypr} is implemented on top of the +The pretty printer {@link erl_prettypr} is implemented on top of the library module {@link prettypr}: this is a powerful and flexible generic -pretty printing library, which is also distributed separately.</p> - -<p>For a short demonstration of parsing and pretty-printing, simply -compile the included module <a -href="../examples/demo.erl"><code>demo.erl</code></a>, and execute -<code>demo:run()</code> from the Erlang shell. It will compile the -remaining modules and give you further instructions.</p> - -<p>Also try the {@link erl_tidy} module, as follows: -<pre> - erl_tidy:dir("any-erlang-source-dir", [test, old_guard_tests]).</pre> -("<code>test</code>" assures that no files are modified).</p> - -<p>News in 1.4: -<ul> - <li>Added support for {@link erl_syntax:cond_expr/1. cond-expressions}, - {@link erl_syntax:try_expr/4. try-expressions} and - {@link erl_syntax:class_qualifier/2. class-qualifier patterns}.</li> - <li>Added support for parameterized modules.</li> - <li>{@link igor. Igor} is officially included.</li> - <li>Quick-parse functionality added to {@link epp_dodger}.</li> -</ul> -</p> - -<p>News in 1.3: -<ul> - <li>Added support for qualified names (as used by "packages").</li> - <li>Various internal changes.</li> -</ul> -</p> +pretty printing library, which is also distributed separately. -<p>News in 1.2: -<ul> - <li>HTML Documentation (generated with EDoc).</li> - <li>A few bug fixes and some minor interface changes (sorry for any - inconvenience).</li> -</ul> -</p> +For a short demonstration of parsing and pretty-printing, simply +compile the included module <a href="../examples/demo.erl">`demo.erl'</a>, +and execute `demo:run()' from the Erlang shell. It will compile the +remaining modules and give you further instructions. -<p>News in 1.1: -<ul> - <li>Module {@link erl_tidy}: check or tidy either a single module, or a - whole directory tree recursively. Rewrites and reformats the code - without losing comments or expanding macros. Safe mode allows - generating reports without modifying files.</li> - <li>Module {@link erl_syntax_lib}: contains support functions for easier - analysis of the source code structure.</li> - <li>Module {@link epp_dodger}: Bypasses the Erlang preprocessor - avoids - macro expansion, file inclusion, conditional compilation, etc. - Allows you to find/modify particular definitions/applications of - macros, and other things previously not possible.</li> -</ul> -</p> +Also try the {@link erl_tidy} module, as follows: +```erl_tidy:dir("any-erlang-source-dir", [test, old_guard_tests]).''' +(the `test' option assures that no files are modified). diff --git a/lib/syntax_tools/doc/src/Makefile b/lib/syntax_tools/doc/src/Makefile index 2502bf877a..b7c599a9b9 100644 --- a/lib/syntax_tools/doc/src/Makefile +++ b/lib/syntax_tools/doc/src/Makefile @@ -50,6 +50,8 @@ XML_REF3_FILES = \ erl_syntax_lib.xml \ erl_tidy.xml \ igor.xml \ + merl.xml \ + merl_transform.xml \ prettypr.xml XML_PART_FILES = part.xml part_notes.xml diff --git a/lib/syntax_tools/doc/src/ref_man.xml b/lib/syntax_tools/doc/src/ref_man.xml index 598f656011..2b114c8528 100644 --- a/lib/syntax_tools/doc/src/ref_man.xml +++ b/lib/syntax_tools/doc/src/ref_man.xml @@ -29,12 +29,11 @@ </header> <description> <p><em>Syntax_Tools</em> contains modules for handling abstract - Erlang syntax trees, in a way that is compatible with the "parse - trees" of the STDLIB module <c>erl_parse</c>, together with - utilities for reading source files in unusual ways and - pretty-printing syntax trees. Also included is an amazing module - merger and renamer called Igor, as well as an automatic - code-cleaner.</p> + Erlang syntax trees, in a way that is compatible with the "external + format" parse trees of the STDLIB module <c>erl_parse</c>, together + with utilities for reading source files, pretty-printing syntax trees, + merging and renaming modules, cleaning up obsolete constructs, and + doing metaprogramming in Erlang.</p> </description> <xi:include href="epp_dodger.xml"/> <xi:include href="erl_comment_scan.xml"/> @@ -44,6 +43,8 @@ <xi:include href="erl_syntax_lib.xml"/> <xi:include href="erl_tidy.xml"/> <xi:include href="igor.xml"/> + <xi:include href="merl.xml"/> + <xi:include href="merl_transform.xml"/> <xi:include href="prettypr.xml"/> </application> diff --git a/lib/syntax_tools/examples/merl/Makefile b/lib/syntax_tools/examples/merl/Makefile new file mode 100644 index 0000000000..13a9703733 --- /dev/null +++ b/lib/syntax_tools/examples/merl/Makefile @@ -0,0 +1,22 @@ +EBIN=../../ebin +INCLUDES=../../include +SOURCES=merl_build.erl lisp.erl lispc.erl basic.erl basicc.erl +HEADERS=$(INCLUDES)/merl.hrl +OBJECTS=$(SOURCES:%.erl=%.beam) +ERLC_FLAGS=+debug_info -I$(INCLUDES) -pa $(EBIN) + +all: $(OBJECTS) test + +%.beam: %.erl $(HEADERS) Makefile + erlc $(ERLC_FLAGS) -o ./ $< + +# additional dependencies due to the parse transform +lispc.beam basicc.beam: $(EBIN)/merl_transform.beam $(EBIN)/merl.beam + +clean: + -rm -f $(OBJECTS) + +test: + erl -noshell -pa $(EBIN) \ + -eval 'eunit:test([lisp, lispc, basic, basicc],[])' \ + -s init stop diff --git a/lib/syntax_tools/examples/merl/basic.erl b/lib/syntax_tools/examples/merl/basic.erl new file mode 100644 index 0000000000..9030059d11 --- /dev/null +++ b/lib/syntax_tools/examples/merl/basic.erl @@ -0,0 +1,77 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012 Richard Carlsson +%% @doc Trivial Basic interpreter in Erlang + +-module(basic). + +-export([run/2]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(INTERPRETED, true). +-include("basic_test.erl"). + +run(N, Prog) -> + ets:new(var, [private, named_table]), + ets:new(line, [private, named_table, ordered_set]), + lists:foreach(fun (T) -> ets:insert(line, T) end, Prog), + goto(N). + +stop(N) -> + ets:delete(var), + ets:delete(line), + N. + +goto('$end_of_table') -> stop(0); +goto(L) -> + L1 = ets:next(line, L), + %% user-supplied line numbers might not exist + case ets:lookup(line, L) of + [{_, X}] -> + stmt(X, L1); + _ -> + goto(L1) + end. + +stmt({print, S, As}, L) -> io:format(S, [expr(A) || A <- As]), goto(L); +stmt({set, V, X}, L) -> ets:insert(var, {V, expr(X)}), goto(L); +stmt({goto, X}, _L) -> goto(expr(X)); +stmt({stop, X}, _L) -> stop(expr(X)); +stmt({iff, X, A, B}, _L) -> + case expr(X) of + 0 -> goto(B); + _ -> goto(A) + end. + +expr(X) when is_number(X) ; is_list(X) -> + X; +expr(X) when is_atom(X) -> + case ets:lookup(var, X) of + [] -> 0; + [{_,V}] -> V + end; +expr({plus, X, Y}) -> + expr(X) + expr(Y); +expr({equal, X, Y}) -> + bool(expr(X) == expr(Y)); +expr({gt, X, Y}) -> + bool(expr(X) > expr(Y)); +expr({knot, X}) -> + case expr(X) of + 0 -> 1; + _ -> 0 + end. + +bool(true) -> 1; +bool(false) -> 0. diff --git a/lib/syntax_tools/examples/merl/basic_test.erl b/lib/syntax_tools/examples/merl/basic_test.erl new file mode 100644 index 0000000000..ff35de6325 --- /dev/null +++ b/lib/syntax_tools/examples/merl/basic_test.erl @@ -0,0 +1,77 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012 Richard Carlsson +%% @doc Tests. For including in another module. + +%-module(basic_test). +%-import(basic, run/1) + +-export([basic_fib/1]). + +-include_lib("eunit/include/eunit.hrl"). + +basics_test_() -> + [?_assertEqual(42, run(1,[{1,{stop, 42}}])), + ?_assertEqual("hello", run(1,[{1,{stop,"hello"}}])), + ?_assertEqual(0, run(1,[{1,{print, "hello ~w", [42]}}])), + ?_assertEqual(5, run(1,[{1,{stop, {plus, 2, 3}}}])), + ?_assertEqual(5, run(1,[{1,{stop,{plus, 8, -3}}}])), + ?_assertEqual(0, run(1,[{1,{stop,{equal, 0, 1}}}])), + ?_assertEqual(1, run(1,[{1,{stop,{equal, 1, 1}}}])), + ?_assertEqual(0, run(1,[{1,{stop,{gt, 0, 1}}}])), + ?_assertEqual(0, run(1,[{1,{stop,{gt, 1, 1}}}])), + ?_assertEqual(1, run(1,[{1,{stop,{gt, 2, 1}}}])), + ?_assertEqual(0, run(1,[{1,{stop,{knot, 42}}}])), + ?_assertEqual(1, run(1,[{1,{stop,{knot, 0}}}])), + ?_assertEqual(42, run(1,[{1,{set, x, 42}}, {2,{stop,x}}])), + ?_assertEqual(17, run(1,[{1,{iff, 1, 2, 3}}, + {2,{stop, 17}}, + {3,{stop, 42}}])), + ?_assertEqual(42, run(1,[{1,{iff, 0, 2, 3}}, + {2,{stop, 17}}, + {3,{stop, 42}}])), + ?_assertEqual(17, run(1,[{1,{iff, 1, 2, 3}}, + {2,{stop, 17}}, + {3,{stop, -1}}])), + ?_assertEqual(42, run(1,[{1,{iff, 0, 2, 3}}, + {2,{stop, -1}}, + {3,{stop, 42}}])) + + + ]. + + +fib_test_() -> + [?_assertEqual(fib(N), basic_fib(N)) || N <- lists:seq(1,15) + ]. + + +fib(N) when N > 1 -> + fib(N-1) + fib(N-2); +fib(_) -> + 1. + +basic_fib(N) -> + run(1, + [{1,{set,x,0}}, + {2,{set,a,1}}, + {3,{set,b,0}}, + {10,{iff, {equal, x, N}, 20, 30}}, + {20,{stop,a}}, + {30,{print,"~w, ~w, ~w\n",[x,a,b]}}, + {31,{set,t,a}}, + {32,{set,a,{plus,a,b}}}, + {33,{set,b,t}}, + {34,{set,x,{plus,x,1}}}, + {40,{goto,10}} + ]). diff --git a/lib/syntax_tools/examples/merl/basicc.erl b/lib/syntax_tools/examples/merl/basicc.erl new file mode 100644 index 0000000000..531ac51538 --- /dev/null +++ b/lib/syntax_tools/examples/merl/basicc.erl @@ -0,0 +1,149 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012 Richard Carlsson +%% @doc Basic compiler in Erlang. + +-module(basicc). + +-export([run/2, make_lines/1, bool/1]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(INTERPRETED, true). +-include("basic_test.erl"). + +-include("merl.hrl"). + +run(N, Prog) -> + compile(Prog, tmp), + tmp:run(N, Prog). + +make_lines(Prog) -> + ets:new(line, [private, named_table, ordered_set]), + lists:foreach(fun ({L,_}) -> ets:insert(line, {L,label(L)}) end, Prog). + +compile(Prog, ModName) -> + make_lines(Prog), + Fs0 = lists:map(fun ({L, X}) -> + {true, label(L), + case stmt(X) of + {Stmt, false} -> + [?Q("() -> _@Stmt")]; + {Stmt, true} -> + Next = case ets:next(line, L) of + '$end_of_table' -> + ?Q("stop(0)"); + L1 -> + Label = label(L1), + ?Q("_@Label@()") + end, + [?Q("() -> _@Stmt, _@Next")] + end} + end, Prog), + ets:delete(line), + Run = ?Q(["(N, Prog) ->", + " ets:new(var, [private, named_table]),", + " basicc:make_lines(Prog),", + " goto(N)" + ]), + Stop = ?Q(["(R) ->", + " ets:delete(var),", + " ets:delete(line),", + " R" + ]), + Goto = ?Q(["(L) ->", + " case ets:lookup(line, L) of", + " [{_, X}] -> apply(tmp, X, []);", + " _ ->", + " case ets:next(line, L) of", + " '$end_of_table' -> stop(0);", + " L1 -> goto(L1)", + " end", + " end"]), + Fs = [{true, run, [Run]}, + {false, stop, [Stop]}, + {true, goto, [Goto]} + | Fs0], + Forms = merl_build:module_forms( + lists:foldl(fun ({X, Name, Cs}, S) -> + merl_build:add_function(X, Name, Cs, S) + end, + merl_build:init_module(ModName), + Fs)), + %% %% Write source to file for debugging + %% file:write_file(lists:concat([ModName, "_gen.erl"]), + %% erl_prettypr:format(erl_syntax:form_list(Forms), + %% [{paper,160},{ribbon,80}])), + merl:compile_and_load(Forms, [verbose]). + +label(L) -> + list_to_atom("label_" ++ integer_to_list(L)). + +stmt({print, S, As}) -> + Exprs = [expr(A) || A <- As], + {[?Q(["io:format(_@S@, [_@Exprs])"])], true}; +stmt({set, V, X}) -> + Expr = expr(X), + {[?Q(["ets:insert(var, {_@V@, _@Expr})"])], true}; +stmt({goto, X}) -> + {[jump(X)], false}; +stmt({stop, X}) -> + Expr = expr(X), + {[?Q(["stop(_@Expr)"])], false}; +stmt({iff, X, A, B}) -> + Cond = expr(X), + True = jump(A), + False = jump(B), + {?Q(["case _@Cond of", + " 0 -> _@False;", + " _ -> _@True", + "end"]), + false}. + +jump(X) -> + case ets:lookup(line, X) of + [{_, F}] -> + ?Q(["_@F@()"]); + true -> + Expr = expr(X), + [?Q(["goto(_@Expr)"])] + end. + +expr(X) when is_number(X) ; is_list(X) -> + ?Q("_@X@"); +expr(X) when is_atom(X) -> + ?Q(["case ets:lookup(var, _@X@) of", + " [] -> 0;", + " [{_,V}] -> V", + "end"]); +expr({plus, X, Y}) -> + ExprX = expr(X), + ExprY = expr(Y), + ?Q("_@ExprX + _@ExprY"); +expr({equal, X, Y}) -> + ExprX = expr(X), + ExprY = expr(Y), + ?Q("basicc:bool(_@ExprX == _@ExprY)"); +expr({gt, X, Y}) -> + ExprX = expr(X), + ExprY = expr(Y), + ?Q("basicc:bool(_@ExprX > _@ExprY)"); +expr({knot, X}) -> + Expr = expr(X), + ?Q(["case _@Expr of", + " 0 -> 1;", + " _ -> 0", + "end"]). + +bool(true) -> 1; +bool(false) -> 0. diff --git a/lib/syntax_tools/examples/merl/lisp.erl b/lib/syntax_tools/examples/merl/lisp.erl new file mode 100644 index 0000000000..371dc6b261 --- /dev/null +++ b/lib/syntax_tools/examples/merl/lisp.erl @@ -0,0 +1,160 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012 Richard Carlsson +%% @doc Trivial Lisp interpreter in Erlang. + +-module(lisp). + +-export([eval/1]). + +-export([init/0, equal/2, gt/2, knot/1]). + +-record(st, {env}). + +-define(INTERPRETED, true). +-include("lisp_test.erl"). + +eval(P) -> + {X, _} = eval(P, init()), + X. + +init() -> + Env = [{print, {builtin, fun do_print/2}} + ,{list, {builtin, fun do_list/2}} + ,{apply, {builtin, fun do_apply/2}} + ,{plus, {builtin, fun do_plus/2}} + ,{equal, {builtin, fun do_equal/2}} + ,{gt, {builtin, fun do_gt/2}} + ,{knot, {builtin, fun do_knot/2}} + ,{y, y()} + ], + #st{env=dict:from_list(Env)}. + +eval([lambda, Ps, B], #st{env=E}=St) when is_list(Ps) -> + case lists:all(fun is_atom/1, Ps) andalso + (length(Ps) =:= length(lists:usort(Ps))) of + true -> {{lambda, Ps, B, E}, St}; + false -> throw(bad_lambda) + end; +eval([lambda | _], _) -> + throw(bad_lambda); +eval([def, A, V, B], #st{env=E0}=St) when is_atom(A) -> + {V1, St1} = eval(V, St), + E1 = bind(A, V1, E0), + {X, St2} = eval(B, St1#st{env=E1}), + {X, St2#st{env=E0}}; +eval([def | _], _) -> + throw(bad_def); +eval([quote, A], St) -> + {A, St}; +eval([quote | _], _) -> + throw(bad_quote); +eval([iff, X, A, B], St) -> + case eval(X, St) of + {[], St1} -> eval(B, St1); + {_, St1} -> eval(A, St1) + end; +eval([do], _St0) -> + throw(bad_do); +eval([do | As], St0) -> + lists:foldl(fun (X, {_,St}) -> eval(X, St) end, {[],St0}, As); +eval([_|_]=L, St) -> + {[F | As], St1} = lists:mapfoldl(fun eval/2, St, L), + call(F, As, St1); +eval(A, St) when is_atom(A) -> + {deref(A, St), St}; +eval(C, St) -> + {C, St}. + +%% UTILITY FUNCTIONS + +deref(A, #st{env=E}) -> + case dict:find(A, E) of + {ok, V} -> V; + error -> throw({undefined, A}) + end. + +bind(A, V, E) -> + dict:store(A, V, E). + +bind_args([P | Ps], [A | As], E) -> + bind_args(Ps, As, dict:store(P, A, E)); +bind_args([], [], E) -> + E; +bind_args(_, _, _) -> + throw(bad_arity). + +call({lambda, Ps, B, E}, As, #st{env=E0}=St) -> + {X, St1} = eval(B, St#st{env=bind_args(Ps, As, E)}), + {X, St1#st{env=E0}}; +call({builtin, F}, As, St) -> + F(As, St); +call(X, _, _) -> + throw({bad_fun, X}). + +bool(true) -> 1; +bool(false) -> []. + +%% BUILTINS + +y() -> + {Y, _} = eval([lambda, [f], + [[lambda, [x], [f, [lambda, [y], [[x, x], y]]]], + [lambda, [x], [f, [lambda, [y], [[x, x], y]]]]]], + #st{env=dict:new()}), + Y. + +do_print([S | Xs], St) -> + io:format(S, Xs), + {[], St}; +do_print(_, _) -> + throw(bad_print). + +do_list(As, St) -> + {As, St}. + +do_apply([F, As], St) -> + call(F, As, St); +do_apply(_, _) -> + throw(bad_apply). + +do_plus([X, Y], St) when is_number(X), is_number(Y) -> + {X + Y, St}; +do_plus(As, _) -> + throw({bad_plus, As}). + +do_equal([X, Y], St) -> + {equal(X, Y), St}; +do_equal(As, _) -> + throw({bad_equal, As}). + +equal(X, Y) -> + bool(X =:= Y). + +do_gt([X, Y], St) -> + {gt(X, Y), St}; +do_gt(As, _) -> + throw({bad_gt, As}). + +gt(X, Y) -> + bool(X > Y). + +do_knot([X], St) -> + {knot(X), St}; +do_knot(As, _) -> + throw({bad_gt, As}). + +knot([]) -> + 1; +knot(_) -> + []. diff --git a/lib/syntax_tools/examples/merl/lisp_test.erl b/lib/syntax_tools/examples/merl/lisp_test.erl new file mode 100644 index 0000000000..cab8134b8f --- /dev/null +++ b/lib/syntax_tools/examples/merl/lisp_test.erl @@ -0,0 +1,98 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012 Richard Carlsson +%% @doc Tests. For including in another module. + +%-module(lisp_test). +%-import(lisp, eval/1) + +-export([fib/1, lisp_fib/1]). + +-include_lib("eunit/include/eunit.hrl"). + +basics_test_() -> + [?_assertEqual(42, eval(42)), + ?_assertEqual("hello", eval([quote, "hello"])), + ?_assertEqual(print, eval([quote, print])), + ?_assertMatch([17,[1,2],42], eval([list,17,[list,1,2],42])), + ?_assertEqual([], eval([print, [quote, "hello ~w"], [list, 42]])), + ?_assertEqual(5, eval([plus, 2, 3])), + ?_assertEqual(5, eval([plus, 8, -3])), + ?_assertEqual([], eval([equal, 0, 1])), + ?_assertEqual(1, eval([equal, 1, 1])), + ?_assertEqual([], eval([gt, 0, 1])), + ?_assertEqual([], eval([gt, 1, 1])), + ?_assertEqual(1, eval([gt, 2, 1])), + ?_assertEqual([], eval([knot, 42])), + ?_assertEqual(1, eval([knot, []])), + ?_assertEqual(42, eval([do, 17, 42])), + ?_assertEqual([], eval([apply, print, [quote, ["~p", [42]]]])), + ?_assertEqual(42, eval([iff, [], 17, 42])), + ?_assertEqual(17, eval([iff, 1, 17, 42])), + ?_assertEqual(42, eval([iff, [], [apply], 42])), + ?_assertEqual(17, eval([iff, 1, 17, [apply]])), + ?_assertEqual(17, eval([def, foo, 17, foo])), + ?_assertEqual(17, eval([def, bar, 42, [def, foo, 17, foo]])), + ?_assertEqual(42, eval([def, bar, 42, [def, foo, 17, bar]])), + ?_assertEqual(17, eval([def, foo, 42, [def, foo, 17, foo]])) + ]. + +-ifdef(INTERPRETED). +interpreter_basics_test_() -> + [?_assertThrow({undefined, foo}, eval(foo)), + ?_assertMatch({builtin,_}, eval(print)), + ?_assertThrow(bad_do, eval([do])), + ?_assertThrow(bad_apply, eval([apply])), + ?_assertThrow({undefined, foo}, eval([def, bar, 17, foo])) + ]. + +interpreter_lambda_test_() -> + [?_assertMatch({lambda,_,_,_}, eval([lambda, [], 42])), + ?_assertMatch({lambda,_,_,_}, eval([lambda, [x], x])), + ?_assertMatch({lambda,_,_,_}, eval([lambda, [x,y], 42])) + ]. +-endif. + +lambda_test_() -> + [?_assertThrow(bad_lambda, eval([lambda])), + ?_assertThrow(bad_lambda, eval([lambda, []])), + ?_assertThrow(bad_lambda, eval([lambda, [], 17, 42])), + ?_assertThrow(bad_lambda, eval([lambda, 17, 42])), + ?_assertThrow(bad_lambda, eval([lambda, [17], 42])), + ?_assertThrow(bad_lambda, eval([lambda, [foo, foo], 42])), + ?_assertEqual(42, eval([[lambda, [x], x], 42])), + ?_assertEqual([42, 17], eval([[lambda, [x], [list, x, 17]], 42])), + ?_assertEqual([42, 17], eval([def, f, [def, y, 42, + [lambda, [x], [list, y, x]]], + [f, 17]])) + ]. + +fib_test_() -> + [?_assertEqual(fib(N), lisp_fib(N)) || N <- lists:seq(1,15) + ]. + + +fib(N) when N > 1 -> + fib(N-1) + fib(N-2); +fib(_) -> + 1. + +lisp_fib(N) -> + eval([def, fib, + [y, [lambda, [f], [lambda, [x], + [iff, [gt, x, 1], + [plus, [f, [plus,x,-1]], [f, [plus,x,-2]]], + 1] + ]]], + [fib, N] + ]). diff --git a/lib/syntax_tools/examples/merl/lispc.erl b/lib/syntax_tools/examples/merl/lispc.erl new file mode 100644 index 0000000000..97072cdab7 --- /dev/null +++ b/lib/syntax_tools/examples/merl/lispc.erl @@ -0,0 +1,102 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012 Richard Carlsson +%% @doc Lisp compiler in Erlang. + +-module(lispc). + +-export([eval/1]). + +-record(st, {}). + +-include("lisp_test.erl"). + +-include("merl.hrl"). + +eval(Lisp) -> + compile(Lisp, tmp), + tmp:eval(). + +compile(Lisp, ModName) -> + {Code, _} = gen(Lisp, #st{}), + Main = ?Q(["() ->", + " __print = fun (S, Xs) -> io:format(S,Xs), [] end,", + " __apply = fun erlang:apply/2,", + " __plus = fun erlang:'+'/2,", + " __equal = fun lisp:equal/2,", + " __gt = fun lisp:gt/2,", + " __knot = fun lisp:knot/1,", + " __y = fun (F) ->", + " (fun (X) -> F(fun (Y) -> (X(X))(Y) end) end)", + " (fun (X) -> F(fun (Y) -> (X(X))(Y) end) end)", + " end,", + " _@Code"]), + Forms = merl_build:module_forms( + merl_build:add_function(true, eval, [Main], + merl_build:init_module(ModName))), + %% %% Write source to file for debugging + %% file:write_file(lists:concat([ModName, "_gen.erl"]), + %% erl_prettypr:format(erl_syntax:form_list(Forms), + %% [{paper,160},{ribbon,80}])), + merl:compile_and_load(Forms, [verbose]). + +var(Atom) -> + merl:var(list_to_atom("__" ++ atom_to_list(Atom))). + +gen([lambda, Ps, B], St) when is_list(Ps) -> + case lists:all(fun is_atom/1, Ps) andalso + (length(Ps) =:= length(lists:usort(Ps))) of + true -> + Vars = [var(P) || P <- Ps], + {Body, St1} = gen(B, St), + {?Q("fun (_@Vars) -> _@Body end"), St1}; + false -> + throw(bad_lambda) + end; +gen([lambda | _], _) -> + throw(bad_lambda); +gen([def, A, V, B], St) when is_atom(A) -> + Var = var(A), + {Val, St1} = gen(V, St), + {Body, St2} = gen(B, St1), + {?Q("(fun (_@Var) -> _@Body end)(_@Val)"), St2}; +gen([def | _], _) -> + throw(bad_def); +gen([quote, A], St) -> + {merl:term(A), St}; +gen([quote | _], _) -> + throw(bad_quote); +gen([iff, X, A, B], St) -> + {Cond, St1} = gen(X, St), + {True, St2} = gen(A, St1), + {False, St3} = gen(B, St2), + {?Q(["case _@Cond of", + " [] -> _@False;", + " _ -> _@True", + "end"]), + St3}; +gen([do], _) -> + throw(bad_do); +gen([do | As], St0) -> + {Body, St1} = lists:mapfoldl(fun gen/2, St0, As), + {?Q("begin _@Body end"), St1}; +gen([list | As], St0) -> + {Elem, St1} = lists:mapfoldl(fun gen/2, St0, As), + {?Q("[ _@Elem ]"), St1}; +gen([_|_]=L, St) -> + {[F | As], St1} = lists:mapfoldl(fun gen/2, St, L), + {?Q("((_@F)(_@As))"), St1}; +gen(A, St) when is_atom(A) -> + {var(A), St}; +gen(C, St) -> + {merl:term(C), St}. diff --git a/lib/syntax_tools/examples/merl/merl_build.erl b/lib/syntax_tools/examples/merl/merl_build.erl new file mode 100644 index 0000000000..c539f8e2af --- /dev/null +++ b/lib/syntax_tools/examples/merl/merl_build.erl @@ -0,0 +1,104 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012 Richard Carlsson +%% @doc Making it simple to build a module with merl + +-module(merl_build). + +-export([init_module/1, module_forms/1, add_function/4, add_record/3, + add_import/3, add_attribute/3, set_file/2]). + +-import(merl, [term/1]). + +-include("merl.hrl"). + +-type filename() :: string(). + +-record(module, { name :: atom() + , file :: filename() + , exports=[] :: [{atom(), integer()}] + , imports=[] :: [{atom(), [{atom(), integer()}]}] + , attributes=[] :: [{filename(), atom(), [term()]}] + , records=[] :: [{filename(), atom(), + [{atom(), merl:tree()}]}] + , functions=[] :: [{filename(), atom(), [merl:tree()]}] + }). + +%% TODO: init module from a list of forms (from various sources) + +%% @doc Create a new module representation, using the given module name. +init_module(Name) when is_atom(Name) -> + %% use the module name as the default file name - better than nothing + #module{name=Name, file=atom_to_list(Name)}. + +%% @doc Get the list of syntax tree forms for a module representation. This can +%% be passed to compile/2. +module_forms(#module{name=Name, + exports=Xs, + imports=Is, + records=Rs, + attributes=As, + functions=Fs}) + when is_atom(Name), Name =/= undefined -> + Module = ?Q("-module('@Name@')."), + Exported = [erl_syntax:arity_qualifier(term(N), term(A)) + || {N,A} <- ordsets:from_list(Xs)], + Export = ?Q("-export(['@_Exported'/1])."), + Imports = [?Q("-import('@M@', ['@_NAs'/1]).") + || {M, Ns} <- Is, + NAs <- [[erl_syntax:arity_qualifier(term(N), term(A)) + || {N,A} <- ordsets:from_list(Ns)]] + ], + Attrs = [?Q("-file(\"'@File@\",1). -'@N@'('@T@').") + || {File, N, T} <- lists:reverse(As)], + Records = [?Q("-file(\"'@File@\",1). -record('@N@',{'@_RFs'=[]}).") + || {File, N, Es} <- lists:reverse(Rs), + RFs <- [[erl_syntax:record_field(term(F), V) + || {F,V} <- Es]] + ], + Functions = [?Q("-file(\"'@File@\",1). '@_F'() -> [].") + || {File, N, Cs} <- lists:reverse(Fs), + F <- [erl_syntax:function(term(N), Cs)]], + lists:flatten([Module, Export, Imports, Attrs, Records, Functions]). + +%% @doc Set the source file name for all subsequently added functions, +%% records, and attributes. +set_file(Filename, #module{}=M) -> + M#module{file=filename:flatten(Filename)}. + +%% @doc Add a function to a module representation. +add_function(Exported, Name, Clauses, + #module{file=File, exports=Xs, functions=Fs}=M) + when is_boolean(Exported), is_atom(Name), Clauses =/= [] -> + Arity = length(erl_syntax:clause_patterns(hd(Clauses))), + Xs1 = case Exported of + true -> [{Name,Arity} | Xs]; + false -> Xs + end, + M#module{exports=Xs1, functions=[{File, Name, Clauses} | Fs]}. + +%% @doc Add a record declaration to a module representation. +add_record(Name, Fields, #module{file=File, records=Rs}=M) + when is_atom(Name) -> + M#module{records=[{File, Name, Fields} | Rs]}. + +%% @doc Add a "wild" attribute, such as `-compile(Opts)' to a module +%% representation. Note that such attributes can only have a single argument. +add_attribute(Name, Term, #module{file=File, attributes=As}=M) + when is_atom(Name) -> + M#module{attributes=[{File, Name, Term} | As]}. + +%% @doc Add an import declaration to a module representation. +add_import(From, Names, #module{imports=Is}=M) + when is_atom(From), is_list(Names) -> + M#module{imports=[{From, Names} | Is]}. diff --git a/lib/syntax_tools/include/merl.hrl b/lib/syntax_tools/include/merl.hrl new file mode 100644 index 0000000000..e44a78dece --- /dev/null +++ b/lib/syntax_tools/include/merl.hrl @@ -0,0 +1,29 @@ +%% --------------------------------------------------------------------- +%% Header file for merl +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-ifndef(MERL_HRL). + + +%% Quoting a piece of code +-define(Q(Text), merl:quote(?LINE, Text)). + +%% Quasi-quoting code, substituting metavariables listed in Env +-define(Q(Text, Env), merl:qquote(?LINE, Text, Env)). + + +-ifndef(MERL_NO_TRANSFORM). +-compile({parse_transform, merl_transform}). +-endif. + + +-endif. diff --git a/lib/syntax_tools/src/Makefile b/lib/syntax_tools/src/Makefile index c9fbad8f9a..2e91adf8af 100644 --- a/lib/syntax_tools/src/Makefile +++ b/lib/syntax_tools/src/Makefile @@ -22,6 +22,9 @@ RELSYSDIR = $(RELEASE_PATH)/lib/syntax_tools-$(VSN) # EBIN = ../ebin +INCLUDE=../include + +ERL_COMPILE_FLAGS += -pa $(EBIN) -pa ./ -I$(INCLUDE) ifeq ($(NATIVE_LIBS_ENABLED),yes) ERL_COMPILE_FLAGS += +native @@ -30,10 +33,15 @@ ERL_COMPILE_FLAGS += +nowarn_shadow_vars +warn_unused_import -Werror # +warn_mis SOURCES=erl_syntax.erl erl_prettypr.erl erl_syntax_lib.erl \ erl_comment_scan.erl erl_recomment.erl erl_tidy.erl \ - epp_dodger.erl prettypr.erl igor.erl + epp_dodger.erl prettypr.erl igor.erl \ + merl.erl merl_transform.erl + +INCLUDE_FILES = merl.hrl OBJECTS=$(SOURCES:%.erl=$(EBIN)/%.$(EMULATOR)) $(APP_TARGET) $(APPUP_TARGET) +INCLUDE_DELIVERABLES = $(INCLUDE_FILES:%=$(INCLUDE)/%) + APP_FILE= syntax_tools.app APP_SRC= $(APP_FILE).src APP_TARGET= $(EBIN)/$(APP_FILE) @@ -52,6 +60,7 @@ all: $(OBJECTS) clean: + rm -f ./merl_transform.beam rm -f $(OBJECTS) rm -f core *~ @@ -64,6 +73,15 @@ realclean: clean $(EBIN)/%.$(EMULATOR):%.erl $(erlc_verbose)erlc -W $(ERL_COMPILE_FLAGS) -o$(EBIN) $< +# special rules and dependencies to apply the transform to itself +$(EBIN)/merl_transform.beam: $(EBIN)/merl.beam ./merl_transform.beam \ + ../include/merl.hrl \ + $(EBIN)/erl_syntax.beam $(EBIN)/erl_syntax_lib.beam +./merl_transform.beam: ./merl_transform.erl $(EBIN)/merl.beam \ + ../include/merl.hrl + $(V_ERLC) -DMERL_NO_TRANSFORM $(ERL_COMPILE_FLAGS) -o ./ $< + + # ---------------------------------------------------- # Special Build Targets # ---------------------------------------------------- @@ -84,6 +102,8 @@ release_spec: opt $(INSTALL_DATA) $(OBJECTS) "$(RELSYSDIR)/ebin" $(INSTALL_DIR) "$(RELSYSDIR)/src" $(INSTALL_DATA) $(SOURCES) "$(RELSYSDIR)/src" + $(INSTALL_DIR) "$(RELSYSDIR)/include" + $(INSTALL_DATA) $(INCLUDE_DELIVERABLES) "$(RELSYSDIR)/include" release_docs_spec: diff --git a/lib/syntax_tools/src/merl.erl b/lib/syntax_tools/src/merl.erl new file mode 100644 index 0000000000..690306c17b --- /dev/null +++ b/lib/syntax_tools/src/merl.erl @@ -0,0 +1,1230 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% Note: EDoc uses @@ and @} as escape sequences, so in the doc text below, +%% `@@' must be written `@@@@' and `@}' must be written `@@}'. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2010-2015 Richard Carlsson +%% +%% @doc Metaprogramming in Erlang. +%% Merl is a more user friendly interface to the `erl_syntax' module, making +%% it easy both to build new ASTs from scratch and to +%% match and decompose existing ASTs. For details that are outside the scope +%% of Merl itself, please see the documentation of {@link erl_syntax}. +%% +%% == Quick start == +%% +%% To enable the full power of Merl, your module needs to include the Merl +%% header file: +%% ```-include_lib("syntax_tools/include/merl.hrl").''' +%% +%% Then, you can use the `?Q(Text)' macros in your code to create ASTs or match +%% on existing ASTs. For example: +%% ```Tuple = ?Q("{foo, 42}"), +%% ?Q("{foo, _@Number}") = Tuple, +%% Call = ?Q("foo:bar(_@Number)")''' +%% +%% Calling `merl:print(Call)' will then print the following code: +%% ```foo:bar(42)''' +%% +%% The `?Q' macros turn the quoted code fragments into ASTs, and lifts +%% metavariables such as `_@Tuple' and `_@Number' to the level of your Erlang +%% code, so you can use the corresponding Erlang variables `Tuple' and `Number' +%% directly. This is the most straightforward way to use Merl, and in many +%% cases it's all you need. +%% +%% You can even write case switches using `?Q' macros as patterns. For example: +%% ```case AST of +%% ?Q("{foo, _@Foo}") -> handle(Foo); +%% ?Q("{bar, _@Bar}") when erl_syntax:is_integer(Bar) -> handle(Bar); +%% _ -> handle_default() +%% end''' +%% +%% These case switches only allow `?Q(...)' or `_' as clause patterns, and the +%% guards may contain any expressions, not just Erlang guard expressions. +%% +%% If the macro `MERL_NO_TRANSFORM' is defined before the `merl.hrl' header +%% file is included, the parse transform used by Merl will be disabled, and in +%% that case, the match expressions `?Q(...) = ...', case switches using +%% `?Q(...)' patterns, and automatic metavariables like `_@Tuple' cannot be +%% used in your code, but the Merl macros and functions still work. To do +%% metavariable substitution, you need to use the `?Q(Text, Map)' macro, e.g.: +%% ```Tuple = ?Q("{foo, _@bar, _@baz}", [{bar, Bar}, {baz,Baz}])''' +%% +%% The text given to a `?Q(Text)' macro can be either a single string, or a +%% list of strings. The latter is useful when you need to split a long +%% expression over multiple lines, e.g.: +%% ```?Q(["case _@Expr of", +%% " {foo, X} -> f(X);", +%% " {bar, X} -> g(X)", +%% " _ -> h(X)" +%% "end"])''' +%% If there is a syntax error somewhere in the text (like the missing semicolon +%% in the second clause above) this allows Merl to generate an error message +%% pointing to the exact line in your source code. (Just remember to +%% comma-separate the strings in the list, otherwise Erlang will concatenate +%% the string fragments as if they were a single string.) +%% +%% == Metavariable syntax == +%% +%% There are several ways to write a metavariable in your quoted code: +%% <ul> +%% <li>Atoms starting with `@', for example `` '@foo' '' or `` '@Foo' ''</li> +%% <li>Variables starting with `_@', for example `_@bar' or `_@Bar'</li> +%% <li>Strings starting with ``"'@'', for example ``"'@File"''</li> +%% <li>Integers starting with 909, for example `9091' or `909123'</li> +%% </ul> +%% Following the prefix, one or more `_' or `0' characters may be used to +%% indicate "lifting" of the variable one or more levels, and after that, a `@' +%% or `9' character indicates a glob metavariable (matching zero or more +%% elements in a sequence) rather than a normal metavariable. For example: +%% <ul> +%% <li>`` '@_foo' '' is lifted one level, and `_@__foo' is lifted two +%% levels</li> +%% <li>`_@@@@bar' is a glob variable, and `_@_@bar' is a lifted glob +%% variable</li> +%% <li>`90901' is a lifted variable,`90991' is a glob variable, and `9090091' +%% is a glob variable lifted two levels</li> +%% </ul> +%% (Note that the last character in the name is never considered to be a lift +%% or glob marker, hence, `_@__' and `90900' are only lifted one level, not +%% two. Also note that globs only matter for matching; when doing +%% substitutions, a non-glob variable can be used to inject a sequence of +%% elements, and vice versa.) +%% +%% If the name after the prefix and any lift and glob markers is `_' or `0', +%% the variable is treated as an anonymous catch-all pattern in matches. For +%% example, `_@_', `_@@@@_', `_@__', or even `_@__@_'. +%% +%% Finally, if the name without any prefixes or lift/glob markers begins with +%% an uppercase character, as in `_@Foo' or `_@_@Foo', it will become a +%% variable on the Erlang level, and can be used to easily deconstruct and +%% construct syntax trees: +%% ```case Input of +%% ?Q("{foo, _@Number}") -> ?Q("foo:bar(_@Number)"); +%% ...''' +%% We refer to these as "automatic metavariables". If in addition the name ends +%% with `@', as in `_@Foo@', the value of the variable as an Erlang term will +%% be automatically converted to the corresponding abstract syntax tree when +%% used to construct a larger tree. For example, in: +%% ```Bar = {bar, 42}, +%% Foo = ?Q("{foo, _@Bar@@}")''' +%% (where Bar is just some term, not a syntax tree) the result `Foo' will be a +%% syntax tree representing `{foo, {bar, 42}}'. This avoids the need for +%% temporary variables in order to inject data, as in +%% ```TmpBar = erl_syntax:abstract(Bar), +%% Foo = ?Q("{foo, _@TmpBar}")''' +%% +%% If the context requires an integer rather than a variable, an atom, or a +%% string, you cannot use the uppercase convention to mark an automatic +%% metavariable. Instead, if the integer (without the `909'-prefix and +%% lift/glob markers) ends in a `9', the integer will become an Erlang-level +%% variable prefixed with `Q', and if it ends with `99' it will also be +%% automatically abstracted. For example, the following will increment the +%% arity of the exported function f: +%% ```case Form of +%% ?Q("-export([f/90919]).") -> +%% Q2 = erl_syntax:concrete(Q1) + 1, +%% ?Q("-export([f/909299])."); +%% ...''' +%% +%% == When to use the various forms of metavariables == +%% +%% Merl can only parse a fragment of text if it follows the basic syntactical +%% rules of Erlang. In most places, a normal Erlang variable can be used as +%% metavariable, for example: +%% ```?Q("f(_@Arg)") = Expr''' +%% but if you want to match on something like the name of a function, you have +%% to use an atom as metavariable: +%% ```?Q("'@Name'() -> _@@@@_." = Function''' +%% (note the anonymous glob variable `_@@@@_' to ignore the function body). +%% +%% In some contexts, only a string or an integer is allowed. For example, the +%% directive `-file(Name, Line)' requires that `Name' is a string literal and +%% `Line' an integer literal: +%% +%% ```?Q("-file(\"'@File\", 9090).") = ?Q("-file(\"foo.erl\", 42).")).''' +%% This will extract the string literal `"foo.erl"' into the variable `Foo'. +%% Note the use of the anonymous variable `9090' to ignore the line number. To +%% match and also bind a metavariable that must be an integer literal, we can +%% use the convention of ending the integer with a 9, turning it into a +%% Q-prefixed variable on the Erlang level (see the previous section). +%% +%% === Globs === +%% +%% Whenever you want to match out a number of elements in a sequence (zero or +%% more) rather than a fixed set of elements, you need to use a glob. For +%% example: +%% ```?Q("{_@@@@Elements}") = ?Q({a, b, c})''' +%% will bind Elements to the list of individual syntax trees representing the +%% atoms `a', `b', and `c'. This can also be used with static prefix and suffix +%% elements in the sequence. For example: +%% ```?Q("{a, b, _@@@@Elements}") = ?Q({a, b, c, d})''' +%% will bind Elements to the list of the `c' and `d' subtrees, and +%% ```?Q("{_@@@@Elements, c, d}") = ?Q({a, b, c, d})''' +%% will bind Elements to the list of the `a' and `b' subtrees. You can even use +%% plain metavariables in the prefix or suffix: +%% ```?Q("{_@First, _@@@@Rest}") = ?Q({a, b, c})''' +%% or +%% ```?Q("{_@@@@_, _@Last}") = ?Q({a, b, c})''' +%% (ignoring all but the last element). You cannot however have two globs as +%% part of the same sequence. +%% +%% === Lifted metavariables === +%% +%% In some cases, the Erlang syntax rules make it impossible to place a +%% metavariable directly where you would like it. For example, you cannot +%% write: +%% ```?Q("-export([_@@@@Name]).")''' +%% to match out all name/arity pairs in the export list, or to insert a list of +%% exports in a declaration, because the Erlang parser only allows elements on +%% the form `A/I' (where `A' is an atom and `I' an integer) in the export list. +%% A variable like the above is not allowed, but neither is a single atom or +%% integer, so `` '@@@@Name' '' or `909919' wouldn't work either. +%% +%% What you have to do in such cases is to write your metavariable in a +%% syntactically valid position, and use lifting markers to denote where it +%% should really apply, as in: +%% ```?Q("-export(['@@_@@Name'/0]).")''' +%% This causes the variable to be lifted (after parsing) to the next higher +%% level in the syntax tree, replacing that entire subtree. In this case, the +%% `` '@@_@@Name'/0 '' will be replaced with `` '@@@@Name' '', and the ``/0'' +%% part was just used as dummy notation and will be discarded. +%% +%% You may even need to apply lifting more than once. To match the entire +%% export list as a single syntax tree, you can write: +%% ```?Q("-export(['@@__Name'/0]).")''' +%% using two underscores, but with no glob marker this time. This will make the +%% entire ``['@@__Name'/0]'' part be replaced with `` '@@Name' ''. +%% +%% Sometimes, the tree structure of a code fragment isn't very obvious, and +%% parts of the structure may be invisible when printed as source code. For +%% instance, a simple function definition like the following: +%% ```zero() -> 0.''' +%% consists of the name (the atom `zero'), and a list of clauses containing the +%% single clause `() -> 0'. The clause consists of an argument list (empty), a +%% guard (empty), and a body (which is always a list of expressions) containing +%% the single expression `0'. This means that to match out the name and the +%% list of clauses of any function, you'll need to use a pattern like +%% ``?Q("'@Name'() -> _@_@Body.")'', using a dummy clause whose body is a glob +%% lifted one level. +%% +%% To visualize the structure of a syntax tree, you can use the function +%% `merl:show(T)', which prints a summary. For example, entering +%% ```merl:show(merl:quote("inc(X, Y) when Y > 0 -> X + Y."))''' +%% in the Erlang shell will print the following (where the `+' signs separate +%% groups of subtrees on the same level): +%% ```function: inc(X, Y) when ... -> X + Y. +%% atom: inc +%% + +%% clause: (X, Y) when ... -> X + Y +%% variable: X +%% variable: Y +%% + +%% disjunction: Y > 0 +%% conjunction: Y > 0 +%% infix_expr: Y > 0 +%% variable: Y +%% + +%% operator: > +%% + +%% integer: 0 +%% + +%% infix_expr: X + Y +%% variable: X +%% + +%% operator: + +%% + +%% variable: Y''' +%% +%% This shows another important non-obvious case: a clause guard, even if it's +%% as simple as `Y > 0', always consists of a single disjunction of one or more +%% conjunctions of tests, much like a tuple of tuples. Thus: +%% <ul> +%% <li>``"when _@Guard ->"'' will only match a guard with exactly one +%% test</li> +%% <li>``"when _@@@@Guard ->"'' will match a guard with one or more +%% comma-separated tests (but no semicolons), binding `Guard' to the list +%% of tests</li> +%% <li>``"when _@_Guard ->"'' will match just like the previous pattern, but +%% binds `Guard' to the conjunction subtree</li> +%% <li>``"when _@_@Guard ->"'' will match an arbitrary nonempty guard, +%% binding `Guard' to the list of conjunction subtrees</li> +%% <li>``"when _@__Guard ->"'' will match like the previous pattern, but +%% binds `Guard' to the whole disjunction subtree</li> +%% <li>and finally, ``"when _@__@Guard ->"'' will match any clause, +%% binding `Guard' to `[]' if the guard is empty and to `[Disjunction]' +%% otherwise</li> +%% </ul> +%% +%% Thus, the following pattern matches all possible clauses: +%% ```"(_@@Args) when _@__@Guard -> _@@Body"''' +%% @end + +-module(merl). + +-export([term/1, var/1, print/1, show/1]). + +-export([quote/1, quote/2, qquote/2, qquote/3]). + +-export([template/1, tree/1, subst/2, tsubst/2, alpha/2, match/2, switch/2]). + +-export([template_vars/1, meta_template/1]). + +-export([compile/1, compile/2, compile_and_load/1, compile_and_load/2]). + +%% NOTE: this module may not include merl.hrl! + +-type tree() :: erl_syntax:syntaxTree(). + +-type tree_or_trees() :: tree() | [tree()]. + +-type pattern() :: tree() | template(). + +-type pattern_or_patterns() :: pattern() | [pattern()]. + +-type env() :: [{Key::id(), pattern_or_patterns()}]. + +-type id() :: atom() | integer(). + +%% A list of strings or binaries is assumed to represent individual lines, +%% while a flat string or binary represents source code containing newlines. +-type text() :: string() | binary() | [string()] | [binary()]. + +-type location() :: erl_anno:location(). + + +%% ------------------------------------------------------------------------ +%% Compiling and loading code directly to memory + +%% @equiv compile(Code, []) +compile(Code) -> + compile(Code, []). + +%% @doc Compile a syntax tree or list of syntax trees representing a module +%% into a binary BEAM object. +%% @see compile_and_load/2 +%% @see compile/1 +compile(Code, Options) when not is_list(Code)-> + case type(Code) of + form_list -> compile(erl_syntax:form_list_elements(Code)); + _ -> compile([Code], Options) + end; +compile(Code, Options0) when is_list(Options0) -> + Forms = [erl_syntax:revert(F) || F <- Code], + Options = [verbose, report_errors, report_warnings, binary | Options0], + compile:noenv_forms(Forms, Options). + + +%% @equiv compile_and_load(Code, []) +compile_and_load(Code) -> + compile_and_load(Code, []). + +%% @doc Compile a syntax tree or list of syntax trees representing a module +%% and load the resulting module into memory. +%% @see compile/2 +%% @see compile_and_load/1 +compile_and_load(Code, Options) -> + case compile(Code, Options) of + {ok, ModuleName, Binary} -> + _ = code:load_binary(ModuleName, "", Binary), + {ok, Binary}; + Other -> Other + end. + + +%% ------------------------------------------------------------------------ +%% Utility functions + + +-spec var(atom()) -> tree(). + +%% @doc Create a variable. + +var(Name) -> + erl_syntax:variable(Name). + + +-spec term(term()) -> tree(). + +%% @doc Create a syntax tree for a constant term. + +term(Term) -> + erl_syntax:abstract(Term). + + +%% @doc Pretty-print a syntax tree or template to the standard output. This +%% is a utility function for development and debugging. + +print(Ts) when is_list(Ts) -> + lists:foreach(fun print/1, Ts); +print(T) -> + io:put_chars(erl_prettypr:format(tree(T))), + io:nl(). + +%% @doc Print the structure of a syntax tree or template to the standard +%% output. This is a utility function for development and debugging. + +show(Ts) when is_list(Ts) -> + lists:foreach(fun show/1, Ts); +show(T) -> + io:put_chars(pp(tree(T), 0)), + io:nl(). + +pp(T, I) -> + [lists:duplicate(I, $\s), + limit(lists:flatten([atom_to_list(type(T)), ": ", + erl_prettypr:format(erl_syntax_lib:limit(T,3))]), + 79-I), + $\n, + pp_1(lists:filter(fun (X) -> X =/= [] end, subtrees(T)), I+2) + ]. + +pp_1([G], I) -> + pp_2(G, I); +pp_1([G | Gs], I) -> + [pp_2(G, I), lists:duplicate(I, $\s), "+\n" | pp_1(Gs, I)]; +pp_1([], _I) -> + []. + +pp_2(G, I) -> + [pp(E, I) || E <- G]. + +%% limit string to N characters, stay on a single line and compact whitespace +limit([$\n | Cs], N) -> limit([$\s | Cs], N); +limit([$\r | Cs], N) -> limit([$\s | Cs], N); +limit([$\v | Cs], N) -> limit([$\s | Cs], N); +limit([$\t | Cs], N) -> limit([$\s | Cs], N); +limit([$\s, $\s | Cs], N) -> limit([$\s | Cs], N); +limit([C | Cs], N) when C < 32 -> limit(Cs, N); +limit([C | Cs], N) when N > 3 -> [C | limit(Cs, N-1)]; +limit([_C1, _C2, _C3, _C4 | _Cs], 3) -> "..."; +limit(Cs, 3) -> Cs; +limit([_C1, _C2, _C3 | _], 2) -> ".."; +limit(Cs, 2) -> Cs; +limit([_C1, _C2 | _], 1) -> "."; +limit(Cs, 1) -> Cs; +limit(_, _) -> []. + +%% ------------------------------------------------------------------------ +%% Parsing and instantiating code fragments + + +-spec qquote(Text::text(), Env::env()) -> tree_or_trees(). + +%% @doc Parse text and substitute meta-variables. +%% +%% @equiv qquote(1, Text, Env) + +qquote(Text, Env) -> + qquote(1, Text, Env). + + +-spec qquote(StartPos::location(), Text::text(), Env::env()) -> tree_or_trees(). + +%% @doc Parse text and substitute meta-variables. Takes an initial scanner +%% starting position as first argument. +%% +%% The macro `?Q(Text, Env)' expands to `merl:qquote(?LINE, Text, Env)'. +%% +%% @see quote/2 + +qquote(StartPos, Text, Env) -> + subst(quote(StartPos, Text), Env). + + +-spec quote(Text::text()) -> tree_or_trees(). + +%% @doc Parse text. +%% +%% @equiv quote(1, Text) + +quote(Text) -> + quote(1, Text). + + +-spec quote(StartPos::location(), Text::text()) -> tree_or_trees(). + +%% @doc Parse text. Takes an initial scanner starting position as first +%% argument. +%% +%% The macro `?Q(Text)' expands to `merl:quote(?LINE, Text, Env)'. +%% +%% @see quote/1 + +quote({Line, Col}, Text) + when is_integer(Line), is_integer(Col) -> + quote_1(Line, Col, Text); +quote(StartPos, Text) when is_integer(StartPos) -> + quote_1(StartPos, undefined, Text). + +quote_1(StartLine, StartCol, Text) -> + %% be backwards compatible as far as R12, ignoring any starting column + StartPos = case erlang:system_info(version) of + "5.6" ++ _ -> StartLine; + "5.7" ++ _ -> StartLine; + "5.8" ++ _ -> StartLine; + _ when StartCol =:= undefined -> StartLine; + _ -> {StartLine, StartCol} + end, + FlatText = flatten_text(Text), + {ok, Ts, _} = erl_scan:string(FlatText, StartPos), + merge_comments(StartLine, erl_comment_scan:string(FlatText), parse_1(Ts)). + +parse_1(Ts) -> + %% if dot tokens are present, it is assumed that the text represents + %% complete forms, not dot-terminated expressions or similar + case split_forms(Ts) of + {ok, Fs} -> parse_forms(Fs); + error -> + parse_2(Ts) + end. + +split_forms(Ts) -> + split_forms(Ts, [], []). + +split_forms([{dot,_}=T|Ts], Fs, As) -> + split_forms(Ts, [lists:reverse(As, [T]) | Fs], []); +split_forms([T|Ts], Fs, As) -> + split_forms(Ts, Fs, [T|As]); +split_forms([], Fs, []) -> + {ok, lists:reverse(Fs)}; +split_forms([], [], _) -> + error; % no dot tokens found - not representing form(s) +split_forms([], _, [T|_]) -> + fail("incomplete form after ~p", [T]). + +parse_forms([Ts | Tss]) -> + case erl_parse:parse_form(Ts) of + {ok, Form} -> [Form | parse_forms(Tss)]; + {error, R} -> parse_error(R) + end; +parse_forms([]) -> + []. + +parse_2(Ts) -> + %% one or more comma-separated expressions? + %% (recall that Ts has no dot tokens if we get to this stage) + case erl_parse:parse_exprs(Ts ++ [{dot,0}]) of + {ok, Exprs} -> Exprs; + {error, E} -> + parse_3(Ts ++ [{'end',0}, {dot,0}], [E]) + end. + +parse_3(Ts, Es) -> + %% try-clause or clauses? + case erl_parse:parse_exprs([{'try',0}, {atom,0,true}, {'catch',0} | Ts]) of + {ok, [{'try',_,_,_,_,_}=X]} -> + %% get the right kind of qualifiers in the clause patterns + erl_syntax:try_expr_handlers(X); + {error, E} -> + parse_4(Ts, [E|Es]) + end. + +parse_4(Ts, Es) -> + %% fun-clause or clauses? (`(a)' is also a pattern, but `(a,b)' isn't, + %% so fun-clauses must be tried before normal case-clauses + case erl_parse:parse_exprs([{'fun',0} | Ts]) of + {ok, [{'fun',_,{clauses,Cs}}]} -> Cs; + {error, E} -> + parse_5(Ts, [E|Es]) + end. + +parse_5(Ts, Es) -> + %% case-clause or clauses? + case erl_parse:parse_exprs([{'case',0}, {atom,0,true}, {'of',0} | Ts]) of + {ok, [{'case',_,_,Cs}]} -> Cs; + {error, E} -> + %% select the best error to report + parse_error(lists:last(lists:sort([E|Es]))) + end. + +-dialyzer({nowarn_function, parse_error/1}). % no local return + +parse_error({L, M, R}) when is_atom(M), is_integer(L) -> + fail("~w: ~s", [L, M:format_error(R)]); +parse_error({{L,C}, M, R}) when is_atom(M), is_integer(L), is_integer(C) -> + fail("~w:~w: ~s", [L,C,M:format_error(R)]); +parse_error({_, M, R}) when is_atom(M) -> + fail(M:format_error(R)); +parse_error(R) -> + fail("unknown parse error: ~p", [R]). + +%% ------------------------------------------------------------------------ +%% Templates, substitution and matching + +%% Leaves are normal syntax trees, and inner nodes are tuples +%% {template,Type,Attrs,Groups} where Groups are lists of lists of nodes. +%% Metavariables are 1-tuples {VarName}, where VarName is an atom or an +%% integer. {'_'} and {0} work as anonymous variables in matching. Glob +%% metavariables are tuples {'*',VarName}, and {'*','_'} and {'*',0} are +%% anonymous globs. + +%% Note that although template() :: tree() | ..., it is implied that these +%% syntax trees are free from metavariables, so pattern() :: tree() | +%% template() is in fact a wider type than template(). + +-type template() :: tree() + | {id()} + | {'*',id()} + | {template, atom(), term(), [[template()]]}. + +-type template_or_templates() :: template() | [template()]. + +-spec template(pattern_or_patterns()) -> template_or_templates(). + +%% @doc Turn a syntax tree or list of trees into a template or templates. +%% Templates can be instantiated or matched against, and reverted back to +%% normal syntax trees using {@link tree/1}. If the input is already a +%% template, it is not modified further. +%% +%% @see subst/2 +%% @see match/2 +%% @see tree/1 + +template(Trees) when is_list(Trees) -> + [template_0(T) || T <- Trees]; +template(Tree) -> + template_0(Tree). + +template_0({template, _, _, _}=Template) -> Template; +template_0({'*',_}=Template) -> Template; +template_0({_}=Template) -> Template; +template_0(Tree) -> + case template_1(Tree) of + false -> Tree; + {Name} when is_list(Name) -> + fail("bad metavariable: '~s'", [tl(Name)]); % drop v/n from name + Template -> Template + end. + +%% returns either a template or a lifted metavariable {String}, or 'false' +%% if Tree contained no metavariables +template_1(Tree) -> + case subtrees(Tree) of + [] -> + case metavar(Tree) of + {"v_"++Cs}=V when Cs =/= [] -> V; % to be lifted + {"n0"++Cs}=V when Cs =/= [] -> V; % to be lifted + {"v@"++Cs} when Cs =/= [] -> {'*',list_to_atom(Cs)}; + {"n9"++Cs} when Cs =/= [] -> {'*',list_to_integer(Cs)}; + {"v"++Cs} -> {list_to_atom(Cs)}; + {"n"++Cs} -> {list_to_integer(Cs)}; + false -> false + end; + Gs -> + case template_2(Gs, [], false) of + Gs1 when is_list(Gs1) -> + {template, type(Tree), erl_syntax:get_attrs(Tree), Gs1}; + Other -> + Other + end + end. + +template_2([G | Gs], As, Bool) -> + case template_3(G, [], false) of + {"v_"++Cs}=V when Cs =/= [] -> V; % lift further + {"n0"++Cs}=V when Cs =/= [] -> V; % lift further + {"v@"++Cs} when Cs =/= [] -> {'*',list_to_atom(Cs)}; % stop + {"n9"++Cs} when Cs =/= [] -> {'*',list_to_integer(Cs)}; % stop + {"v"++Cs} when is_list(Cs) -> {list_to_atom(Cs)}; % stop + {"n"++Cs} when is_list(Cs) -> {list_to_integer(Cs)}; % stop + false -> template_2(Gs, [G | As], Bool); + G1 -> template_2(Gs, [G1 | As], true) + end; +template_2([], _As, false) -> false; +template_2([], As, true) -> lists:reverse(As). + +template_3([T | Ts], As, Bool) -> + case template_1(T) of + {"v_"++Cs} when Cs =/= [] -> {"v"++Cs}; % lift + {"n0"++Cs} when Cs =/= [] -> {"n"++Cs}; % lift + false -> template_3(Ts, [T | As], Bool); + T1 -> template_3(Ts, [T1 | As], true) + end; +template_3([], _As, false) -> false; +template_3([], As, true) -> lists:reverse(As). + + +%% @doc Turn a template into a syntax tree representing the template. +%% Meta-variables in the template are turned into normal Erlang variables if +%% their names (after the metavariable prefix characters) begin with an +%% uppercase character. E.g., `_@Foo' in the template becomes the variable +%% `Foo' in the meta-template. Furthermore, variables ending with `@' are +%% automatically wrapped in a call to merl:term/1, so e.g. `_@Foo@ in the +%% template becomes `merl:term(Foo)' in the meta-template. + +-spec meta_template(template_or_templates()) -> tree_or_trees(). + +meta_template(Templates) when is_list(Templates) -> + [meta_template_1(T) || T <- Templates]; +meta_template(Template) -> + meta_template_1(Template). + +meta_template_1({template, Type, Attrs, Groups}) -> + erl_syntax:tuple( + [erl_syntax:atom(template), + erl_syntax:atom(Type), + erl_syntax:abstract(Attrs), + erl_syntax:list([erl_syntax:list([meta_template_1(T) || T <- G]) + || G <- Groups])]); +meta_template_1({Var}=V) -> + meta_template_2(Var, V); +meta_template_1({'*',Var}=V) -> + meta_template_2(Var, V); +meta_template_1(Leaf) -> + erl_syntax:abstract(Leaf). + +meta_template_2(Var, V) when is_atom(Var) -> + case atom_to_list(Var) of + [C|_]=Name when C >= $A, C =< $Z ; C >= $À, C =< $Þ, C /= $× -> + case lists:reverse(Name) of + "@"++([_|_]=RevRealName) -> % don't allow empty RealName + RealName = lists:reverse(RevRealName), + erl_syntax:application(erl_syntax:atom(merl), + erl_syntax:atom(term), + [erl_syntax:variable(RealName)]); + _ -> + %% plain automatic metavariable + erl_syntax:variable(Name) + end; + _ -> + erl_syntax:abstract(V) + end; +meta_template_2(Var, V) when is_integer(Var) -> + if Var > 9, (Var rem 10) =:= 9 -> + %% at least 2 digits, ends in 9: make it a Q-variable + if Var > 99, (Var rem 100) =:= 99 -> + %% at least 3 digits, ends in 99: wrap in merl:term/1 + Name = "Q" ++ integer_to_list(Var div 100), + erl_syntax:application(erl_syntax:atom(merl), + erl_syntax:atom(term), + [erl_syntax:variable(Name)]); + true -> + %% plain automatic Q-variable + Name = integer_to_list(Var div 10), + erl_syntax:variable("Q" ++ Name) + end; + true -> + erl_syntax:abstract(V) + end. + + + +-spec template_vars(template_or_templates()) -> [id()]. + +%% @doc Return an ordered list of the metavariables in the template. + +template_vars(Template) -> + template_vars(Template, []). + +template_vars(Templates, Vars) when is_list(Templates) -> + lists:foldl(fun template_vars_1/2, Vars, Templates); +template_vars(Template, Vars) -> + template_vars_1(Template, Vars). + +template_vars_1({template, _, _, Groups}, Vars) -> + lists:foldl(fun (G, V) -> lists:foldl(fun template_vars_1/2, V, G) end, + Vars, Groups); +template_vars_1({Var}, Vars) -> + ordsets:add_element(Var, Vars); +template_vars_1({'*',Var}, Vars) -> + ordsets:add_element(Var, Vars); +template_vars_1(_, Vars) -> + Vars. + + +-spec tree(template_or_templates()) -> tree_or_trees(). + +%% @doc Revert a template to a normal syntax tree. Any remaining +%% metavariables are turned into `@'-prefixed atoms or `909'-prefixed +%% integers. +%% @see template/1 + +tree(Templates) when is_list(Templates) -> + [tree_1(T) || T <- Templates]; +tree(Template) -> + tree_1(Template). + +tree_1({template, Type, Attrs, Groups}) -> + %% flattening here is needed for templates created via source transforms + Gs = [lists:flatten([tree_1(T) || T <- G]) || G <- Groups], + erl_syntax:set_attrs(make_tree(Type, Gs), Attrs); +tree_1({Var}) when is_atom(Var) -> + erl_syntax:atom(list_to_atom("@"++atom_to_list(Var))); +tree_1({Var}) when is_integer(Var) -> + erl_syntax:integer(list_to_integer("909"++integer_to_list(Var))); +tree_1({'*',Var}) when is_atom(Var) -> + erl_syntax:atom(list_to_atom("@@"++atom_to_list(Var))); +tree_1({'*',Var}) when is_integer(Var) -> + erl_syntax:integer(list_to_integer("9099"++integer_to_list(Var))); +tree_1(Leaf) -> + Leaf. % any syntax tree, not necessarily atomic (due to substitutions) + + +-spec subst(pattern_or_patterns(), env()) -> tree_or_trees(). + +%% @doc Substitute metavariables in a pattern or list of patterns, yielding +%% a syntax tree or list of trees as result. Both for normal metavariables +%% and glob metavariables, the substituted value may be a single element or +%% a list of elements. For example, if a list representing `1, 2, 3' is +%% substituted for `var' in either of `[foo, _@var, bar]' or `[foo, _@@var, +%% bar]', the result represents `[foo, 1, 2, 3, bar]'. + +subst(Trees, Env) when is_list(Trees) -> + [subst_0(T, Env) || T <- Trees]; +subst(Tree, Env) -> + subst_0(Tree, Env). + +subst_0(Tree, Env) -> + tree_1(subst_1(template(Tree), Env)). + + +-spec tsubst(pattern_or_patterns(), env()) -> template_or_templates(). + +%% @doc Like subst/2, but does not convert the result from a template back +%% to a tree. Useful if you want to do multiple separate substitutions. +%% @see subst/2 +%% @see tree/1 + +tsubst(Trees, Env) when is_list(Trees) -> + [subst_1(template(T), Env) || T <- Trees]; +tsubst(Tree, Env) -> + subst_1(template(Tree), Env). + +subst_1({template, Type, Attrs, Groups}, Env) -> + Gs1 = [lists:flatten([subst_1(T, Env) || T <- G]) || G <- Groups], + {template, Type, Attrs, Gs1}; +subst_1({Var}=V, Env) -> + case lists:keyfind(Var, 1, Env) of + {Var, TreeOrTrees} -> TreeOrTrees; + false -> V + end; +subst_1({'*',Var}=V, Env) -> + case lists:keyfind(Var, 1, Env) of + {Var, TreeOrTrees} -> TreeOrTrees; + false -> V + end; +subst_1(Leaf, _Env) -> + Leaf. + + +-spec alpha(pattern_or_patterns(), [{id(), id()}]) -> template_or_templates(). + +%% @doc Alpha converts a pattern (renames variables). Similar to tsubst/1, +%% but only renames variables (including globs). +%% @see tsubst/2 + +alpha(Trees, Env) when is_list(Trees) -> + [alpha_1(template(T), Env) || T <- Trees]; +alpha(Tree, Env) -> + alpha_1(template(Tree), Env). + +alpha_1({template, Type, Attrs, Groups}, Env) -> + Gs1 = [lists:flatten([alpha_1(T, Env) || T <- G]) || G <- Groups], + {template, Type, Attrs, Gs1}; +alpha_1({Var}=V, Env) -> + case lists:keyfind(Var, 1, Env) of + {Var, NewVar} -> {NewVar}; + false -> V + end; +alpha_1({'*',Var}=V, Env) -> + case lists:keyfind(Var, 1, Env) of + {Var, NewVar} -> {'*',NewVar}; + false -> V + end; +alpha_1(Leaf, _Env) -> + Leaf. + + +-spec match(pattern_or_patterns(), tree_or_trees()) -> + {ok, env()} | error. + +%% @doc Match a pattern against a syntax tree (or patterns against syntax +%% trees) returning an environment mapping variable names to subtrees; the +%% environment is always sorted on keys. Note that multiple occurrences of +%% metavariables in the pattern is not allowed, but is not checked. +%% +%% @see template/1 +%% @see switch/2 + +match(Patterns, Trees) when is_list(Patterns), is_list(Trees) -> + try {ok, match_1(Patterns, Trees, [])} + catch + error -> error + end; +match(Patterns, Tree) when is_list(Patterns) -> match(Patterns, [Tree]); +match(Pattern, Trees) when is_list(Trees) -> match([Pattern], Trees); +match(Pattern, Tree) -> + try {ok, match_template(template(Pattern), Tree, [])} + catch + error -> error + end. + +match_1([P|Ps], [T | Ts], Dict) -> + match_1(Ps, Ts, match_template(template(P), T, Dict)); +match_1([], [], Dict) -> + Dict; +match_1(_, _, _Dict) -> + erlang:error(merl_match_arity). + +%% match a template against a syntax tree +match_template({template, Type, _, Gs}, Tree, Dict) -> + case type(Tree) of + Type -> match_template_1(Gs, subtrees(Tree), Dict); + _ -> throw(error) % type mismatch + end; +match_template({Var}, _Tree, Dict) + when Var =:= '_' ; Var =:= 0 -> + Dict; % anonymous variable +match_template({Var}, Tree, Dict) -> + orddict:store(Var, Tree, Dict); +match_template(Tree1, Tree2, Dict) -> + %% if Tree1 is not a template, Tree1 and Tree2 are both syntax trees + case compare_trees(Tree1, Tree2) of + true -> Dict; + false -> throw(error) % different trees + end. + +match_template_1([G1 | Gs1], [G2 | Gs2], Dict) -> + match_template_2(G1, G2, match_template_1(Gs1, Gs2, Dict)); +match_template_1([], [], Dict) -> + Dict; +match_template_1(_, _, _Dict) -> + throw(error). % shape mismatch + +match_template_2([{Var} | Ts1], [_ | Ts2], Dict) + when Var =:= '_' ; Var =:= 0 -> + match_template_2(Ts1, Ts2, Dict); % anonymous variable +match_template_2([{Var} | Ts1], [Tree | Ts2], Dict) -> + match_template_2(Ts1, Ts2, orddict:store(Var, Tree, Dict)); +match_template_2([{'*',Var} | Ts1], Ts2, Dict) -> + match_glob(lists:reverse(Ts1), lists:reverse(Ts2), Var, Dict); +match_template_2([T1 | Ts1], [T2 | Ts2], Dict) -> + match_template_2(Ts1, Ts2, match_template(T1, T2, Dict)); +match_template_2([], [], Dict) -> + Dict; +match_template_2(_, _, _Dict) -> + throw(error). % shape mismatch + +%% match the tails in reverse order; no further globs allowed +match_glob([{'*',Var} | _], _, _, _) -> + fail("multiple glob variables in same match group: ~w", [Var]); +match_glob([T1 | Ts1], [T2 | Ts2], Var, Dict) -> + match_glob(Ts1, Ts2, Var, match_template(T1, T2, Dict)); +match_glob([], _Group, Var, Dict) when Var =:= '_' ; Var =:= 0 -> + Dict; % anonymous glob variable +match_glob([], Group, Var, Dict) -> + orddict:store(Var, lists:reverse(Group), Dict); +match_glob(_, _, _, _Dict) -> + throw(error). % shape mismatch + + +%% compare two syntax trees for equivalence +compare_trees(T1, T2) -> + Type1 = type(T1), + case type(T2) of + Type1 -> + case subtrees(T1) of + [] -> + case subtrees(T2) of + [] -> compare_leaves(Type1, T1, T2); + _Gs2 -> false % shape mismatch + end; + Gs1 -> + case subtrees(T2) of + [] -> false; % shape mismatch + Gs2 -> compare_trees_1(Gs1, Gs2) + end + end; + _Type2 -> + false % different tree types + end. + +compare_trees_1([G1 | Gs1], [G2 | Gs2]) -> + compare_trees_2(G1, G2) andalso compare_trees_1(Gs1, Gs2); +compare_trees_1([], []) -> + true; +compare_trees_1(_, _) -> + false. % shape mismatch + +compare_trees_2([T1 | Ts1], [T2 | Ts2]) -> + compare_trees(T1, T2) andalso compare_trees_2(Ts1, Ts2); +compare_trees_2([], []) -> + true; +compare_trees_2(_, _) -> + false. % shape mismatch + +compare_leaves(Type, T1, T2) -> + case Type of + atom -> + erl_syntax:atom_value(T1) + =:= erl_syntax:atom_value(T2); + char -> + erl_syntax:char_value(T1) + =:= erl_syntax:char_value(T2); + float -> + erl_syntax:float_value(T1) + =:= erl_syntax:float_value(T2); + integer -> + erl_syntax:integer_value(T1) + =:= erl_syntax:integer_value(T2); + string -> + erl_syntax:string_value(T1) + =:= erl_syntax:string_value(T2); + operator -> + erl_syntax:operator_name(T1) + =:= erl_syntax:operator_name(T2); + text -> + erl_syntax:text_string(T1) + =:= erl_syntax:text_string(T2); + variable -> + erl_syntax:variable_name(T1) + =:= erl_syntax:variable_name(T2); + _ -> + true % trivially equal nodes + end. + + +%% @doc Match against one or more clauses with patterns and optional guards. +%% +%% Note that clauses following a default action will be ignored. +%% +%% @see match/2 + +-type switch_clause() :: + {pattern_or_patterns(), guarded_actions()} + | {pattern_or_patterns(), guard_test(), switch_action()} + | default_action(). + +-type guarded_actions() :: guarded_action() | [guarded_action()]. + +-type guarded_action() :: switch_action() | {guard_test(), switch_action()}. + +-type switch_action() :: fun( (env()) -> any() ). + +-type guard_test() :: fun( (env()) -> boolean() ). + +-type default_action() :: fun( () -> any() ). + + +-spec switch(tree_or_trees(), [switch_clause()]) -> any(). + +switch(Trees, [{Patterns, GuardedActions} | Cs]) when is_list(GuardedActions) -> + switch_1(Trees, Patterns, GuardedActions, Cs); +switch(Trees, [{Patterns, GuardedAction} | Cs]) -> + switch_1(Trees, Patterns, [GuardedAction], Cs); +switch(Trees, [{Patterns, Guard, Action} | Cs]) -> + switch_1(Trees, Patterns, [{Guard, Action}], Cs); +switch(_Trees, [Default | _Cs]) when is_function(Default, 0) -> + Default(); +switch(_Trees, []) -> + erlang:error(merl_switch_clause); +switch(_Tree, _) -> + erlang:error(merl_switch_badarg). + +switch_1(Trees, Patterns, GuardedActions, Cs) -> + case match(Patterns, Trees) of + {ok, Env} -> + switch_2(Env, GuardedActions, Trees, Cs); + error -> + switch(Trees, Cs) + end. + +switch_2(Env, [{Guard, Action} | Bs], Trees, Cs) + when is_function(Guard, 1), is_function(Action, 1) -> + case Guard(Env) of + true -> Action(Env); + false -> switch_2(Env, Bs, Trees, Cs) + end; +switch_2(Env, [Action | _Bs], _Trees, _Cs) when is_function(Action, 1) -> + Action(Env); +switch_2(_Env, [], Trees, Cs) -> + switch(Trees, Cs); +switch_2(_Env, _, _Trees, _Cs) -> + erlang:error(merl_switch_badarg). + + +%% ------------------------------------------------------------------------ +%% Internal utility functions + +-dialyzer({nowarn_function, fail/1}). % no local return + +fail(Text) -> + fail(Text, []). + +fail(Fs, As) -> + throw({error, lists:flatten(io_lib:format(Fs, As))}). + +flatten_text([L | _]=Lines) when is_list(L) -> + lists:foldr(fun(S, T) -> S ++ [$\n | T] end, "", Lines); +flatten_text([B | _]=Lines) when is_binary(B) -> + lists:foldr(fun(S, T) -> binary_to_list(S) ++ [$\n | T] end, "", Lines); +flatten_text(Text) when is_binary(Text) -> + binary_to_list(Text); +flatten_text(Text) -> + Text. + +-spec metavar(tree()) -> {string()} | false. + +%% Check if a syntax tree represents a metavariable. If not, 'false' is +%% returned; otherwise, this returns a 1-tuple with a string containing the +%% variable name including lift/glob prefixes but without any leading +%% metavariable prefix, and instead prefixed with "v" for a variable or "i" +%% for an integer. +%% +%% Metavariables are atoms starting with @, variables starting with _@, +%% strings starting with "'@, or integers starting with 909. Following the +%% prefix, one or more _ or 0 characters (unless it's the last character in +%% the name) may be used to indicate "lifting" of the variable one or more +%% levels , and after that, a @ or 9 character indicates a glob metavariable +%% rather than a normal metavariable. If the name after the prefix is _ or +%% 0, the variable is treated as an anonymous catch-all pattern in matches. + +metavar(Tree) -> + case type(Tree) of + atom -> + case erl_syntax:atom_name(Tree) of + "@" ++ Cs when Cs =/= [] -> {"v"++Cs}; + _ -> false + end; + variable -> + case erl_syntax:variable_literal(Tree) of + "_@" ++ Cs when Cs =/= [] -> {"v"++Cs}; + _ -> false + end; + integer -> + case erl_syntax:integer_value(Tree) of + N when N >= 9090 -> + case integer_to_list(N) of + "909" ++ Cs -> {"n"++Cs}; + _ -> false + end; + _ -> false + end; + string -> + case erl_syntax:string_value(Tree) of + "'@" ++ Cs -> {"v"++Cs}; + _ -> false + end; + _ -> + false + end. + +%% wrappers around erl_syntax functions to provide more uniform shape of +%% generic subtrees (maybe this can be fixed in syntax_tools one day) + +type(T) -> + case erl_syntax:type(T) of + nil -> list; + Type -> Type + end. + +subtrees(T) -> + case erl_syntax:type(T) of + tuple -> + [erl_syntax:tuple_elements(T)]; %% don't treat {} as a leaf + nil -> + [[], []]; %% don't treat [] as a leaf, but as a list + list -> + case erl_syntax:list_suffix(T) of + none -> + [erl_syntax:list_prefix(T), []]; + S -> + [erl_syntax:list_prefix(T), [S]] + end; + binary_field -> + [[erl_syntax:binary_field_body(T)], + erl_syntax:binary_field_types(T)]; + clause -> + case erl_syntax:clause_guard(T) of + none -> + [erl_syntax:clause_patterns(T), [], + erl_syntax:clause_body(T)]; + G -> + [erl_syntax:clause_patterns(T), [G], + erl_syntax:clause_body(T)] + end; + receive_expr -> + case erl_syntax:receive_expr_timeout(T) of + none -> + [erl_syntax:receive_expr_clauses(T), [], []]; + E -> + [erl_syntax:receive_expr_clauses(T), [E], + erl_syntax:receive_expr_action(T)] + end; + record_expr -> + case erl_syntax:record_expr_argument(T) of + none -> + [[], [erl_syntax:record_expr_type(T)], + erl_syntax:record_expr_fields(T)]; + V -> + [[V], [erl_syntax:record_expr_type(T)], + erl_syntax:record_expr_fields(T)] + end; + record_field -> + case erl_syntax:record_field_value(T) of + none -> + [[erl_syntax:record_field_name(T)], []]; + V -> + [[erl_syntax:record_field_name(T)], [V]] + end; + _ -> + erl_syntax:subtrees(T) + end. + +make_tree(list, [P, []]) -> erl_syntax:list(P); +make_tree(list, [P, [S]]) -> erl_syntax:list(P, S); +make_tree(tuple, [E]) -> erl_syntax:tuple(E); +make_tree(binary_field, [[B], Ts]) -> erl_syntax:binary_field(B, Ts); +make_tree(clause, [P, [], B]) -> erl_syntax:clause(P, none, B); +make_tree(clause, [P, [G], B]) -> erl_syntax:clause(P, G, B); +make_tree(receive_expr, [C, [], _A]) -> erl_syntax:receive_expr(C); +make_tree(receive_expr, [C, [E], A]) -> erl_syntax:receive_expr(C, E, A); +make_tree(record_expr, [[], [T], F]) -> erl_syntax:record_expr(T, F); +make_tree(record_expr, [[E], [T], F]) -> erl_syntax:record_expr(E, T, F); +make_tree(record_field, [[N], []]) -> erl_syntax:record_field(N); +make_tree(record_field, [[N], [E]]) -> erl_syntax:record_field(N, E); +make_tree(Type, Groups) -> + erl_syntax:make_tree(Type, Groups). + +merge_comments(_StartLine, [], [T]) -> T; +merge_comments(_StartLine, [], Ts) -> Ts; +merge_comments(StartLine, Comments, Ts) -> + merge_comments(StartLine, Comments, Ts, []). + +merge_comments(_StartLine, [], [], [T]) -> T; +merge_comments(_StartLine, [], [T], []) -> T; +merge_comments(_StartLine, [], Ts, Acc) -> + lists:reverse(Acc, Ts); +merge_comments(StartLine, Cs, [], Acc) -> + merge_comments(StartLine, [], [], + [erl_syntax:set_pos( + erl_syntax:comment(Indent, Text), + StartLine + Line - 1) + || {Line, _, Indent, Text} <- Cs] ++ Acc); +merge_comments(StartLine, [C|Cs], [T|Ts], Acc) -> + {Line, _Col, Indent, Text} = C, + CommentLine = StartLine + Line - 1, + case erl_syntax:get_pos(T) of + Pos when Pos < CommentLine -> + %% TODO: traverse sub-tree rather than only the top level nodes + merge_comments(StartLine, [C|Cs], Ts, [T|Acc]); + CommentLine -> + Tc = erl_syntax:add_postcomments( + [erl_syntax:comment(Indent, Text)], T), + merge_comments(StartLine, Cs, [Tc|Ts], Acc); + _ -> + Tc = erl_syntax:add_precomments( + [erl_syntax:comment(Indent, Text)], T), + merge_comments(StartLine, Cs, [Tc|Ts], Acc) + end. diff --git a/lib/syntax_tools/src/merl_tests.erl b/lib/syntax_tools/src/merl_tests.erl new file mode 100644 index 0000000000..c1aae3100e --- /dev/null +++ b/lib/syntax_tools/src/merl_tests.erl @@ -0,0 +1,539 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012-2015 Richard Carlsson +%% @doc Unit tests for merl. +%% @private + +-module(merl_tests). + +%-define(MERL_NO_TRANSFORM, true). +-include("merl.hrl"). + +-include_lib("eunit/include/eunit.hrl"). + + +%% utilities + +f(Ts) when is_list(Ts) -> + lists:flatmap(fun erl_prettypr:format/1, Ts); +f(T) -> + erl_prettypr:format(T). + +fe(Env) -> [{Key, f(T)} || {Key, T} <- Env]. + +g_exported_() -> + %% for testing the parse transform, autoexported to avoid complaints + {ok, merl:quote(?LINE, "42")}. + + +ok({ok, X}) -> X. + + +%% +%% tests +%% + +parse_error_test_() -> + [?_assertThrow({error, "1: syntax error before: '{'" ++ _}, + f(merl:quote("{"))) + ]. + +term_test_() -> + [?_assertEqual(tuple, erl_syntax:type(merl:term({}))), + ?_assertEqual("{foo, 42}", f(merl:term({foo, 42}))) + ]. + +quote_form_test_() -> + [?_assertEqual("f(X) -> {ok, X}.", + f(?Q("f(X) -> {ok, X}."))), + ?_assertEqual("-module(foo).", + f(?Q("-module(foo)."))), + ?_assertEqual("-import(bar, [f/1, g/2]).", + f(?Q("-import(bar, [f/1, g/2])."))), + ?_assertEqual(("-module(foo)." + "-export([f/1])." + "f(X) -> {ok, X}."), + f(?Q(["-module(foo).", + "-export([f/1]).", + "f(X) -> {ok, X}."]))) + ]. + +quote_term_test_() -> + [?_assertEqual("foo", + f(?Q("foo"))), + ?_assertEqual("42", + f(?Q("42"))), + ?_assertEqual("{foo, 42}", + f(?Q("{foo, 42}"))), + ?_assertEqual(("1" ++ "2" ++ "3"), + f(?Q("1, 2, 3"))), + ?_assertEqual(("foo" "42" "{}" "true"), + f(?Q("foo, 42, {}, (true)"))) + ]. + +quote_expr_test_() -> + [?_assertEqual("2 + 2", + f(?Q("2 + 2"))), + ?_assertEqual("f(foo, 42)", + f(?Q("f(foo, 42)"))), + ?_assertEqual("case X of\n a -> 1;\n b -> 2\nend", + f(?Q("case X of a -> 1; b -> 2 end"))), + ?_assertEqual(("2 + 2" ++ "f(42)" ++ "catch 22"), + f(?Q("2 + 2, f(42), catch 22"))) + ]. + +quote_try_clause_test_() -> + [?_assertEqual("(error:R) when R =/= foo -> ok", + f(?Q("error:R when R =/= foo -> ok"))), + %% note that without any context, clauses are printed as fun-clauses + ?_assertEqual(("(error:badarg) -> badarg" + "(exit:normal) -> normal" + "(_) -> other"), + f(?Q(["error:badarg -> badarg;", + "exit:normal -> normal;" + "_ -> other"]))) + ]. + +quote_fun_clause_test_() -> + [?_assertEqual("(X, Y) when X < Y -> {ok, X}", + f(?Q("(X, Y) when X < Y -> {ok, X}"))), + ?_assertEqual(("(X, Y) when X < Y -> less" + "(X, Y) when X > Y -> greater" + "(_, _) -> equal"), + f(?Q(["(X, Y) when X < Y -> less;", + "(X, Y) when X > Y -> greater;" + "(_, _) -> equal"])))]. + +quote_case_clause_test_() -> + [?_assertEqual("({X, Y}) when X < Y -> X", + f(?Q("{X, Y} when X < Y -> X"))), + ?_assertEqual(("({X, Y}) when X < Y -> -1" + "({X, Y}) when X > Y -> 1" + "(_) -> 0"), + f(?Q(["{X, Y} when X < Y -> -1;", + "{X, Y} when X > Y -> 1;" + "_ -> 0"])))]. + +quote_comment_test_() -> + [?_assertEqual("%% comment preserved\n" + "{foo, 42}", + f(?Q(["%% comment preserved", + "{foo, 42}"]))), + ?_assertEqual("{foo, 42}" + "%% comment preserved\n", + f(?Q(["{foo, 42}", + "%% comment preserved"]))), + ?_assertEqual(" % just a comment (with indent)\n", + f(?Q(" % just a comment (with indent)"))) + ]. + +metavar_test_() -> + [?_assertEqual("'@foo'", f(merl:tree(merl:template(?Q("'@foo'"))))), + ?_assertEqual("'@foo'", f(merl:tree(merl:template(?Q("_@foo"))))), + ?_assertEqual("'@foo'", f(merl:tree(merl:template(?Q("\"'@foo\""))))), + ?_assertEqual("{'@foo'}", f(merl:tree(merl:template(?Q("{_@foo}"))))), + ?_assertEqual("'@foo'", f(merl:tree(merl:template(?Q("{_@_foo}"))))), + ?_assertEqual("909123", f(merl:tree(merl:template(?Q("{9090123}"))))), + ?_assertEqual("{'@foo'}", + f(merl:tree(merl:template(?Q("{{{_@__foo}}}"))))), + ?_assertEqual("{909123}", + f(merl:tree(merl:template(?Q("{{{90900123}}}"))))), + ?_assertEqual("{'@@foo'}", + f(merl:tree(merl:template(?Q("{{{_@__@foo}}}"))))), + ?_assertEqual("{9099123}", + f(merl:tree(merl:template(?Q("{{{909009123}}}"))))) + ]. + +subst_test_() -> + [?_assertEqual("42", + f(merl:subst(?Q("_@foo"), [{foo, merl:term(42)}]))), + ?_assertEqual("'@foo'", + f(merl:subst(?Q("_@foo"), []))), + ?_assertEqual("{42}", + f(merl:subst(?Q("{_@foo}"), + [{foo, merl:term(42)}]))), + ?_assertEqual("{'@foo'}", + f(merl:subst(?Q("{_@foo}"), []))), + ?_assertEqual("fun bar/0", + f(merl:subst(merl:template(?Q("fun '@foo'/0")), + [{foo, merl:term(bar)}]))), + ?_assertEqual("fun foo/3", + f(merl:subst(merl:template(?Q("fun foo/9091")), + [{1, merl:term(3)}]))), + ?_assertEqual("[42]", + f(merl:subst(merl:template(?Q("[_@foo]")), + [{foo, merl:term(42)}]))), + ?_assertEqual("[foo, bar]", + f(merl:subst(merl:template(?Q("[_@foo]")), + [{foo, [merl:term(foo),merl:term(bar)]}]))), + ?_assertEqual("{fee, fie, foe, fum}", + f(merl:subst(merl:template(?Q("{fee, _@foo, fum}")), + [{foo, [merl:term(fie),merl:term(foe)]}]))), + ?_assertEqual("[foo, bar]", + f(merl:subst(merl:template(?Q("[_@@foo]")), + [{foo, [merl:term(foo),merl:term(bar)]}]))), + ?_assertEqual("{fee, fie, foe, fum}", + f(merl:subst(merl:template(?Q("{fee, _@@foo, fum}")), + [{foo, [merl:term(fie),merl:term(foe)]}]))), + ?_assertEqual("['@@foo']", + f(merl:subst(merl:template(?Q("[_@@foo]")), []))), + ?_assertEqual("foo", + f(merl:subst(merl:template(?Q("[_@_foo]")), + [{foo, merl:term(foo)}]))), + ?_assertEqual("{'@foo'}", + f(merl:subst(merl:template(?Q("{[_@_foo]}")), []))), + ?_assertEqual("{'@@foo'}", + f(merl:subst(merl:template(?Q("{[_@_@foo]}")), []))), + ?_assertEqual("-export([foo/1, bar/2]).", + f(merl:subst(merl:template(?Q("-export(['@_@foo'/0]).")), + [{foo, [erl_syntax:arity_qualifier( + merl:term(foo), + merl:term(1)), + erl_syntax:arity_qualifier( + merl:term(bar), + merl:term(2)) + ]} + ]))) + ]. + +match_test_() -> + [?_assertEqual({ok, []}, merl:match(?Q("foo"), ?Q("foo"))), + ?_assertEqual(error, merl:match(?Q("foo"), ?Q("bar"))), + ?_assertEqual({ok,[]}, merl:match(?Q("{foo,42}"), ?Q("{foo,42}"))), + ?_assertEqual(error, merl:match(?Q("{foo,42}"), ?Q("{foo,bar}"))), + ?_assertEqual({ok,[]}, merl:match(?Q("[foo,[42]]"), ?Q("[foo,[42]]"))), + ?_assertEqual(error, merl:match(?Q("[foo,[42]]"), ?Q("[foo,{42}]"))), + ?_assertEqual({ok,[]}, merl:match(?Q("[foo,[_@_]]"), + ?Q("[foo,[42]]"))), + ?_assertEqual({ok,[]}, merl:match(?Q("[foo,[9090]]"), + ?Q("[foo,[42]]"))), + ?_assertEqual({ok,[]}, merl:match(?Q("{_@_,[_@_,2]}"), + ?Q("{foo,[1,2]}"))), + ?_assertEqual(error, merl:match(?Q("{_@_,[_@_,2]}"), + ?Q("{foo,[1,3]}"))), + ?_assertEqual({ok,[]}, merl:match(?Q("[foo,[9090,9090]]"), + ?Q("[foo,[1,2]]"))), + ?_assertEqual(error, merl:match(?Q("[foo,[9090,9090]]"), + ?Q("[foo,[1,2,3]]"))), + ?_assertEqual([{foo,"42"}], + fe(ok(merl:match(?Q("_@foo"), ?Q("42"))))), + ?_assertEqual([{foo,"42"}], + fe(ok(merl:match(?Q("{_@foo}"), ?Q("{42}"))))), + ?_assertEqual([{1,"0"},{foo,"bar"}], + fe(ok(merl:match(?Q("fun '@foo'/9091"), + ?Q("fun bar/0"))))), + ?_assertEqual([{line,"17"},{text,"\"hello\""}], + fe(ok(merl:match(?Q("{_@line, _@text}"), + ?Q("{17, \"hello\"}"))))), + ?_assertEqual([{line,"17"},{text,"\"hello\""}], + fe(ok(merl:match(?Q("foo(_@line, _@text)"), + ?Q("foo(17, \"hello\")"))))), + ?_assertEqual([{foo,""}], + fe(ok(merl:match(?Q("f(_@@foo)"), + ?Q("f()"))))), + ?_assertEqual([{foo,"fee"}], + fe(ok(merl:match(?Q("f(_@@foo)"), + ?Q("f(fee)"))))), + ?_assertEqual([{foo,"feefiefum"}], + fe(ok(merl:match(?Q("f(_@@foo)"), + ?Q("f(fee, fie, fum)"))))), + ?_assertEqual([{foo,""}], + fe(ok(merl:match(?Q("[_@@foo]"), + ?Q("[]"))))), + ?_assertEqual([{foo,"fee"}], + fe(ok(merl:match(?Q("[_@@foo]"), + ?Q("[fee]"))))), + ?_assertEqual([{foo,"feefiefoefum"}], + fe(ok(merl:match(?Q("[_@@foo]"), + ?Q("[fee, fie, foe, fum]"))))), + ?_assertEqual([{foo,""}], + fe(ok(merl:match(?Q("{_@@foo}"), + ?Q("{}"))))), + ?_assertEqual([{foo,"fee"}], + fe(ok(merl:match(?Q("{_@@foo}"), + ?Q("{fee}"))))), + ?_assertEqual([{foo,"feefiefoefum"}], + fe(ok(merl:match(?Q("{_@@foo}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertEqual([{foo,"fie"}], + fe(ok(merl:match(?Q("{fee, _@@foo}"), + ?Q("{fee, fie}"))))), + ?_assertEqual([{foo,"fiefoefum"}], + fe(ok(merl:match(?Q("{fee, _@@foo}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertEqual([{foo,"fie"}], + fe(ok(merl:match(?Q("{_@@foo, foe, fum}"), + ?Q("{fie, foe, fum}"))))), + ?_assertEqual([{foo,"feefie"}], + fe(ok(merl:match(?Q("{_@@foo, foe, fum}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertEqual([{foo,"fie"}], + fe(ok(merl:match(?Q("{fee, _@@foo, fum}"), + ?Q("{fee, fie, fum}"))))), + ?_assertEqual([{foo,"fiefoe"}], + fe(ok(merl:match(?Q("{fee, _@@foo, fum}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertEqual([{foo,"fiefoe"},{post,"fum"},{pre,"fee"}], + fe(ok(merl:match(?Q("{_@pre, _@@foo, _@post}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertThrow({error, "multiple glob variables"++_}, + fe(ok(merl:match(?Q("{_@@foo, _@@bar}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertEqual([], + fe(ok(merl:match(?Q("{fee, _@@_}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertEqual([], + fe(ok(merl:match(?Q("{_@@_, foe, fum}"), + ?Q("{fee, fie, foe, fum}"))))), + ?_assertEqual([{post,"fum"},{pre,"fee"}], + fe(ok(merl:match(?Q("{_@pre, _@@_, _@post}"), + ?Q("{fee, fie, foe, fum}"))))) + ]. + +switch_test_() -> + [?_assertEqual(42, merl:switch(?Q("foo"), [fun () -> 42 end])), + ?_assertEqual(17, merl:switch(?Q("foo"), [fun () -> 17 end, + fun () -> 42 end])), + ?_assertEqual(17, merl:switch(?Q("foo"), [{?Q("foo"), + fun ([]) -> 17 end}, + fun () -> 42 end])), + ?_assertEqual(17, + merl:switch(?Q("foo"), [{?Q("bar"), fun ([]) -> 0 end}, + {?Q("foo"), fun ([]) -> 17 end}, + fun () -> 42 end])), + ?_assertEqual([{foo,"17"}], + merl:switch(?Q("{foo,17}"), + [{?Q("{bar, _@foo}"), fun (_) -> 0 end}, + {?Q("{foo, _@foo}"), fun fe/1}, + fun () -> 42 end])), + ?_assertEqual(17, + merl:switch(?Q("{foo, 17}"), + [{?Q("{foo, _@foo}"), + fun ([{foo, X}]) -> f(X) =:= "17" end, + fun (_) -> 17 end}, + fun () -> 42 end])), + ?_assertEqual([{foo,"17"}], + merl:switch(?Q("{foo, 17}"), + [{?Q("{foo, _@foo}"), + fun ([{foo, X}]) -> f(X) =:= "42" end, + fun (_) -> 0 end}, + {?Q("{foo, _@foo}"), fun fe/1}, + fun () -> 42 end])), + ?_assertEqual(17, + merl:switch(?Q("{foo, 17}"), + [{?Q("{foo, _@foo}"), + [{fun ([{foo, X}]) -> f(X) =:= "17" end, + fun (_) -> 17 end}, + fun (_) -> 0 end]}, + fun () -> 42 end])), + ?_assertEqual([{foo,"17"}], + merl:switch(?Q("{foo, 17}"), + [{?Q("{foo, _@foo}"), + [{fun ([{foo, X}]) -> f(X) =:= "42" end, + fun (_) -> 0 end}, + fun fe/1]}, + fun () -> 42 end])) + ]. + +-ifndef(MERL_NO_TRANSFORM). + +inline_meta_test_() -> + [?_assertEqual("{foo}", + f(begin + Foo = ?Q("foo"), + ?Q("{_@Foo}") + end)), + ?_assertEqual("{foo, '@bar'}", + f(begin + Foo = ?Q("foo"), + ?Q("{_@Foo,_@bar}") + end)), + ?_assertEqual("{foo, '@bar'}", + f(begin + Q1 = ?Q("foo"), + ?Q("{90919,_@bar}") + end)) + ]. + +inline_meta_autoabstract_test_() -> + [?_assertEqual("{foo}", + f(begin + Foo = foo, + ?Q("{_@Foo@}") + end)), + ?_assertEqual("{foo, '@bar@'}", + f(begin + Foo = foo, + ?Q("{_@Foo@,_@bar@}") + end)), + ?_assertEqual("{foo, '@bar@'}", + f(begin + Q1 = foo, + ?Q("{909199,_@bar@}") + end)) + ]. + +meta_match_test_() -> + [?_assertEqual("{[bar], baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + ?Q("{foo, _@Bar, '@Baz'}") = Tree, + ?Q("{_@Bar, _@Baz}") + end)), + ?_assertEqual("{[bar], baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + ?Q("{foo, 90919, 90929}") = Tree, + ?Q("{_@Q1, _@Q2}") + end)), + ?_assertError({badmatch,error}, + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + ?Q("{fie, _@Bar, '@Baz'}") = Tree, + ?Q("{_@Bar, _@Baz}") + end)) + ]. + +meta_case_test_() -> + [?_assertEqual("{[bar], baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + case Tree of + ?Q("{foo, _@Bar, '@Baz'}") -> ?Q("{_@Bar, _@Baz}") + end + end)), + ?_assertEqual("{foo, [bar], baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + case Tree of + ?Q("{fie, _@Bar, '@Baz'}") -> ?Q("{_@Bar, _@Baz}"); + _ -> Tree + end + end)), + ?_assertError(merl_switch_clause, + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + case Tree of + ?Q("{fie, _@Bar, '@Baz'}") -> ?Q("{_@Bar, _@Baz}") + end + end)), + ?_assertEqual("{foo, 4}", + f(begin + Tree = ?Q("{foo, 3}"), + case Tree of + ?Q("{foo, _@N}") -> + N1 = erl_syntax:concrete(N) + 1, + ?Q("{foo, _@N1@}"); + _ -> Tree + end + end)), + ?_assertEqual("-export([f/4]).", + f(begin + Tree = ?Q("-export([f/3])."), + case Tree of + ?Q("-export([f/90919]).") -> + Q2 = erl_syntax:concrete(Q1) + 1, + ?Q("-export([f/909299])."); + _ -> Tree + end + end)), + ?_assertEqual("{1, [bar], baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + case Tree of + ?Q("{foo, _@Bar, '@Baz'}") -> + ?Q("{1, _@Bar, _@Baz}"); + ?Q("{fie, _@Bar, '@Baz'}") -> + ?Q("{2, _@Bar, _@Baz}"); + _ -> Tree + end + end)), + ?_assertEqual("{2, [bar], baz()}", + f(begin + Tree = ?Q("{fie, [bar], baz()}"), + case Tree of + ?Q("{foo, _@Bar, '@Baz'}") -> + ?Q("{1, _@Bar, _@Baz}"); + ?Q("{fie, _@Bar, '@Baz'}") -> + ?Q("{2, _@Bar, _@Baz}"); + _ -> Tree + end + end)), + ?_assertEqual("{2, baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + case Tree of + ?Q("{foo, [_@Bar], '@Baz'}") + when erl_syntax:is_atom(Bar, foo) -> + ?Q("{1, _@Baz}"); + ?Q("{foo, [_@Bar], '@Baz'}") + when erl_syntax:is_atom(Bar, bar) -> + ?Q("{2, _@Baz}"); + ?Q("{foo, [_@Bar], '@Baz'}") -> + ?Q("{3, _@Baz}"); + _ -> Tree + end + end)), + ?_assertEqual("{2, 42}", + f(begin + Tree = ?Q("{foo, [bar], 42}"), + case Tree of + ?Q("{foo, [_@Bar], '@Baz'}") + when erl_syntax:is_atom(Bar, bar), + erl_syntax:is_integer(Baz, 17) -> + ?Q("{1, _@Bar}"); + ?Q("{foo, [_@Bar], '@Baz'}") + when erl_syntax:is_atom(Bar, bar), + erl_syntax:is_integer(Baz, 42) -> + ?Q("{2, _@Baz}"); + ?Q("{foo, [_@Bar], '@Baz'}") -> + ?Q("{3, _@Baz}"); + _ -> Tree + end + end)), + ?_assertEqual("{2, 42}", + f(begin + Tree = ?Q("{foo, [baz], 42}"), + case Tree of + ?Q("{foo, [_@Bar], '@Baz'}") + when erl_syntax:is_atom(Bar, bar), + erl_syntax:is_integer(Baz, 17) + ; erl_syntax:is_atom(Bar, baz), + erl_syntax:is_integer(Baz, 17) -> + ?Q("{1, _@Bar}"); + ?Q("{foo, [_@Bar], '@Baz'}") + when erl_syntax:is_atom(Bar, bar), + erl_syntax:is_integer(Baz, 42) + ; erl_syntax:is_atom(Bar, baz), + erl_syntax:is_integer(Baz, 42) -> + ?Q("{2, _@Baz}"); + ?Q("{foo, [_@Bar], '@Baz'}") -> + ?Q("{3, _@Baz}"); + _ -> Tree + end + end)), + ?_assertEqual("{2, foo, Bar, Baz, Bar(), Baz()}", + f(begin + Tree = ?Q("foo(Bar, Baz) -> Bar(), Baz()."), + case Tree of + ?Q("'@Func'(_@Args) -> _@Body.") -> + ?Q("{1, _@Func, _@Args, _@Body}"); + ?Q("'@Func'(_@@Args) -> _@@Body.") -> + ?Q("{2, _@Func, _@Args, _@Body}"); + ?Q("'@Func'(_@Args, Baz) -> _@Body1, _@Body2.") -> + ?Q("{3, _@Func, _@Args, _@Body1, _@Body2}") + end + end)) + ]. + +-endif. diff --git a/lib/syntax_tools/src/merl_transform.erl b/lib/syntax_tools/src/merl_transform.erl new file mode 100644 index 0000000000..66b06c8137 --- /dev/null +++ b/lib/syntax_tools/src/merl_transform.erl @@ -0,0 +1,262 @@ +%% --------------------------------------------------------------------- +%% Licensed under the Apache License, Version 2.0 (the "License"); you may +%% not use this file except in compliance with the License. You may obtain +%% a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0> +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @author Richard Carlsson <[email protected]> +%% @copyright 2012-2015 Richard Carlsson +%% @doc Parse transform for merl. Enables the use of automatic metavariables +%% and using quasi-quotes in matches and case switches. Also optimizes calls +%% to functions in `merl' by partially evaluating them, turning strings to +%% templates, etc., at compile-time. +%% +%% Using `-include_lib("syntax_tools/include/merl.hrl").' enables this +%% transform, unless the macro `MERL_NO_TRANSFORM' is defined first. + +-module(merl_transform). + +-export([parse_transform/2]). + +%% NOTE: We cannot use inline metavariables or any other parse transform +%% features in this module, because it must be possible to compile it with +%% the parse transform disabled! +-include("merl.hrl"). + +%% TODO: unroll calls to switch? it will probably get messy + +%% TODO: use Igor to make resulting code independent of merl at runtime? + +parse_transform(Forms, _Options) -> + erl_syntax:revert_forms(expand(erl_syntax:form_list(Forms))). + +expand(Tree0) -> + Tree = pre(Tree0), + post(case erl_syntax:subtrees(Tree) of + [] -> + Tree; + Gs -> + erl_syntax:update_tree(Tree, + [[expand(T) || T <- G] || G <- Gs]) + end). + +pre(T) -> + merl:switch( + T, + [{?Q("merl:quote(_@line, _@text) = _@expr"), + fun ([{expr,_}, {line,Line}, {text,Text}]) -> + erl_syntax:is_literal(Text) andalso erl_syntax:is_literal(Line) + end, + fun ([{expr,Expr}, {line,Line}, {text,Text}]) -> + pre_expand_match(Expr, erl_syntax:concrete(Line), + erl_syntax:concrete(Text)) + end}, + {?Q(["case _@expr of", + " merl:quote(_@_, _@text) when _@__@_ -> _@@_; _@_@_ -> 0", + "end"]), + fun case_guard/1, + fun (As) -> case_body(As, T) end}, + fun () -> T end + ]). + +case_guard([{expr,_}, {text,Text}]) -> + erl_syntax:is_literal(Text). + +case_body([{expr,Expr}, {text,_Text}], T) -> + pre_expand_case(Expr, erl_syntax:case_expr_clauses(T), + erl_syntax:get_pos(T)). + +post(T) -> + merl:switch( + T, + [{?Q("merl:_@function(_@@args)"), + [{fun ([{args, As}, {function, F}]) -> + lists:all(fun erl_syntax:is_literal/1, [F|As]) + end, + fun ([{args, As}, {function, F}]) -> + Line = erl_syntax:get_pos(F), + [F1|As1] = lists:map(fun erl_syntax:concrete/1, [F|As]), + eval_call(Line, F1, As1, T) + end}, + fun ([{args, As}, {function, F}]) -> + merl:switch( + F, + [{?Q("qquote"), fun ([]) -> expand_qquote(As, T, 1) end}, + {?Q("subst"), fun ([]) -> expand_template(F, As, T) end}, + {?Q("match"), fun ([]) -> expand_template(F, As, T) end}, + fun () -> T end + ]) + end]}, + fun () -> T end]). + +expand_qquote([Line, Text, Env], T, _) -> + case erl_syntax:is_literal(Line) of + true -> + expand_qquote([Text, Env], T, erl_syntax:concrete(Line)); + false -> + T + end; +expand_qquote([Text, Env], T, Line) -> + case erl_syntax:is_literal(Text) of + true -> + As = [Line, erl_syntax:concrete(Text)], + %% expand further if possible + expand(merl:qquote(Line, "merl:subst(_@tree, _@env)", + [{tree, eval_call(Line, quote, As, T)}, + {env, Env}])); + false -> + T + end; +expand_qquote(_As, T, _StartPos) -> + T. + +expand_template(F, [Pattern | Args], T) -> + case erl_syntax:is_literal(Pattern) of + true -> + Line = erl_syntax:get_pos(Pattern), + As = [erl_syntax:concrete(Pattern)], + merl:qquote(Line, "merl:_@function(_@pattern, _@args)", + [{function, F}, + {pattern, eval_call(Line, template, As, T)}, + {args, Args}]); + false -> + T + end; +expand_template(_F, _As, T) -> + T. + +eval_call(Line, F, As, T) -> + try apply(merl, F, As) of + T1 when F =:= quote -> + %% lift metavariables in a template to Erlang variables + Template = merl:template(T1), + Vars = merl:template_vars(Template), + case lists:any(fun is_inline_metavar/1, Vars) of + true when is_list(T1) -> + merl:qquote(Line, "merl:tree([_@template])", + [{template, merl:meta_template(Template)}]); + true -> + merl:qquote(Line, "merl:tree(_@template)", + [{template, merl:meta_template(Template)}]); + false -> + merl:term(T1) + end; + T1 -> + merl:term(T1) + catch + throw:_Reason -> T + end. + +pre_expand_match(Expr, Line, Text) -> + {Template, Out, _Vars} = rewrite_pattern(Line, Text), + merl:qquote(Line, "{ok, _@out} = merl:match(_@template, _@expr)", + [{expr, Expr}, + {out, Out}, + {template, erl_syntax:abstract(Template)}]). + +rewrite_pattern(Line, Text) -> + %% we must rewrite the metavariables in the pattern to use lowercase, + %% and then use real matching to bind the Erlang-level variables + T0 = merl:template(merl:quote(Line, Text)), + Vars = [V || V <- merl:template_vars(T0), is_inline_metavar(V)], + {merl:alpha(T0, [{V, var_to_tag(V)} || V <- Vars]), + erl_syntax:list([erl_syntax:tuple([erl_syntax:abstract(var_to_tag(V)), + erl_syntax:variable(var_name(V))]) + || V <- Vars]), + Vars}. + +var_name(V) when is_integer(V) -> + V1 = if V > 99, (V rem 100) =:= 99 -> + V div 100; + V > 9, (V rem 10) =:= 9 -> + V div 10; + true -> V + end, + list_to_atom("Q" ++ integer_to_list(V1)); +var_name(V) -> V. + +var_to_tag(V) when is_integer(V) -> V; +var_to_tag(V) -> + list_to_atom(string:to_lower(atom_to_list(V))). + +pre_expand_case(Expr, Clauses, Line) -> + merl:qquote(Line, "merl:switch(_@expr, _@clauses)", + [{clauses, erl_syntax:list([pre_expand_case_clause(C) + || C <- Clauses])}, + {expr, Expr}]). + +pre_expand_case_clause(T) -> + %% note that the only allowed non ``?Q(...) -> ...'' clause is ``_ -> ...'' + merl:switch( + T, + [{?Q("(merl:quote(_@line, _@text)) when _@__@guard -> _@@body"), + fun ([{body,_}, {guard,_}, {line,Line}, {text,Text}]) -> + erl_syntax:is_literal(Text) andalso erl_syntax:is_literal(Line) + end, + fun ([{body,Body}, {guard,Guard}, {line,Line}, {text,Text}]) -> + pre_expand_case_clause(Body, Guard, erl_syntax:concrete(Line), + erl_syntax:concrete(Text)) + end}, + {?Q("_ -> _@@body"), + fun (Env) -> merl:qquote("fun () -> _@body end", Env) end} + ]). + +pre_expand_case_clause(Body, Guard, Line, Text) -> + %% this is similar to a meta-match ``?Q("...") = Term'' + %% (note that the guards may in fact be arbitrary expressions) + {Template, Out, Vars} = rewrite_pattern(Line, Text), + GuardExprs = rewrite_guard(Guard), + Param = [{body, Body}, + {guard,GuardExprs}, + {out, Out}, + {template, erl_syntax:abstract(Template)}, + {unused, dummy_uses(Vars)}], + case GuardExprs of + [] -> + merl:qquote(Line, ["{_@template, ", + " fun (_@out) -> _@unused, _@body end}"], + Param); + _ -> + merl:qquote(Line, ["{_@template, ", + " fun (_@out) -> _@unused, _@guard end, ", + " fun (_@out) -> _@unused, _@body end}"], + Param) + end. + +%% We have to insert dummy variable uses at the beginning of the "guard" and +%% "body" function bodies to avoid warnings for unused variables in the +%% generated code. (Expansions at the Erlang level can't be marked up as +%% compiler generated to allow later compiler stages to ignore them.) +dummy_uses(Vars) -> + [?Q("_ = _@var", [{var, erl_syntax:variable(var_name(V))}]) + || V <- Vars]. + +rewrite_guard([]) -> []; +rewrite_guard([D]) -> [make_orelse(erl_syntax:disjunction_body(D))]. + +make_orelse([]) -> []; +make_orelse([C]) -> make_andalso(erl_syntax:conjunction_body(C)); +make_orelse([C | Cs]) -> + ?Q("_@expr orelse _@rest", + [{expr, make_andalso(erl_syntax:conjunction_body(C))}, + {rest, make_orelse(Cs)}]). + +make_andalso([E]) -> E; +make_andalso([E | Es]) -> + ?Q("_@expr andalso _@rest", [{expr, E}, {rest, make_andalso(Es)}]). + +is_inline_metavar(Var) when is_atom(Var) -> + is_erlang_var(atom_to_list(Var)); +is_inline_metavar(Var) when is_integer(Var) -> + Var > 9 andalso (Var rem 10) =:= 9; +is_inline_metavar(_) -> false. + +is_erlang_var([C|_]) when C >= $A, C =< $Z ; C >= $À, C =< $Þ, C /= $× -> + true; +is_erlang_var(_) -> + false. diff --git a/lib/syntax_tools/src/syntax_tools.app.src b/lib/syntax_tools/src/syntax_tools.app.src index 83dcb5fe23..dd4ac46055 100644 --- a/lib/syntax_tools/src/syntax_tools.app.src +++ b/lib/syntax_tools/src/syntax_tools.app.src @@ -11,8 +11,11 @@ erl_syntax_lib, erl_tidy, igor, + merl, + merl_transform, prettypr]}, {registered,[]}, {applications, [stdlib]}, {env, []}, - {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]}]}. + {runtime_dependencies, + ["compiler-6.0","erts-6.0","kernel-3.0","stdlib-2.5"]}]}. diff --git a/lib/syntax_tools/test/Makefile b/lib/syntax_tools/test/Makefile index f67e3f8984..569c044b1a 100644 --- a/lib/syntax_tools/test/Makefile +++ b/lib/syntax_tools/test/Makefile @@ -6,7 +6,8 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk # ---------------------------------------------------- MODULES= \ - syntax_tools_SUITE + syntax_tools_SUITE \ + merl_SUITE ERL_FILES= $(MODULES:%=%.erl) diff --git a/lib/syntax_tools/test/merl_SUITE.erl b/lib/syntax_tools/test/merl_SUITE.erl new file mode 100644 index 0000000000..08b0f7a696 --- /dev/null +++ b/lib/syntax_tools/test/merl_SUITE.erl @@ -0,0 +1,91 @@ +%% ``The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved via the world wide web at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% The Initial Developer of the Original Code is Ericsson Utvecklings AB. +%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings +%% AB. All Rights Reserved.'' +%% +-module(merl_SUITE). + +-include_lib("test_server/include/test_server.hrl"). + +%% include the Merl header file +-include_lib("syntax_tools/include/merl.hrl"). + +%% for assert macros +-include_lib("eunit/include/eunit.hrl"). + +%% Test server specific exports +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2]). + +%% Test cases +-export([merl_smoke_test/1]). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [merl_smoke_test]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + +-define(tokens2str(X), ??X). + +merl_smoke_test(Config) when is_list(Config) -> + ?assertThrow({error, "1: syntax error before: '{'" ++ _}, + f(merl:quote("{"))), + ?assertEqual(tuple, erl_syntax:type(merl:term({}))), + ?assertEqual("{foo, 42}", f(merl:term({foo, 42}))), + ?assertEqual("f(X) -> {ok, X}.", f(?Q("f(X) -> {ok, X}."))), + ?assertEqual("{foo, 42}", f(?Q("{foo, 42}"))), + ?assertEqual("2 + 2", f(?Q("2 + 2"))), + ?assertEqual("%% comment preserved\n{foo, 42}", + f(?Q(["%% comment preserved", "{foo, 42}"]))), + ?assertEqual("'@foo'", f(merl:tree(merl:template(?Q("'@foo'"))))), + ?assertEqual("42", f(merl:subst(?Q("_@foo"), [{foo, merl:term(42)}]))), + ?assertEqual({ok, []}, merl:match(?Q("foo"), ?Q("foo"))), + ?assertEqual(42, merl:switch(?Q("foo"), [fun () -> 42 end])), + ?assertEqual("{foo}", f(begin Foo = ?Q("foo"), ?Q("{_@Foo}") end)), + ?assertEqual("{foo}", f(begin Foo = foo, ?Q("{_@Foo@}") end)), + ?assertEqual("{[bar], baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + ?Q("{foo, _@Bar, '@Baz'}") = Tree, + ?Q("{_@Bar, _@Baz}") + end)), + ?assertEqual("{[bar], baz()}", + f(begin + Tree = ?Q("{foo, [bar], baz()}"), + case Tree of + ?Q("{foo, _@Bar, '@Baz'}") -> ?Q("{_@Bar, _@Baz}") + end + end)), + ok. + +%% utilities + +f(Ts) when is_list(Ts) -> + lists:flatmap(fun erl_prettypr:format/1, Ts); +f(T) -> + erl_prettypr:format(T). diff --git a/lib/syntax_tools/vsn.mk b/lib/syntax_tools/vsn.mk index 1c42ef0ddb..403e90196e 100644 --- a/lib/syntax_tools/vsn.mk +++ b/lib/syntax_tools/vsn.mk @@ -1 +1 @@ -SYNTAX_TOOLS_VSN = 1.6.18 +SYNTAX_TOOLS_VSN = 1.7 diff --git a/lib/test_server/include/test_server.hrl b/lib/test_server/include/test_server.hrl index 36e7e1f83d..f206374116 100644 --- a/lib/test_server/include/test_server.hrl +++ b/lib/test_server/include/test_server.hrl @@ -21,7 +21,7 @@ -line_trace(true). -define(line, io:format(lists:concat([?MODULE,",",integer_to_list(?LINE),": ~p"]), - [erlang:now()]),). + [erlang:monotonic_time()-erlang:system_info(start_time)]),). -else. -define(line,). -endif. diff --git a/lib/test_server/src/erl2html2.erl b/lib/test_server/src/erl2html2.erl index 2e443c7b8b..b0b5c40965 100644 --- a/lib/test_server/src/erl2html2.erl +++ b/lib/test_server/src/erl2html2.erl @@ -109,27 +109,26 @@ parse_file(File, InclPath) -> Error end. -parse_preprocessed_file(Epp,File,InCorrectFile) -> +parse_preprocessed_file(Epp, File, InCorrectFile) -> case epp:parse_erl_form(Epp) of {ok,Form} -> case Form of {attribute,_,file,{File,_}} -> - parse_preprocessed_file(Epp,File,true); + parse_preprocessed_file(Epp, File, true); {attribute,_,file,{_OtherFile,_}} -> - parse_preprocessed_file(Epp,File,false); + parse_preprocessed_file(Epp, File, false); {function,L,F,A,Cs} when InCorrectFile -> {CLs,LastCL} = find_clause_lines(Cs, []), - Clauses = [{clause,get_line(CL)} || - {clause,CL,_,_,_} <- tl(CLs)], - [{atom_to_list(F),A,get_line(L),get_line(LastCL)} | Clauses] ++ - parse_preprocessed_file(Epp,File,true); + %% tl(CLs) cause we know the start line already + [{atom_to_list(F),A,get_line(L),LastCL} | tl(CLs)] ++ + parse_preprocessed_file(Epp, File, true); _ -> - parse_preprocessed_file(Epp,File,InCorrectFile) + parse_preprocessed_file(Epp, File, InCorrectFile) end; {error,Reason={_L,epp,{undefined,_Macro,none}}} -> throw({error,Reason,InCorrectFile}); {error,_Reason} -> - parse_preprocessed_file(Epp,File,InCorrectFile); + parse_preprocessed_file(Epp, File, InCorrectFile); {eof,_Location} -> [] end. @@ -150,9 +149,8 @@ parse_non_preprocessed_file(Epp, File, Location) -> try erl_syntax:revert(Tree) of {function,L,F,A,Cs} -> {CLs,LastCL} = find_clause_lines(Cs, []), - Clauses = [{clause,get_line(CL)} || - {clause,CL,_,_,_} <- tl(CLs)], - [{atom_to_list(F),A,get_line(L),get_line(LastCL)} | Clauses] ++ + %% tl(CLs) cause we know the start line already + [{atom_to_list(F),A,get_line(L),LastCL} | tl(CLs)] ++ parse_non_preprocessed_file(Epp, File, Location1); _ -> parse_non_preprocessed_file(Epp, File, Location1) @@ -172,17 +170,21 @@ get_line(Anno) -> %%% Find the line number of the last expression in the function find_clause_lines([{clause,CL,_Params,_Op,Exprs}], CLs) -> % last clause try tuple_to_list(lists:last(Exprs)) of - [_Type,ExprLine | _] -> - {lists:reverse([{clause,CL}|CLs]), ExprLine}; + [_Type,ExprLine | _] when is_integer(ExprLine) -> + {lists:reverse([{clause,get_line(CL)}|CLs]), get_line(ExprLine)}; + [tree,_ | Exprs1] -> + find_clause_lines([{clause,CL,undefined,undefined,Exprs1}], CLs); + [macro,{_var,ExprLine,_MACRO} | _] when is_integer(ExprLine) -> + {lists:reverse([{clause,get_line(CL)}|CLs]), get_line(ExprLine)}; _ -> - {lists:reverse([{clause,CL}|CLs]), CL} + {lists:reverse([{clause,get_line(CL)}|CLs]), get_line(CL)} catch _:_ -> - {lists:reverse([{clause,CL}|CLs]), CL} + {lists:reverse([{clause,get_line(CL)}|CLs]), get_line(CL)} end; find_clause_lines([{clause,CL,_Params,_Op,_Exprs} | Cs], CLs) -> - find_clause_lines(Cs, [{clause,CL}|CLs]). + find_clause_lines(Cs, [{clause,get_line(CL)}|CLs]). %%%----------------------------------------------------------------- %%% Add a link target for each line and one for each function definition. @@ -190,18 +192,18 @@ build_html(SFd,DFd,Encoding,FuncsAndCs) -> build_html(SFd,DFd,Encoding,file:read_line(SFd),1,FuncsAndCs, false,undefined). -%% function start line found -build_html(SFd,DFd,Enc,{ok,Str},L0,[{F,A,L0,LastL}|FuncsAndCs], - _IsFuncDef,_FAndLastL) -> - FALink = test_server_ctrl:uri_encode(F++"-"++integer_to_list(A),utf8), - file:write(DFd,["<a name=\"",to_raw_list(FALink,Enc),"\"/>"]), - build_html(SFd,DFd,Enc,{ok,Str},L0,FuncsAndCs,true,{F,LastL}); %% line of last expression in function found build_html(SFd,DFd,Enc,{ok,Str},LastL,FuncsAndCs,_IsFuncDef,{F,LastL}) -> LastLineLink = test_server_ctrl:uri_encode(F++"-last_expr",utf8), file:write(DFd,["<a name=\"", to_raw_list(LastLineLink,Enc),"\"/>"]), build_html(SFd,DFd,Enc,{ok,Str},LastL,FuncsAndCs,true,undefined); +%% function start line found +build_html(SFd,DFd,Enc,{ok,Str},L0,[{F,A,L0,LastL}|FuncsAndCs], + _IsFuncDef,_FAndLastL) -> + FALink = test_server_ctrl:uri_encode(F++"-"++integer_to_list(A),utf8), + file:write(DFd,["<a name=\"",to_raw_list(FALink,Enc),"\"/>"]), + build_html(SFd,DFd,Enc,{ok,Str},L0,FuncsAndCs,true,{F,LastL}); build_html(SFd,DFd,Enc,{ok,Str},L,[{clause,L}|FuncsAndCs], _IsFuncDef,FAndLastL) -> build_html(SFd,DFd,Enc,{ok,Str},L,FuncsAndCs,true,FAndLastL); diff --git a/lib/test_server/src/test_server.app.src b/lib/test_server/src/test_server.app.src index 5538e8b851..bdd9d28444 100644 --- a/lib/test_server/src/test_server.app.src +++ b/lib/test_server/src/test_server.app.src @@ -32,7 +32,7 @@ test_server_break_process]}, {applications, [kernel,stdlib]}, {env, []}, - {runtime_dependencies, ["tools-2.6.14","stdlib-2.0","runtime_tools-1.8.14", - "observer-2.0","kernel-3.0","inets-5.10", - "syntax_tools-1.6.16","erts-7.0"]}]}. + {runtime_dependencies, ["tools-2.8","stdlib-2.5","runtime_tools-1.8.16", + "observer-2.1","kernel-4.0","inets-6.0", + "syntax_tools-1.7","erts-7.0"]}]}. diff --git a/lib/test_server/src/ts.erl b/lib/test_server/src/ts.erl index 469593e947..85f97656ff 100644 --- a/lib/test_server/src/ts.erl +++ b/lib/test_server/src/ts.erl @@ -262,18 +262,28 @@ run_all(_Vars) -> run_some([], _Opts) -> ok; -run_some([{App,Mod}|Apps], Opts) -> +run_some(Apps, Opts) -> + case proplists:get_value(test_category, Opts) of + bench -> + check_and_run(fun(Vars) -> ts_benchmark:run(Apps, Opts, Vars) end); + _Other -> + run_some1(Apps, Opts) + end. + +run_some1([], _Opts) -> + ok; +run_some1([{App,Mod}|Apps], Opts) -> case run(App, Mod, Opts) of ok -> ok; Error -> io:format("~p: ~p~n",[{App,Mod},Error]) end, - run_some(Apps, Opts); -run_some([App|Apps], Opts) -> + run_some1(Apps, Opts); +run_some1([App|Apps], Opts) -> case run(App, Opts) of ok -> ok; Error -> io:format("~p: ~p~n",[App,Error]) end, - run_some(Apps, Opts). + run_some1(Apps, Opts). %% This can be used from command line. Both App and %% TestCategory must be specified. App may be 'all' diff --git a/lib/test_server/test/erl2html2_SUITE.erl b/lib/test_server/test/erl2html2_SUITE.erl index 908985c879..796b84dedd 100644 --- a/lib/test_server/test/erl2html2_SUITE.erl +++ b/lib/test_server/test/erl2html2_SUITE.erl @@ -130,15 +130,7 @@ groups() -> %% @end %%-------------------------------------------------------------------- all() -> - [m1]. - -%%-------------------------------------------------------------------- -%% @spec TestCase() -> Info -%% Info = [tuple()] -%% @end -%%-------------------------------------------------------------------- -m1() -> - []. + [macros_defined, macros_undefined]. %%-------------------------------------------------------------------- %% @spec TestCase(Config0) -> @@ -149,19 +141,29 @@ m1() -> %% Comment = term() %% @end %%-------------------------------------------------------------------- -m1(Config) -> - {Src,Dst} = convert_module("m1",Config), +macros_defined(Config) -> + %% let erl2html2 use epp as parser + DataDir = ?config(data_dir,Config), + InclDir = filename:join(DataDir, "include"), + {Src,Dst} = convert_module("m1",[InclDir],Config), {true,L} = check_line_numbers(Src,Dst), - ok = check_link_targets(Src,Dst,L,[{baz,0}]), + ok = check_link_targets(Src,Dst,L,[{baz,0}],[]), ok. -convert_module(Mod,Config) -> +macros_undefined(Config) -> + %% let erl2html2 use epp_dodger as parser + {Src,Dst} = convert_module("m1",[],Config), + {true,L} = check_line_numbers(Src,Dst), + ok = check_link_targets(Src,Dst,L,[{baz,0}],[{quux,0}]), + ok. + +convert_module(Mod,InclDirs,Config) -> DataDir = ?config(data_dir,Config), PrivDir = ?config(priv_dir,Config), Src = filename:join(DataDir,Mod++".erl"), Dst = filename:join(PrivDir,Mod++".erl.html"), io:format("<a href=\"~s\">~s</a>\n",[Src,filename:basename(Src)]), - ok = erl2html2:convert(Src, Dst, [], "<html><body>"), + ok = erl2html2:convert(Src, Dst, InclDirs, "<html><body>"), io:format("<a href=\"~s\">~s</a>\n",[Dst,filename:basename(Dst)]), {Src,Dst}. @@ -229,36 +231,46 @@ check_line_number(Last,Line,OrigLine) -> %% function. %% The test module has -compile(export_all), so all functions are %% found by listing the exported ones. -check_link_targets(Src,Dst,L,RmFncs) -> +check_link_targets(Src,Dst,L,RmFncs,ShouldRemain) -> Mod = list_to_atom(filename:basename(filename:rootname(Src))), Exports = Mod:module_info(exports)--[{module_info,0},{module_info,1}|RmFncs], - {ok,{[],L},_} = xmerl_sax_parser:file(Dst, - [{event_fun,fun sax_event/3}, - {event_state,{Exports,0}}]), + LastExprFuncs = [Func || {Func,_A} <- Exports], + {ok,{FAs,Fs,L},_} = + xmerl_sax_parser:file(Dst, + [{event_fun,fun sax_event/3}, + {event_state,{Exports,LastExprFuncs,0}}]), + true = (length(FAs) == length(ShouldRemain)), + [] = [FA || FA <- FAs, not lists:member(FA,ShouldRemain)], + [] = [F || F <- Fs, not lists:keymember(F,1,ShouldRemain)], ok. sax_event(Event,_Loc,State) -> sax_event(Event,State). -sax_event({startElement,_Uri,"a",_QN,Attrs},{Exports,PrevLine}) -> +sax_event({startElement,_Uri,"a",_QN,Attrs},{Exports,LastExprFuncs,PrevLine}) -> {_,_,"name",Name} = lists:keyfind("name",3,Attrs), case catch list_to_integer(Name) of Line when is_integer(Line) -> case PrevLine + 1 of Line -> -% erlang:display({found_line,Line}), - {Exports,Line}; + {Exports,LastExprFuncs,Line}; Other -> ct:fail({unexpected_line_number_target,Other}) end; {'EXIT',_} -> - {match,[FStr,AStr]} = - re:run(Name,"^(.*)-([0-9]+)$",[{capture,all_but_first,list}]), + {match,[FStr,EndStr]} = + re:run(Name,"^(.*)-(last_expr|[0-9]+)$", + [{capture,all_but_first,list}]), F = list_to_atom(http_uri:decode(FStr)), - A = list_to_integer(AStr), -% erlang:display({found_fnc,F,A}), - A = proplists:get_value(F,Exports), - {lists:delete({F,A},Exports),PrevLine} + case EndStr of + "last_expr" -> + true = lists:member(F,LastExprFuncs), + {Exports,lists:delete(F,LastExprFuncs),PrevLine}; + _ -> + A = list_to_integer(EndStr), + A = proplists:get_value(F,Exports), + {lists:delete({F,A},Exports),LastExprFuncs,PrevLine} + end end; sax_event(_,State) -> State. diff --git a/lib/test_server/test/erl2html2_SUITE_data/include/header3.hrl b/lib/test_server/test/erl2html2_SUITE_data/include/header3.hrl new file mode 100644 index 0000000000..2a20850a3a --- /dev/null +++ b/lib/test_server/test/erl2html2_SUITE_data/include/header3.hrl @@ -0,0 +1 @@ +-define(EPP_SWITCH, on). diff --git a/lib/test_server/test/erl2html2_SUITE_data/m1.erl b/lib/test_server/test/erl2html2_SUITE_data/m1.erl index 156f1d0a51..1d405963a5 100644 --- a/lib/test_server/test/erl2html2_SUITE_data/m1.erl +++ b/lib/test_server/test/erl2html2_SUITE_data/m1.erl @@ -7,9 +7,15 @@ -include("header1.hrl"). -include("header2.hrl"). +-include("header3.hrl"). -define(MACRO1,value). +%% This macro is used to select parser in erl2html2. +%% If EPP_SWITCH is defined epp is used, else epp_dodger. +epp_switch() -> + ?EPP_SWITCH. + %%% Comment foo(x) -> %% Comment diff --git a/lib/test_server/vsn.mk b/lib/test_server/vsn.mk index 2a2ed2b3b0..fd9e4e6d74 100644 --- a/lib/test_server/vsn.mk +++ b/lib/test_server/vsn.mk @@ -1 +1 @@ -TEST_SERVER_VSN = 3.8.1 +TEST_SERVER_VSN = 3.9 diff --git a/lib/tools/doc/src/cprof.xml b/lib/tools/doc/src/cprof.xml index 553597837e..bfddb9f5a8 100644 --- a/lib/tools/doc/src/cprof.xml +++ b/lib/tools/doc/src/cprof.xml @@ -66,7 +66,7 @@ <func> <name>analyse() -> {AllCallCount, ModAnalysisList}</name> <name>analyse(Limit) -> {AllCallCount, ModAnalysisList}</name> - <name>analyse(Mod) -> ModAnlysis</name> + <name>analyse(Mod) -> ModAnalysis</name> <name>analyse(Mod, Limit) -> ModAnalysis</name> <fsummary>Collect and analyse call counters.</fsummary> <type> diff --git a/lib/tools/src/tools.app.src b/lib/tools/src/tools.app.src index a4e5d85f92..8458941761 100644 --- a/lib/tools/src/tools.app.src +++ b/lib/tools/src/tools.app.src @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2012. All Rights Reserved. +%% Copyright Ericsson AB 1996-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -40,7 +40,7 @@ {env, [{file_util_search_methods,[{"", ""}, {"ebin", "esrc"}, {"ebin", "src"}]} ] }, - {runtime_dependencies, ["webtool-0.8.10","stdlib-2.0","runtime_tools-1.8.14", + {runtime_dependencies, ["webtool-0.8.10","stdlib-2.5","runtime_tools-1.8.14", "kernel-3.0","inets-5.10","erts-7.0", "compiler-5.0"]} ] diff --git a/lib/tools/test/lcnt_SUITE.erl b/lib/tools/test/lcnt_SUITE.erl index 010dffe138..de68486b1b 100644 --- a/lib/tools/test/lcnt_SUITE.erl +++ b/lib/tools/test/lcnt_SUITE.erl @@ -97,12 +97,12 @@ t_conflicts_file([File|Files]) -> {ok, _} = lcnt:start(), ok = lcnt:load(File), ok = lcnt:conflicts(), - THs = [-1, 0, 100, 1000], + THs = [-1, 5], Print = [name , id , type , entry , tries , colls , ratio , time , duration], Opts = [ [{sort, Sort}, {reverse, Rev}, {max_locks, ML}, {combine, Combine}, {thresholds, [TH]}, {print, [Print]}] || - Sort <- [name , id , type , tries , colls , ratio , time , entry], - ML <- [none, 1 , 32, 4096], + Sort <- [name , type , tries , colls , ratio , time], + ML <- [none, 32], Combine <- [true, false], TH <- [{tries, Tries} || Tries <- THs] ++ [{colls, Colls} || Colls <- THs] ++ [{time, Time} || Time <- THs], Rev <- [true, false] @@ -131,12 +131,12 @@ t_locations_file([File|Files]) -> {ok, _} = lcnt:start(), ok = lcnt:load(File), ok = lcnt:locations(), - THs = [-1, 0, 100, 1000], + THs = [-1, 0, 100], Print = [name , id , type , entry , tries , colls , ratio , time , duration], Opts = [ [{full_id, Id}, {sort, Sort}, {max_locks, ML}, {combine, Combine}, {thresholds, [TH]}, {print, Print}] || Sort <- [name , id , type , tries , colls , ratio , time , entry], - ML <- [none, 1 , 64], + ML <- [none, 64], Combine <- [true, false], TH <- [{tries, Tries} || Tries <- THs] ++ [{colls, Colls} || Colls <- THs] ++ [{time, Time} || Time <- THs], Id <- [true, false] diff --git a/lib/tools/vsn.mk b/lib/tools/vsn.mk index 3b3202d38b..68c3f6e29c 100644 --- a/lib/tools/vsn.mk +++ b/lib/tools/vsn.mk @@ -1 +1 @@ -TOOLS_VSN = 2.7.2 +TOOLS_VSN = 2.8 diff --git a/lib/typer/vsn.mk b/lib/typer/vsn.mk index ce658e257b..74c0ccfc59 100644 --- a/lib/typer/vsn.mk +++ b/lib/typer/vsn.mk @@ -1 +1 @@ -TYPER_VSN = 0.9.8 +TYPER_VSN = 0.9.9 diff --git a/lib/webtool/vsn.mk b/lib/webtool/vsn.mk index a79c273d9f..4a701ae6e0 100644 --- a/lib/webtool/vsn.mk +++ b/lib/webtool/vsn.mk @@ -1 +1 @@ -WEBTOOL_VSN=0.8.10 +WEBTOOL_VSN=0.9 diff --git a/lib/wx/c_src/wxe_driver.c b/lib/wx/c_src/wxe_driver.c index ea52737fa2..ec1ba7f566 100644 --- a/lib/wx/c_src/wxe_driver.c +++ b/lib/wx/c_src/wxe_driver.c @@ -146,7 +146,12 @@ wxe_driver_stop(ErlDrvData handle) if(sd->port_handle != WXE_DRV_PORT_HANDLE) { // fprintf(stderr, "%s:%d: STOP \r\n", __FILE__,__LINE__); meta_command(DELETE_PORT,sd); - free(handle); + } else { + // fprintf(stderr, "%s:%d: STOP \r\n", __FILE__,__LINE__); + stop_native_gui(wxe_master); + unload_native_gui(); + free(wxe_master); + wxe_master = NULL; } } @@ -154,10 +159,6 @@ static void wxe_driver_unload(void) { // fprintf(stderr, "%s:%d: UNLOAD \r\n", __FILE__,__LINE__); - stop_native_gui(wxe_master); - unload_native_gui(); - free(wxe_master); - wxe_master = NULL; } static ErlDrvSSizeT diff --git a/lib/wx/c_src/wxe_impl.cpp b/lib/wx/c_src/wxe_impl.cpp index ef648e008c..2fd5f0c52c 100644 --- a/lib/wx/c_src/wxe_impl.cpp +++ b/lib/wx/c_src/wxe_impl.cpp @@ -89,7 +89,7 @@ void push_command(int op,char * buf,int len, wxe_data *sd) } void meta_command(int what, wxe_data *sd) { - if(what == PING_PORT) { + if(what == PING_PORT && wxe_status == WXE_INITIATED) { erl_drv_mutex_lock(wxe_batch_locker_m); if(wxe_batch_caller > 0) { wxe_queue->Add(WXE_DEBUG_PING, NULL, 0, sd); @@ -98,9 +98,12 @@ void meta_command(int what, wxe_data *sd) { wxWakeUpIdle(); erl_drv_mutex_unlock(wxe_batch_locker_m); } else { - if(sd) { + if(sd && wxe_status == WXE_INITIATED) { wxeMetaCommand Cmd(sd, what); wxTheApp->AddPendingEvent(Cmd); + if(what == DELETE_PORT) { + free(sd); + } } } } @@ -169,6 +172,7 @@ void WxeApp::MacOpenFile(const wxString &filename) { #endif void WxeApp::shutdown(wxeMetaCommand& Ecmd) { + wxe_status = WXE_EXITING; ExitMainLoop(); delete wxe_queue; delete wxe_queue_cb_saved; @@ -200,6 +204,10 @@ void handle_event_callback(ErlDrvPort port, ErlDrvTermData process) { WxeApp * app = (WxeApp *) wxTheApp; ErlDrvMonitor monitor; + + if(wxe_status != WXE_INITIATED) + return; + // Is thread safe if pdl have been incremented if(driver_monitor_process(port, process, &monitor) == 0) { // Should we be able to handle commands when recursing? probably @@ -217,6 +225,8 @@ void handle_event_callback(ErlDrvPort port, ErlDrvTermData process) void WxeApp::dispatch_cmds() { + if(wxe_status != WXE_INITIATED) + return; erl_drv_mutex_lock(wxe_batch_locker_m); recurse_level++; int level = dispatch(wxe_queue_cb_saved, 0, WXE_STORED); diff --git a/lib/wx/c_src/wxe_impl.h b/lib/wx/c_src/wxe_impl.h index a0a1c84718..b251d5f0f9 100644 --- a/lib/wx/c_src/wxe_impl.h +++ b/lib/wx/c_src/wxe_impl.h @@ -46,7 +46,8 @@ typedef wxString wxeLocaleC; #define WXE_NOT_INITIATED 0 #define WXE_INITIATED 1 -#define WXE_EXITED 2 +#define WXE_EXITING 2 +#define WXE_EXITED 3 #define WXE_ERROR -1 void send_msg(const char *, const wxString *); // For debugging and error msgs diff --git a/lib/wx/vsn.mk b/lib/wx/vsn.mk index 942d4c0d6f..09fb9f384c 100644 --- a/lib/wx/vsn.mk +++ b/lib/wx/vsn.mk @@ -1 +1 @@ -WX_VSN = 1.3.3 +WX_VSN = 1.4 diff --git a/lib/xmerl/src/xmerl.app.src b/lib/xmerl/src/xmerl.app.src index 45cfe9d250..aed9cf176f 100644 --- a/lib/xmerl/src/xmerl.app.src +++ b/lib/xmerl/src/xmerl.app.src @@ -40,5 +40,5 @@ {registered, []}, {env, []}, {applications, [kernel, stdlib]}, - {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-6.0"]} + {runtime_dependencies, ["stdlib-2.5","kernel-3.0","erts-6.0"]} ]}. |