Merge branch 'master' of super:otp into bmk/inets/inets58_integration

8 files changed, 313 insertions, 48 deletions

diff --git a/lib/diameter/doc/src/depend.sed b/lib/diameter/doc/src/depend.sed
index 5973c4586e..42de597f15 100644
--- a/lib/diameter/doc/src/depend.sed
+++ b/lib/diameter/doc/src/depend.sed
@@ -21,14 +21,18 @@
 # massaged in Makefile.
 #
 
-/^<com>\([^<]*\)<\/com>/b rf
-/^<module>\([^<]*\)<\/module>/b rf
+/^<com>/b c
+/^<module>/b c
 
 /^<chapter>/!d
 
+# Chapter: html basename is same as xml.
 s@@$(HTMLDIR)/%FILE%.html: %FILE%.xml@
 q
 
-:rf
-s@@$(HTMLDIR)/\1.html: %FILE%.xml@
+# Reference: html basename is from contents of com/module element.
+:c
+s@^[^>]*>@@
+s@<.*@@
+s@.*@$(HTMLDIR)/&.html: %FILE%.xml@
 q
diff --git a/lib/diameter/doc/src/diameter.xml b/lib/diameter/doc/src/diameter.xml
index 36b6cbf0cf..43c497f50a 100644
--- a/lib/diameter/doc/src/diameter.xml
+++ b/lib/diameter/doc/src/diameter.xml
@@ -277,6 +277,10 @@ callback.</p>
 
 </taglist>
 
+<p>
+An invalid option will cause <seealso marker="#call">call/4</seealso>
+to fail.</p>
+
 <marker id="capability"/>
 </item>
 
@@ -363,6 +367,19 @@ capabilities exchange message.
 Optional, defaults to the empty list.</p>
 </item>
 
+<tag><c>{'Inband-Security-Id', [Unsigned32()]}</c></tag>
+<item>
+<p>
+Values of Inband-Security-Id AVPs sent in an outgoing
+capabilities exchange message.
+Optional, defaults to the empty list, which is equivalent to a
+list containing only 0 (= NO_INBAND_SECURITY).</p>
+
+<p>
+If 1 (= TLS) is specified then TLS is selected if the CER/CEA received
+from the peer offers it.</p>
+</item>
+
 <tag><c>{'Acct-Application-Id', [Unsigned32()]}</c></tag>
 <item>
 <p>
@@ -405,6 +422,8 @@ sense.</p>
 <code>
 eval([{M,F,A} | T]) ->
     apply(M, F, T ++ A);
+eval([[F|A] | T]) ->
+    eval([F | T ++ A]);
 eval([F|A]) ->
     apply(F, A);
 eval(F) ->
@@ -461,14 +480,14 @@ or any peer if the request does not contain
 a <c>Destination-Realm</c> AVP.</p>
 </item>
 
-<tag><c>{host, any|UTF8String()}</c></tag>
+<tag><c>{host, any|DiameterIdentity()}</c></tag>
 <item>
 <p>
 Matches only those peers whose <c>Origin-Host</c> has the
 specified value, or all peers if the atom <c>any</c>.</p>
 </item>
 
-<tag><c>{realm, any|UTF8String()</c></tag>
+<tag><c>{realm, any|DiameterIdentity()</c></tag>
 <item>
 <p>
 Matches only those peers whose <c>Origin-Realm</c> has the
@@ -478,8 +497,9 @@ value, or all peers if the atom <c>any</c>.</p>
 <tag><c>{eval, evaluable()}</c></tag>
 <item>
 <p>
-Matches only those peers for which the specified evaluable() evaluates
-to true on the peer's <c>diameter_caps</c> record.</p>
+Matches only those peers for which the specified evaluable() returns
+<c>true</c> on the connection's <c>diameter_caps</c> record.
+Any other return value or exception is equivalent to <c>false</c>.</p>
 </item>
 
 <tag><c>{neg, peer_filter()}</c></tag>
@@ -503,6 +523,21 @@ specified list.</p>
 
 </taglist>
 
+<p>
+Note that the <c>host</c> and <c>realm</c> filters examine the
+outgoing request as passed to <seealso marker="#call">call/4</seealso>,
+assuming that this is a record- or list-valued message() as documented
+in <seealso marker="diameter_app">diameter_app(3)</seealso>, and that
+the message contains at most one of each AVP.
+If this is not the case then the <c>{host|realm, DiameterIdentity()}</c>
+filters must be used to achieve the desired result.
+Note also that an empty host/realm (which should not be typical)
+is equivalent to an unspecified one for the purposes of filtering.</p>
+
+<p>
+An invalid filter is equivalent to <c>{any, []}</c>, a filter
+that matches no peer.</p>
+
 <marker id="service_event"/>
 </item>
 
@@ -661,6 +696,14 @@ in question.</p>
 AVP's used to construct outgoing CER/CEA messages.
 Any AVP specified takes precedence over a corresponding value specified
 for the service in question.</p>
+
+<p>
+Specifying a capability as a transport option
+may be particularly appropriate for Inband-Security-Id in case
+TLS is desired over TCP as implemented by
+<seealso marker="diameter_tcp">diameter_tcp(3)</seealso> but
+not over SCTP as implemented by
+<seealso marker="diameter_sctp">diameter_sctp(3)</seealso>.</p>
 </item>
 
 <tag><c>{watchdog_timer, TwInit}</c></tag>
@@ -787,7 +830,7 @@ transports.</p>
 <type>
 <v>SvcName = service_name()</v>
 <v>App = application_alias()</v>
-<v>Request = diameter_app:message()</v>
+<v>Request = diameter_app:message() | term()</v>
 <v>Answer = term()</v>
 <v>Options = [call_opt()]</v>
 </type>
@@ -819,9 +862,8 @@ If there are no suitable peers, or if
 <seealso marker="diameter_app#pick_peer">pick_peer/4</seealso>
 rejects them by returning 'false', then <c>{error, no_connection}</c>
 is returned.
-If <seealso marker="diameter_app#pick_peer">pick_peer/4</seealso>
-selects a candidate peer then a request process is spawned for the
-outgoing request, in which there is a
+Otherwise <seealso marker="diameter_app#pick_peer">pick_peer/4</seealso>
+is followed by a
 <seealso
 marker="diameter_app#prepare_request">prepare_request/3</seealso>
 callback, the message is encoded and sent.</p>
diff --git a/lib/diameter/doc/src/diameter_app.xml b/lib/diameter/doc/src/diameter_app.xml
index fc359b9d1d..a9ae0ebbec 100644
--- a/lib/diameter/doc/src/diameter_app.xml
+++ b/lib/diameter/doc/src/diameter_app.xml
@@ -269,7 +269,12 @@ The candidate peers list will only include those
 which are selected by any <c>filter</c> option specified in the call to
 <seealso marker="diameter#call">diameter:call/4</seealso>, and only
 those which have indicated support for the Diameter application in
-question.</p>
+question.
+The order of the elements is unspecified except that any
+peers whose Origin-Host and Origin-Realm matches that of the
+outgoing request (in the sense of a <c>{filter, {all, [host, realm]}}</c>
+option to <seealso marker="diameter#call">diameter:call/4</seealso>)
+will be placed at the head of the list.</p>
 
 <p>
 The return values <c>false</c> and <c>{false, State}</c> are
@@ -467,11 +472,11 @@ callback returned false.</p>
 <v>Packet  = packet()</v>
 <v>SvcName = term()</v>
 <v>Peer    = peer()</v>
-<v>Action  = Reply | {relay, Opts} | discard | {eval, Action, ContF}</v>
+<v>Action  = Reply | {relay, Opts} | discard | {eval, Action, PostF}</v>
 <v>Reply   = {reply, message()}
            | {protocol_error, 3000..3999}</v>
 <v>Opts    = diameter:call_opts()</v>
-<v>ContF   = diameter:evaluable()</v>
+<v>PostF   = diameter:evaluable()</v>
 </type>
 <desc>
 <p>
@@ -559,26 +564,28 @@ will cause the request process in question to fail.</p>
 <tag><c>{relay, Opts}</c></tag>
 <item>
 <p>
-Relay a request to another peer.
-The appropriate Route-Record AVP will be added to the relayed request
-by diameter and <seealso marker="#pick_peer">pick_peer/4</seealso>
-and <seealso marker="#prepare_request">prepare_request/3</seealso>
-callback will take place just as if <seealso
+Relay a request to another peer in the role of a Diameter relay agent.
+If a routing loop is detected then the request is answered with
+3005 (DIAMETER_LOOP_DETECTED).
+Otherwise a Route-Record AVP (containing the sending peer's Origin-Host) is
+added to the request and <seealso marker="#pick_peer">pick_peer/4</seealso>
+and subsequent callbacks take place just as if <seealso
 marker="diameter#call">diameter:call/4</seealso> had been called
 explicitly.
-However, returning a <c>relay</c> tuple also causes the End-to-End
-Identifier to be preserved in the header of the relayed request as
-required by RFC 3588.</p>
+The End-to-End Identifier of the incoming request is preserved in the
+header of the relayed request.</p>
 
 <p>
-The returned <c>Opts</c> should not specify <c>detach</c> and
-the <seealso marker="#handle_answer">handle_answer/4</seealso>
-callback following from a relayed request must return its first
+The returned <c>Opts</c> should not specify <c>detach</c>.
+A subsequent <seealso marker="#handle_answer">handle_answer/4</seealso>
+callback for the relayed request must return its first
 argument, the <c>diameter_packet</c> record containing the answer
 message.
 Note that the <c>extra</c> option can be specified to supply arguments
-that can distinguish the relay case from others if so desired,
-although the form of the request message may be sufficient.</p>
+that can distinguish the relay case from others if so desired.
+Any other return value (for example, from a
+<seealso marker="#handle_error">handle_error/4</seealso> callback)
+causes the request to be answered with 3002 (DIAMETER_UNABLE_TO_DELIVER).</p>
 </item>
 
 <tag><c>discard</c></tag>
@@ -587,18 +594,18 @@ although the form of the request message may be sufficient.</p>
 Discard the request.</p>
 </item>
 
-<tag><c>{eval, Action, ContF}</c></tag>
+<tag><c>{eval, Action, PostF}</c></tag>
 <item>
 <p>
 Handle the request as if <c>Action</c> has been returned and then
-evaluate <c>ContF</c> in the request process.</p>
+evaluate <c>PostF</c> in the request process.</p>
 </item>
 
 </taglist>
 
 <p>
-Note that diameter will respond to protocol errors in an incoming
-request without invoking <c>handle_request/3</c>.</p>
+Note that protocol errors detected by diameter will result in an
+answer message without <c>handle_request/3</c> being invoked.</p>
 
 </desc>
 </func>
diff --git a/lib/diameter/doc/src/diameter_sctp.xml b/lib/diameter/doc/src/diameter_sctp.xml
index d0377f4b38..c1e839b8e1 100644
--- a/lib/diameter/doc/src/diameter_sctp.xml
+++ b/lib/diameter/doc/src/diameter_sctp.xml
@@ -74,11 +74,12 @@ marker="diameter_transport#start">diameter_transport(3)</seealso>.</p>
 <p>
 The only diameter_sctp-specific argument is the options list.
 Options <c>raddr</c> and <c>rport</c> specify the remote address
-and port for a connector and not valid for a listener.
+and port for a connecting transport and not valid for a listening
+transport.
 The former is required while latter defaults to 3868 if unspecified.
 More than one <c>raddr</c> option can be specified, in which case the
-connector in question attempts each in sequence until an association
-is established.
+connecting transport in question attempts each in sequence until
+an association is established.
 Remaining options are any accepted by gen_sctp:open/1, with the exception
 of options <c>mode</c>, <c>binary</c>, <c>list</c>, <c>active</c>
 and <c>sctp_events</c>.
@@ -89,7 +90,8 @@ and port respectively.</p>
 Multiple <c>ip</c> options can be specified for a multihomed peer.
 If none are specified then the values of Host-IP-Address
 on the service are used. (In particular, one of these must be specified.)
-Option <c>port</c> defaults to 3868 for a listener and 0 for a connector.</p>
+Option <c>port</c> defaults to 3868 for a listening transport and 0 for a
+connecting transport.</p>
 
 <p>
 diameter_sctp uses the <c>transport_data</c> field of
diff --git a/lib/diameter/doc/src/diameter_soc.xml b/lib/diameter/doc/src/diameter_soc.xml
index 4f8581a904..6b9ef9f756 100644
--- a/lib/diameter/doc/src/diameter_soc.xml
+++ b/lib/diameter/doc/src/diameter_soc.xml
@@ -57,9 +57,13 @@ including the P Flag in the AVP header.</p>
 
 <item>
 <p>
-There is no TLS support.
-It's unclear (aka uninvestigated) how TLS would impact
-diameter but IPsec can be used without it needing to know.</p>
+There is no TLS support over SCTP.
+RFC 3588 requires that a Diameter server support TLS but in
+practise this seems to mean TLS over SCTP since there are limitations
+with running over SCTP: see RFC 6083 (DTLS over SCTP), which is a
+response to RFC 3436 (TLS over SCTP).
+The current RFC 3588 draft acknowledges this by equating
+TLS with TLS/TCP and DTLS/SCTP but we do not yet support DTLS.</p>
 </item>
 
 <item>
diff --git a/lib/diameter/doc/src/diameter_tcp.xml b/lib/diameter/doc/src/diameter_tcp.xml
index 5d6e07b1b8..e6b53383c0 100644
--- a/lib/diameter/doc/src/diameter_tcp.xml
+++ b/lib/diameter/doc/src/diameter_tcp.xml
@@ -43,7 +43,14 @@ It can be specified as the value of a transport_module option to
 <seealso
 marker="diameter#add_transport">diameter:add_transport/2</seealso>
 and implements the behaviour documented in
-<seealso marker="diameter_transport">diameter_transport(3)</seealso>.</p>
+<seealso marker="diameter_transport">diameter_transport(3)</seealso>.
+TLS security is supported, both as an upgrade following
+capabilities exchange as specified by RFC 3588 and
+at connection establishment as in the current draft standard.</p>
+
+<p>
+Note that the ssl application is required for TLS and must be started
+before configuring TLS capability on diameter transports.</p>
 
 <marker id="start"/>
 </description>
@@ -60,10 +67,15 @@ and implements the behaviour documented in
 <v>Type = connect | accept</v>
 <v>Ref = reference()</v>
 <v>Svc = #diameter_service{}</v>
-<v>Opt = {raddr, ip_address()} | {rport, integer()} | term()</v>
+<v>Opt = OwnOpt | SslOpt | OtherOpt</v>
 <v>Pid = pid()</v>
 <v>LAddr = ip_address()</v>
 <v>Reason = term()</v>
+<v>OwnOpt = {raddr, ip_address()}
+          | {rport, integer()}
+          | {port, integer()}</v>
+<v>SslOpt = {ssl_options, true | list()}</v>
+<v>OtherOpt = term()</v>
 </type>
 <desc>
 
@@ -74,16 +86,42 @@ marker="diameter_transport#start">diameter_transport(3)</seealso>.</p>
 <p>
 The only diameter_tcp-specific argument is the options list.
 Options <c>raddr</c> and <c>rport</c> specify the remote address
-and port for a connector and not valid for a listener.
-Remaining options are any accepted by gen_tcp:connect/3 for
-a connector, or gen_tcp:listen/2 for a listener, with the exception
-of <c>binary</c>, <c>packet</c> and <c>active</c>.
-Also, option <c>port</c> can be specified for a listener to specify the
-local listening port, the default being the standardized 3868 if
-unspecified.
+and port for a connecting transport and are not valid for a listening
+transport.
+Option <c>ssl_options</c> must be specified for a transport
+that must be able to support TLS: a value of <c>true</c> results in a
+TLS handshake immediately upon connection establishment while
+list() specifies options to be passed to ssl:connect/2 of ssl:ssl_accept/2
+after capabilities exchange if TLS is negotiated.
+Remaining options are any accepted by ssl:connect/3 or gen_tcp:connect/3 for
+a connecting transport, or ssl:listen/3 or gen_tcp:listen/2 for
+a listening transport, depending on whether or not <c>{ssl_options, true}</c>
+has been specified.
+Options <c>binary</c>, <c>packet</c> and <c>active</c> cannot be specified.
+Also, option <c>port</c> can be specified for a listening transport
+to specify the local listening port, the default being the standardized
+3868 if unspecified.
 Note that option <c>ip</c> specifies the local address.</p>
 
 <p>
+An <c>ssl_options</c> list must be specified if and only if
+the transport in question has specified an Inband-Security-Id
+AVP with value TLS on the relevant call to
+<seealso
+marker="diameter#start_service">start_service/2</seealso> or
+<seealso
+marker="diameter#add_transport">add_transport/2</seealso>,
+so that the transport process will receive notification of
+whether or not to commence with a TLS handshake following capabilities
+exchange.
+Failing to specify an options list on a TLS-capable transport
+for which TLS is negotiated will cause TLS handshake to fail.
+Failing to specify TLS capability when <c>ssl_options</c> has been
+specified will cause the transport process to wait for a notification
+that will not be forthcoming, which will eventually cause the RFC 3539
+watchdog to take down the connection.</p>
+
+<p>
 If the service specifies more than one Host-IP-Address and
 option <c>ip</c> is unspecified then then the
 first of the service's addresses is used as the local address.</p>
@@ -103,6 +141,7 @@ The returned local address list has length one.</p>
 <title>SEE ALSO</title>
 
 <p>
+<seealso marker="diameter">diameter(3)</seealso>,
 <seealso marker="diameter_transport">diameter_transport(3)</seealso></p>
 
 </section>
diff --git a/lib/diameter/doc/src/diameter_transport.xml b/lib/diameter/doc/src/diameter_transport.xml
index 37cc871e75..087a90b099 100644
--- a/lib/diameter/doc/src/diameter_transport.xml
+++ b/lib/diameter/doc/src/diameter_transport.xml
@@ -143,6 +143,34 @@ connection.
 Pid is the pid() of the parent process.</p>
 </item>
 
+<tag><c>{diameter, {tls, Ref, Type, Bool}}</c></tag>
+<item>
+<p>
+Indication of whether or not capabilities exchange has selected
+inband security using TLS.
+Ref is a reference() that must be included in the
+<c>{diameter, {tls, Ref}}</c> reply message to the transport's
+parent process (see below).
+Type is either <c>connect</c> or <c>accept</c> depending on
+whether the process has been started for a connecting or listening
+transport respectively.
+Bool is a boolean() indicating whether or not the transport connection
+should be upgraded to TLS.</p>
+
+<p>
+If TLS is requested (Bool = true) then a connecting process should
+initiate a TLS handshake with the peer and an accepting process should
+prepare to accept a handshake.
+A successful handshake should be followed by a <c>{diameter, {tls, Ref}}</c>
+message to the parent process.
+A failed handshake should cause the process to exit.</p>
+
+<p>
+This message is only sent to a transport process over whose
+<c>Inband-Security-Id</c> configuration has indicated support for
+TLS.</p>
+</item>
+
 </taglist>
 
 <p>
@@ -184,6 +212,16 @@ How the <c>transport_data</c> is used/interpreted is up to the
 transport module.</p>
 </item>
 
+<tag><c>{diameter, {tls, Ref}}</c></tag>
+<item>
+<p>
+Acknowledgment of a successful TLS handshake.
+Ref is the reference() received in the
+<c>{diameter, {tls, Ref, Type, Bool}}</c> message in response
+to which the reply is sent.
+A transport must exit if a handshake is not successful.</p>
+</item>
+
 </taglist>
 
 </section>
diff --git a/lib/diameter/doc/src/notes.xml b/lib/diameter/doc/src/notes.xml
index eafddd7d1e..e2723f3e99 100644
--- a/lib/diameter/doc/src/notes.xml
+++ b/lib/diameter/doc/src/notes.xml
@@ -36,6 +36,135 @@ first.</p>
 
 <!-- ===================================================================== -->
 
+<section><title>Diameter 0.10</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+          <p>
+	    Handle #sctp_paddr_change and #sctp_pdapi_event from
+	    gen_sctp.</p>
+          <p>
+	    The events are enabled by default but diameter_sctp
+	    neither disabled nor dealt with them. Reception of such
+	    an event caused a transport process to crash.</p>
+          <p>
+	    Own Id: OTP-9538</p>
+        </item>
+        <item>
+          <p>
+	    Fix header folding bug.</p>
+          <p>
+	    A prepare_request callback from diameter can return a
+	    diameter_header record in order to set values in the
+	    header of an outgoing request. A fault in
+	    diameter_lib:fold_tuple/3 caused the subsequent encode of
+	    the outgoing request to fail.</p>
+          <p>
+	    Own Id: OTP-9577</p>
+        </item>
+        <item>
+          <p>
+	    Fix bugs in sending of answer-message replies.</p>
+          <p>
+	    3001 (DIAMETER_COMMAND_UNSUPPORTED) was not sent since
+	    the decode placed the AVP list in the wrong field of the
+	    diameter_packet, causing the subsequent encode to fail.
+	    Session-Id was also set improperly, causing encode to
+	    fail even in this case.</p>
+          <p>
+	    Own Id: OTP-9578</p>
+        </item>
+        <item>
+          <p>
+	    Fix improper use of error_logger:info_report/2.</p>
+          <p>
+	    Function doesn't take a format string and arguments as it
+	    was called. Instead use error_logger:info_report/1 and
+	    use the same report format as used for warning and error
+	    reports.</p>
+          <p>
+	    Own Id: OTP-9579</p>
+        </item>
+        <item>
+          <p>
+	    Fix and clarify semantics of peer filters.</p>
+          <p>
+	    An eval filter returning a non-true value caused the call
+	    process to fail and the doc was vague on how an exception
+	    was treated. Clarify that the non-tuple host/realm
+	    filters assume messages of a certain form.</p>
+          <p>
+	    Own Id: OTP-9580</p>
+        </item>
+        <item>
+          <p>
+	    Fix and clarify relay behaviour.</p>
+          <p>
+	    Implicit filtering of the sending peer in relaying a
+	    request could cause loop detection to be preempted in a
+	    manner not specified by RFC3588. Reply with 3002
+	    (DIAMETER_UNABLE_TO_DELIVER) on anything but an answer to
+	    a relayed request.</p>
+          <p>
+	    Own Id: OTP-9583</p>
+        </item>
+      </list>
+    </section>
+
+
+    <section><title>Improvements and New Features</title>
+      <list>
+        <item>
+          <p>
+	    @id required in dictionary files only when @messages is
+	    specified.</p>
+          <p>
+	    @id defines an application identifier and this is used
+	    only when sending or receiving messages. A dictionary can
+	    define only AVP's however, to be included by other
+	    dictionaries using @inherits, in which case it makes no
+	    sense to require @id.</p>
+          <p>
+	    Note that message definitions are not inherited with
+	    @inherits, only AVP's</p>
+          <p>
+	    Own Id: OTP-9467</p>
+        </item>
+        <item>
+          <p>
+	    Allow @enum when AVP is defined in an inherited
+	    dictionary.</p>
+          <p>
+	    3GPP standards (for one) extend the values allowed for
+	    RFC 3588 AVP's of type Enumerated. Previously, extending
+	    an AVP was only possible by completely redefining the
+	    AVP.</p>
+          <p>
+	    Own Id: OTP-9469</p>
+        </item>
+        <item>
+          <p>
+	    Migrate testsuites to pure common test and add both
+	    suites and testcases.</p>
+          <p>
+	    Own Id: OTP-9553</p>
+        </item>
+        <item>
+          <p>
+	    Requests of arbitrary form.</p>
+          <p>
+	    diameter:call/4 can be passed anything, as long as the
+	    subsequent prepare_request callback returns a term that
+	    can be encoded.</p>
+          <p>
+	    Own Id: OTP-9581</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
 <section>
 <title>diameter 0.9</title>