Add ssl_crl_hash_dir module

This module is an implementation of the ssl_crl_cache_api behaviour.
It can be used when there is a directory containing CRLs for all
relevant CAs, in the form used by e.g. Apache.  The module assumes
that the directory is being updated through an external process.

6 files changed, 355 insertions, 20 deletions

diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml
index a1fba5fbff..31f88f3285 100644
--- a/lib/ssl/doc/src/ssl.xml
+++ b/lib/ssl/doc/src/ssl.xml
@@ -360,15 +360,59 @@ marker="public_key:public_key#pkix_path_validation-3">public_key:pkix_path_valid
 	<p>Specify how to perform lookup and caching of certificate revocation lists.
 	<c>Module</c> defaults to <seealso marker="ssl:ssl_crl_cache">ssl_crl_cache</seealso>
 	with <c> DbHandle </c> being <c>internal</c> and an
-	empty argument list. The following arguments may be specified for the internal cache:</p>
+	empty argument list.</p>
+
+	<p>There are two implementations available:</p>
+
 	<taglist>
-	  <tag><c>{http, timeout()}</c></tag>
-	  <item><p>
-	    Enables fetching of CRLs specified as http URIs in<seealso 
-	    marker="public_key:public_key_records"> X509 certificate extensions.</seealso>
-	    Requires the OTP inets application.</p>
-	  </item>	    
-	</taglist>    
+	  <tag><c>ssl_crl_cache</c></tag>
+	  <item>
+	    <p>This module maintains a cache of CRLs.  CRLs can be
+	    added to the cache using the function <seealso
+	    marker="ssl:ssl_crl_cache#insert-1">ssl_crl_cache:insert/1</seealso>,
+	    and optionally automatically fetched through HTTP if the
+	    following argument is specified:</p>
+
+	    <taglist>
+	      <tag><c>{http, timeout()}</c></tag>
+	      <item><p>
+		Enables fetching of CRLs specified as http URIs in<seealso
+		marker="public_key:public_key_records">X509 certificate extensions</seealso>.
+		Requires the OTP inets application.</p>
+	      </item>
+	    </taglist>
+	  </item>
+
+	  <tag><c>ssl_crl_hash_dir</c></tag>
+	  <item>
+	    <p>This module makes use of a directory where CRLs are
+	    stored in files named by the hash of the issuer name.</p>
+
+	    <p>The file names consist of eight hexadecimal digits
+	    followed by <c>.rN</c>, where <c>N</c> is an integer,
+	    e.g. <c>1a2b3c4d.r0</c>.  For the first version of the
+	    CRL, <c>N</c> starts at zero, and for each new version,
+	    <c>N</c> is incremented by one.  The OpenSSL utility
+	    <c>c_rehash</c> creates symlinks according to this
+	    pattern.</p>
+
+	    <p>For a given hash value, this module finds all
+	    consecutive <c>.r*</c> files starting from zero, and those
+	    files taken together make up the revocation list.  CRL
+	    files whose <c>nextUpdate</c> fields are in the past, or
+	    that are issued by a different CA that happens to have the
+	    same name hash, are excluded.</p>
+
+	    <p>The following argument is required:</p>
+
+	    <taglist>
+	      <tag><c>{dir, string()}</c></tag>
+	      <item><p>Specifies the directory in which the CRLs can be found.</p></item>
+	    </taglist>
+
+	  </item>
+	</taglist>
+
       </item>
 
       <tag><c>{partial_chain, fun(Chain::[DerCert]) -> {trusted_ca, DerCert} |
diff --git a/lib/ssl/src/Makefile b/lib/ssl/src/Makefile
index 7a7a373487..b625db0656 100644
--- a/lib/ssl/src/Makefile
+++ b/lib/ssl/src/Makefile
@@ -70,6 +70,7 @@ MODULES= \
 	ssl_session_cache \
 	ssl_crl\
 	ssl_crl_cache \
+	ssl_crl_hash_dir \
 	ssl_socket \
 	ssl_listen_tracker_sup \
 	tls_record \
diff --git a/lib/ssl/src/ssl.app.src b/lib/ssl/src/ssl.app.src
index 1a2bf90ccf..5aa399201a 100644
--- a/lib/ssl/src/ssl.app.src
+++ b/lib/ssl/src/ssl.app.src
@@ -44,6 +44,7 @@
 	       ssl_crl,
 	       ssl_crl_cache, 
 	       ssl_crl_cache_api,
+	       ssl_crl_hash_dir,
 	       %% App structure
 	       ssl_app,
 	       ssl_sup,
diff --git a/lib/ssl/src/ssl_crl_hash_dir.erl b/lib/ssl/src/ssl_crl_hash_dir.erl
new file mode 100644
index 0000000000..bb62737232
--- /dev/null
+++ b/lib/ssl/src/ssl_crl_hash_dir.erl
@@ -0,0 +1,106 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2016-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+
+-module(ssl_crl_hash_dir).
+
+-include_lib("public_key/include/public_key.hrl").
+
+-behaviour(ssl_crl_cache_api).
+
+-export([lookup/3, select/2, fresh_crl/2]).
+
+lookup(#'DistributionPoint'{cRLIssuer = CRLIssuer} = DP, CertIssuer, CRLDbInfo) ->
+    Issuer =
+	case CRLIssuer of
+	    asn1_NOVALUE ->
+		%% If the distribution point extension doesn't
+		%% indicate a CRL issuer, use the certificate issuer.
+		CertIssuer;
+	    _ ->
+		CRLIssuer
+	end,
+    %% Find all CRLs for this issuer, and return those that match the
+    %% given distribution point.
+    AllCRLs = select(Issuer, CRLDbInfo),
+    lists:filter(fun(DER) ->
+			 public_key:pkix_match_dist_point(DER, DP)
+		 end, AllCRLs).
+
+fresh_crl(#'DistributionPoint'{}, CurrentCRL) ->
+    CurrentCRL.
+
+select(Issuer, {_DbHandle, [{dir, Dir}]}) ->
+    case find_crls(Issuer, Dir) of
+        [_|_] = DERs ->
+	    DERs;
+        [] ->
+            %% That's okay, just report that we didn't find any CRL.
+            %% If the crl_check setting is best_effort, ssl_handshake
+            %% is happy with that, but if it's true, this is an error.
+            [];
+        {error, Error} ->
+            error_logger:error_report(
+              [{cannot_find_crl, Error},
+               {dir, Dir},
+               {module, ?MODULE},
+               {line, ?LINE}]),
+            []
+    end.
+
+find_crls(Issuer, Dir) ->
+    case filelib:is_dir(Dir) of
+        true ->
+	    Hash = public_key:short_name_hash(Issuer),
+	    find_crls(Issuer, Hash, Dir, 0, []);
+        false ->
+            {error, not_a_directory}
+    end.
+
+find_crls(Issuer, Hash, Dir, N, Acc) ->
+    Filename = filename:join(Dir, Hash ++ ".r" ++ integer_to_list(N)),
+    case file:read_file(Filename) of
+	{error, enoent} ->
+	    Acc;
+	{ok, Bin} ->
+	    try maybe_parse_pem(Bin) of
+		DER when is_binary(DER) ->
+		    %% Found one file.  Let's see if there are more.
+		    find_crls(Issuer, Hash, Dir, N + 1, [DER] ++ Acc)
+	    catch
+		error:Error ->
+		    %% Something is wrong with the file.  Report
+		    %% it, and try the next one.
+		    error_logger:error_report(
+		      [{crl_parse_error, Error},
+		       {filename, Filename},
+		       {module, ?MODULE},
+		       {line, ?LINE}]),
+		    find_crls(Issuer, Hash, Dir, N + 1, Acc)
+	    end
+    end.
+
+maybe_parse_pem(<<"-----BEGIN", _/binary>> = PEM) ->
+    %% It's a PEM encoded file.  Need to extract the DER
+    %% encoded data.
+    [{'CertificateList', DER, not_encrypted}] = public_key:pem_decode(PEM),
+    DER;
+maybe_parse_pem(DER) when is_binary(DER) ->
+    %% Let's assume it's DER-encoded.
+    DER.
+
diff --git a/lib/ssl/test/make_certs.erl b/lib/ssl/test/make_certs.erl
index 5eebf773a7..ed1a6523ed 100644
--- a/lib/ssl/test/make_certs.erl
+++ b/lib/ssl/test/make_certs.erl
@@ -172,11 +172,15 @@ revoke(Root, CA, User, C) ->
     gencrl(Root, CA, C).
 
 gencrl(Root, CA, C) ->
+    %% By default, the CRL is valid for 24 hours from now.
+    gencrl(Root, CA, C, 24).
+
+gencrl(Root, CA, C, CrlHours) ->
     CACnfFile = filename:join([Root, CA, "ca.cnf"]),
     CACRLFile = filename:join([Root, CA, "crl.pem"]),
     Cmd = [C#config.openssl_cmd, " ca"
 	   " -gencrl ",
-	   " -crlhours 24",
+	   " -crlhours ", integer_to_list(CrlHours),
 	   " -out ", CACRLFile,
 	   " -config ", CACnfFile],
     Env = [{"ROOTDIR", filename:absname(Root)}], 
diff --git a/lib/ssl/test/ssl_crl_SUITE.erl b/lib/ssl/test/ssl_crl_SUITE.erl
index 5b86027210..27005682e9 100644
--- a/lib/ssl/test/ssl_crl_SUITE.erl
+++ b/lib/ssl/test/ssl_crl_SUITE.erl
@@ -41,20 +41,26 @@ groups() ->
     [
      {check_true, [],  [{group, v2_crl},
 			{group, v1_crl},
-			{group, idp_crl}]},
+			{group, idp_crl},
+                        {group, crl_hash_dir}]},
      {check_peer, [],   [{group, v2_crl},
 			 {group, v1_crl},
-			 {group, idp_crl}]},
+			 {group, idp_crl},
+                         {group, crl_hash_dir}]},
      {check_best_effort, [], [{group, v2_crl},
 			       {group, v1_crl},
-			      {group, idp_crl}]},
+			      {group, idp_crl},
+			      {group, crl_hash_dir}]},
      {v2_crl,  [], basic_tests()},
      {v1_crl,  [], basic_tests()},
-     {idp_crl, [], basic_tests()}].
+     {idp_crl, [], basic_tests()},
+     {crl_hash_dir, [], basic_tests() ++ crl_hash_dir_tests()}].
 
 basic_tests() ->
     [crl_verify_valid, crl_verify_revoked, crl_verify_no_crl].
 
+crl_hash_dir_tests() ->
+    [crl_hash_dir_collision, crl_hash_dir_expired].
 
 init_per_suite(Config) ->
     case os:find_executable("openssl") of
@@ -101,7 +107,24 @@ init_per_group(Group, Config0) ->
 	    CertDir = filename:join(?config(priv_dir, Config0), Group),
 	    {CertOpts, Config} = init_certs(CertDir, Group, Config0),
 	    {ok, _} =  make_certs:all(DataDir, CertDir, CertOpts),
-	    [{cert_dir, CertDir}, {idp_crl, false} | Config]
+	    case Group of
+		crl_hash_dir ->
+		    CrlDir = filename:join(CertDir, "crls"),
+		    %% Copy CRLs to their hashed filenames.
+		    %% Find the hashes with 'openssl crl -noout -hash -in crl.pem'.
+		    populate_crl_hash_dir(CertDir, CrlDir,
+					  [{"erlangCA", "d6134ed3"},
+					   {"otpCA", "d4c8d7e5"}],
+					  replace),
+		    CrlCacheOpts = [{crl_cache,
+				     {ssl_crl_hash_dir,
+				      {internal, [{dir, CrlDir}]}}}];
+		_ ->
+		    CrlCacheOpts = []
+	    end,
+	    [{crl_cache_opts, CrlCacheOpts},
+	     {cert_dir, CertDir},
+	     {idp_crl, false} | Config]
     end.
 
 end_per_group(_GroupName, Config) ->
@@ -164,9 +187,10 @@ crl_verify_valid(Config) when is_list(Config) ->
 			   {crl_cache, {ssl_crl_cache, {internal, [{http, 5000}]}}},
 			   {verify, verify_peer}];
 		      false ->
-			  [{cacertfile, filename:join([PrivDir, "server", "cacerts.pem"])},
-			   {crl_check, Check},
-			   {verify, verify_peer}]
+			  ?config(crl_cache_opts, Config) ++
+			      [{cacertfile, filename:join([PrivDir, "server", "cacerts.pem"])},
+			       {crl_check, Check},
+			       {verify, verify_peer}]
 		  end,			  
     {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config),
 
@@ -196,9 +220,10 @@ crl_verify_revoked(Config)  when is_list(Config) ->
 			   {crl_check, Check},
 			   {verify, verify_peer}];
 		      false ->
-			  [{cacertfile, filename:join([PrivDir, "revoked", "cacerts.pem"])},
-			   {crl_check, Check},
-			   {verify, verify_peer}]
+			  ?config(crl_cache_opts, Config) ++
+			      [{cacertfile, filename:join([PrivDir, "revoked", "cacerts.pem"])},
+			       {crl_check, Check},
+			       {verify, verify_peer}]
 		  end,	
     
     crl_verify_error(Hostname, ServerNode, ServerOpts, ClientNode, ClientOpts,
@@ -251,6 +276,132 @@ crl_verify_no_crl(Config) when is_list(Config) ->
             crl_verify_valid(Hostname, ServerNode, ServerOpts, ClientNode, ClientOpts)
     end.
 
+crl_hash_dir_collision() ->
+    [{doc,"Verify ssl_crl_hash_dir behaviour with hash collisions"}].
+crl_hash_dir_collision(Config) when is_list(Config) ->
+    PrivDir = ?config(cert_dir, Config),
+    Check = ?config(crl_check, Config),
+
+    %% Create two CAs whose names hash to the same value
+    CA1 = "hash-collision-0000000000",
+    CA2 = "hash-collision-0258497583",
+    CertsConfig = make_certs:make_config([]),
+    make_certs:intermediateCA(PrivDir, CA1, "erlangCA", CertsConfig),
+    make_certs:intermediateCA(PrivDir, CA2, "erlangCA", CertsConfig),
+
+    make_certs:enduser(PrivDir, CA1, "collision-client-1", CertsConfig),
+    make_certs:enduser(PrivDir, CA2, "collision-client-2", CertsConfig),
+
+    [ServerOpts1, ServerOpts2] =
+	[
+	 [{keyfile, filename:join([PrivDir, EndUser, "key.pem"])},
+	  {certfile, filename:join([PrivDir, EndUser, "cert.pem"])},
+	  {cacertfile, filename:join([PrivDir, EndUser, "cacerts.pem"])}]
+	 || EndUser <- ["collision-client-1", "collision-client-2"]],
+
+    %% Add CRLs for our new CAs into the CRL hash directory.
+    %% Find the hashes with 'openssl crl -noout -hash -in crl.pem'.
+    CrlDir = filename:join(PrivDir, "crls"),
+    populate_crl_hash_dir(PrivDir, CrlDir,
+			  [{CA1, "b68fc624"},
+			   {CA2, "b68fc624"}],
+			 replace),
+
+    ClientOpts = ?config(crl_cache_opts, Config) ++
+	[{cacertfile, filename:join([PrivDir, "erlangCA", "cacerts.pem"])},
+	 {crl_check, Check},
+	 {verify, verify_peer}],
+
+    {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config),
+
+    %% Neither certificate revoked; both succeed.
+    crl_verify_valid(Hostname, ServerNode, ServerOpts1, ClientNode, ClientOpts),
+    crl_verify_valid(Hostname, ServerNode, ServerOpts2, ClientNode, ClientOpts),
+
+    make_certs:revoke(PrivDir, CA1, "collision-client-1", CertsConfig),
+    populate_crl_hash_dir(PrivDir, CrlDir,
+			  [{CA1, "b68fc624"},
+			   {CA2, "b68fc624"}],
+			 replace),
+
+    %% First certificate revoked; first fails, second succeeds.
+    crl_verify_error(Hostname, ServerNode, ServerOpts1, ClientNode, ClientOpts,
+                     "certificate revoked"),
+    crl_verify_valid(Hostname, ServerNode, ServerOpts2, ClientNode, ClientOpts),
+
+    make_certs:revoke(PrivDir, CA2, "collision-client-2", CertsConfig),
+    populate_crl_hash_dir(PrivDir, CrlDir,
+			  [{CA1, "b68fc624"},
+			   {CA2, "b68fc624"}],
+			 replace),
+
+    %% Second certificate revoked; both fail.
+    crl_verify_error(Hostname, ServerNode, ServerOpts1, ClientNode, ClientOpts,
+                     "certificate revoked"),
+    crl_verify_error(Hostname, ServerNode, ServerOpts2, ClientNode, ClientOpts,
+                     "certificate revoked"),
+
+    ok.
+
+crl_hash_dir_expired() ->
+    [{doc,"Verify ssl_crl_hash_dir behaviour with expired CRLs"}].
+crl_hash_dir_expired(Config) when is_list(Config) ->
+    PrivDir = ?config(cert_dir, Config),
+    Check = ?config(crl_check, Config),
+
+    CA = "CRL-maybe-expired-CA",
+    %% Add "issuing distribution point", to ensure that verification
+    %% fails if there is no valid CRL.
+    CertsConfig = make_certs:make_config([{issuing_distribution_point, true}]),
+    make_certs:intermediateCA(PrivDir, CA, "erlangCA", CertsConfig),
+    EndUser = "CRL-maybe-expired",
+    make_certs:enduser(PrivDir, CA, EndUser, CertsConfig),
+
+    ServerOpts =  [{keyfile, filename:join([PrivDir, EndUser, "key.pem"])},
+		   {certfile, filename:join([PrivDir, EndUser, "cert.pem"])},
+		   {cacertfile, filename:join([PrivDir, EndUser, "cacerts.pem"])}],
+    ClientOpts = ?config(crl_cache_opts, Config) ++
+	[{cacertfile, filename:join([PrivDir, CA, "cacerts.pem"])},
+	 {crl_check, Check},
+	 {verify, verify_peer}],
+    {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config),
+
+    %% First make a CRL that expired yesterday.
+    make_certs:gencrl(PrivDir, CA, CertsConfig, -24),
+    CrlDir = filename:join(PrivDir, "crls"),
+    populate_crl_hash_dir(PrivDir, CrlDir,
+			  [{CA, "1627b4b0"}],
+			  replace),
+
+    %% Since the CRL has expired, it's treated as missing, and the
+    %% outcome depends on the crl_check setting.
+    case Check of
+        true ->
+            %% The error "revocation status undetermined" gets turned
+            %% into "bad certificate".
+            crl_verify_error(Hostname, ServerNode, ServerOpts, ClientNode, ClientOpts,
+                             "bad certificate");
+        peer ->
+            crl_verify_error(Hostname, ServerNode, ServerOpts, ClientNode, ClientOpts,
+                             "bad certificate");
+        best_effort ->
+            %% In "best effort" mode, we consider the certificate not
+            %% to be revoked if we can't find the appropriate CRL.
+            crl_verify_valid(Hostname, ServerNode, ServerOpts, ClientNode, ClientOpts)
+    end,
+
+    %% Now make a CRL that expires tomorrow.
+    make_certs:gencrl(PrivDir, CA, CertsConfig, 24),
+    CrlDir = filename:join(PrivDir, "crls"),
+    populate_crl_hash_dir(PrivDir, CrlDir,
+			  [{CA, "1627b4b0"}],
+			  add),
+
+    %% With a valid CRL, verification should always pass.
+    crl_verify_valid(Hostname, ServerNode, ServerOpts, ClientNode, ClientOpts),
+
+    ok.
+
 crl_verify_valid(Hostname, ServerNode, ServerOpts, ClientNode, ClientOpts) ->
     Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, 
 					{from, self()}, 
@@ -311,3 +462,31 @@ make_dir_path(PathComponents) ->
 
 rename_crl(Filename) ->
     file:rename(Filename, Filename ++ ".notfound").
+
+populate_crl_hash_dir(CertDir, CrlDir, CAsHashes, AddOrReplace) ->
+    ok = filelib:ensure_dir(filename:join(CrlDir, "crls")),
+    case AddOrReplace of
+	replace ->
+	    %% Delete existing files, so we can override them.
+	    [ok = file:delete(FileToDelete) ||
+		{_CA, Hash} <- CAsHashes,
+		FileToDelete <- filelib:wildcard(
+				  filename:join(CrlDir, Hash ++ ".r*"))];
+	add ->
+	    ok
+    end,
+    %% Create new files, incrementing suffix if needed to find unique names.
+    [{ok, _} =
+	 file:copy(filename:join([CertDir, CA, "crl.pem"]),
+		   find_free_name(CrlDir, Hash, 0))
+     || {CA, Hash} <- CAsHashes],
+    ok.
+
+find_free_name(CrlDir, Hash, N) ->
+    Name = filename:join(CrlDir, Hash ++ ".r" ++ integer_to_list(N)),
+    case filelib:is_file(Name) of
+	true ->
+	    find_free_name(CrlDir, Hash, N + 1);
+	false ->
+	    Name
+    end.