Merge branch 'cr/md2-With-RSA-Encryption' into dev

* cr/md2-With-RSA-Encryption:
  Document crypto:sha_mac_96/2 to compute an SHA MAC, not MD5
  Support md2WithRSAEncryption certificates in public_key
  Support 'md2' hash in crypto:rsa_sign/3 and crypto:rsa_verify/4

OTP-9554

4 files changed, 56 insertions, 40 deletions

diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c
index c781ccb302..83772d9023 100644
--- a/lib/crypto/c_src/crypto.c
+++ b/lib/crypto/c_src/crypto.c
@@ -43,6 +43,7 @@
 #include <openssl/aes.h>
 #include <openssl/md5.h>
 #include <openssl/md4.h>
+#include <openssl/md2.h>
 #include <openssl/sha.h>
 #include <openssl/bn.h>
 #include <openssl/objects.h>
@@ -267,6 +268,7 @@ static ERL_NIF_TERM atom_true;
 static ERL_NIF_TERM atom_false;
 static ERL_NIF_TERM atom_sha;
 static ERL_NIF_TERM atom_md5;
+static ERL_NIF_TERM atom_md2;
 static ERL_NIF_TERM atom_ripemd160;
 static ERL_NIF_TERM atom_error;
 static ERL_NIF_TERM atom_rsa_pkcs1_padding;
@@ -337,6 +339,7 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
     atom_false = enif_make_atom(env,"false");
     atom_sha = enif_make_atom(env,"sha");
     atom_md5 = enif_make_atom(env,"md5");
+    atom_md2 = enif_make_atom(env,"md2");
     atom_ripemd160 = enif_make_atom(env,"ripemd160");
     atom_error = enif_make_atom(env,"error");
     atom_rsa_pkcs1_padding = enif_make_atom(env,"rsa_pkcs1_padding");
@@ -1047,16 +1050,28 @@ static ERL_NIF_TERM dss_verify(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
     return(i > 0) ? atom_true : atom_false;
 }
 
+struct hash_def {
+    int type;
+    unsigned int m_len;
+    unsigned char * (*func) (const unsigned char *d, size_t n, unsigned char *md);
+};
+
+static const struct hash_def md2_hash_def  = { NID_md2,  MD2_DIGEST_LENGTH, &MD2};
+static const struct hash_def md5_hash_def  = { NID_md5,  MD5_DIGEST_LENGTH, &MD5};
+static const struct hash_def sha1_hash_def = { NID_sha1, SHA_DIGEST_LENGTH, &SHA1};
+
 static ERL_NIF_TERM rsa_verify(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {/* (Type, Data, Signature, Key=[E,N]) */
     ErlNifBinary data_bin, sign_bin;
     unsigned char hmacbuf[SHA_DIGEST_LENGTH];
     ERL_NIF_TERM head, tail, ret;
-    int i, is_sha;
+    int i;
     RSA* rsa = RSA_new();
+    const struct hash_def *hash_def = NULL;
 
-    if (argv[0] == atom_sha) is_sha = 1;
-    else if (argv[0] == atom_md5) is_sha = 0;
+    if      (argv[0] == atom_sha)  hash_def = &sha1_hash_def;
+    else if (argv[0] == atom_md5)  hash_def = &md5_hash_def;
+    else if (argv[0] == atom_md2)  hash_def = &md2_hash_def;
     else goto badarg;
 
     if (!inspect_mpint(env, argv[1], &data_bin)
@@ -1070,16 +1085,9 @@ static ERL_NIF_TERM rsa_verify(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv
 	ret = enif_make_badarg(env);
     }
     else {
-	if (is_sha) {
-	    SHA1(data_bin.data+4, data_bin.size-4, hmacbuf);
-	    i = RSA_verify(NID_sha1, hmacbuf, SHA_DIGEST_LENGTH,
-			   sign_bin.data+4, sign_bin.size-4, rsa);
-	}
-	else {
-	    MD5(data_bin.data+4, data_bin.size-4, hmacbuf);
-	    i = RSA_verify(NID_md5, hmacbuf, MD5_DIGEST_LENGTH,
-			   sign_bin.data+4, sign_bin.size-4, rsa);
-	}
+	(void) *hash_def->func(data_bin.data+4, data_bin.size-4, hmacbuf);
+	i = RSA_verify(hash_def->type, hmacbuf, hash_def->m_len,
+		       sign_bin.data+4, sign_bin.size-4, rsa);
 	ret = (i==1 ? atom_true : atom_false);
     }
     RSA_free(rsa);
@@ -1221,10 +1229,12 @@ static ERL_NIF_TERM rsa_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
     unsigned char hmacbuf[SHA_DIGEST_LENGTH];
     unsigned rsa_s_len;
     RSA *rsa = RSA_new();
-    int i, is_sha;
+    int i;
+    const struct hash_def *hash_def = NULL;
 
-    if (argv[0] == atom_sha) is_sha = 1;
-    else if (argv[0] == atom_md5) is_sha = 0;
+    if      (argv[0] == atom_sha)  hash_def = &sha1_hash_def;
+    else if (argv[0] == atom_md5)  hash_def = &md5_hash_def;
+    else if (argv[0] == atom_md2)  hash_def = &md2_hash_def;
     else goto badarg;
 
     if (!inspect_mpint(env,argv[1],&data_bin)
@@ -1240,18 +1250,10 @@ static ERL_NIF_TERM rsa_sign_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar
 	return enif_make_badarg(env);
     }
     enif_alloc_binary(RSA_size(rsa), &ret_bin);
-    if (is_sha) {
-	SHA1(data_bin.data+4, data_bin.size-4, hmacbuf);
-	ERL_VALGRIND_ASSERT_MEM_DEFINED(hmacbuf, SHA_DIGEST_LENGTH);
-	i =  RSA_sign(NID_sha1, hmacbuf, SHA_DIGEST_LENGTH,
-		      ret_bin.data, &rsa_s_len, rsa);
-    }
-    else {
-	MD5(data_bin.data+4, data_bin.size-4, hmacbuf);
-	ERL_VALGRIND_ASSERT_MEM_DEFINED(hmacbuf, MD5_DIGEST_LENGTH);
-	i = RSA_sign(NID_md5, hmacbuf,MD5_DIGEST_LENGTH,
-		     ret_bin.data, &rsa_s_len, rsa);     
-    }
+    (void) *hash_def->func(data_bin.data+4, data_bin.size-4, hmacbuf);
+    ERL_VALGRIND_ASSERT_MEM_DEFINED(hmacbuf, hash_def->m_len);
+    i =  RSA_sign(hash_def->type, hmacbuf, hash_def->m_len,
+		  ret_bin.data, &rsa_s_len, rsa);
     RSA_free(rsa);
     if (i) {
 	ERL_VALGRIND_MAKE_MEM_DEFINED(ret_bin.data, rsa_s_len);
diff --git a/lib/crypto/doc/src/crypto.xml b/lib/crypto/doc/src/crypto.xml
index 96c4a072e1..b593958264 100644
--- a/lib/crypto/doc/src/crypto.xml
+++ b/lib/crypto/doc/src/crypto.xml
@@ -347,7 +347,7 @@ Mpint() = <![CDATA[<<ByteLen:32/integer-big, Bytes:ByteLen/binary>>]]>
     </func>
     <func>
       <name>sha_mac_96(Key, Data) -> Mac</name>
-      <fsummary>Compute an <c>MD5 MAC</c>message authentification code</fsummary>
+      <fsummary>Compute an <c>SHA MAC</c>message authentification code</fsummary>
       <type>
         <v>Key = Data = iolist() | binary()</v>
         <v>Mac = binary()</v>
@@ -795,7 +795,7 @@ Mpint() = <![CDATA[<<ByteLen:32/integer-big, Bytes:ByteLen/binary>>]]>
         <v>E, N, D = Mpint</v>
         <d>Where <c>E</c> is the public exponent, <c>N</c> is public modulus and 
 	  <c>D</c> is the private exponent.</d>
-	<v>DigestType = md5 | sha</v>
+	<v>DigestType = md2 | md5 | sha</v>
 	<d>The default <c>DigestType</c> is sha.</d>
         <v>Mpint = binary()</v>
         <v>Signature = binary()</v>
@@ -817,7 +817,7 @@ Mpint() = <![CDATA[<<ByteLen:32/integer-big, Bytes:ByteLen/binary>>]]>
         <v>Key = [E, N]</v>
         <v>E, N = Mpint</v>
         <d>Where <c>E</c> is the public exponent and <c>N</c> is public modulus.</d>
-	<v>DigestType = md5 | sha</v>
+	<v>DigestType = md2 | md5 | sha</v>
 	<d> The default <c>DigestType</c> is sha.</d>
         <v>Mpint = binary()</v>
       </type>
diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl
index c3e13d6b91..ddad00f4b4 100644
--- a/lib/crypto/src/crypto.erl
+++ b/lib/crypto/src/crypto.erl
@@ -91,7 +91,7 @@
                     aes_ctr_stream_init, aes_ctr_stream_encrypt, aes_ctr_stream_decrypt,
 		    info_lib]).
 
--type rsa_digest_type() :: 'md5' | 'sha'.
+-type rsa_digest_type() :: 'md2' | 'md5' | 'sha'.
 -type dss_digest_type() :: 'none' | 'sha'.
 -type crypto_integer() :: binary() | integer().
 
diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl
index 8d2f42469b..2fa058c852 100644
--- a/lib/crypto/test/crypto_SUITE.erl
+++ b/lib/crypto/test/crypto_SUITE.erl
@@ -1082,16 +1082,30 @@ rsa_sign_test(Config) when is_list(Config) ->
     
     PrivKey = [crypto:mpint(PubEx), crypto:mpint(Mod), crypto:mpint(PrivEx)],
     PubKey  = [crypto:mpint(PubEx), crypto:mpint(Mod)],
-    ?line Sig1 = crypto:rsa_sign(sized_binary(Msg), PrivKey),
-    ?line m(crypto:rsa_verify(sized_binary(Msg), sized_binary(Sig1),PubKey), true),
+    ?line Sig = crypto:rsa_sign(sized_binary(Msg), PrivKey),
+    ?line m(crypto:rsa_verify(sized_binary(Msg), sized_binary(Sig),PubKey), true),
     
-    ?line Sig2 = crypto:rsa_sign(md5, sized_binary(Msg), PrivKey),
-    ?line m(crypto:rsa_verify(md5, sized_binary(Msg), sized_binary(Sig2),PubKey), true),
-    
-    ?line m(Sig1 =:= Sig2, false),
-    ?line m(crypto:rsa_verify(md5, sized_binary(Msg), sized_binary(Sig1),PubKey), false),
-    ?line m(crypto:rsa_verify(sha, sized_binary(Msg), sized_binary(Sig1),PubKey), true),
+    ?line Sig_md2 = crypto:rsa_sign(md2, sized_binary(Msg), PrivKey),
+    ?line Sig_md5 = crypto:rsa_sign(md5, sized_binary(Msg), PrivKey),
+    ?line Sig_sha = crypto:rsa_sign(sha, sized_binary(Msg), PrivKey),
+
+    ?line m(Sig =:= Sig_sha, true),
+    ?line m(Sig_md2 =:= Sig_md5, false),
+    ?line m(Sig_md2 =:= Sig_sha, false),
+    ?line m(Sig_md5 =:= Sig_sha, false),
   
+    ?line m(crypto:rsa_verify(md2, sized_binary(Msg), sized_binary(Sig_md2),PubKey), true),
+    ?line m(crypto:rsa_verify(md2, sized_binary(Msg), sized_binary(Sig_md5),PubKey), false),
+    ?line m(crypto:rsa_verify(md2, sized_binary(Msg), sized_binary(Sig_sha),PubKey), false),
+
+    ?line m(crypto:rsa_verify(md5, sized_binary(Msg), sized_binary(Sig_md2),PubKey), false),
+    ?line m(crypto:rsa_verify(md5, sized_binary(Msg), sized_binary(Sig_md5),PubKey), true),
+    ?line m(crypto:rsa_verify(md5, sized_binary(Msg), sized_binary(Sig_sha),PubKey), false),
+
+    ?line m(crypto:rsa_verify(sha, sized_binary(Msg), sized_binary(Sig_md2),PubKey), false),
+    ?line m(crypto:rsa_verify(sha, sized_binary(Msg), sized_binary(Sig_md5),PubKey), false),
+    ?line m(crypto:rsa_verify(sha, sized_binary(Msg), sized_binary(Sig_sha),PubKey), true),
+
     ok.
     
 dsa_sign_test(doc) ->