crypto: Update RSA options to match specs and different OpenSSL versions

 - Put rsa_pkcs1_oaep_padding in supports/0
 - #ifdef updates
 - Refine PKCS1_OAEP defines

2 files changed, 88 insertions, 33 deletions

diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c
index d40d285f86..b2d8123f00 100644
--- a/lib/crypto/c_src/crypto.c
+++ b/lib/crypto/c_src/crypto.c
@@ -189,6 +189,10 @@
 # define HAVE_GCM
 # define HAVE_CCM
 # define HAVE_CMAC
+# if defined(RSA_PKCS1_OAEP_PADDING)
+#   define HAVE_RSA_OAEP_PADDING
+# endif
+# define HAVE_RSA_MGF1_MD
 # if OPENSSL_VERSION_NUMBER < PACKED_OPENSSL_VERSION(1,0,1,'d')
 #  define HAVE_GCM_EVP_DECRYPT_BUG
 # endif
@@ -1340,7 +1344,7 @@ static ERL_NIF_TERM algo_mac[3]; /* increase when extending the list */
 static int algo_curve_cnt, algo_curve_fips_cnt;
 static ERL_NIF_TERM algo_curve[87]; /* increase when extending the list */
 static int algo_rsa_opts_cnt, algo_rsa_opts_fips_cnt;
-static ERL_NIF_TERM algo_rsa_opts[10]; /* increase when extending the list */
+static ERL_NIF_TERM algo_rsa_opts[11]; /* increase when extending the list */
 
 static void init_algorithms_types(ErlNifEnv* env)
 {
@@ -1562,7 +1566,12 @@ static void init_algorithms_types(ErlNifEnv* env)
     algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_pkcs1_pss_padding");
     algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_pss_saltlen");
 # endif
+# ifdef HAVE_RSA_MGF1_MD
     algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_mgf1_md");
+# endif
+# ifdef HAVE_RSA_OAEP_PADDING
+    algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_pkcs1_oaep_padding");
+# endif
 # ifdef HAVE_RSA_OAEP_MD
     algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_oaep_label");
     algo_rsa_opts[algo_rsa_opts_cnt++] = enif_make_atom(env,"rsa_oaep_md");
@@ -4693,16 +4702,16 @@ printf("\r\n");
 
     if (argv[0] == atom_rsa) {
 	if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) <= 0) goto badarg;
-#ifdef HAVE_RSA_PKCS1_PSS_PADDING
+# ifdef HAVE_RSA_PKCS1_PSS_PADDING
 	if (sig_opt.rsa_padding == RSA_PKCS1_PSS_PADDING) {
             if (sig_opt.rsa_mgf1_md != NULL) {
-#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,1)
+# ifdef HAVE_RSA_MGF1_MD
 		if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, sig_opt.rsa_mgf1_md) <= 0) goto badarg;
-#else
+# else
                 EVP_PKEY_CTX_free(ctx);
                 EVP_PKEY_free(pkey);
                 return atom_notsup;
-#endif
+# endif
             }
 	    if (sig_opt.rsa_pss_saltlen > -2
 		&& EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, sig_opt.rsa_pss_saltlen) <= 0)
@@ -4833,13 +4842,13 @@ static ERL_NIF_TERM pkey_verify_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM
 	if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) <= 0) goto badarg;
 	if (sig_opt.rsa_padding == RSA_PKCS1_PSS_PADDING) {
             if (sig_opt.rsa_mgf1_md != NULL) {
-#if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,0,1)
+# ifdef HAVE_RSA_MGF1_MD
 		if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, sig_opt.rsa_mgf1_md) <= 0) goto badarg;
-#else
+# else
                 EVP_PKEY_CTX_free(ctx);
                 EVP_PKEY_free(pkey);
                 return atom_notsup;
-#endif
+# endif
             }
 	    if (sig_opt.rsa_pss_saltlen > -2
 		&& EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, sig_opt.rsa_pss_saltlen) <= 0)
@@ -4932,8 +4941,10 @@ static int get_pkey_crypt_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NI
                     ) {
 		    if (tpl_terms[1] == atom_rsa_pkcs1_padding) {
 			opt->rsa_padding = RSA_PKCS1_PADDING;
+#ifdef HAVE_RSA_OAEP_PADDING
 		    } else if (tpl_terms[1] == atom_rsa_pkcs1_oaep_padding) {
 			opt->rsa_padding = RSA_PKCS1_OAEP_PADDING;
+#endif
 #ifdef HAVE_RSA_SSLV23_PADDING
 		    } else if (tpl_terms[1] == atom_rsa_sslv23_padding) {
 			opt->rsa_padding = RSA_SSLV23_PADDING;
@@ -4952,7 +4963,7 @@ static int get_pkey_crypt_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NI
 		    }
 		    opt->signature_md = opt_md;
 		} else if (tpl_terms[0] == atom_rsa_mgf1_md && enif_is_atom(env, tpl_terms[1])) {
-#ifndef HAVE_RSA_OAEP_MD
+#ifndef HAVE_RSA_MGF1_MD
 		    if (tpl_terms[1] != atom_sha)
 			return PKEY_NOTSUP;
 #endif
diff --git a/lib/crypto/doc/src/algorithm_details.xml b/lib/crypto/doc/src/algorithm_details.xml
index 68ad264df7..4d58d26970 100644
--- a/lib/crypto/doc/src/algorithm_details.xml
+++ b/lib/crypto/doc/src/algorithm_details.xml
@@ -63,9 +63,9 @@
 
 	<row><cell><c>aes_ige256</c></cell><cell>16</cell><cell>32</cell><cell>16</cell></row>
 	<row><cell><c>blowfish_cbc</c></cell> <cell>4-56</cell> <cell>8</cell> <cell>8</cell></row>
-	<row><cell><c>blowfish_cfb64</c></cell> <cell>1-</cell> <cell>8</cell> <cell>any</cell></row>
-	<row><cell><c>blowfish_ecb</c></cell><cell>1-</cell><cell> </cell><cell>8</cell></row>
-	<row><cell><c>blowfish_ofb64</c></cell><cell>1-</cell><cell>8</cell><cell>any</cell></row>
+	<row><cell><c>blowfish_cfb64</c></cell> <cell>&#8805;1</cell> <cell>8</cell> <cell>any</cell></row>
+	<row><cell><c>blowfish_ecb</c></cell><cell>&#8805;1</cell><cell> </cell><cell>8</cell></row>
+	<row><cell><c>blowfish_ofb64</c></cell><cell>&#8805;1</cell><cell>8</cell><cell>any</cell></row>
 
 	<row><cell><c>des3_cbc</c><br/><i>(=DES EDE3 CBC)</i></cell><cell>[8,8,8]</cell><cell>8</cell><cell>8</cell></row>
 	<row><cell><c>des3_cfb</c><br/><i>(=DES EDE3 CFB)</i></cell><cell>[8,8,8]</cell><cell>8</cell><cell>any</cell></row>
@@ -74,7 +74,7 @@
 	<row><cell><c>des_cfb</c></cell><cell>8</cell><cell>8</cell><cell>any</cell></row>
 	<row><cell><c>des_ecb</c></cell><cell>8</cell><cell> </cell><cell>8</cell></row>
 	<row><cell><c>des_ede3</c><br/><i>(=DES EDE3 CBC)</i></cell><cell>[8,8,8]</cell><cell>8</cell><cell>8</cell></row>
-	<row><cell><c>rc2_cbc</c></cell><cell>1-</cell><cell>8</cell><cell>8</cell></row>
+	<row><cell><c>rc2_cbc</c></cell><cell>&#8805;1</cell><cell>8</cell><cell>8</cell></row>
 	<tcaption>Block cipher key lengths</tcaption>
       </table>
     </section>
@@ -90,9 +90,9 @@
       </p>
       <table>
 	<row><cell><strong>Cipher and Mode</strong></cell><cell><strong>Key length</strong><br/><strong>[bytes]</strong></cell><cell><strong>IV length</strong><br/><strong>[bytes]</strong></cell><cell><strong>AAD length</strong><br/><strong>[bytes]</strong></cell><cell><strong>Tag length</strong><br/><strong>[bytes]</strong></cell><cell><strong>Block size</strong><br/><strong>[bytes]</strong></cell><cell><strong>Supported with</strong><br/><strong>OpenSSL versions</strong></cell></row>
-	<row><cell><c>aes_ccm</c></cell> <cell>16,24,32</cell> <cell>7-13</cell> <cell>any</cell> <cell>even 4-16<br/>default: 12</cell> <cell>any</cell><cell>1.1.0 -</cell></row>
-	<row><cell><c>aes_gcm</c></cell> <cell>16,24,32</cell> <cell>1-</cell> <cell>any</cell> <cell>1-16<br/>default: 16</cell> <cell>any</cell><cell>1.1.0 -</cell></row>
-	<row><cell><c>chacha20_poly1305</c></cell><cell>32</cell> <cell>1-16</cell> <cell>any</cell> <cell>16</cell> <cell>any</cell><cell>1.1.0 -</cell></row>
+	<row><cell><c>aes_ccm</c></cell> <cell>16,24,32</cell> <cell>7-13</cell> <cell>any</cell> <cell>even 4-16<br/>default: 12</cell> <cell>any</cell><cell>&#8805;1.1.0</cell></row>
+	<row><cell><c>aes_gcm</c></cell> <cell>16,24,32</cell> <cell>&#8805;1</cell> <cell>any</cell> <cell>1-16<br/>default: 16</cell> <cell>any</cell><cell>&#8805;1.1.0</cell></row>
+	<row><cell><c>chacha20_poly1305</c></cell><cell>32</cell> <cell>1-16</cell> <cell>any</cell> <cell>16</cell> <cell>any</cell><cell>&#8805;1.1.0</cell></row>
 	<tcaption>AEAD cipher key lengths</tcaption>
       </table>
     </section>
@@ -108,8 +108,8 @@
       </p>
       <table>
 	<row><cell><strong>Cipher and Mode</strong></cell><cell><strong>Key length</strong><br/><strong>[bytes]</strong></cell><cell><strong>IV length</strong><br/><strong>[bytes]</strong></cell><cell><strong>Supported with</strong><br/><strong>OpenSSL versions</strong></cell></row>
-	<row><cell><c>aes_ctr</c></cell><cell>16, 24, 32</cell><cell>16</cell><cell>1.0.1 -</cell></row>
-	<row><cell><c>rc4</c></cell><cell>1-</cell><cell> </cell> <cell>all</cell></row>
+	<row><cell><c>aes_ctr</c></cell><cell>16, 24, 32</cell><cell>16</cell><cell>&#8805;1.0.1</cell></row>
+	<row><cell><c>rc4</c></cell><cell>&#8805;1</cell><cell> </cell> <cell>all</cell></row>
 	<tcaption>Stream cipher key lengths</tcaption>
       </table>
     </section>
@@ -141,9 +141,9 @@
 	<row><cell><c>aes_cfb8</c></cell>  <cell>16</cell><cell>1</cell></row>
 
 	<row><cell><c>blowfish_cbc</c></cell> <cell>4-56</cell> <cell>8</cell></row>
-	<row><cell><c>blowfish_cfb64</c></cell> <cell>1-</cell> <cell>1</cell></row>
-	<row><cell><c>blowfish_ecb</c></cell><cell>1-</cell>    <cell>8</cell></row>
-	<row><cell><c>blowfish_ofb64</c></cell><cell>1-</cell>  <cell>1</cell></row>
+	<row><cell><c>blowfish_cfb64</c></cell> <cell>&#8805;1</cell> <cell>1</cell></row>
+	<row><cell><c>blowfish_ecb</c></cell><cell>&#8805;1</cell>    <cell>8</cell></row>
+	<row><cell><c>blowfish_ofb64</c></cell><cell>&#8805;1</cell>  <cell>1</cell></row>
 
 	<row><cell><c>des3_cbc</c><br/><i>(=DES EDE3 CBC)</i></cell><cell>[8,8,8]</cell><cell>8</cell></row>
 	<row><cell><c>des3_cfb</c><br/><i>(=DES EDE3 CFB)</i></cell><cell>[8,8,8]</cell><cell>1</cell></row>
@@ -152,7 +152,7 @@
 
 	<row><cell><c>des_cfb</c></cell><cell>8</cell><cell>1</cell></row>
 	<row><cell><c>des_ecb</c></cell><cell>8</cell><cell>1</cell></row>
-	<row><cell><c>rc2_cbc</c></cell><cell>1-</cell><cell>8</cell></row>
+	<row><cell><c>rc2_cbc</c></cell><cell>&#8805;1</cell><cell>8</cell></row>
 	<tcaption>CMAC cipher key lengths</tcaption>
       </table>
     </section>
@@ -195,7 +195,7 @@
       </row>
       <row><cell>SHA1</cell><cell>sha</cell><cell>all</cell></row>
       <row><cell>SHA2</cell><cell>sha224, sha256, sha384, sha512</cell><cell>all</cell></row>
-      <row><cell>SHA3</cell><cell>sha3_224, sha3_256, sha3_384, sha3_512</cell><cell>1.1.1 -</cell></row>
+      <row><cell>SHA3</cell><cell>sha3_224, sha3_256, sha3_384, sha3_512</cell><cell>&#8805;1.1.1</cell></row>
       <row><cell>MD4</cell><cell>md4</cell><cell>all</cell></row>
       <row><cell>MD5</cell><cell>md5</cell><cell>all</cell></row>
       <row><cell>RIPEMD</cell><cell>ripemd160</cell><cell>all</cell></row>
@@ -221,18 +221,62 @@
 	without prior notice.</p>
       </warning>
       <table>
-	<row><cell><strong>Option</strong></cell> <cell><strong>sign/verify</strong></cell> <cell><strong>encrypt/decrypt</strong></cell> <cell><strong>Supported with</strong><br/><strong>OpenSSL versions</strong></cell> </row>
-	<row><cell>{rsa_mgf1_md,atom()}</cell>                <cell>x</cell> <cell>x</cell>  <cell>1.0.1</cell></row>
-	<row><cell>{rsa_oaep_label, binary()}</cell>          <cell> </cell> <cell>x</cell>  <cell></cell></row>
-	<row><cell>{rsa_oaep_md, atom()}</cell>               <cell> </cell> <cell>x</cell>  <cell></cell></row>
-	<row><cell>{rsa_padding,rsa_pkcs1_pss_padding}</cell> <cell>x</cell> <cell> </cell>  <cell>1.0.0</cell></row>
-	<row><cell>{rsa_pss_saltlen, -2..}</cell>             <cell>x</cell> <cell> </cell>  <cell>1.0.0</cell></row>
-	<row><cell>{rsa_padding,rsa_no_padding}</cell>        <cell>x</cell> <cell>x</cell>  <cell></cell></row>
-	<row><cell>{rsa_padding,rsa_pkcs1_padding}</cell>     <cell>x</cell> <cell>x</cell>  <cell></cell></row>
-	<row><cell>{rsa_padding,rsa_sslv23_padding}</cell>    <cell> </cell> <cell>x</cell>  <cell></cell></row>
-	<row><cell>{rsa_padding,rsa_x931_padding}</cell>      <cell>x</cell> <cell> </cell>  <cell></cell></row>
+	<row><cell><strong>Option</strong></cell>
+	     <cell><strong>sign/verify</strong></cell>
+	     <cell><strong>public encrypt</strong><br/><strong>private decrypt</strong></cell>
+	     <cell><strong>private encrypt</strong><br/><strong>public decrypt</strong></cell>
+	</row>
+	<row><cell>{rsa_padding,rsa_x931_padding}</cell>
+             <cell>x</cell>
+             <cell></cell>
+             <cell>x</cell>
+	</row>
+	<row><cell>{rsa_padding,rsa_pkcs1_padding}</cell>
+	     <cell>x</cell>
+             <cell>x</cell>
+             <cell>x</cell>
+	</row>
+	<row><cell>{rsa_padding,rsa_pkcs1_pss_padding}<br/>
+	           {rsa_pss_saltlen, -2..}<br/>
+		   {rsa_mgf1_md, atom()}
+             </cell>
+	     <cell>x (2)<br/>
+	           x (2)<br/>
+	           x (2)</cell>
+	     <cell></cell>
+             <cell></cell>
+	</row>
+	<row><cell>{rsa_padding,rsa_pkcs1_oaep_padding}<br/>
+                   {rsa_mgf1_md, atom()}<br/>
+	           {rsa_oaep_label, binary()}}<br/>
+		   {rsa_oaep_md, atom()}
+	     </cell>
+	     <cell></cell>
+             <cell>x (2)<br/>
+	           x (2)<br/>
+	           x (3)<br/>
+	           x (3)
+	     </cell>
+	     <cell></cell>
+	</row>
+	<row><cell>{rsa_padding,rsa_no_padding}</cell>
+             <cell>x (1)</cell>
+	     <cell></cell>
+             <cell></cell>
+	</row>
+	<!-- row><cell>{rsa_padding,rsa_sslv23_padding}</cell>
+	     <cell></cell>
+             <cell></cell>
+             <cell></cell>
+	</row -->
 	<tcaption></tcaption>
       </table>
+      <p>Notes:</p>
+      <list type="ordered">
+	<item>(1)  OpenSSL &#8804; 1.0.0</item>
+	<item>(2)  OpenSSL &#8805; 1.0.1</item>
+	<item>(3)  OpenSSL &#8805; 1.1.0</item>
+      </list>
     </section>
 
     <section>