Merge branch 'hans/crypto/rsa_opts/ERL-675/PR1899/OTP-15212' into maint

* hans/crypto/rsa_opts/ERL-675/PR1899/OTP-15212:
  crypto: Added tests for rsa opts from PR838
  Fixes ERL-675

2 files changed, 90 insertions, 16 deletions

diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c
index f8d1778bb9..a9fef65e03 100644
--- a/lib/crypto/c_src/crypto.c
+++ b/lib/crypto/c_src/crypto.c
@@ -179,6 +179,7 @@
 #if OPENSSL_VERSION_NUMBER >= PACKED_OPENSSL_VERSION_PLAIN(1,1,0)
 # ifndef HAS_LIBRESSL
 #  define HAVE_CHACHA20_POLY1305
+#  define HAVE_RSA_OAEP_MD
 # endif
 #endif
 
diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl
index 943062f56f..aa2278dbdd 100644
--- a/lib/crypto/test/crypto_SUITE.erl
+++ b/lib/crypto/test/crypto_SUITE.erl
@@ -840,26 +840,69 @@ negative_verify(Type, Hash, Msg, Signature, Public, Options) ->
 	    ok
     end.
 
+-define(PUB_PRIV_ENC_DEC_CATCH(Type,Padding),
+        CC:EE ->
+               ct:log("~p:~p in ~p:~p/~p, line ~p.~n"
+                      "Type = ~p~nPadding = ~p",
+                     [CC,EE,?MODULE,?FUNCTION_NAME,?FUNCTION_ARITY,?LINE,(Type),(Padding)]),
+               MaybeUnsupported =
+                   case crypto:info_lib() of
+                       [{<<"OpenSSL">>,_,_}] ->
+                           is_list(Padding) andalso
+                               lists:any(fun(P) -> lists:member(P,(Padding)) end,
+                                         [{rsa_padding, rsa_pkcs1_oaep_padding},
+                                          {rsa_padding, rsa_sslv23_padding},
+                                          {rsa_padding, rsa_x931_padding}]);
+                       _ ->
+                           false
+                   end,
+               case CC of
+                   error when MaybeUnsupported ->
+                       ct:comment("Padding unsupported?",[]);
+                   _ ->
+                       ct:fail({?FUNCTION_NAME,CC,EE,(Type),(Padding)})
+               end
+       ).
+
 do_public_encrypt({Type, Public, Private, Msg, Padding}) ->
-    PublicEcn = (catch crypto:public_encrypt(Type, Msg, Public, Padding)),
-    case crypto:private_decrypt(Type, PublicEcn, Private, Padding) of
-	Msg ->
-	    ok;
-	Other ->
-	    ct:fail({{crypto, private_decrypt, [Type, PublicEcn, Private, Padding]}, {expected, Msg}, {got, Other}})
+    try
+        crypto:public_encrypt(Type, Msg, Public, Padding)
+    of
+        PublicEcn ->
+            try
+                crypto:private_decrypt(Type, PublicEcn, Private, Padding)
+            of
+                Msg ->
+                    ok;
+                Other ->
+                    ct:fail({{crypto, private_decrypt, [Type, PublicEcn, Private, Padding]}, {expected, Msg}, {got, Other}})
+            catch
+                ?PUB_PRIV_ENC_DEC_CATCH(Type, Padding)
+            end
+    catch
+        ?PUB_PRIV_ENC_DEC_CATCH(Type, Padding)
     end. 
 
-do_private_encrypt({_Type, _Public, _Private, _Msg, rsa_pkcs1_oaep_padding}) ->
-    ok; %% Not supported by openssl
+
 do_private_encrypt({Type, Public, Private, Msg, Padding}) ->
-    PrivEcn = (catch crypto:private_encrypt(Type, Msg, Private, Padding)),
-    case crypto:public_decrypt(Type, PrivEcn, Public, Padding) of
-	Msg ->
-	    ok;
-	Other ->
-	    ct:fail({{crypto, public_decrypt, [Type, PrivEcn, Public, Padding]}, {expected, Msg}, {got, Other}})
+    try
+        crypto:private_encrypt(Type, Msg, Private, Padding)
+    of
+        PrivEcn ->
+            try
+                crypto:public_decrypt(Type, PrivEcn, Public, Padding)
+            of
+                Msg ->
+                    ok;
+                Other ->
+                    ct:fail({{crypto, public_decrypt, [Type, PrivEcn, Public, Padding]}, {expected, Msg}, {got, Other}})
+            catch
+                ?PUB_PRIV_ENC_DEC_CATCH(Type, Padding)
+            end
+    catch
+        ?PUB_PRIV_ENC_DEC_CATCH(Type, Padding)
     end.
-     
+
 do_generate_compute({srp = Type, UserPrivate, UserGenParams, UserComParams,
 		     HostPublic, HostPrivate, HostGenParams, HostComParam, SessionKey}) ->
     {UserPublic, UserPrivate} = crypto:generate_key(Type, UserGenParams, UserPrivate),
@@ -1245,7 +1288,12 @@ group_config(rsa = Type, Config) ->
         end,
     MsgPubEnc = <<"7896345786348 Asldi">>,
     PubPrivEnc = [{rsa, PublicS, PrivateS, MsgPubEnc, rsa_pkcs1_padding},
+                  {rsa, PublicS, PrivateS, MsgPubEnc, [{rsa_padding, rsa_pkcs1_padding}]},
+                  {rsa, PublicS, PrivateS, MsgPubEnc, [{rsa_padding, rsa_sslv23_padding}]},
+                  {rsa, PublicS, PrivateS, MsgPubEnc, [{rsa_padding, rsa_x931_padding}]},
                   rsa_oaep(),
+		  rsa_oaep_label(),
+		  rsa_oaep256(),
                   no_padding()
                  ],
     Generate = [{rsa, 1024, 3},  {rsa, 2048, 17},  {rsa, 3072, 65537}],
@@ -2338,7 +2386,32 @@ rsa_oaep() ->
 			 hexstr2bin("4f456c502493bdc0ed2ab756a3a6ed4d67352a697d4216e93212b127a63d5411ce6fa98d5dbefd73263e3728142743818166ed7dd63687dd2a8ca1d2f4fbd8e1")],
     %%Msg = hexstr2bin("6628194e12073db03ba94cda9ef9532397d50dba79b987004afefe34"),
     Msg =  hexstr2bin("750c4047f547e8e41411856523298ac9bae245efaf1397fbe56f9dd5"),
-    {rsa, Public, Private, Msg, rsa_pkcs1_oaep_padding}.
+    {rsa, Public, Private, Msg, [{rsa_padding, rsa_pkcs1_oaep_padding}]}.
+
+rsa_oaep_label() ->
+    Public = [hexstr2bin("010001"),
+	      hexstr2bin("a8b3b284af8eb50b387034a860f146c4919f318763cd6c5598c8ae4811a1e0abc4c7e0b082d693a5e7fced675cf4668512772c0cbc64a742c6c630f533c8cc72f62ae833c40bf25842e984bb78bdbf97c0107d55bdb662f5c4e0fab9845cb5148ef7392dd3aaff93ae1e6b667bb3d4247616d4f5ba10d4cfd226de88d39f16fb")],
+    Private = Public ++ [hexstr2bin("53339cfdb79fc8466a655c7316aca85c55fd8f6dd898fdaf119517ef4f52e8fd8e258df93fee180fa0e4ab29693cd83b152a553d4ac4d1812b8b9fa5af0e7f55fe7304df41570926f3311f15c4d65a732c483116ee3d3d2d0af3549ad9bf7cbfb78ad884f84d5beb04724dc7369b31def37d0cf539e9cfcdd3de653729ead5d1"),
+			 hexstr2bin("d32737e7267ffe1341b2d5c0d150a81b586fb3132bed2f8d5262864a9cb9f30af38be448598d413a172efb802c21acf1c11c520c2f26a471dcad212eac7ca39d"),
+			 hexstr2bin("cc8853d1d54da630fac004f471f281c7b8982d8224a490edbeb33d3e3d5cc93c4765703d1dd791642f1f116a0dd852be2419b2af72bfe9a030e860b0288b5d77"),
+			 hexstr2bin("0e12bf1718e9cef5599ba1c3882fe8046a90874eefce8f2ccc20e4f2741fb0a33a3848aec9c9305fbecbd2d76819967d4671acc6431e4037968db37878e695c1"),
+			 hexstr2bin("95297b0f95a2fa67d00707d609dfd4fc05c89dafc2ef6d6ea55bec771ea333734d9251e79082ecda866efef13c459e1a631386b7e354c899f5f112ca85d71583"),
+			 hexstr2bin("4f456c502493bdc0ed2ab756a3a6ed4d67352a697d4216e93212b127a63d5411ce6fa98d5dbefd73263e3728142743818166ed7dd63687dd2a8ca1d2f4fbd8e1")],
+    Msg = hexstr2bin("750c4047f547e8e41411856523298ac9bae245efaf1397fbe56f9dd5"),
+    Lbl = hexstr2bin("1332a67ca7088f75c9b8fb5e3d072882"),
+    {rsa, Public, Private, Msg, [{rsa_padding, rsa_pkcs1_oaep_padding}, {rsa_oaep_label, Lbl}]}.
+
+rsa_oaep256() ->
+    Public = [hexstr2bin("010001"),
+	      hexstr2bin("a8b3b284af8eb50b387034a860f146c4919f318763cd6c5598c8ae4811a1e0abc4c7e0b082d693a5e7fced675cf4668512772c0cbc64a742c6c630f533c8cc72f62ae833c40bf25842e984bb78bdbf97c0107d55bdb662f5c4e0fab9845cb5148ef7392dd3aaff93ae1e6b667bb3d4247616d4f5ba10d4cfd226de88d39f16fb")],
+    Private = Public ++ [hexstr2bin("53339cfdb79fc8466a655c7316aca85c55fd8f6dd898fdaf119517ef4f52e8fd8e258df93fee180fa0e4ab29693cd83b152a553d4ac4d1812b8b9fa5af0e7f55fe7304df41570926f3311f15c4d65a732c483116ee3d3d2d0af3549ad9bf7cbfb78ad884f84d5beb04724dc7369b31def37d0cf539e9cfcdd3de653729ead5d1"),
+			 hexstr2bin("d32737e7267ffe1341b2d5c0d150a81b586fb3132bed2f8d5262864a9cb9f30af38be448598d413a172efb802c21acf1c11c520c2f26a471dcad212eac7ca39d"),
+			 hexstr2bin("cc8853d1d54da630fac004f471f281c7b8982d8224a490edbeb33d3e3d5cc93c4765703d1dd791642f1f116a0dd852be2419b2af72bfe9a030e860b0288b5d77"),
+			 hexstr2bin("0e12bf1718e9cef5599ba1c3882fe8046a90874eefce8f2ccc20e4f2741fb0a33a3848aec9c9305fbecbd2d76819967d4671acc6431e4037968db37878e695c1"),
+			 hexstr2bin("95297b0f95a2fa67d00707d609dfd4fc05c89dafc2ef6d6ea55bec771ea333734d9251e79082ecda866efef13c459e1a631386b7e354c899f5f112ca85d71583"),
+			 hexstr2bin("4f456c502493bdc0ed2ab756a3a6ed4d67352a697d4216e93212b127a63d5411ce6fa98d5dbefd73263e3728142743818166ed7dd63687dd2a8ca1d2f4fbd8e1")],
+    Msg = hexstr2bin("750c4047f547e8e41411856523298ac9bae245efaf1397fbe56f9dd5"),
+    {rsa, Public, Private, Msg, [{rsa_padding, rsa_pkcs1_oaep_padding}, {rsa_oaep_md, sha256}]}.
 
 ecc() ->
 %%               http://point-at-infinity.org/ecc/nisttv