Merge branch 'hans/crypto/bad_ret_fips/OTP-15634' into maint

* hans/crypto/bad_ret_fips/OTP-15634:
  crypto: Fix bad return value for aes_cfb8 and aes_cfb128 if FIPS_SUPPORT
  crypto: Fix bad return code for eddsa if FIPS_SUPPORT

2 files changed, 35 insertions, 28 deletions

diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c
index df607732bf..194a3d30e9 100644
--- a/lib/crypto/c_src/crypto.c
+++ b/lib/crypto/c_src/crypto.c
@@ -177,7 +177,8 @@
     && !defined(HAS_LIBRESSL) \
     && defined(HAVE_EC)
 # define HAVE_ED_CURVE_DH
-# if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1))
+# if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1)) \
+    && !defined(FIPS_SUPPORT)
 #   define HAVE_EDDSA
 # endif
 #endif
@@ -1425,8 +1426,6 @@ static void init_algorithms_types(ErlNifEnv* env)
 #endif
     algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cbc");
     algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cbc128");
-    algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cfb8");
-    algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cfb128");
     algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cbc256");
     algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_ctr");
     algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_ecb");
@@ -1441,6 +1440,8 @@ static void init_algorithms_types(ErlNifEnv* env)
 #ifdef HAVE_AES_IGE
     algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"aes_ige256");
 #endif
+    algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cfb8");
+    algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cfb128");
 #ifndef OPENSSL_NO_DES
     algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"des_cbc");
     algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"des_cfb");
@@ -2325,21 +2326,24 @@ static ERL_NIF_TERM block_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM
         return enif_raise_exception(env, atom_notsup);
     }
 
-    if (argv[0] == atom_aes_cfb8
-        && (key.size == 24 || key.size == 32)) {
-        /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes?
-         * Fall back on low level API
-         */
-        return aes_cfb_8_crypt(env, argc-1, argv+1);
+    if (argv[0] == atom_aes_cfb8) {
+        CHECK_NO_FIPS_MODE();
+        if ((key.size == 24 || key.size == 32)) {
+            /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes?
+             * Fall back on low level API
+             */
+            return aes_cfb_8_crypt(env, argc-1, argv+1);
+        }
+    }
+    else if (argv[0] == atom_aes_cfb128) {
+        CHECK_NO_FIPS_MODE();
+        if ((key.size == 24 || key.size == 32)) {
+            /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes?
+             * Fall back on low level API
+             */
+            return aes_cfb_128_crypt_nif(env, argc-1, argv+1);
+        }
     }
-    else if (argv[0] == atom_aes_cfb128
-        && (key.size == 24 || key.size == 32)) {
-        /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes?
-         * Fall back on low level API
-         */
-        return aes_cfb_128_crypt_nif(env, argc-1, argv+1);
-   }
-
     ivec_size  = EVP_CIPHER_iv_length(cipher);
 
 #ifdef HAVE_ECB_IVEC_BUG
@@ -4357,8 +4361,11 @@ static int get_pkey_digest_type(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_
     *md = NULL;
 
     if (type == atom_none && algorithm == atom_rsa) return PKEY_OK;
+    if (algorithm == atom_eddsa)
 #ifdef HAVE_EDDSA
-    if (algorithm == atom_eddsa) return PKEY_OK;
+        return PKEY_OK;
+#else
+        return PKEY_NOTSUP;
 #endif
     digp = get_digest_type(type);
     if (!digp) return PKEY_BADARG;
diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl
index 72cb9aabfd..bc8b124b10 100644
--- a/lib/crypto/src/crypto.erl
+++ b/lib/crypto/src/crypto.erl
@@ -512,17 +512,17 @@ block_encrypt(Type, Key, Ivec, PlainText) when Type =:= des_cbc;
                                                Type =:= aes_cbc256;
                                                Type =:= aes_cbc;
                                                Type =:= rc2_cbc ->
-    block_crypt_nif(Type, Key, Ivec, PlainText, true);
+    notsup_to_error(block_crypt_nif(Type, Key, Ivec, PlainText, true));
 block_encrypt(Type, Key0, Ivec, PlainText) when Type =:= des3_cbc;
                                                 Type =:= des_ede3 ->
     Key = check_des3_key(Key0),
-    block_crypt_nif(des_ede3_cbc, Key, Ivec, PlainText, true);
+    notsup_to_error(block_crypt_nif(des_ede3_cbc, Key, Ivec, PlainText, true));
 block_encrypt(des3_cbf, Key0, Ivec, PlainText) -> % cfb misspelled
     Key = check_des3_key(Key0),
-    block_crypt_nif(des_ede3_cbf, Key, Ivec, PlainText, true);
+    notsup_to_error(block_crypt_nif(des_ede3_cbf, Key, Ivec, PlainText, true));
 block_encrypt(des3_cfb, Key0, Ivec, PlainText) ->
     Key = check_des3_key(Key0),
-    block_crypt_nif(des_ede3_cfb, Key, Ivec, PlainText, true);
+    notsup_to_error(block_crypt_nif(des_ede3_cfb, Key, Ivec, PlainText, true));
 block_encrypt(aes_ige256, Key, Ivec, PlainText) ->
     notsup_to_error(aes_ige_crypt_nif(Key, Ivec, PlainText, true));
 block_encrypt(Type, Key, Ivec, {AAD, PlainText}) when Type =:= aes_gcm;
@@ -549,17 +549,17 @@ block_decrypt(Type, Key, Ivec, Data) when Type =:= des_cbc;
                                           Type =:= aes_cfb128;
                                           Type =:= aes_cbc256;
                                           Type =:= rc2_cbc ->
-    block_crypt_nif(Type, Key, Ivec, Data, false);
+    notsup_to_error(block_crypt_nif(Type, Key, Ivec, Data, false));
 block_decrypt(Type, Key0, Ivec, Data) when Type =:= des3_cbc;
                                            Type =:= des_ede3 ->
     Key = check_des3_key(Key0),
-    block_crypt_nif(des_ede3_cbc, Key, Ivec, Data, false);
+    notsup_to_error(block_crypt_nif(des_ede3_cbc, Key, Ivec, Data, false));
 block_decrypt(des3_cbf, Key0, Ivec, Data) -> % cfb misspelled
     Key = check_des3_key(Key0),
-    block_crypt_nif(des_ede3_cbf, Key, Ivec, Data, false);
+    notsup_to_error(block_crypt_nif(des_ede3_cbf, Key, Ivec, Data, false));
 block_decrypt(des3_cfb, Key0, Ivec, Data) ->
     Key = check_des3_key(Key0),
-    block_crypt_nif(des_ede3_cfb, Key, Ivec, Data, false);
+    notsup_to_error(block_crypt_nif(des_ede3_cfb, Key, Ivec, Data, false));
 block_decrypt(aes_ige256, Key, Ivec, Data) ->
     notsup_to_error(aes_ige_crypt_nif(Key, Ivec, Data, false));
 block_decrypt(Type, Key, Ivec, {AAD, Data, Tag}) when Type =:= aes_gcm;
@@ -571,13 +571,13 @@ block_decrypt(Type, Key, Ivec, {AAD, Data, Tag}) when Type =:= aes_gcm;
 -spec block_encrypt(Type::block_cipher_without_iv(), Key::key(), PlainText::iodata()) -> binary().
 
 block_encrypt(Type, Key, PlainText) ->
-    block_crypt_nif(Type, Key, PlainText, true).
+    notsup_to_error(block_crypt_nif(Type, Key, PlainText, true)).
 
 
 -spec block_decrypt(Type::block_cipher_without_iv(), Key::key(), Data::iodata()) -> binary().
 
 block_decrypt(Type, Key, Data) ->
-    block_crypt_nif(Type, Key, Data, false).
+    notsup_to_error(block_crypt_nif(Type, Key, Data, false)).
 
 
 -spec next_iv(Type:: cbc_cipher(), Data) -> NextIVec when % Type :: cbc_cipher(), %des_cbc | des3_cbc | aes_cbc | aes_ige,