Merge branch 'hans/crypto/FIPS_macs/OTP-13872' into maint

* hans/crypto/FIPS_macs/OTP-13872:
  crypto: Fix FIPS flags for MACs
  crypto: Fix FIPS flags for digests in HMAC

1 files changed, 39 insertions, 6 deletions

diff --git a/lib/crypto/c_src/mac.c b/lib/crypto/c_src/mac.c
index ed09dae8e4..149975ba9d 100644
--- a/lib/crypto/c_src/mac.c
+++ b/lib/crypto/c_src/mac.c
@@ -34,6 +34,7 @@ struct mac_type_t {
 	const char*  str;        /* before init, NULL for end-of-table */
 	ERL_NIF_TERM atom;       /* after init, 'false' for end-of-table */
     }name;
+    unsigned flags;
     union {
         const int pkey_type;
     }alg;
@@ -41,6 +42,9 @@ struct mac_type_t {
     size_t key_len;      /* != 0 to also match on key_len */
 };
 
+/* masks in the flags field if mac_type_t */
+#define NO_FIPS_MAC 1
+
 #define NO_mac 0
 #define HMAC_mac 1
 #define CMAC_mac 2
@@ -48,7 +52,7 @@ struct mac_type_t {
 
 static struct mac_type_t mac_types[] =
 {
-    {{"poly1305"},
+    {{"poly1305"}, NO_FIPS_MAC,
 #ifdef HAVE_POLY1305
      /* If we have POLY then we have EVP_PKEY */
      {EVP_PKEY_POLY1305}, POLY1305_mac, 32
@@ -57,7 +61,7 @@ static struct mac_type_t mac_types[] =
 #endif
     },
 
-    {{"hmac"},
+    {{"hmac"}, 0,
 #ifdef HAS_EVP_PKEY_CTX
      {EVP_PKEY_HMAC}, HMAC_mac, 0
 #else
@@ -66,7 +70,7 @@ static struct mac_type_t mac_types[] =
 #endif
     },
 
-    {{"cmac"},
+    {{"cmac"}, 0,
 #ifdef HAVE_CMAC
      /* If we have CMAC then we have EVP_PKEY */
      {EVP_PKEY_CMAC}, CMAC_mac, 0
@@ -76,12 +80,21 @@ static struct mac_type_t mac_types[] =
     },
 
     /*==== End of list ==== */
-    {{NULL},
+    {{NULL}, 0,
      {0}, NO_mac, 0
     }
 };
 
 
+#ifdef FIPS_SUPPORT
+/* May have FIPS support, must check dynamically if it is enabled */
+# define MAC_FORBIDDEN_IN_FIPS(P) (((P)->flags & NO_FIPS_MAC) && FIPS_mode())
+#else
+/* No FIPS support since the symbol FIPS_SUPPORT is undefined */
+# define MAC_FORBIDDEN_IN_FIPS(P) 0
+#endif
+
+
 /***************************
  Mandatory prototypes
 ***************************/
@@ -219,6 +232,12 @@ ERL_NIF_TERM mac_one_time(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
             goto err;
         }
 
+    if (MAC_FORBIDDEN_IN_FIPS(macp))
+        {
+            return_term = EXCP_NOTSUP(env, "MAC algorithm forbidden in FIPS");
+            goto err;
+        }
+
     /*--------------------------------------------------
       Algorithm dependent indata checking and computation.
       If EVP_PKEY is available, only set the pkey variable
@@ -245,7 +264,11 @@ ERL_NIF_TERM mac_one_time(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
                     return_term = EXCP_NOTSUP(env, "Unsupported digest algorithm");
                     goto err;
                 }
-
+            if (DIGEST_FORBIDDEN_IN_FIPS(digp))
+                {
+                    return_term = EXCP_NOTSUP(env, "Digest algorithm for HMAC forbidden in FIPS");
+                    goto err;
+                }
             md = digp->md.p;
 
 #ifdef HAS_EVP_PKEY_CTX
@@ -496,6 +519,12 @@ ERL_NIF_TERM mac_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
             goto err;
         }
 
+    if (MAC_FORBIDDEN_IN_FIPS(macp))
+        {
+            return_term = EXCP_NOTSUP(env, "MAC algorithm forbidden in FIPS");
+            goto err;
+        }
+
     /*--------------------------------------------------
       Algorithm dependent indata checking and computation.
       If EVP_PKEY is available, only set the pkey variable
@@ -522,7 +551,11 @@ ERL_NIF_TERM mac_init_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
                     return_term = EXCP_NOTSUP(env, "Unsupported digest algorithm");
                     goto err;
                 }
-
+            if (DIGEST_FORBIDDEN_IN_FIPS(digp))
+                {
+                    return_term = EXCP_NOTSUP(env, "Digest algorithm for HMAC forbidden in FIPS");
+                    goto err;
+                }
             md = digp->md.p;
 
 # ifdef HAVE_PKEY_new_raw_private_key