Revamp evp_generate_key_nif()

* Bug fix: Some EVP_PKEY_* calls return <= 0 on failure and 1 on success.

1 files changed, 44 insertions, 24 deletions

diff --git a/lib/crypto/c_src/evp.c b/lib/crypto/c_src/evp.c
index 49e563f039..7aa4be1f9f 100644
--- a/lib/crypto/c_src/evp.c
+++ b/lib/crypto/c_src/evp.c
@@ -106,38 +106,58 @@ ERL_NIF_TERM evp_generate_key_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM a
     int type;
     EVP_PKEY_CTX *ctx = NULL;
     EVP_PKEY *pkey = NULL;
-    ERL_NIF_TERM ret_pub, ret_prv;
+    ERL_NIF_TERM ret_pub, ret_prv, ret;
     size_t key_len;
+    unsigned char *out_pub = NULL, *out_priv = NULL;
 
-    if (argv[0] == atom_x25519) type = EVP_PKEY_X25519;
-    else if (argv[0] == atom_x448) type = EVP_PKEY_X448;
-    else return enif_make_badarg(env);
+    if (argc != 1)
+        goto bad_arg;
+
+    if (argv[0] == atom_x25519)
+        type = EVP_PKEY_X25519;
+    else if (argv[0] == atom_x448)
+        type = EVP_PKEY_X448;
+    else
+        goto bad_arg;
+
+    if ((ctx = EVP_PKEY_CTX_new_id(type, NULL)) == NULL)
+        goto bad_arg;
 
-    if (!(ctx = EVP_PKEY_CTX_new_id(type, NULL))) return enif_make_badarg(env);
+    if (EVP_PKEY_keygen_init(ctx) != 1)
+        goto err;
+    if (EVP_PKEY_keygen(ctx, &pkey) != 1)
+        goto err;
+
+    if (EVP_PKEY_get_raw_public_key(pkey, NULL, &key_len) != 1)
+        goto err;
+    if ((out_pub = enif_make_new_binary(env, key_len, &ret_pub)) == NULL)
+        goto err;
+    if (EVP_PKEY_get_raw_public_key(pkey, out_pub, &key_len) != 1)
+        goto err;
 
-    if (!EVP_PKEY_keygen_init(ctx)) goto return_error;
-    if (!EVP_PKEY_keygen(ctx, &pkey)) goto return_error;
+    if (EVP_PKEY_get_raw_private_key(pkey, NULL, &key_len) != 1)
+        goto err;
+    if ((out_priv = enif_make_new_binary(env, key_len, &ret_prv)) == NULL)
+        goto err;
+    if (EVP_PKEY_get_raw_private_key(pkey, out_priv, &key_len) != 1)
+        goto err;
 
-    if (!EVP_PKEY_get_raw_public_key(pkey, NULL, &key_len)) goto return_error;
-    if (!EVP_PKEY_get_raw_public_key(pkey,
-                                     enif_make_new_binary(env, key_len, &ret_pub),
-                                     &key_len))
-        goto return_error;
+    ret = enif_make_tuple2(env, ret_pub, ret_prv);
+    goto done;
 
-    if (!EVP_PKEY_get_raw_private_key(pkey, NULL, &key_len)) goto return_error;
-    if (!EVP_PKEY_get_raw_private_key(pkey,
-                                      enif_make_new_binary(env, key_len, &ret_prv),
-                                      &key_len))
-        goto return_error;
+ bad_arg:
+    ret = enif_make_badarg(env);
+    goto done;
 
-    EVP_PKEY_free(pkey);
-    EVP_PKEY_CTX_free(ctx);
-    return enif_make_tuple2(env, ret_pub, ret_prv);
+ err:
+    ret = atom_error;
 
-return_error:
-    if (pkey) EVP_PKEY_free(pkey);
-    if (ctx)  EVP_PKEY_CTX_free(ctx);
-    return atom_error;
+ done:
+    if (pkey)
+        EVP_PKEY_free(pkey);
+    if (ctx)
+        EVP_PKEY_CTX_free(ctx);
+    return ret;
 
 #else
     return atom_notsup;