diff options
Diffstat (limited to 'lib/crypto/c_src/pkey.c')
| -rw-r--r-- | lib/crypto/c_src/pkey.c | 1564 |
1 files changed, 925 insertions, 639 deletions
diff --git a/lib/crypto/c_src/pkey.c b/lib/crypto/c_src/pkey.c index bd56b2d977..393358d173 100644 --- a/lib/crypto/c_src/pkey.c +++ b/lib/crypto/c_src/pkey.c @@ -68,13 +68,19 @@ static int get_pkey_digest_type(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_ struct digest_type_t *digp = NULL; *md = NULL; - if (type == atom_none && algorithm == atom_rsa) return PKEY_OK; + if (type == atom_none && algorithm == atom_rsa) + return PKEY_OK; + if (algorithm == atom_eddsa) { #ifdef HAVE_EDDSA - if (algorithm == atom_eddsa) return PKEY_OK; + if (!FIPS_mode()) return PKEY_OK; +#else + return PKEY_NOTSUP; #endif - digp = get_digest_type(type); - if (!digp) return PKEY_BADARG; - if (!digp->md.p) return PKEY_NOTSUP; + } + if ((digp = get_digest_type(type)) == NULL) + return PKEY_BADARG; + if (digp->md.p == NULL) + return PKEY_NOTSUP; *md = digp->md.p; return PKEY_OK; @@ -85,67 +91,83 @@ static int get_pkey_sign_digest(ErlNifEnv *env, ERL_NIF_TERM algorithm, unsigned char *md_value, const EVP_MD **mdp, unsigned char **tbsp, size_t *tbslenp) { - int i; + int i, ret; const ERL_NIF_TERM *tpl_terms; int tpl_arity; ErlNifBinary tbs_bin; - EVP_MD_CTX *mdctx; - const EVP_MD *md = *mdp; - unsigned char *tbs = *tbsp; - size_t tbslen = *tbslenp; + EVP_MD_CTX *mdctx = NULL; + const EVP_MD *md; + unsigned char *tbs; + size_t tbslen; unsigned int tbsleni; - if ((i = get_pkey_digest_type(env, algorithm, type, &md)) != PKEY_OK) { - return i; - } + md = *mdp; + tbs = *tbsp; + tbslen = *tbslenp; + + if ((i = get_pkey_digest_type(env, algorithm, type, &md)) != PKEY_OK) + return i; + if (enif_get_tuple(env, data, &tpl_arity, &tpl_terms)) { - if (tpl_arity != 2 || tpl_terms[0] != atom_digest - || !enif_inspect_binary(env, tpl_terms[1], &tbs_bin) - || (md != NULL && tbs_bin.size != EVP_MD_size(md))) { - return PKEY_BADARG; - } + if (tpl_arity != 2) + goto bad_arg; + if (tpl_terms[0] != atom_digest) + goto bad_arg; + if (!enif_inspect_iolist_as_binary(env, tpl_terms[1], &tbs_bin)) + goto bad_arg; + if (tbs_bin.size > INT_MAX) + goto bad_arg; + if (md != NULL) { + if ((int)tbs_bin.size != EVP_MD_size(md)) + goto bad_arg; + } + /* We have a digest (= hashed text) in tbs_bin */ tbs = tbs_bin.data; tbslen = tbs_bin.size; } else if (md == NULL) { - if (!enif_inspect_binary(env, data, &tbs_bin)) { - return PKEY_BADARG; - } + if (!enif_inspect_iolist_as_binary(env, data, &tbs_bin)) + goto bad_arg; + /* md == NULL, that is no hashing because DigestType argument was atom_none */ tbs = tbs_bin.data; tbslen = tbs_bin.size; } else { - if (!enif_inspect_binary(env, data, &tbs_bin)) { - return PKEY_BADARG; - } + if (!enif_inspect_iolist_as_binary(env, data, &tbs_bin)) + goto bad_arg; + /* We have the cleartext in tbs_bin and the hash algo info in md */ tbs = md_value; - mdctx = EVP_MD_CTX_create(); - if (!mdctx) { - return PKEY_BADARG; - } + + if ((mdctx = EVP_MD_CTX_create()) == NULL) + goto err; + /* Looks well, now hash the plain text into a digest according to md */ - if (EVP_DigestInit_ex(mdctx, md, NULL) <= 0) { - EVP_MD_CTX_destroy(mdctx); - return PKEY_BADARG; - } - if (EVP_DigestUpdate(mdctx, tbs_bin.data, tbs_bin.size) <= 0) { - EVP_MD_CTX_destroy(mdctx); - return PKEY_BADARG; - } - if (EVP_DigestFinal_ex(mdctx, tbs, &tbsleni) <= 0) { - EVP_MD_CTX_destroy(mdctx); - return PKEY_BADARG; - } - tbslen = (size_t)(tbsleni); - EVP_MD_CTX_destroy(mdctx); + if (EVP_DigestInit_ex(mdctx, md, NULL) != 1) + goto err; + if (EVP_DigestUpdate(mdctx, tbs_bin.data, tbs_bin.size) != 1) + goto err; + if (EVP_DigestFinal_ex(mdctx, tbs, &tbsleni) != 1) + goto err; + + tbslen = (size_t)tbsleni; } *mdp = md; *tbsp = tbs; *tbslenp = tbslen; - return PKEY_OK; + ret = PKEY_OK; + goto done; + + bad_arg: + err: + ret = PKEY_BADARG; + + done: + if (mdctx) + EVP_MD_CTX_destroy(mdctx); + return ret; } static int get_pkey_sign_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM options, @@ -155,11 +177,9 @@ static int get_pkey_sign_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF const ERL_NIF_TERM *tpl_terms; int tpl_arity; const EVP_MD *opt_md; - int i; - if (!enif_is_list(env, options)) { - return PKEY_BADARG; - } + if (!enif_is_list(env, options)) + goto bad_arg; /* defaults */ if (algorithm == atom_rsa) { @@ -168,533 +188,711 @@ static int get_pkey_sign_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF opt->rsa_pss_saltlen = -2; } - if (enif_is_empty_list(env, options)) { + if (enif_is_empty_list(env, options)) return PKEY_OK; - } - if (algorithm == atom_rsa) { - tail = options; - while (enif_get_list_cell(env, tail, &head, &tail)) { - if (enif_get_tuple(env, head, &tpl_arity, &tpl_terms) && tpl_arity == 2) { - if (tpl_terms[0] == atom_rsa_mgf1_md && enif_is_atom(env, tpl_terms[1])) { - i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); - if (i != PKEY_OK) { - return i; - } - opt->rsa_mgf1_md = opt_md; - } else if (tpl_terms[0] == atom_rsa_padding) { - if (tpl_terms[1] == atom_rsa_pkcs1_padding) { - opt->rsa_padding = RSA_PKCS1_PADDING; - } else if (tpl_terms[1] == atom_rsa_pkcs1_pss_padding) { + if (algorithm != atom_rsa) + goto bad_arg; + + tail = options; + while (enif_get_list_cell(env, tail, &head, &tail)) { + if (!enif_get_tuple(env, head, &tpl_arity, &tpl_terms)) + goto bad_arg; + if (tpl_arity != 2) + goto bad_arg; + + if (tpl_terms[0] == atom_rsa_mgf1_md && enif_is_atom(env, tpl_terms[1])) { + int result; + + result = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); + if (result != PKEY_OK) + return result; + + opt->rsa_mgf1_md = opt_md; + + } else if (tpl_terms[0] == atom_rsa_padding) { + if (tpl_terms[1] == atom_rsa_pkcs1_padding) { + opt->rsa_padding = RSA_PKCS1_PADDING; + + } else if (tpl_terms[1] == atom_rsa_pkcs1_pss_padding) { #ifdef HAVE_RSA_PKCS1_PSS_PADDING - opt->rsa_padding = RSA_PKCS1_PSS_PADDING; - if (opt->rsa_mgf1_md == NULL) { - opt->rsa_mgf1_md = md; - } + opt->rsa_padding = RSA_PKCS1_PSS_PADDING; + if (opt->rsa_mgf1_md == NULL) + opt->rsa_mgf1_md = md; #else - return PKEY_NOTSUP; + return PKEY_NOTSUP; #endif - } else if (tpl_terms[1] == atom_rsa_x931_padding) { - opt->rsa_padding = RSA_X931_PADDING; - } else if (tpl_terms[1] == atom_rsa_no_padding) { - opt->rsa_padding = RSA_NO_PADDING; - } else { - return PKEY_BADARG; - } - } else if (tpl_terms[0] == atom_rsa_pss_saltlen) { - if (!enif_get_int(env, tpl_terms[1], &(opt->rsa_pss_saltlen)) - || opt->rsa_pss_saltlen < -2) { - return PKEY_BADARG; - } - } else { - return PKEY_BADARG; - } - } else { - return PKEY_BADARG; - } - } - } else { - return PKEY_BADARG; + + } else if (tpl_terms[1] == atom_rsa_x931_padding) { + opt->rsa_padding = RSA_X931_PADDING; + + } else if (tpl_terms[1] == atom_rsa_no_padding) { + opt->rsa_padding = RSA_NO_PADDING; + + } else { + goto bad_arg; + } + + } else if (tpl_terms[0] == atom_rsa_pss_saltlen) { + if (!enif_get_int(env, tpl_terms[1], &(opt->rsa_pss_saltlen))) + goto bad_arg; + if (opt->rsa_pss_saltlen < -2) + goto bad_arg; + + } else { + goto bad_arg; + } } return PKEY_OK; + + bad_arg: + return PKEY_BADARG; } static int get_pkey_private_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM key, EVP_PKEY **pkey) { + EVP_PKEY *result = NULL; + RSA *rsa = NULL; + DSA *dsa = NULL; +#if defined(HAVE_EC) + EC_KEY *ec = NULL; +#endif + char *id = NULL; + char *password = NULL; + if (enif_is_map(env, key)) { #ifdef HAS_ENGINE_SUPPORT /* Use key stored in engine */ ENGINE *e; - char *id = NULL; - char *password; if (!get_engine_and_key_id(env, key, &id, &e)) - return PKEY_BADARG; + goto err; + password = get_key_password(env, key); - *pkey = ENGINE_load_private_key(e, id, NULL, password); - if (password) enif_free(password); - enif_free(id); - if (!*pkey) - return PKEY_BADARG; + result = ENGINE_load_private_key(e, id, NULL, password); + #else return PKEY_BADARG; #endif - } - else if (algorithm == atom_rsa) { - RSA *rsa = RSA_new(); - - if (!get_rsa_private_key(env, key, rsa)) { - RSA_free(rsa); - return PKEY_BADARG; - } - - *pkey = EVP_PKEY_new(); - if (!EVP_PKEY_assign_RSA(*pkey, rsa)) { - EVP_PKEY_free(*pkey); - RSA_free(rsa); - return PKEY_BADARG; - } + } else if (algorithm == atom_rsa) { + if ((rsa = RSA_new()) == NULL) + goto err; + + if (!get_rsa_private_key(env, key, rsa)) + goto err; + if ((result = EVP_PKEY_new()) == NULL) + goto err; + if (EVP_PKEY_assign_RSA(result, rsa) != 1) + goto err; + /* On success, result owns rsa */ + rsa = NULL; + } else if (algorithm == atom_ecdsa) { #if defined(HAVE_EC) - EC_KEY *ec = NULL; const ERL_NIF_TERM *tpl_terms; int tpl_arity; - if (enif_get_tuple(env, key, &tpl_arity, &tpl_terms) && tpl_arity == 2 - && enif_is_tuple(env, tpl_terms[0]) && enif_is_binary(env, tpl_terms[1]) - && get_ec_key(env, tpl_terms[0], tpl_terms[1], atom_undefined, &ec)) { - - *pkey = EVP_PKEY_new(); - if (!EVP_PKEY_assign_EC_KEY(*pkey, ec)) { - EVP_PKEY_free(*pkey); - EC_KEY_free(ec); - return PKEY_BADARG; - } - } else { - return PKEY_BADARG; - } + if (!enif_get_tuple(env, key, &tpl_arity, &tpl_terms)) + goto err; + if (tpl_arity != 2) + goto err; + if (!enif_is_tuple(env, tpl_terms[0])) + goto err; + if (!enif_is_binary(env, tpl_terms[1])) + goto err; + if (!get_ec_key(env, tpl_terms[0], tpl_terms[1], atom_undefined, &ec)) + goto err; + + if ((result = EVP_PKEY_new()) == NULL) + goto err; + if (EVP_PKEY_assign_EC_KEY(result, ec) != 1) + goto err; + /* On success, result owns ec */ + ec = NULL; + #else return PKEY_NOTSUP; #endif } else if (algorithm == atom_eddsa) { -#if defined(HAVE_EDDSA) - if (!get_eddsa_key(env, 0, key, pkey)) { - return PKEY_BADARG; - } +#ifdef HAVE_EDDSA + if (!FIPS_mode()) + { + if (!get_eddsa_key(env, 0, key, &result)) + goto err; + else + goto done; // Not nice.... + } #else - return PKEY_NOTSUP; + return PKEY_NOTSUP; #endif } else if (algorithm == atom_dss) { - DSA *dsa = DSA_new(); + if ((dsa = DSA_new()) == NULL) + goto err; + if (!get_dss_private_key(env, key, dsa)) + goto err; + + if ((result = EVP_PKEY_new()) == NULL) + goto err; + if (EVP_PKEY_assign_DSA(result, dsa) != 1) + goto err; + /* On success, result owns dsa */ + dsa = NULL; - if (!get_dss_private_key(env, key, dsa)) { - DSA_free(dsa); - return PKEY_BADARG; - } - - *pkey = EVP_PKEY_new(); - if (!EVP_PKEY_assign_DSA(*pkey, dsa)) { - EVP_PKEY_free(*pkey); - DSA_free(dsa); - return PKEY_BADARG; - } } else { return PKEY_BADARG; } - return PKEY_OK; + goto done; + + err: + if (result) + EVP_PKEY_free(result); + result = NULL; + + done: + if (password) + enif_free(password); + if (id) + enif_free(id); + if (rsa) + RSA_free(rsa); + if (dsa) + DSA_free(dsa); +#ifdef HAVE_EC + if (ec) + EC_KEY_free(ec); +#endif + + if (result == NULL) { + return PKEY_BADARG; + } else { + *pkey = result; + return PKEY_OK; + } } static int get_pkey_public_key(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM key, EVP_PKEY **pkey) { + EVP_PKEY *result = NULL; + RSA *rsa = NULL; + DSA *dsa = NULL; +#if defined(HAVE_EC) + EC_KEY *ec = NULL; +#endif + char *id = NULL; + char *password = NULL; + if (enif_is_map(env, key)) { #ifdef HAS_ENGINE_SUPPORT /* Use key stored in engine */ ENGINE *e; - char *id = NULL; - char *password; if (!get_engine_and_key_id(env, key, &id, &e)) - return PKEY_BADARG; + goto err; + password = get_key_password(env, key); - *pkey = ENGINE_load_public_key(e, id, NULL, password); - if (password) enif_free(password); - enif_free(id); - if (!pkey) - return PKEY_BADARG; + result = ENGINE_load_public_key(e, id, NULL, password); + #else return PKEY_BADARG; #endif } else if (algorithm == atom_rsa) { - RSA *rsa = RSA_new(); - - if (!get_rsa_public_key(env, key, rsa)) { - RSA_free(rsa); - return PKEY_BADARG; - } - - *pkey = EVP_PKEY_new(); - if (!EVP_PKEY_assign_RSA(*pkey, rsa)) { - EVP_PKEY_free(*pkey); - RSA_free(rsa); - return PKEY_BADARG; - } + if ((rsa = RSA_new()) == NULL) + goto err; + + if (!get_rsa_public_key(env, key, rsa)) + goto err; + + if ((result = EVP_PKEY_new()) == NULL) + goto err; + if (EVP_PKEY_assign_RSA(result, rsa) != 1) + goto err; + /* On success, result owns rsa */ + rsa = NULL; + } else if (algorithm == atom_ecdsa) { #if defined(HAVE_EC) - EC_KEY *ec = NULL; const ERL_NIF_TERM *tpl_terms; int tpl_arity; - if (enif_get_tuple(env, key, &tpl_arity, &tpl_terms) && tpl_arity == 2 - && enif_is_tuple(env, tpl_terms[0]) && enif_is_binary(env, tpl_terms[1]) - && get_ec_key(env, tpl_terms[0], atom_undefined, tpl_terms[1], &ec)) { - - *pkey = EVP_PKEY_new(); - if (!EVP_PKEY_assign_EC_KEY(*pkey, ec)) { - EVP_PKEY_free(*pkey); - EC_KEY_free(ec); - return PKEY_BADARG; - } - } else { - return PKEY_BADARG; - } + if (!enif_get_tuple(env, key, &tpl_arity, &tpl_terms)) + goto err; + if (tpl_arity != 2) + goto err; + if (!enif_is_tuple(env, tpl_terms[0])) + goto err; + if (!enif_is_binary(env, tpl_terms[1])) + goto err; + if (!get_ec_key(env, tpl_terms[0], atom_undefined, tpl_terms[1], &ec)) + goto err; + + if ((result = EVP_PKEY_new()) == NULL) + goto err; + + if (EVP_PKEY_assign_EC_KEY(result, ec) != 1) + goto err; + /* On success, result owns ec */ + ec = NULL; + #else return PKEY_NOTSUP; #endif } else if (algorithm == atom_eddsa) { -#if defined(HAVE_EDDSA) - if (!get_eddsa_key(env, 1, key, pkey)) { - return PKEY_BADARG; +#ifdef HAVE_EDDSA + if (!FIPS_mode()) { + if (!get_eddsa_key(env, 1, key, &result)) + goto err; } #else - return PKEY_NOTSUP; + return PKEY_NOTSUP; #endif } else if (algorithm == atom_dss) { - DSA *dsa = DSA_new(); - - if (!get_dss_public_key(env, key, dsa)) { - DSA_free(dsa); - return PKEY_BADARG; - } - - *pkey = EVP_PKEY_new(); - if (!EVP_PKEY_assign_DSA(*pkey, dsa)) { - EVP_PKEY_free(*pkey); - DSA_free(dsa); - return PKEY_BADARG; - } + if ((dsa = DSA_new()) == NULL) + goto err; + + if (!get_dss_public_key(env, key, dsa)) + goto err; + + if ((result = EVP_PKEY_new()) == NULL) + goto err; + if (EVP_PKEY_assign_DSA(result, dsa) != 1) + goto err; + /* On success, result owns dsa */ + dsa = NULL; + } else { return PKEY_BADARG; } - return PKEY_OK; + goto done; + + err: + if (result) + EVP_PKEY_free(result); + result = NULL; + + done: + if (password) + enif_free(password); + if (id) + enif_free(id); + if (rsa) + RSA_free(rsa); + if (dsa) + DSA_free(dsa); +#ifdef HAVE_EC + if (ec) + EC_KEY_free(ec); +#endif + + if (result == NULL) { + return PKEY_BADARG; + } else { + *pkey = result; + return PKEY_OK; + } } ERL_NIF_TERM pkey_sign_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {/* (Algorithm, Type, Data|{digest,Digest}, Key|#{}, Options) */ int i; + int sig_bin_alloc = 0; + ERL_NIF_TERM ret; const EVP_MD *md = NULL; unsigned char md_value[EVP_MAX_MD_SIZE]; - EVP_PKEY *pkey; + EVP_PKEY *pkey = NULL; +#ifdef HAVE_EDDSA + EVP_MD_CTX *mdctx = NULL; +#endif #ifdef HAS_EVP_PKEY_CTX - EVP_PKEY_CTX *ctx; + EVP_PKEY_CTX *ctx = NULL; size_t siglen; #else - unsigned len, siglen; + int len; + unsigned int siglen; #endif PKeySignOptions sig_opt; ErlNifBinary sig_bin; /* signature */ unsigned char *tbs; /* data to be signed */ size_t tbslen; + RSA *rsa = NULL; + DSA *dsa = NULL; +#if defined(HAVE_EC) + EC_KEY *ec = NULL; +#endif /*char buf[1024]; enif_get_atom(env,argv[0],buf,1024,ERL_NIF_LATIN1); printf("algo=%s ",buf); enif_get_atom(env,argv[1],buf,1024,ERL_NIF_LATIN1); printf("hash=%s ",buf); -printf("\r\n"); */ #ifndef HAS_ENGINE_SUPPORT - if (enif_is_map(env, argv[3])) { + if (enif_is_map(env, argv[3])) return atom_notsup; - } #endif i = get_pkey_sign_digest(env, argv[0], argv[1], argv[2], md_value, &md, &tbs, &tbslen); - if (i != PKEY_OK) { - if (i == PKEY_NOTSUP) - return atom_notsup; - else - return enif_make_badarg(env); + switch (i) { + case PKEY_OK: + break; + case PKEY_NOTSUP: + goto notsup; + default: + goto bad_arg; } i = get_pkey_sign_options(env, argv[0], argv[4], md, &sig_opt); - if (i != PKEY_OK) { - if (i == PKEY_NOTSUP) - return atom_notsup; - else - return enif_make_badarg(env); + switch (i) { + case PKEY_OK: + break; + case PKEY_NOTSUP: + goto notsup; + default: + goto bad_arg; } - if (get_pkey_private_key(env, argv[0], argv[3], &pkey) != PKEY_OK) { - return enif_make_badarg(env); - } + if (get_pkey_private_key(env, argv[0], argv[3], &pkey) != PKEY_OK) + goto bad_arg; #ifdef HAS_EVP_PKEY_CTX - ctx = EVP_PKEY_CTX_new(pkey, NULL); - if (!ctx) goto badarg; + if ((ctx = EVP_PKEY_CTX_new(pkey, NULL)) == NULL) + goto err; if (argv[0] != atom_eddsa) { - if (EVP_PKEY_sign_init(ctx) <= 0) goto badarg; - if (md != NULL && EVP_PKEY_CTX_set_signature_md(ctx, md) <= 0) goto badarg; + if (EVP_PKEY_sign_init(ctx) != 1) + goto err; + if (md != NULL) { + if (EVP_PKEY_CTX_set_signature_md(ctx, md) != 1) + goto err; + } } if (argv[0] == atom_rsa) { - if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) <= 0) goto badarg; + if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) != 1) + goto err; # ifdef HAVE_RSA_PKCS1_PSS_PADDING if (sig_opt.rsa_padding == RSA_PKCS1_PSS_PADDING) { if (sig_opt.rsa_mgf1_md != NULL) { # ifdef HAVE_RSA_MGF1_MD - if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, sig_opt.rsa_mgf1_md) <= 0) goto badarg; + if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, sig_opt.rsa_mgf1_md) != 1) + goto err; # else - EVP_PKEY_CTX_free(ctx); - EVP_PKEY_free(pkey); - return atom_notsup; + goto notsup; # endif } - if (sig_opt.rsa_pss_saltlen > -2 - && EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, sig_opt.rsa_pss_saltlen) <= 0) - goto badarg; - } + if (sig_opt.rsa_pss_saltlen > -2) { + if (EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, sig_opt.rsa_pss_saltlen) != 1) + goto err; + } + } #endif } if (argv[0] == atom_eddsa) { #ifdef HAVE_EDDSA - EVP_MD_CTX* mdctx = EVP_MD_CTX_new(); - if (!EVP_DigestSignInit(mdctx, NULL, NULL, NULL, pkey)) { - if (mdctx) EVP_MD_CTX_free(mdctx); - goto badarg; - } - - if (!EVP_DigestSign(mdctx, NULL, &siglen, tbs, tbslen)) { - EVP_MD_CTX_free(mdctx); - goto badarg; - } - enif_alloc_binary(siglen, &sig_bin); - - if (!EVP_DigestSign(mdctx, sig_bin.data, &siglen, tbs, tbslen)) { - EVP_MD_CTX_free(mdctx); - goto badarg; + if (!FIPS_mode()) { + if ((mdctx = EVP_MD_CTX_new()) == NULL) + goto err; + + if (EVP_DigestSignInit(mdctx, NULL, NULL, NULL, pkey) != 1) + goto err; + if (EVP_DigestSign(mdctx, NULL, &siglen, tbs, tbslen) != 1) + goto err; + if (!enif_alloc_binary(siglen, &sig_bin)) + goto err; + sig_bin_alloc = 1; + + if (EVP_DigestSign(mdctx, sig_bin.data, &siglen, tbs, tbslen) != 1) + goto bad_key; } - EVP_MD_CTX_free(mdctx); -#else - goto badarg; + else #endif - } - else - { - if (EVP_PKEY_sign(ctx, NULL, &siglen, tbs, tbslen) <= 0) goto badarg; - enif_alloc_binary(siglen, &sig_bin); + goto notsup; + } else { + if (EVP_PKEY_sign(ctx, NULL, &siglen, tbs, tbslen) != 1) + goto err; + if (!enif_alloc_binary(siglen, &sig_bin)) + goto err; + sig_bin_alloc = 1; if (md != NULL) { ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md)); } - i = EVP_PKEY_sign(ctx, sig_bin.data, &siglen, tbs, tbslen); + if (EVP_PKEY_sign(ctx, sig_bin.data, &siglen, tbs, tbslen) != 1) + goto bad_key; } - - EVP_PKEY_CTX_free(ctx); #else /*printf("Old interface\r\n"); */ if (argv[0] == atom_rsa) { - RSA *rsa = EVP_PKEY_get1_RSA(pkey); - enif_alloc_binary(RSA_size(rsa), &sig_bin); - len = EVP_MD_size(md); - ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); - i = RSA_sign(md->type, tbs, len, sig_bin.data, &siglen, rsa); - RSA_free(rsa); + if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) + goto err; + if ((len = RSA_size(rsa)) < 0) + goto err; + if (!enif_alloc_binary((size_t)len, &sig_bin)) + goto err; + sig_bin_alloc = 1; + + if ((len = EVP_MD_size(md)) < 0) + goto err; + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); + + if (RSA_sign(md->type, tbs, (unsigned int)len, sig_bin.data, &siglen, rsa) != 1) + goto bad_key; } else if (argv[0] == atom_dss) { - DSA *dsa = EVP_PKEY_get1_DSA(pkey); - enif_alloc_binary(DSA_size(dsa), &sig_bin); - len = EVP_MD_size(md); - ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); - i = DSA_sign(md->type, tbs, len, sig_bin.data, &siglen, dsa); - DSA_free(dsa); + if ((dsa = EVP_PKEY_get1_DSA(pkey)) == NULL) + goto err; + if ((len = DSA_size(dsa)) < 0) + goto err; + if (!enif_alloc_binary((size_t)len, &sig_bin)) + goto err; + sig_bin_alloc = 1; + + if ((len = EVP_MD_size(md)) < 0) + goto err; + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); + + if (DSA_sign(md->type, tbs, len, sig_bin.data, &siglen, dsa) != 1) + goto bad_key; } else if (argv[0] == atom_ecdsa) { #if defined(HAVE_EC) - EC_KEY *ec = EVP_PKEY_get1_EC_KEY(pkey); - enif_alloc_binary(ECDSA_size(ec), &sig_bin); - len = EVP_MD_size(md); - ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); - i = ECDSA_sign(md->type, tbs, len, sig_bin.data, &siglen, ec); - EC_KEY_free(ec); + if ((ec = EVP_PKEY_get1_EC_KEY(pkey)) == NULL) + goto err; + if ((len = ECDSA_size(ec)) < 0) + goto err; + if (!enif_alloc_binary((size_t)len, &sig_bin)) + goto err; + sig_bin_alloc = 1; + + len = EVP_MD_size(md); + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, len); + + if (ECDSA_sign(md->type, tbs, len, sig_bin.data, &siglen, ec) != 1) + goto bad_key; #else - EVP_PKEY_free(pkey); - return atom_notsup; + goto notsup; #endif } else { - goto badarg; + goto bad_arg; } #endif - EVP_PKEY_free(pkey); - if (i == 1) { - ERL_VALGRIND_MAKE_MEM_DEFINED(sig_bin.data, siglen); - if (siglen != sig_bin.size) { - enif_realloc_binary(&sig_bin, siglen); - ERL_VALGRIND_ASSERT_MEM_DEFINED(sig_bin.data, siglen); - } - return enif_make_binary(env, &sig_bin); - } else { - enif_release_binary(&sig_bin); - return atom_error; + ERL_VALGRIND_MAKE_MEM_DEFINED(sig_bin.data, siglen); + if (siglen != sig_bin.size) { + if (!enif_realloc_binary(&sig_bin, siglen)) + goto err; + ERL_VALGRIND_ASSERT_MEM_DEFINED(sig_bin.data, siglen); } - - badarg: + ret = enif_make_binary(env, &sig_bin); + sig_bin_alloc = 0; + goto done; + + bad_key: + ret = atom_error; + goto done; + + notsup: + ret = atom_notsup; + goto done; + + bad_arg: + err: + ret = enif_make_badarg(env); + goto done; + + done: + if (sig_bin_alloc) + enif_release_binary(&sig_bin); + if (rsa) + RSA_free(rsa); + if (dsa) + DSA_free(dsa); +#ifdef HAVE_EC + if (ec) + EC_KEY_free(ec); +#endif #ifdef HAS_EVP_PKEY_CTX - EVP_PKEY_CTX_free(ctx); + if (ctx) + EVP_PKEY_CTX_free(ctx); #endif - EVP_PKEY_free(pkey); - return enif_make_badarg(env); + if (pkey) + EVP_PKEY_free(pkey); + + return ret; } ERL_NIF_TERM pkey_verify_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {/* (Algorithm, Type, Data|{digest,Digest}, Signature, Key, Options) */ int i; + int result; const EVP_MD *md = NULL; unsigned char md_value[EVP_MAX_MD_SIZE]; - EVP_PKEY *pkey; + EVP_PKEY *pkey = NULL; #ifdef HAS_EVP_PKEY_CTX - EVP_PKEY_CTX *ctx; + EVP_PKEY_CTX *ctx = NULL; #else #endif PKeySignOptions sig_opt; ErlNifBinary sig_bin; /* signature */ unsigned char *tbs; /* data to be signed */ size_t tbslen; + ERL_NIF_TERM ret; + RSA *rsa = NULL; + DSA *dsa = NULL; +#ifdef HAVE_EC + EC_KEY *ec = NULL; +#endif +#ifdef HAVE_EDDSA + EVP_MD_CTX *mdctx = NULL; +#endif #ifndef HAS_ENGINE_SUPPORT - if (enif_is_map(env, argv[4])) { + if (enif_is_map(env, argv[4])) return atom_notsup; - } #endif - if (!enif_inspect_binary(env, argv[3], &sig_bin)) { + if (!enif_inspect_binary(env, argv[3], &sig_bin)) return enif_make_badarg(env); - } i = get_pkey_sign_digest(env, argv[0], argv[1], argv[2], md_value, &md, &tbs, &tbslen); - if (i != PKEY_OK) { - if (i == PKEY_NOTSUP) - return atom_notsup; - else - return enif_make_badarg(env); + switch (i) { + case PKEY_OK: + break; + case PKEY_NOTSUP: + goto notsup; + default: + goto bad_arg; } i = get_pkey_sign_options(env, argv[0], argv[5], md, &sig_opt); - if (i != PKEY_OK) { - if (i == PKEY_NOTSUP) - return atom_notsup; - else - return enif_make_badarg(env); + switch (i) { + case PKEY_OK: + break; + case PKEY_NOTSUP: + goto notsup; + default: + goto bad_arg; } if (get_pkey_public_key(env, argv[0], argv[4], &pkey) != PKEY_OK) { - return enif_make_badarg(env); + goto bad_arg; } #ifdef HAS_EVP_PKEY_CTX /* printf("EVP interface\r\n"); */ - ctx = EVP_PKEY_CTX_new(pkey, NULL); - if (!ctx) goto badarg; + if ((ctx = EVP_PKEY_CTX_new(pkey, NULL)) == NULL) + goto err; if (argv[0] != atom_eddsa) { - if (EVP_PKEY_verify_init(ctx) <= 0) goto badarg; - if (md != NULL && EVP_PKEY_CTX_set_signature_md(ctx, md) <= 0) goto badarg; + if (EVP_PKEY_verify_init(ctx) != 1) + goto err; + if (md != NULL) { + if (EVP_PKEY_CTX_set_signature_md(ctx, md) != 1) + goto err; + } } if (argv[0] == atom_rsa) { - if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) <= 0) goto badarg; - if (sig_opt.rsa_padding == RSA_PKCS1_PSS_PADDING) { + if (EVP_PKEY_CTX_set_rsa_padding(ctx, sig_opt.rsa_padding) != 1) + goto err; + if (sig_opt.rsa_padding == RSA_PKCS1_PSS_PADDING) { if (sig_opt.rsa_mgf1_md != NULL) { # ifdef HAVE_RSA_MGF1_MD - if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, sig_opt.rsa_mgf1_md) <= 0) goto badarg; + if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, sig_opt.rsa_mgf1_md) != 1) + goto err; # else - EVP_PKEY_CTX_free(ctx); - EVP_PKEY_free(pkey); - return atom_notsup; + goto notsup; # endif } - if (sig_opt.rsa_pss_saltlen > -2 - && EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, sig_opt.rsa_pss_saltlen) <= 0) - goto badarg; - } + if (sig_opt.rsa_pss_saltlen > -2) { + if (EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, sig_opt.rsa_pss_saltlen) != 1) + goto err; + } + } } - if (argv[0] == atom_eddsa) { + if (argv[0] == atom_eddsa) { #ifdef HAVE_EDDSA - EVP_MD_CTX* mdctx = EVP_MD_CTX_create(); - - if (!EVP_DigestVerifyInit(mdctx, NULL, NULL, NULL, pkey)) { - if (mdctx) EVP_MD_CTX_destroy(mdctx); - goto badarg; - } + if (!FIPS_mode()) { + if ((mdctx = EVP_MD_CTX_new()) == NULL) + goto err; - i = EVP_DigestVerify(mdctx, sig_bin.data, sig_bin.size, tbs, tbslen); - EVP_MD_CTX_destroy(mdctx); -#else - goto badarg; -#endif + if (EVP_DigestVerifyInit(mdctx, NULL, NULL, NULL, pkey) != 1) + goto err; + + result = EVP_DigestVerify(mdctx, sig_bin.data, sig_bin.size, tbs, tbslen); } - else - { - if (md != NULL) { - ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md)); - } - i = EVP_PKEY_verify(ctx, sig_bin.data, sig_bin.size, tbs, tbslen); + else +#endif + goto notsup; + } else { + if (md != NULL) { + ERL_VALGRIND_ASSERT_MEM_DEFINED(tbs, EVP_MD_size(md)); } - - EVP_PKEY_CTX_free(ctx); + result = EVP_PKEY_verify(ctx, sig_bin.data, sig_bin.size, tbs, tbslen); + } #else /*printf("Old interface\r\n"); */ + if (tbslen > INT_MAX) + goto bad_arg; + if (sig_bin.size > INT_MAX) + goto bad_arg; if (argv[0] == atom_rsa) { - RSA *rsa = EVP_PKEY_get1_RSA(pkey); - i = RSA_verify(md->type, tbs, tbslen, sig_bin.data, sig_bin.size, rsa); - RSA_free(rsa); + if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) + goto err; + result = RSA_verify(md->type, tbs, (unsigned int)tbslen, sig_bin.data, (unsigned int)sig_bin.size, rsa); } else if (argv[0] == atom_dss) { - DSA *dsa = EVP_PKEY_get1_DSA(pkey); - i = DSA_verify(0, tbs, tbslen, sig_bin.data, sig_bin.size, dsa); - DSA_free(dsa); + if ((dsa = EVP_PKEY_get1_DSA(pkey)) == NULL) + goto err; + result = DSA_verify(0, tbs, (int)tbslen, sig_bin.data, (int)sig_bin.size, dsa); } else if (argv[0] == atom_ecdsa) { #if defined(HAVE_EC) - EC_KEY *ec = EVP_PKEY_get1_EC_KEY(pkey); - i = ECDSA_verify(EVP_MD_type(md), tbs, tbslen, sig_bin.data, sig_bin.size, ec); - EC_KEY_free(ec); + if ((ec = EVP_PKEY_get1_EC_KEY(pkey)) == NULL) + goto err; + result = ECDSA_verify(EVP_MD_type(md), tbs, (int)tbslen, sig_bin.data, (int)sig_bin.size, ec); #else - EVP_PKEY_free(pkey); - return atom_notsup; + goto notsup; #endif } else { - goto badarg; + goto bad_arg; } #endif - EVP_PKEY_free(pkey); - if (i == 1) { - return atom_true; - } else { - return atom_false; - } + ret = (result == 1 ? atom_true : atom_false); + goto done; - badarg: + bad_arg: + err: + ret = enif_make_badarg(env); + goto done; + + notsup: + ret = atom_notsup; + + done: #ifdef HAS_EVP_PKEY_CTX - EVP_PKEY_CTX_free(ctx); + if (ctx) + EVP_PKEY_CTX_free(ctx); +#endif +#ifdef HAVE_EDDSA + if (mdctx) + EVP_MD_CTX_free(mdctx); #endif - EVP_PKEY_free(pkey); - return enif_make_badarg(env); + if (pkey) + EVP_PKEY_free(pkey); + if (rsa) + RSA_free(rsa); + if (dsa) + DSA_free(dsa); +#ifdef HAVE_EC + if (ec) + EC_KEY_free(ec); +#endif + + return ret; } static int get_pkey_crypt_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_TERM options, @@ -704,11 +902,9 @@ static int get_pkey_crypt_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NI const ERL_NIF_TERM *tpl_terms; int tpl_arity; const EVP_MD *opt_md; - int i; - if (!enif_is_list(env, options)) { - return PKEY_BADARG; - } + if (!enif_is_list(env, options)) + goto bad_arg; /* defaults */ if (algorithm == atom_rsa) { @@ -720,98 +916,124 @@ static int get_pkey_crypt_options(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NI opt->signature_md = NULL; } - if (enif_is_empty_list(env, options)) { - return PKEY_OK; - } + if (enif_is_empty_list(env, options)) + return PKEY_OK; + + if (algorithm != atom_rsa) + goto bad_arg; + + tail = options; + while (enif_get_list_cell(env, tail, &head, &tail)) { + if (!enif_get_tuple(env, head, &tpl_arity, &tpl_terms)) + goto bad_arg; + if (tpl_arity != 2) + goto bad_arg; + + if (tpl_terms[0] == atom_rsa_padding + || tpl_terms[0] == atom_rsa_pad /* Compatibility */ + ) { + if (tpl_terms[1] == atom_rsa_pkcs1_padding) { + opt->rsa_padding = RSA_PKCS1_PADDING; - if (algorithm == atom_rsa) { - tail = options; - while (enif_get_list_cell(env, tail, &head, &tail)) { - if (enif_get_tuple(env, head, &tpl_arity, &tpl_terms) && tpl_arity == 2) { - if (tpl_terms[0] == atom_rsa_padding - || tpl_terms[0] == atom_rsa_pad /* Compatibility */ - ) { - if (tpl_terms[1] == atom_rsa_pkcs1_padding) { - opt->rsa_padding = RSA_PKCS1_PADDING; #ifdef HAVE_RSA_OAEP_PADDING - } else if (tpl_terms[1] == atom_rsa_pkcs1_oaep_padding) { - opt->rsa_padding = RSA_PKCS1_OAEP_PADDING; + } else if (tpl_terms[1] == atom_rsa_pkcs1_oaep_padding) { + opt->rsa_padding = RSA_PKCS1_OAEP_PADDING; #endif + #ifdef HAVE_RSA_SSLV23_PADDING - } else if (tpl_terms[1] == atom_rsa_sslv23_padding) { - opt->rsa_padding = RSA_SSLV23_PADDING; + } else if (tpl_terms[1] == atom_rsa_sslv23_padding) { + opt->rsa_padding = RSA_SSLV23_PADDING; #endif - } else if (tpl_terms[1] == atom_rsa_x931_padding) { - opt->rsa_padding = RSA_X931_PADDING; - } else if (tpl_terms[1] == atom_rsa_no_padding) { - opt->rsa_padding = RSA_NO_PADDING; - } else { - return PKEY_BADARG; - } - } else if (tpl_terms[0] == atom_signature_md && enif_is_atom(env, tpl_terms[1])) { - i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); - if (i != PKEY_OK) { - return i; - } - opt->signature_md = opt_md; - } else if (tpl_terms[0] == atom_rsa_mgf1_md && enif_is_atom(env, tpl_terms[1])) { + + } else if (tpl_terms[1] == atom_rsa_x931_padding) { + opt->rsa_padding = RSA_X931_PADDING; + + } else if (tpl_terms[1] == atom_rsa_no_padding) { + opt->rsa_padding = RSA_NO_PADDING; + + } else { + goto bad_arg; + } + + } else if (tpl_terms[0] == atom_signature_md && enif_is_atom(env, tpl_terms[1])) { + int i; + i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); + if (i != PKEY_OK) { + return i; + } + opt->signature_md = opt_md; + + } else if (tpl_terms[0] == atom_rsa_mgf1_md && enif_is_atom(env, tpl_terms[1])) { + int i; #ifndef HAVE_RSA_MGF1_MD - if (tpl_terms[1] != atom_sha) - return PKEY_NOTSUP; + if (tpl_terms[1] != atom_sha) + return PKEY_NOTSUP; #endif - i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); - if (i != PKEY_OK) { - return i; - } - opt->rsa_mgf1_md = opt_md; - } else if (tpl_terms[0] == atom_rsa_oaep_label - && enif_inspect_binary(env, tpl_terms[1], &(opt->rsa_oaep_label))) { + i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); + if (i != PKEY_OK) { + return i; + } + opt->rsa_mgf1_md = opt_md; + + } else if (tpl_terms[0] == atom_rsa_oaep_label + && enif_inspect_binary(env, tpl_terms[1], &(opt->rsa_oaep_label))) { #ifdef HAVE_RSA_OAEP_MD - continue; + continue; #else - return PKEY_NOTSUP; + return PKEY_NOTSUP; #endif - } else if (tpl_terms[0] == atom_rsa_oaep_md && enif_is_atom(env, tpl_terms[1])) { + + } else if (tpl_terms[0] == atom_rsa_oaep_md && enif_is_atom(env, tpl_terms[1])) { + int i; #ifndef HAVE_RSA_OAEP_MD - if (tpl_terms[1] != atom_sha) - return PKEY_NOTSUP; + if (tpl_terms[1] != atom_sha) + return PKEY_NOTSUP; #endif - i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); - if (i != PKEY_OK) { - return i; - } - opt->rsa_oaep_md = opt_md; - } else { - return PKEY_BADARG; - } - } else { - return PKEY_BADARG; - } - } - } else { - return PKEY_BADARG; + i = get_pkey_digest_type(env, algorithm, tpl_terms[1], &opt_md); + if (i != PKEY_OK) { + return i; + } + opt->rsa_oaep_md = opt_md; + + } else { + goto bad_arg; + } } return PKEY_OK; + + bad_arg: + return PKEY_BADARG; } static size_t size_of_RSA(EVP_PKEY *pkey) { - size_t tmplen; - RSA *rsa = EVP_PKEY_get1_RSA(pkey); - if (rsa == NULL) return 0; - tmplen = RSA_size(rsa); - RSA_free(rsa); - return tmplen; + int ret = 0; + RSA *rsa = NULL; + + if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) + goto err; + ret = RSA_size(rsa); + + err: + if (rsa) + RSA_free(rsa); + + return (ret < 0) ? 0 : (size_t)ret; } ERL_NIF_TERM pkey_crypt_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {/* (Algorithm, Data, PublKey=[E,N]|[E,N,D]|[E,N,D,P1,P2,E1,E2,C], Options, IsPrivate, IsEncrypt) */ + ERL_NIF_TERM ret; int i; - EVP_PKEY *pkey; + int result = 0; + int tmp_bin_alloc = 0; + int out_bin_alloc = 0; + EVP_PKEY *pkey = NULL; #ifdef HAS_EVP_PKEY_CTX - EVP_PKEY_CTX *ctx; + EVP_PKEY_CTX *ctx = NULL; #else - RSA *rsa; + int len; + RSA *rsa = NULL; #endif PKeyCryptOptions crypt_opt; ErlNifBinary in_bin, out_bin, tmp_bin; @@ -819,164 +1041,174 @@ ERL_NIF_TERM pkey_crypt_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) #ifdef HAVE_RSA_SSLV23_PADDING size_t tmplen; #endif - int is_private = (argv[4] == atom_true), - is_encrypt = (argv[5] == atom_true); + int is_private, is_encrypt; int algo_init = 0; + unsigned char *label_copy = NULL; + + ASSERT(argc == 6); + + is_private = (argv[4] == atom_true); + is_encrypt = (argv[5] == atom_true); /* char algo[1024]; */ #ifndef HAS_ENGINE_SUPPORT - if (enif_is_map(env, argv[2])) { + if (enif_is_map(env, argv[2])) return atom_notsup; - } #endif - if (!enif_inspect_binary(env, argv[1], &in_bin)) { - return enif_make_badarg(env); - } + if (!enif_inspect_binary(env, argv[1], &in_bin)) + goto bad_arg; i = get_pkey_crypt_options(env, argv[0], argv[3], &crypt_opt); - if (i != PKEY_OK) { - if (i == PKEY_NOTSUP) - return atom_notsup; - else - return enif_make_badarg(env); + switch (i) { + case PKEY_OK: + break; + case PKEY_NOTSUP: + goto notsup; + default: + goto bad_arg; } if (is_private) { - if (get_pkey_private_key(env, argv[0], argv[2], &pkey) != PKEY_OK) { - return enif_make_badarg(env); - } + if (get_pkey_private_key(env, argv[0], argv[2], &pkey) != PKEY_OK) + goto bad_arg; } else { - if (get_pkey_public_key(env, argv[0], argv[2], &pkey) != PKEY_OK) { - return enif_make_badarg(env); - } + if (get_pkey_public_key(env, argv[0], argv[2], &pkey) != PKEY_OK) + goto bad_arg; } - out_bin.data = NULL; - out_bin.size = 0; - tmp_bin.data = NULL; - tmp_bin.size = 0; - #ifdef HAS_EVP_PKEY_CTX - ctx = EVP_PKEY_CTX_new(pkey, NULL); - if (!ctx) goto badarg; + if ((ctx = EVP_PKEY_CTX_new(pkey, NULL)) == NULL) + goto err; /* enif_get_atom(env,argv[0],algo,1024,ERL_NIF_LATIN1); */ if (is_private) { if (is_encrypt) { /* private encrypt */ - if ((algo_init=EVP_PKEY_sign_init(ctx)) <= 0) { - /* fprintf(stderr,"BADARG %s private encrypt algo_init=%d %s:%d\r\n", algo, algo_init, __FILE__, __LINE__); */ - goto badarg; - } + if ((algo_init = EVP_PKEY_sign_init(ctx)) != 1) + goto bad_arg; } else { /* private decrypt */ - if ((algo_init=EVP_PKEY_decrypt_init(ctx)) <= 0) { - /* fprintf(stderr,"BADARG %s private decrypt algo_init=%d %s:%d\r\n", algo, algo_init, __FILE__, __LINE__); */ - goto badarg; - } + if ((algo_init = EVP_PKEY_decrypt_init(ctx)) != 1) + goto bad_arg; } } else { if (is_encrypt) { /* public encrypt */ - if ((algo_init=EVP_PKEY_encrypt_init(ctx)) <= 0) { - /* fprintf(stderr,"BADARG %s public encrypt algo_init=%d %s:%d\r\n", algo,algo_init,__FILE__, __LINE__); */ - goto badarg; - } + if ((algo_init = EVP_PKEY_encrypt_init(ctx)) != 1) + goto bad_arg; } else { /* public decrypt */ - if ((algo_init=EVP_PKEY_verify_recover_init(ctx)) <= 0) { - /* fprintf(stderr,"BADARG %s public decrypt algo_init=%d %s:%d\r\n", algo,algo_init,__FILE__, __LINE__); */ - goto badarg; - } + if ((algo_init = EVP_PKEY_verify_recover_init(ctx)) != 1) + goto bad_arg; } } if (argv[0] == atom_rsa) { - if (crypt_opt.signature_md != NULL - && EVP_PKEY_CTX_set_signature_md(ctx, crypt_opt.signature_md) <= 0) - goto badarg; + if (crypt_opt.signature_md != NULL) { + if (EVP_PKEY_CTX_set_signature_md(ctx, crypt_opt.signature_md) != 1) + goto bad_arg; + } + #ifdef HAVE_RSA_SSLV23_PADDING - if (crypt_opt.rsa_padding == RSA_SSLV23_PADDING) { - if (is_encrypt) { + if (crypt_opt.rsa_padding == RSA_SSLV23_PADDING) { + if (is_encrypt) { tmplen = size_of_RSA(pkey); - if (tmplen == 0) goto badarg; - if (!enif_alloc_binary(tmplen, &tmp_bin)) goto badarg; - if (RSA_padding_add_SSLv23(tmp_bin.data, tmplen, in_bin.data, in_bin.size) <= 0) - goto badarg; - in_bin = tmp_bin; - } - if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_NO_PADDING) <= 0) goto badarg; - } else + if (tmplen < 1 || tmplen > INT_MAX) + goto err; + if (!enif_alloc_binary(tmplen, &tmp_bin)) + goto err; + tmp_bin_alloc = 1; + if (in_bin.size > INT_MAX) + goto err; + if (!RSA_padding_add_SSLv23(tmp_bin.data, (int)tmplen, in_bin.data, (int)in_bin.size)) + goto err; + in_bin = tmp_bin; + } + if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_NO_PADDING) != 1) + goto err; + } else #endif - { - if (EVP_PKEY_CTX_set_rsa_padding(ctx, crypt_opt.rsa_padding) <= 0) goto badarg; + { + if (EVP_PKEY_CTX_set_rsa_padding(ctx, crypt_opt.rsa_padding) != 1) + goto err; } + #ifdef HAVE_RSA_OAEP_MD - if (crypt_opt.rsa_padding == RSA_PKCS1_OAEP_PADDING) { - if (crypt_opt.rsa_oaep_md != NULL - && EVP_PKEY_CTX_set_rsa_oaep_md(ctx, crypt_opt.rsa_oaep_md) <= 0) - goto badarg; - if (crypt_opt.rsa_mgf1_md != NULL - && EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, crypt_opt.rsa_mgf1_md) <= 0) goto badarg; - if (crypt_opt.rsa_oaep_label.data != NULL && crypt_opt.rsa_oaep_label.size > 0) { - unsigned char *label_copy = NULL; - label_copy = OPENSSL_malloc(crypt_opt.rsa_oaep_label.size); - if (label_copy == NULL) goto badarg; - memcpy((void *)(label_copy), (const void *)(crypt_opt.rsa_oaep_label.data), - crypt_opt.rsa_oaep_label.size); - if (EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, label_copy, - crypt_opt.rsa_oaep_label.size) <= 0) { - OPENSSL_free(label_copy); - label_copy = NULL; - goto badarg; - } - } - } + if (crypt_opt.rsa_padding == RSA_PKCS1_OAEP_PADDING) { + if (crypt_opt.rsa_oaep_md != NULL) { + if (EVP_PKEY_CTX_set_rsa_oaep_md(ctx, crypt_opt.rsa_oaep_md) != 1) + goto err; + } + + if (crypt_opt.rsa_mgf1_md != NULL) { + if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, crypt_opt.rsa_mgf1_md) != 1) + goto err; + } + + if (crypt_opt.rsa_oaep_label.data != NULL && crypt_opt.rsa_oaep_label.size > 0) { + if (crypt_opt.rsa_oaep_label.size > INT_MAX) + goto err; + if ((label_copy = OPENSSL_malloc(crypt_opt.rsa_oaep_label.size)) == NULL) + goto err; + + memcpy((void *)(label_copy), (const void *)(crypt_opt.rsa_oaep_label.data), + crypt_opt.rsa_oaep_label.size); + + if (EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, label_copy, + (int)crypt_opt.rsa_oaep_label.size) != 1) + goto err; + /* On success, label_copy is owned by ctx */ + label_copy = NULL; + } + } #endif } if (is_private) { - if (is_encrypt) { - /* private_encrypt */ - i = EVP_PKEY_sign(ctx, NULL, &outlen, in_bin.data, in_bin.size); - } else { - /* private_decrypt */ - i = EVP_PKEY_decrypt(ctx, NULL, &outlen, in_bin.data, in_bin.size); - } + if (is_encrypt) { + /* private_encrypt */ + result = EVP_PKEY_sign(ctx, NULL, &outlen, in_bin.data, in_bin.size); + } else { + /* private_decrypt */ + result = EVP_PKEY_decrypt(ctx, NULL, &outlen, in_bin.data, in_bin.size); + } } else { - if (is_encrypt) { - /* public_encrypt */ - i = EVP_PKEY_encrypt(ctx, NULL, &outlen, in_bin.data, in_bin.size); - } else { - /* public_decrypt */ - i = EVP_PKEY_verify_recover(ctx, NULL, &outlen, in_bin.data, in_bin.size); - } + if (is_encrypt) { + /* public_encrypt */ + result = EVP_PKEY_encrypt(ctx, NULL, &outlen, in_bin.data, in_bin.size); + } else { + /* public_decrypt */ + result = EVP_PKEY_verify_recover(ctx, NULL, &outlen, in_bin.data, in_bin.size); + } } /* fprintf(stderr,"i = %d %s:%d\r\n", i, __FILE__, __LINE__); */ - if (i != 1) goto badarg; + if (result != 1) + goto err; - enif_alloc_binary(outlen, &out_bin); + if (!enif_alloc_binary(outlen, &out_bin)) + goto err; + out_bin_alloc = 1; if (is_private) { - if (is_encrypt) { - /* private_encrypt */ - i = EVP_PKEY_sign(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); - } else { - /* private_decrypt */ - i = EVP_PKEY_decrypt(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); - } + if (is_encrypt) { + /* private_encrypt */ + result = EVP_PKEY_sign(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); + } else { + /* private_decrypt */ + result = EVP_PKEY_decrypt(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); + } } else { - if (is_encrypt) { - /* public_encrypt */ - i = EVP_PKEY_encrypt(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); - } else { - /* public_decrypt */ - i = EVP_PKEY_verify_recover(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); - } + if (is_encrypt) { + /* public_encrypt */ + result = EVP_PKEY_encrypt(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); + } else { + /* public_decrypt */ + result = EVP_PKEY_verify_recover(ctx, out_bin.data, &outlen, in_bin.data, in_bin.size); + } } #else @@ -984,149 +1216,187 @@ ERL_NIF_TERM pkey_crypt_nif(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) if (argv[0] != atom_rsa) { algo_init = -2; /* exitcode: notsup */ - goto badarg; + goto bad_arg; } - rsa = EVP_PKEY_get1_RSA(pkey); - enif_alloc_binary(RSA_size(rsa), &out_bin); + if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) + goto err; + if ((len = RSA_size(rsa)) < 0) + goto err; + if (!enif_alloc_binary((size_t)len, &out_bin)) + goto err; + out_bin_alloc = 1; + + if (in_bin.size > INT_MAX) + goto err; if (is_private) { if (is_encrypt) { /* non-evp rsa private encrypt */ ERL_VALGRIND_ASSERT_MEM_DEFINED(in_bin.data,in_bin.size); - i = RSA_private_encrypt(in_bin.size, in_bin.data, + result = RSA_private_encrypt((int)in_bin.size, in_bin.data, out_bin.data, rsa, crypt_opt.rsa_padding); - if (i > 0) { - ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, i); + if (result > 0) { + ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, result); } } else { /* non-evp rsa private decrypt */ - i = RSA_private_decrypt(in_bin.size, in_bin.data, + result = RSA_private_decrypt((int)in_bin.size, in_bin.data, out_bin.data, rsa, crypt_opt.rsa_padding); - if (i > 0) { - ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, i); - enif_realloc_binary(&out_bin, i); + if (result > 0) { + ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, result); + if (!enif_realloc_binary(&out_bin, (size_t)result)) + goto err; } } } else { if (is_encrypt) { /* non-evp rsa public encrypt */ ERL_VALGRIND_ASSERT_MEM_DEFINED(in_bin.data,in_bin.size); - i = RSA_public_encrypt(in_bin.size, in_bin.data, + result = RSA_public_encrypt((int)in_bin.size, in_bin.data, out_bin.data, rsa, crypt_opt.rsa_padding); - if (i > 0) { - ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, i); - } + if (result > 0) { + ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, result); + } } else { /* non-evp rsa public decrypt */ - i = RSA_public_decrypt(in_bin.size, in_bin.data, + result = RSA_public_decrypt((int)in_bin.size, in_bin.data, out_bin.data, rsa, crypt_opt.rsa_padding); - if (i > 0) { - ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, i); - enif_realloc_binary(&out_bin, i); + if (result > 0) { + ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, result); + if (!enif_realloc_binary(&out_bin, (size_t)result)) + goto err; } } } - outlen = i; - RSA_free(rsa); + outlen = (size_t)result; #endif - if ((i > 0) && argv[0] == atom_rsa && !is_encrypt) { + if ((result > 0) && argv[0] == atom_rsa && !is_encrypt) { #ifdef HAVE_RSA_SSLV23_PADDING - if (crypt_opt.rsa_padding == RSA_SSLV23_PADDING) { - unsigned char *p; + if (crypt_opt.rsa_padding == RSA_SSLV23_PADDING) { + unsigned char *p; + tmplen = size_of_RSA(pkey); - if (tmplen == 0) goto badarg; - if (!enif_alloc_binary(tmplen, &tmp_bin)) - goto badarg; - p = out_bin.data; - p++; - i = RSA_padding_check_SSLv23(tmp_bin.data, tmplen, p, out_bin.size - 1, tmplen); - if (i >= 0) { - outlen = i; - in_bin = out_bin; - out_bin = tmp_bin; - tmp_bin = in_bin; - i = 1; - } - } + if (tmplen < 1 || tmplen > INT_MAX) + goto err; + if (!enif_alloc_binary(tmplen, &tmp_bin)) + goto err; + tmp_bin_alloc = 1; + if (out_bin.size > INT_MAX) + goto err; + + p = out_bin.data; + p++; + + result = RSA_padding_check_SSLv23(tmp_bin.data, (int)tmplen, p, (int)out_bin.size - 1, (int)tmplen); + if (result >= 0) { + outlen = (size_t)result; + in_bin = out_bin; + out_bin = tmp_bin; + tmp_bin = in_bin; + result = 1; + } + } #endif } - if (tmp_bin.data != NULL) { - enif_release_binary(&tmp_bin); - } - -#ifdef HAS_EVP_PKEY_CTX - EVP_PKEY_CTX_free(ctx); -#else -#endif - EVP_PKEY_free(pkey); - if (i > 0) { - ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, outlen); - if (outlen != out_bin.size) { - enif_realloc_binary(&out_bin, outlen); - ERL_VALGRIND_ASSERT_MEM_DEFINED(out_bin.data, outlen); - } - return enif_make_binary(env, &out_bin); + if (result > 0) { + ERL_VALGRIND_MAKE_MEM_DEFINED(out_bin.data, outlen); + if (outlen != out_bin.size) { + if (!enif_realloc_binary(&out_bin, outlen)) + goto err; + ERL_VALGRIND_ASSERT_MEM_DEFINED(out_bin.data, outlen); + } + ret = enif_make_binary(env, &out_bin); + out_bin_alloc = 0; } else { - enif_release_binary(&out_bin); - return atom_error; + ret = atom_error; } + goto done; + + notsup: + ret = atom_notsup; + goto done; + + bad_arg: + err: + if (algo_init == -2) + ret = atom_notsup; + else + ret = enif_make_badarg(env); + + done: + if (out_bin_alloc) + enif_release_binary(&out_bin); + if (tmp_bin_alloc) + enif_release_binary(&tmp_bin); - badarg: - if (out_bin.data != NULL) { - enif_release_binary(&out_bin); - } - if (tmp_bin.data != NULL) { - enif_release_binary(&tmp_bin); - } #ifdef HAS_EVP_PKEY_CTX - EVP_PKEY_CTX_free(ctx); + if (ctx) + EVP_PKEY_CTX_free(ctx); #else + if (rsa) + RSA_free(rsa); #endif - EVP_PKEY_free(pkey); - if (algo_init == -2) - return atom_notsup; - else - return enif_make_badarg(env); + if (pkey) + EVP_PKEY_free(pkey); + + if (label_copy) + OPENSSL_free(label_copy); + + return ret; } ERL_NIF_TERM privkey_to_pubkey_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { /* (Algorithm, PrivKey | KeyMap) */ - EVP_PKEY *pkey; - ERL_NIF_TERM alg = argv[0]; + ERL_NIF_TERM ret; + EVP_PKEY *pkey = NULL; + RSA *rsa = NULL; + DSA *dsa = NULL; ERL_NIF_TERM result[8]; - if (get_pkey_private_key(env, alg, argv[1], &pkey) != PKEY_OK) { - return enif_make_badarg(env); - } - if (alg == atom_rsa) { + ASSERT(argc == 2); + + if (get_pkey_private_key(env, argv[0], argv[1], &pkey) != PKEY_OK) + goto bad_arg; + + if (argv[0] == atom_rsa) { const BIGNUM *n = NULL, *e = NULL, *d = NULL; - RSA *rsa = EVP_PKEY_get1_RSA(pkey); - if (rsa) { - RSA_get0_key(rsa, &n, &e, &d); - result[0] = bin_from_bn(env, e); // Exponent E - result[1] = bin_from_bn(env, n); // Modulus N = p*q - RSA_free(rsa); - EVP_PKEY_free(pkey); - return enif_make_list_from_array(env, result, 2); - } + + if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) + goto err; + + RSA_get0_key(rsa, &n, &e, &d); + + // Exponent E + if ((result[0] = bin_from_bn(env, e)) == atom_error) + goto err; + // Modulus N = p*q + if ((result[1] = bin_from_bn(env, n)) == atom_error) + goto err; + + ret = enif_make_list_from_array(env, result, 2); } else if (argv[0] == atom_dss) { const BIGNUM *p = NULL, *q = NULL, *g = NULL, *pub_key = NULL; - DSA *dsa = EVP_PKEY_get1_DSA(pkey); - if (dsa) { - DSA_get0_pqg(dsa, &p, &q, &g); - DSA_get0_key(dsa, &pub_key, NULL); - result[0] = bin_from_bn(env, p); - result[1] = bin_from_bn(env, q); - result[2] = bin_from_bn(env, g); - result[3] = bin_from_bn(env, pub_key); - DSA_free(dsa); - EVP_PKEY_free(pkey); - return enif_make_list_from_array(env, result, 4); - } + + if ((dsa = EVP_PKEY_get1_DSA(pkey)) == NULL) + goto err; + + DSA_get0_pqg(dsa, &p, &q, &g); + DSA_get0_key(dsa, &pub_key, NULL); + + if ((result[0] = bin_from_bn(env, p)) == atom_error) + goto err; + if ((result[1] = bin_from_bn(env, q)) == atom_error) + goto err; + if ((result[2] = bin_from_bn(env, g)) == atom_error) + goto err; + if ((result[3] = bin_from_bn(env, pub_key)) == atom_error) + goto err; + + ret = enif_make_list_from_array(env, result, 4); } else if (argv[0] == atom_ecdsa) { #if defined(HAVE_EC) @@ -1163,8 +1433,24 @@ ERL_NIF_TERM privkey_to_pubkey_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM return enif_make_list_from_array(env, ..., ...); */ #endif + goto bad_arg; + } else { + goto bad_arg; } - if (pkey) EVP_PKEY_free(pkey); - return enif_make_badarg(env); + goto done; + + bad_arg: + err: + ret = enif_make_badarg(env); + + done: + if (rsa) + RSA_free(rsa); + if (dsa) + DSA_free(dsa); + if (pkey) + EVP_PKEY_free(pkey); + + return ret; } |