Merge branch 'sverk/crypto-ec-mem-leak/OTP-11999' into maint

* sverk/crypto-ec-mem-leak/OTP-11999:
  crypto: Fix memory leak of EC "keys" and "points"
  crypto: Fix crypto for debug and valgrind

2 files changed, 15 insertions, 4 deletions

diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c
index 6642183cb8..e55a03d26a 100644
--- a/lib/crypto/c_src/crypto.c
+++ b/lib/crypto/c_src/crypto.c
@@ -507,6 +507,15 @@ static int init_ose_crypto() {
 #endif
 
 #ifdef HAVE_DYNAMIC_CRYPTO_LIB
+
+# if defined(DEBUG)
+static char crypto_callback_name[] = "crypto_callback.debug";
+# elif defined(VALGRIND)
+static char crypto_callback_name[] = "crypto_callback.valgrind";
+# else
+static char crypto_callback_name[] = "crypto_callback";
+# endif
+
 static int change_basename(ErlNifBinary* bin, char* buf, int bufsz, const char* newfile)
 {
     int i;
@@ -613,7 +622,7 @@ static int init(ErlNifEnv* env, ERL_NIF_TERM load_info)
 #ifdef HAVE_DYNAMIC_CRYPTO_LIB
     {
 	void* handle;
-	if (!change_basename(&lib_bin, lib_buf, sizeof(lib_buf), "crypto_callback")) {
+	if (!change_basename(&lib_bin, lib_buf, sizeof(lib_buf), crypto_callback_name)) {
 	    return 0;
 	}
 	if (!(handle = enif_dlopen(lib_buf, &error_handler, NULL))) {
@@ -3244,6 +3253,7 @@ out:
     if (bn_order) BN_free(bn_order);
     if (cofactor) BN_free(cofactor);
     if (group) EC_GROUP_free(group);
+    if (point) EC_POINT_free(point);
 
     return key;
 }
@@ -3406,8 +3416,11 @@ static ERL_NIF_TERM ec_key_generate(ErlNifEnv* env, int argc, const ERL_NIF_TERM
 	EC_KEY_free(key);
 	return enif_make_tuple2(env, pub_key, priv_key);
     }
-    else
+    else {
+	if (key)
+	    EC_KEY_free(key);
 	return enif_make_badarg(env);
+    }
 #else
     return atom_notsup;
 #endif
diff --git a/lib/crypto/c_src/crypto_callback.c b/lib/crypto/c_src/crypto_callback.c
index a08dcec463..b4c175ae43 100644
--- a/lib/crypto/c_src/crypto_callback.c
+++ b/lib/crypto/c_src/crypto_callback.c
@@ -107,8 +107,6 @@ static INLINE void locking(int mode, ErlNifRWLock* lock)
 
 static void locking_function(int mode, int n, const char *file, int line)
 {
-    ASSERT(n>=0 && n<CRYPTO_num_locks());
-
     locking(mode, lock_vec[n]);
 }