crypto: Refactor mod_exp_nif

As a preparation for the new mod_exp_prime and in the quest to reduce
to use of the old mpint format.

2 files changed, 60 insertions, 43 deletions

diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c
index e77e5fb8f0..8139b331bf 100644
--- a/lib/crypto/c_src/crypto.c
+++ b/lib/crypto/c_src/crypto.c
@@ -304,7 +304,7 @@ static ErlNifFunc nif_funcs[] = {
     {"rand_bytes", 3, rand_bytes_3},
     {"strong_rand_mpint_nif", 3, strong_rand_mpint_nif},
     {"rand_uniform_nif", 2, rand_uniform_nif},
-    {"mod_exp_nif", 3, mod_exp_nif},
+    {"mod_exp_nif", 4, mod_exp_nif},
     {"dss_verify", 4, dss_verify},
     {"rsa_verify_nif", 4, rsa_verify_nif},
     {"aes_cbc_crypt", 4, aes_cbc_crypt},
@@ -1543,16 +1543,19 @@ static ERL_NIF_TERM rand_uniform_nif(ErlNifEnv* env, int argc, const ERL_NIF_TER
 }
 
 static ERL_NIF_TERM mod_exp_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
-{/* (Base,Exponent,Modulo) */
+{/* (Base,Exponent,Modulo,bin_hdr) */
     BIGNUM *bn_base=NULL, *bn_exponent=NULL, *bn_modulo, *bn_result;
     BN_CTX *bn_ctx;
     unsigned char* ptr;
     unsigned dlen;      
+    unsigned bin_hdr; /* return type: 0=plain binary, 4: mpint */
+    unsigned extra_byte;
     ERL_NIF_TERM ret;
 
-    if (!get_bn_from_mpint(env, argv[0], &bn_base)
-	|| !get_bn_from_mpint(env, argv[1], &bn_exponent)
-	|| !get_bn_from_mpint(env, argv[2], &bn_modulo)) {
+    if (!get_bn_from_bin(env, argv[0], &bn_base)
+	|| !get_bn_from_bin(env, argv[1], &bn_exponent)
+	|| !get_bn_from_bin(env, argv[2], &bn_modulo)
+	|| !enif_get_uint(env,argv[3],&bin_hdr) || (bin_hdr & ~4)) {
 
 	if (bn_base) BN_free(bn_base);
 	if (bn_exponent) BN_free(bn_exponent);
@@ -1562,9 +1565,14 @@ static ERL_NIF_TERM mod_exp_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg
     bn_ctx = BN_CTX_new();
     BN_mod_exp(bn_result, bn_base, bn_exponent, bn_modulo, bn_ctx);
     dlen = BN_num_bytes(bn_result);
-    ptr = enif_make_new_binary(env, dlen+4, &ret);
-    put_int32(ptr, dlen);
-    BN_bn2bin(bn_result, ptr+4);
+    extra_byte = bin_hdr && BN_is_bit_set(bn_result, dlen*8-1);
+    ptr = enif_make_new_binary(env, bin_hdr+extra_byte+dlen, &ret);
+    if (bin_hdr) {	
+	put_int32(ptr, extra_byte+dlen);
+	ptr[4] = 0; /* extra zeroed byte to ensure a positive mpint */
+	ptr += bin_hdr + extra_byte;
+    }
+    BN_bn2bin(bn_result, ptr);
     BN_free(bn_result);
     BN_CTX_free(bn_ctx);
     BN_free(bn_modulo);
diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl
index 1328a95e87..f5c1cd2845 100644
--- a/lib/crypto/src/crypto.erl
+++ b/lib/crypto/src/crypto.erl
@@ -787,17 +787,15 @@ rand_uniform_nif(_From,_To) -> ?nif_stub.
 %%
 mod_exp(Base, Exponent, Modulo)
   when is_integer(Base), is_integer(Exponent), is_integer(Modulo) ->
-    erlint(mod_exp(mpint(Base), mpint(Exponent), mpint(Modulo)));
+    bin_to_int(mod_exp_nif(int_to_bin(Base), int_to_bin(Exponent), int_to_bin(Modulo), 0));
 
 mod_exp(Base, Exponent, Modulo) ->
-    case mod_exp_nif(Base,Exponent,Modulo) of
-	<<Len:32/integer, MSB, Rest/binary>> when MSB > 127 ->
-	    <<(Len + 1):32/integer, 0, MSB, Rest/binary>>;
-	Whatever ->
-	    Whatever
+    mod_exp_nif(mpint_to_bin(Base),mpint_to_bin(Exponent),mpint_to_bin(Modulo), 4).
+    
     end.
 
-mod_exp_nif(_Base,_Exp,_Mod) -> ?nif_stub.    
+
+mod_exp_nif(_Base,_Exp,_Mod,_bin_hdr) -> ?nif_stub.    
 
 %%
 %% DSS, RSA - verify
@@ -1071,43 +1069,54 @@ dh_compute_key_nif(_OthersPublicKey, _MyPrivateKey, _DHParameters) -> ?nif_stub.
 
 %% large integer in a binary with 32bit length
 %% MP representaion  (SSH2)
-mpint(X) when X < 0 ->
-    case X of
-	-1 ->
-	    <<0,0,0,1,16#ff>>;	    
-       _ ->
-	    mpint_neg(X,0,[])
-    end;
-mpint(X) ->
-    case X of 
-	0 ->
-	    <<0,0,0,0>>;
-	_ ->
-	    mpint_pos(X,0,[])
-    end.
+mpint(X) when X < 0 -> mpint_neg(X);
+mpint(X) -> mpint_pos(X).
 
 -define(UINT32(X),   X:32/unsigned-big-integer).
 
-mpint_neg(-1,I,Ds=[MSB|_]) ->
-    if MSB band 16#80 =/= 16#80 ->
-	    <<?UINT32((I+1)), (list_to_binary([255|Ds]))/binary>>;
-       true ->
-	    (<<?UINT32(I), (list_to_binary(Ds))/binary>>)
-    end;
-mpint_neg(X,I,Ds)  ->
-    mpint_neg(X bsr 8,I+1,[(X band 255)|Ds]).
+
+mpint_neg(X) ->
+    Bin = int_to_bin_neg(X, []),
+    Sz = byte_size(Bin),
+    <<?UINT32(Sz), Bin/binary>>.
     
-mpint_pos(0,I,Ds=[MSB|_]) ->
+mpint_pos(X) ->
+    Bin = int_to_bin_pos(X, []),
+    <<MSB,_/binary>> = Bin,
+    Sz = byte_size(Bin),
     if MSB band 16#80 == 16#80 ->
-	    <<?UINT32((I+1)), (list_to_binary([0|Ds]))/binary>>;
+	    <<?UINT32((Sz+1)), 0, Bin/binary>>;
        true ->
-	    (<<?UINT32(I), (list_to_binary(Ds))/binary>>)
-    end;
-mpint_pos(X,I,Ds) ->
-    mpint_pos(X bsr 8,I+1,[(X band 255)|Ds]).
+	    <<?UINT32(Sz), Bin/binary>>
+    end.
+
+int_to_bin(X) when X < 0 -> int_to_bin_neg(X, []);
+int_to_bin(X) -> int_to_bin_pos(X, []).
+
+int_to_bin_pos(X) when X >= 0 ->
+    int_to_bin_pos(X, []).
+
+int_to_bin_pos(0,Ds=[_|_]) ->
+    list_to_binary(Ds);
+int_to_bin_pos(X,Ds) ->
+    int_to_bin_pos(X bsr 8, [(X band 255)|Ds]).
+
+int_to_bin_neg(-1, Ds=[MSB|_]) when MSB >= 16#80 ->
+    list_to_binary(Ds);
+int_to_bin_neg(X,Ds) ->
+    int_to_bin_neg(X bsr 8, [(X band 255)|Ds]).
+
+
+bin_to_int(Bin) ->
+    Bits = bit_size(Bin),
+    <<Integer:Bits/integer>> = Bin,
+    Integer.
 
 %% int from integer in a binary with 32bit length
 erlint(<<MPIntSize:32/integer,MPIntValue/binary>>) ->
     Bits= MPIntSize * 8,
     <<Integer:Bits/integer>> = MPIntValue,
     Integer.
+
+mpint_to_bin(<<Len:32, Bin:Len/binary>>) ->
+    Bin.