Fix NIF stack recursion bug and enforce a limit

Fix recursion bug when decoding Constructed value within another
value - here the allowed buffer for the recursed decode shall
only be the size of the enclosing value, not the whole buffer.

Return ASN1_ERROR if BER decode recurses more than about 8 kWords.

2 files changed, 50 insertions, 5 deletions

diff --git a/lib/asn1/c_src/asn1_erl_nif.c b/lib/asn1/c_src/asn1_erl_nif.c
index d5aaadb89b..797be6d4f8 100644
--- a/lib/asn1/c_src/asn1_erl_nif.c
+++ b/lib/asn1/c_src/asn1_erl_nif.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2017. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2018. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -949,6 +949,12 @@ static int ber_decode_value(ErlNifEnv* env, ERL_NIF_TERM *value, unsigned char *
     unsigned char *tmp_out_buff;
     ERL_NIF_TERM term = 0, curr_head = 0;
 
+    /* Recursion depth limitation, borrow a signed int: maybe_ret */
+    maybe_ret = (int) (ErlNifSInt) ((char *)value - (char *)ib_index);
+    maybe_ret = maybe_ret < 0 ? -maybe_ret : maybe_ret;
+    if (maybe_ret >= sizeof(void *) * 8192) /* 8 k pointer words */
+        return ASN1_ERROR;
+
     if (((in_buf[*ib_index]) & 0x80) == ASN1_SHORT_DEFINITE_LENGTH) {
 	len = in_buf[*ib_index];
     } else if (in_buf[*ib_index] == ASN1_INDEFINITE_LENGTH) {
@@ -993,7 +999,7 @@ static int ber_decode_value(ErlNifEnv* env, ERL_NIF_TERM *value, unsigned char *
 	while (*ib_index < end_index) {
 
 	    if ((maybe_ret = ber_decode(env, &term, in_buf, ib_index,
-		    in_buf_len)) <= ASN1_ERROR
+                   *ib_index + len)) <= ASN1_ERROR
 	    )
 		return maybe_ret;
 	    curr_head = enif_make_list_cell(env, term, curr_head);
diff --git a/lib/asn1/test/asn1_SUITE.erl b/lib/asn1/test/asn1_SUITE.erl
index bfeffa969f..31b41a4ba6 100644
--- a/lib/asn1/test/asn1_SUITE.erl
+++ b/lib/asn1/test/asn1_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2001-2017. All Rights Reserved.
+%% Copyright Ericsson AB 2001-2018. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -132,7 +132,8 @@ groups() ->
        testSeq2738,
        % Uses 'Constructed'
        {group, [], [constructed,
-                    ber_decode_error]},
+                    ber_decode_error,
+                    otp_14440]},
        testSeqSetIndefinite,
        testChoiceIndefinite,
        per_open_type,
@@ -737,6 +738,36 @@ ber_decode_error(Config, Rule, Opts) ->
     asn1_test_lib:compile("Constructed", Config, [Rule|Opts]),
     ber_decode_error:run(Opts).
 
+otp_14440(_Config) ->
+    Args = " -pa \"" ++ filename:dirname(code:which(?MODULE)) ++ "\"",
+    {ok,N} = slave:start(hostname(), otp_14440, Args),
+    Result = rpc:call(N, ?MODULE, otp_14440_decode, []),
+    io:format("Decode result = ~p~n", [Result]),
+    case Result of
+        {exit,{error,{asn1,{invalid_value,5}}}} ->
+            ok = slave:stop(N);
+        %% We get this if stack depth limit kicks in:
+        {exit,{error,{asn1,{unknown,_}}}} ->
+            ok = slave:stop(N);
+        _ ->
+            _ = slave:stop(N),
+            ?t:fail(Result)
+    end.
+%%
+otp_14440_decode() ->
+    Data =
+        iolist_to_binary(
+          lists:duplicate(
+            32, list_to_binary(lists:duplicate(1024, 16#7f)))),
+    try asn1rt_nif:decode_ber_tlv(Data) of
+        Result ->
+            {unexpected_return,Result}
+    catch
+        Class:Reason ->
+            {Class,Reason}
+    end.
+
+
 h323test(Config) -> test(Config, fun h323test/3).
 h323test(Config, Rule, Opts) ->
     Files = ["H235-SECURITY-MESSAGES", "H323-MESSAGES",
@@ -1351,7 +1382,7 @@ xref_export_all(_Config) ->
     {ok,_} = xref:q(S, Def),
     {ok,Unused} = xref:q(S, "X - Called - range (closure E | Called)"),
     xref:stop(S),
-    case Unused of
+    case Unused -- [{?MODULE,otp_14440_decode,0}] of
         [] ->
             ok;
         [_|_] ->
@@ -1386,3 +1417,11 @@ all_called_1([F|T]) when is_atom(F) ->
     L ++ all_called_1(T);
 all_called_1([]) ->
     [].
+
+hostname() ->
+    hostname(atom_to_list(node())).
+
+hostname([$@ | Hostname]) ->
+    list_to_atom(Hostname);
+hostname([_C | Cs]) ->
+    hostname(Cs).