Ensure NIF term creation disallows illegal values

Add a check to enif_make_double to see if its double argument is
infinity or NaN, returning a badarg exception if it is. Change the
erl_nif documentation to specify that enif_make_double returns a
badarg exception if its double argument is either infinity or NaN. Add
tests to nif_SUITE for this change.

Add checks to the enif_make* functions for atoms to prevent the
creation of atoms whose name lengths are greater than the allowed
maximum atom length. The enif_make_atom and enif_make_atom_len
functions now return a badarg exception if the input string is too
long. The enif_make_existing_atom and enif_make_existing_atom_len
functions return false if the input string is too long. Change the
erl_nif documentation to reflect the changes to these functions. Add
tests to nif_SUITE for these changes.

Add a field to ErlNifEnv to track that a NIF has raised an exception
via enif_make_badarg. If a NIF calls enif_make_badarg but then ignores
its return value and instead tries to return a non-exception term as
its return value, the runtime still raises a badarg. This is needed to
prevent enif_make_badarg values resulting from calls to
enif_make_double, enif_make_atom, or enif_make_atom_len from being
erroneously stored within other terms and returned from a NIF. Calling
enif_make_badarg but not returning its return value has been
documented as being illegal ever since enif_make_badarg was added, but
the runtime has not enforced it until now. Add tests for regular and
dirty NIFs to ensure that calls to enif_make_badarg result in badarg
exceptions even if a NIF fails to return the result of
enif_make_badarg as its return value. Add documentation to
enif_make_badarg to specify that calling it raises a badarg even if a
NIF ignores its return value.

1 files changed, 77 insertions, 5 deletions

diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index 4560077a51..c11a6ad7c5 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -39,7 +39,8 @@
 	 get_length/1, make_atom/1, make_string/1, reverse_list_test/1,
 	 otp_9828/1,
 	 otp_9668/1, consume_timeslice/1, dirty_nif/1, dirty_nif_send/1,
-	 dirty_nif_exception/1, nif_schedule/1
+	 dirty_nif_exception/1, nif_schedule/1,
+	 nif_exception/1, nif_nan_and_inf/1, nif_atom_too_long/1
 	]).
 
 -export([many_args_100/100]).
@@ -68,7 +69,8 @@ all() ->
      make_string,reverse_list_test,
      otp_9828,
      otp_9668, consume_timeslice,
-     nif_schedule, dirty_nif, dirty_nif_send, dirty_nif_exception
+     nif_schedule, dirty_nif, dirty_nif_send, dirty_nif_exception,
+     nif_exception, nif_nan_and_inf, nif_atom_too_long
     ].
 
 groups() -> 
@@ -1595,11 +1597,27 @@ dirty_nif_exception(Config) when is_list(Config) ->
 	N when is_integer(N) ->
 	    ensure_lib_loaded(Config),
 	    try
-	        call_dirty_nif_exception(),
+		%% this checks that the expected exception
+		%% occurs when the NIF returns the result
+		%% of enif_make_badarg directly
+	        call_dirty_nif_exception(1),
 	        ?t:fail(expected_badarg)
 	    catch
 	        error:badarg ->
-		    [{?MODULE,call_dirty_nif_exception,[],_}|_] =
+		    [{?MODULE,call_dirty_nif_exception,[1],_}|_] =
+			erlang:get_stacktrace(),
+		    ok
+	    end,
+	    try
+		%% this checks that the expected exception
+		%% occurs when the NIF calls enif_make_badarg
+		%% at some point but then returns a value that
+		%% isn't an exception
+	        call_dirty_nif_exception(0),
+	        ?t:fail(expected_badarg)
+	    catch
+	        error:badarg ->
+		    [{?MODULE,call_dirty_nif_exception,[0],_}|_] =
 			erlang:get_stacktrace(),
 		    ok
 	    end
@@ -1608,6 +1626,57 @@ dirty_nif_exception(Config) when is_list(Config) ->
 	    {skipped,"No dirty scheduler support"}
     end.
 
+nif_exception(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+    try
+	call_nif_exception(),
+	?t:fail(expected_badarg)
+    catch
+	error:badarg ->
+	    ok
+    end.
+
+nif_nan_and_inf(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+    try
+	call_nif_nan_or_inf(nan),
+	?t:fail(expected_badarg)
+    catch
+	error:badarg ->
+	    ok
+    end,
+    try
+	call_nif_nan_or_inf(inf),
+	?t:fail(expected_badarg)
+    catch
+	error:badarg ->
+	    ok
+    end,
+    try
+	call_nif_nan_or_inf(tuple),
+	?t:fail(expected_badarg)
+    catch
+	error:badarg ->
+	    ok
+    end.
+
+nif_atom_too_long(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+    try
+	call_nif_atom_too_long(all),
+	?t:fail(expected_badarg)
+    catch
+	error:badarg ->
+	    ok
+    end,
+    try
+	call_nif_atom_too_long(len),
+	?t:fail(expected_badarg)
+    catch
+	error:badarg ->
+	    ok
+    end.
+
 next_msg(_Pid) ->
     receive
 	M -> M
@@ -1741,8 +1810,11 @@ consume_timeslice_nif(_,_) -> ?nif_stub.
 call_nif_schedule(_,_) -> ?nif_stub.
 call_dirty_nif(_,_,_) -> ?nif_stub.
 send_from_dirty_nif(_) -> ?nif_stub.
-call_dirty_nif_exception() -> ?nif_stub.
+call_dirty_nif_exception(_) -> ?nif_stub.
 call_dirty_nif_zero_args() -> ?nif_stub.
+call_nif_exception() -> ?nif_stub.
+call_nif_nan_or_inf(_) -> ?nif_stub.
+call_nif_atom_too_long(_) -> ?nif_stub.
 
 %% maps
 is_map_nif(_) -> ?nif_stub.