Ensure NIF term creation disallows illegal values

Add a check to enif_make_double to see if its double argument is
infinity or NaN, returning a badarg exception if it is. Change the
erl_nif documentation to specify that enif_make_double returns a
badarg exception if its double argument is either infinity or NaN. Add
tests to nif_SUITE for this change.

Add checks to the enif_make* functions for atoms to prevent the
creation of atoms whose name lengths are greater than the allowed
maximum atom length. The enif_make_atom and enif_make_atom_len
functions now return a badarg exception if the input string is too
long. The enif_make_existing_atom and enif_make_existing_atom_len
functions return false if the input string is too long. Change the
erl_nif documentation to reflect the changes to these functions. Add
tests to nif_SUITE for these changes.

Add a field to ErlNifEnv to track that a NIF has raised an exception
via enif_make_badarg. If a NIF calls enif_make_badarg but then ignores
its return value and instead tries to return a non-exception term as
its return value, the runtime still raises a badarg. This is needed to
prevent enif_make_badarg values resulting from calls to
enif_make_double, enif_make_atom, or enif_make_atom_len from being
erroneously stored within other terms and returned from a NIF. Calling
enif_make_badarg but not returning its return value has been
documented as being illegal ever since enif_make_badarg was added, but
the runtime has not enforced it until now. Add tests for regular and
dirty NIFs to ensure that calls to enif_make_badarg result in badarg
exceptions even if a NIF fails to return the result of
enif_make_badarg as its return value. Add documentation to
enif_make_badarg to specify that calling it raises a badarg even if a
NIF ignores its return value.

1 files changed, 12 insertions, 4 deletions

diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index adc3520ebb..ec82ef251e 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -127,6 +127,7 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif)
     env->heap_frag = NULL;
     env->fpe_was_unmasked = erts_block_fpe();
     env->tmp_obj_list = NULL;
+    env->exception_thrown = 0;
 }
 
 static void pre_nif_noproc(ErlNifEnv* env, struct erl_module_nif* mod_nif)
@@ -742,6 +743,7 @@ Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term,
 
 Eterm enif_make_badarg(ErlNifEnv* env)
 {
+    env->exception_thrown = 1;
     BIF_ERROR(env->proc, BADARG);
 }
 
@@ -964,7 +966,10 @@ ERL_NIF_TERM enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i)
 
 ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d)
 {
-    Eterm* hp = alloc_heap(env,FLOAT_SIZE_OBJECT);
+    Eterm* hp;
+    if (!isfinite(d))
+        return enif_make_badarg(env);
+    hp = alloc_heap(env,FLOAT_SIZE_OBJECT);
     FloatDef f;
     f.fd = d;
     PUT_DOUBLE(f, hp);
@@ -978,6 +983,8 @@ ERL_NIF_TERM enif_make_atom(ErlNifEnv* env, const char* name)
 
 ERL_NIF_TERM enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len)
 {
+    if (len > MAX_ATOM_CHARACTERS)
+        return enif_make_badarg(env);
     return erts_atom_put((byte*)name, len, ERTS_ATOM_ENC_LATIN1, 1);
 }
 
@@ -991,6 +998,8 @@ int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len,
 				ERL_NIF_TERM* atom, ErlNifCharEncoding encoding)
 {
     ASSERT(encoding == ERL_NIF_LATIN1);
+    if (len > MAX_ATOM_CHARACTERS)
+        return 0;
     return erts_atom_get(name, len, atom, ERTS_ATOM_ENC_LATIN1);
 }
 
@@ -1754,14 +1763,13 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     ASSERT(ep);
     if (ep->fp)
 	fp = NULL;
-    if (is_non_value(result)) {
+    if (is_non_value(result) || env->exception_thrown) {
 	if (proc->freason != TRAP) {
-	    ASSERT(proc->freason == BADARG);
 	    return init_nif_sched_data(env, dirty_nif_exception, fp, 0, argc, argv);
 	} else {
 	    if (ep->fp == NULL)
 		restore_nif_mfa(proc, ep, 1);
-	    return result;
+	    return THE_NON_VALUE;
 	}
     }
     else