Merge branch 'sverk/pr632/prevent-illegal-nif-terms/OTP-12655'

* sverk/pr632/prevent-illegal-nif-terms/OTP-12655:
  erts: Reject non-finite float terms in erl_drv_output_term
  erts: Remove old docs about experimental NIF versions.
  erts: Add enif_has_pending_exception
  erts: Clearify erl_nif documentation about badarg exception
  erts: Fix compile warning in enif_make_double
  erts: Fix divide by zero compile error in nif_SUITE.c
  erts: Fix isfinite for windows
  Ensure NIF term creation disallows illegal values

11 files changed, 209 insertions, 17 deletions

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index bb7b799950..fce4fd498a 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3516,6 +3516,8 @@ do {								\
 		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]);
 		reg[0] = r(0);
 		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
+		if (env.exception_thrown)
+		    nif_bif_result = THE_NON_VALUE;
 		erts_post_nif(&env);
 	    }
 	    ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result));
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 776bbf6719..25caaa4e44 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -128,6 +128,7 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif)
     env->heap_frag = NULL;
     env->fpe_was_unmasked = erts_block_fpe();
     env->tmp_obj_list = NULL;
+    env->exception_thrown = 0;
 }
 
 static void pre_nif_noproc(ErlNifEnv* env, struct erl_module_nif* mod_nif)
@@ -736,9 +737,15 @@ Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term,
 
 Eterm enif_make_badarg(ErlNifEnv* env)
 {
+    env->exception_thrown = 1;
     BIF_ERROR(env->proc, BADARG);
 }
 
+int enif_has_pending_exception(ErlNifEnv* env)
+{
+    return env->exception_thrown;
+}
+
 int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len,
 		  ErlNifCharEncoding encoding)
 {
@@ -958,8 +965,12 @@ ERL_NIF_TERM enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i)
 
 ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d)
 {
-    Eterm* hp = alloc_heap(env,FLOAT_SIZE_OBJECT);
+    Eterm* hp;
     FloatDef f;
+
+    if (!erts_isfinite(d))
+        return enif_make_badarg(env);
+    hp = alloc_heap(env,FLOAT_SIZE_OBJECT);
     f.fd = d;
     PUT_DOUBLE(f, hp);
     return make_float(hp);
@@ -972,6 +983,8 @@ ERL_NIF_TERM enif_make_atom(ErlNifEnv* env, const char* name)
 
 ERL_NIF_TERM enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len)
 {
+    if (len > MAX_ATOM_CHARACTERS)
+        return enif_make_badarg(env);
     return erts_atom_put((byte*)name, len, ERTS_ATOM_ENC_LATIN1, 1);
 }
 
@@ -985,6 +998,8 @@ int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len,
 				ERL_NIF_TERM* atom, ErlNifCharEncoding encoding)
 {
     ASSERT(encoding == ERL_NIF_LATIN1);
+    if (len > MAX_ATOM_CHARACTERS)
+        return 0;
     return erts_atom_get(name, len, atom, ERTS_ATOM_ENC_LATIN1);
 }
 
@@ -1748,14 +1763,13 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     ASSERT(ep);
     if (ep->fp)
 	fp = NULL;
-    if (is_non_value(result)) {
+    if (is_non_value(result) || env->exception_thrown) {
 	if (proc->freason != TRAP) {
-	    ASSERT(proc->freason == BADARG);
 	    return init_nif_sched_data(env, dirty_nif_exception, fp, 0, argc, argv);
 	} else {
 	    if (ep->fp == NULL)
 		restore_nif_mfa(proc, ep, 1);
-	    return result;
+	    return THE_NON_VALUE;
 	}
     }
     else
diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h
index 9b2b90c82d..c4fdfd4187 100644
--- a/erts/emulator/beam/erl_nif.h
+++ b/erts/emulator/beam/erl_nif.h
@@ -46,9 +46,10 @@
 **           remove enif_schedule_dirty_nif, enif_schedule_dirty_nif_finalizer, enif_dirty_nif_finalizer
 **           add ErlNifEntry options
 **           add ErlNifFunc flags
+** 2.8: 18.0 add enif_has_pending_exception
 */
 #define ERL_NIF_MAJOR_VERSION 2
-#define ERL_NIF_MINOR_VERSION 7
+#define ERL_NIF_MINOR_VERSION 8
 
 /*
  * The emulator will refuse to load a nif-lib with a major version
diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h
index 630cefae93..bdcbb32c46 100644
--- a/erts/emulator/beam/erl_nif_api_funcs.h
+++ b/erts/emulator/beam/erl_nif_api_funcs.h
@@ -156,6 +156,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_next, (ErlNifEnv *env, ErlNifMapIte
 ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_prev, (ErlNifEnv *env, ErlNifMapIterator *iter));
 ERL_NIF_API_FUNC_DECL(int, enif_map_iterator_get_pair, (ErlNifEnv *env, ErlNifMapIterator *iter, ERL_NIF_TERM *key, ERL_NIF_TERM *value));
 ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_schedule_nif,(ErlNifEnv*,const char*,int,ERL_NIF_TERM (*)(ErlNifEnv*,int,const ERL_NIF_TERM[]),int,const ERL_NIF_TERM[]));
+ERL_NIF_API_FUNC_DECL(int, enif_has_pending_exception, (ErlNifEnv *env));
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment) !!!
@@ -305,6 +306,7 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*));
 #  define enif_map_iterator_prev ERL_NIF_API_FUNC_MACRO(enif_map_iterator_prev)
 #  define enif_map_iterator_get_pair ERL_NIF_API_FUNC_MACRO(enif_map_iterator_get_pair)
 #  define enif_schedule_nif ERL_NIF_API_FUNC_MACRO(enif_schedule_nif)
+#  define enif_has_pending_exception ERL_NIF_API_FUNC_MACRO(enif_has_pending_exception)
 
 /*
 ** ADD NEW ENTRIES HERE (before this comment)
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 634fe533d0..40b043d1cc 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -52,6 +52,7 @@ struct enif_environment_t /* ErlNifEnv */
     ErlHeapFragment* heap_frag;
     int fpe_was_unmasked;
     struct enif_tmp_obj_t* tmp_obj_list;
+    int exception_thrown; /* boolean */
 };
 extern void erts_pre_nif(struct enif_environment_t*, Process*,
 			 struct erl_module_nif*);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 8cb185cb2b..dec92be40a 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -5559,7 +5559,9 @@ driver_deliver_term(Eterm to, ErlDrvTermData* data, int len)
 
 	    mess = make_float(hp);
 	    f.fd = *((double *) ptr[0]);
-	    PUT_DOUBLE(f, hp);
+            if (!erts_isfinite(f.fd))
+                ERTS_DDT_FAIL;
+            PUT_DOUBLE(f, hp);
 	    hp += FLOAT_SIZE_OBJECT;
 	    ptr++;
 	    break;
diff --git a/erts/emulator/sys/ose/erl_ose_sys.h b/erts/emulator/sys/ose/erl_ose_sys.h
index cd66d95c26..f6526a4714 100644
--- a/erts/emulator/sys/ose/erl_ose_sys.h
+++ b/erts/emulator/sys/ose/erl_ose_sys.h
@@ -112,6 +112,8 @@ extern clock_t sys_times(SysTimes *buffer);
 /* No use in having other resolutions than 1 Ms. */
 #define SYS_CLOCK_RESOLUTION 1
 
+#define erts_isfinite finite
+
 #ifdef NO_FPE_SIGNALS
 
 #define erts_get_current_fp_exception() NULL
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index 8fc5a3ca49..94adcc00c8 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -338,6 +338,8 @@ extern void sys_stop_cat(void);
 #  define HAVE_ISFINITE
 #endif
 
+#define erts_isfinite isfinite
+
 #ifdef NO_FPE_SIGNALS
 
 #define erts_get_current_fp_exception() NULL
diff --git a/erts/emulator/sys/win32/erl_win_sys.h b/erts/emulator/sys/win32/erl_win_sys.h
index 5181d6b584..714e7357d4 100644
--- a/erts/emulator/sys/win32/erl_win_sys.h
+++ b/erts/emulator/sys/win32/erl_win_sys.h
@@ -267,6 +267,8 @@ extern volatile int erl_fp_exception;
 int _finite(double x);
 #endif
 
+#define erts_isfinite _finite
+
 /*#define NO_FPE_SIGNALS*/
 #define erts_get_current_fp_exception() NULL
 #define __ERTS_FP_CHECK_INIT(fpexnp) do {} while (0)
diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl
index b0624fb8c1..502ada95a1 100644
--- a/erts/emulator/test/nif_SUITE.erl
+++ b/erts/emulator/test/nif_SUITE.erl
@@ -39,7 +39,8 @@
 	 get_length/1, make_atom/1, make_string/1, reverse_list_test/1,
 	 otp_9828/1,
 	 otp_9668/1, consume_timeslice/1, dirty_nif/1, dirty_nif_send/1,
-	 dirty_nif_exception/1, nif_schedule/1
+	 dirty_nif_exception/1, nif_schedule/1,
+	 nif_exception/1, nif_nan_and_inf/1, nif_atom_too_long/1
 	]).
 
 -export([many_args_100/100]).
@@ -68,7 +69,8 @@ all() ->
      make_string,reverse_list_test,
      otp_9828,
      otp_9668, consume_timeslice,
-     nif_schedule, dirty_nif, dirty_nif_send, dirty_nif_exception
+     nif_schedule, dirty_nif, dirty_nif_send, dirty_nif_exception,
+     nif_exception, nif_nan_and_inf, nif_atom_too_long
     ].
 
 groups() -> 
@@ -1595,11 +1597,27 @@ dirty_nif_exception(Config) when is_list(Config) ->
 	N when is_integer(N) ->
 	    ensure_lib_loaded(Config),
 	    try
-	        call_dirty_nif_exception(),
+		%% this checks that the expected exception
+		%% occurs when the NIF returns the result
+		%% of enif_make_badarg directly
+	        call_dirty_nif_exception(1),
 	        ?t:fail(expected_badarg)
 	    catch
 	        error:badarg ->
-		    [{?MODULE,call_dirty_nif_exception,[],_}|_] =
+		    [{?MODULE,call_dirty_nif_exception,[1],_}|_] =
+			erlang:get_stacktrace(),
+		    ok
+	    end,
+	    try
+		%% this checks that the expected exception
+		%% occurs when the NIF calls enif_make_badarg
+		%% at some point but then returns a value that
+		%% isn't an exception
+	        call_dirty_nif_exception(0),
+	        ?t:fail(expected_badarg)
+	    catch
+	        error:badarg ->
+		    [{?MODULE,call_dirty_nif_exception,[0],_}|_] =
 			erlang:get_stacktrace(),
 		    ok
 	    end
@@ -1608,6 +1626,57 @@ dirty_nif_exception(Config) when is_list(Config) ->
 	    {skipped,"No dirty scheduler support"}
     end.
 
+nif_exception(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+    try
+	call_nif_exception(),
+	?t:fail(expected_badarg)
+    catch
+	error:badarg ->
+	    ok
+    end.
+
+nif_nan_and_inf(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+    try
+	call_nif_nan_or_inf(nan),
+	?t:fail(expected_badarg)
+    catch
+	error:badarg ->
+	    ok
+    end,
+    try
+	call_nif_nan_or_inf(inf),
+	?t:fail(expected_badarg)
+    catch
+	error:badarg ->
+	    ok
+    end,
+    try
+	call_nif_nan_or_inf(tuple),
+	?t:fail(expected_badarg)
+    catch
+	error:badarg ->
+	    ok
+    end.
+
+nif_atom_too_long(Config) when is_list(Config) ->
+    ensure_lib_loaded(Config),
+    try
+	call_nif_atom_too_long(all),
+	?t:fail(expected_badarg)
+    catch
+	error:badarg ->
+	    ok
+    end,
+    try
+	call_nif_atom_too_long(len),
+	?t:fail(expected_badarg)
+    catch
+	error:badarg ->
+	    ok
+    end.
+
 next_msg(_Pid) ->
     receive
 	M -> M
@@ -1741,8 +1810,11 @@ consume_timeslice_nif(_,_) -> ?nif_stub.
 call_nif_schedule(_,_) -> ?nif_stub.
 call_dirty_nif(_,_,_) -> ?nif_stub.
 send_from_dirty_nif(_) -> ?nif_stub.
-call_dirty_nif_exception() -> ?nif_stub.
+call_dirty_nif_exception(_) -> ?nif_stub.
 call_dirty_nif_zero_args() -> ?nif_stub.
+call_nif_exception() -> ?nif_stub.
+call_nif_nan_or_inf(_) -> ?nif_stub.
+call_nif_atom_too_long(_) -> ?nif_stub.
 
 %% maps
 is_map_nif(_) -> ?nif_stub.
diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
index 5a3be84825..3cc9f51ef8 100644
--- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
+++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c
@@ -380,7 +380,8 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
     ErlNifSInt64 sint64;
     ErlNifUInt64 uint64;
     double d;
-    ERL_NIF_TERM atom, ref1, ref2;
+    ERL_NIF_TERM atom, ref1, ref2, term;
+    size_t len;
 
     sint = INT_MIN;
     do {
@@ -502,6 +503,7 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
 	    goto error;
 	}
     }
+
     ref1 = enif_make_ref(env);
     ref2 = enif_make_ref(env);
     if (!enif_is_ref(env,ref1) || !enif_is_ref(env,ref2) 
@@ -890,6 +892,7 @@ static ERL_NIF_TERM check_is_exception(ErlNifEnv* env, int argc, const ERL_NIF_T
     ERL_NIF_TERM badarg = enif_make_badarg(env);
     if (enif_is_exception(env, error_atom)) return error_atom;
     if (!enif_is_exception(env, badarg)) return error_atom;
+    if (!enif_has_pending_exception(env)) return error_atom;
     return badarg;
 }
 
@@ -1608,16 +1611,26 @@ static ERL_NIF_TERM send_from_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_
 static ERL_NIF_TERM call_dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     switch (argc) {
-    case 0: {
+    case 1: {
 	ERL_NIF_TERM args[255];
 	int i;
-	for (i = 0; i < 255; i++)
+	args[0] = argv[0];
+	for (i = 1; i < 255; i++)
 	    args[i] = enif_make_int(env, i);
 	return enif_schedule_nif(env, "call_dirty_nif_exception", ERL_NIF_DIRTY_JOB_CPU_BOUND,
 				 call_dirty_nif_exception, 255, argv);
     }
-    case 1:
-	return enif_make_badarg(env);
+    case 2: {
+        int return_badarg_directly;
+        enif_get_int(env, argv[0], &return_badarg_directly);
+        assert(return_badarg_directly == 1 || return_badarg_directly == 0);
+        if (return_badarg_directly)
+            return enif_make_badarg(env);
+        else {
+            /* ignore return value */ enif_make_badarg(env);
+            return enif_make_atom(env, "ok");
+        }
+    }
     default:
 	return enif_schedule_nif(env, "call_dirty_nif_exception", ERL_NIF_DIRTY_JOB_CPU_BOUND,
 				 call_dirty_nif_exception, argc-1, argv);
@@ -1637,6 +1650,82 @@ static ERL_NIF_TERM call_dirty_nif_zero_args(ErlNifEnv* env, int argc, const ERL
 }
 #endif
 
+/*
+ * Call enif_make_badarg, but don't return its return value. Instead,
+ * return ok.  Result should still be a badarg exception for the erlang
+ * caller.
+ */
+static ERL_NIF_TERM call_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    /* ignore return value */ enif_make_badarg(env);
+    return enif_make_atom(env, "ok");
+}
+
+#if !defined(NAN) || !defined(INFINITY)
+double zero(void)
+{
+    return 0.0;
+}
+#endif
+
+static ERL_NIF_TERM call_nif_nan_or_inf(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    double val;
+    char arg[6];
+    ERL_NIF_TERM res;
+
+    assert(argc == 1);
+    enif_get_atom(env, argv[0], arg, sizeof arg, ERL_NIF_LATIN1);
+    if (strcmp(arg, "nan") == 0) {
+        /* Verify that enif_make_double raises a badarg for NaN */
+#ifdef NAN
+        val = NAN;
+#else
+        val = 0.0/zero();
+#endif
+    } else {
+        /* Verify that enif_make_double raises a badarg for NaN and infinity */
+#ifdef INFINITY
+        val = INFINITY;
+#else
+        val = 1.0/zero();
+#endif
+    }
+    res = enif_make_double(env, val);
+    assert(enif_is_exception(env, res));
+    assert(enif_has_pending_exception(env));
+    if (strcmp(arg, "tuple") == 0) {
+        return enif_make_tuple2(env, argv[0], res);
+    } else {
+        return res;
+    }
+}
+
+static ERL_NIF_TERM call_nif_atom_too_long(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+{
+    char str[257];
+    char arg[4];
+    size_t len;
+    int i;
+    ERL_NIF_TERM res;
+
+    assert(argc == 1);
+    enif_get_atom(env, argv[0], arg, sizeof arg, ERL_NIF_LATIN1);
+    /* Verify that creating an atom from a string that's too long results in a badarg */
+    for (i = 0; i < sizeof str; ++i) {
+        str[i] = 'a';
+    }
+    str[256] = '\0';
+    if (strcmp(arg, "len") == 0) {
+        len = strlen(str);
+        res = enif_make_atom_len(env, str, len);
+    } else {
+        res = enif_make_atom(env, str);
+    }
+    assert(enif_is_exception(env, res));
+    return res;
+}
+
 static ERL_NIF_TERM is_map_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
     return enif_make_int(env, enif_is_map(env,argv[0]));
@@ -1821,9 +1910,12 @@ static ErlNifFunc nif_funcs[] =
 #ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT
     {"call_dirty_nif", 3, call_dirty_nif},
     {"send_from_dirty_nif", 1, send_from_dirty_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND},
-    {"call_dirty_nif_exception", 0, call_dirty_nif_exception, ERL_NIF_DIRTY_JOB_IO_BOUND},
+    {"call_dirty_nif_exception", 1, call_dirty_nif_exception, ERL_NIF_DIRTY_JOB_IO_BOUND},
     {"call_dirty_nif_zero_args", 0, call_dirty_nif_zero_args, ERL_NIF_DIRTY_JOB_CPU_BOUND},
 #endif
+    {"call_nif_exception", 0, call_nif_exception},
+    {"call_nif_nan_or_inf", 1, call_nif_nan_or_inf},
+    {"call_nif_atom_too_long", 1, call_nif_atom_too_long},
     {"is_map_nif", 1, is_map_nif},
     {"get_map_size_nif", 1, get_map_size_nif},
     {"make_new_map_nif", 0, make_new_map_nif},