Do not accept Nan and Infinity values

Erlang does not accept these values, so we return an error in the C
interface rather than letting them through to the Erlang VM, which
rejects the message with a somewhat cryptic "bad external term".

2 files changed, 22 insertions, 0 deletions

diff --git a/lib/erl_interface/src/encode/encode_double.c b/lib/erl_interface/src/encode/encode_double.c
index 148a49f73a..a19a871eda 100644
--- a/lib/erl_interface/src/encode/encode_double.c
+++ b/lib/erl_interface/src/encode/encode_double.c
@@ -18,6 +18,9 @@
  */
 #include <stdio.h>
 #include <string.h>
+#if defined(HAVE_ISFINITE)
+#include <math.h>
+#endif
 #include "eidef.h"
 #include "eiext.h"
 #include "putget.h"
@@ -27,6 +30,15 @@ int ei_encode_double(char *buf, int *index, double p)
   char *s = buf + *index;
   char *s0 = s;
 
+  /* Erlang does not handle Inf and NaN, so we return an error rather
+   * than letting the Erlang VM complain about a bad external
+   * term. */
+#if defined(HAVE_ISFINITE)
+  if(!isfinite(p)) {
+      return -1;
+  }
+#endif
+
   if (!buf)
     s += 9;
   else {
diff --git a/lib/erl_interface/src/legacy/erl_eterm.c b/lib/erl_interface/src/legacy/erl_eterm.c
index 636d26b24b..c167fd1f78 100644
--- a/lib/erl_interface/src/legacy/erl_eterm.c
+++ b/lib/erl_interface/src/legacy/erl_eterm.c
@@ -26,6 +26,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <ctype.h>
+#include <math.h>
 
 #include "ei_locking.h"
 #include "ei_resolve.h"
@@ -125,6 +126,15 @@ ETERM *erl_mk_float (double d)
 {
     ETERM *ep;
 
+    /* Erlang does not handle Inf and NaN, so we return an error
+     * rather than letting the Erlang VM complain about a bad external
+     * term. */
+    switch(fpclassify(d)) {
+        case FP_NAN:
+        case FP_INFINITE:
+            return NULL;
+    }
+
     ep = erl_alloc_eterm(ERL_FLOAT);
     ERL_COUNT(ep) = 1;
     ERL_FLOAT_VALUE(ep) = d;