compact IEEE 754 double encoding in external binary format for ei

Implement the compact IEEE 754 double encoding in external binary
format for ei. Encoding for ei now always produces the NEW_FLOAT_EXT
format. Decoding and term printing handle both the old ERL_FLOAT_EXT
encoding and the new NEW_FLOAT_EXT encoding. Legacy erl_interface code
also handles the new encoding, but still produces the ERL_FLOAT_EXT
encoding by default.

Also enable the DFLAG_NEW_FLOATS distribution flag.

Reduce the number of copies of the code for encoding and decoding
doubles throughout ei and erl_interface by instead calling the ei
encoding and decoding functions wherever possible.

Restore commented-out float tests in ei_decode_SUITE and
ei_encode_SUITE in lib/erl_interface/test. Modify them to make them
match the style of other tests in the same suites.

These changes are based on an ei float patch from Serge Aleynikov
originally submitted against R12B-2 in July 2008.

16 files changed, 103 insertions, 63 deletions

diff --git a/lib/erl_interface/include/ei.h b/lib/erl_interface/include/ei.h
index d1a697615a..03219c6d1b 100644
--- a/lib/erl_interface/include/ei.h
+++ b/lib/erl_interface/include/ei.h
@@ -110,6 +110,7 @@
 #define ERL_SMALL_INTEGER_EXT 'a'
 #define ERL_INTEGER_EXT       'b'
 #define ERL_FLOAT_EXT         'c'
+#define NEW_FLOAT_EXT         'F'
 #define ERL_ATOM_EXT          'd'
 #define ERL_REFERENCE_EXT     'e'
 #define ERL_NEW_REFERENCE_EXT 'r'
diff --git a/lib/erl_interface/src/connect/ei_connect.c b/lib/erl_interface/src/connect/ei_connect.c
index d2d0a7e7c1..df3c08f412 100644
--- a/lib/erl_interface/src/connect/ei_connect.c
+++ b/lib/erl_interface/src/connect/ei_connect.c
@@ -1323,7 +1323,8 @@ static int send_name_or_challenge(int fd, char *nodename,
     put32be(s, (DFLAG_EXTENDED_REFERENCES
 		| DFLAG_EXTENDED_PIDS_PORTS
 		| DFLAG_FUN_TAGS
-		| DFLAG_NEW_FUN_TAGS));
+		| DFLAG_NEW_FUN_TAGS
+                | DFLAG_NEW_FLOATS));
     if (f_chall)
 	put32be(s, challenge);
     memcpy(s, nodename, strlen(nodename));
@@ -1393,6 +1394,11 @@ static int recv_challenge(int fd, unsigned *challenge,
 	goto error;
     }
 	    
+    if (!(*flags & DFLAG_NEW_FLOATS)) {
+	EI_TRACE_ERR0("recv_challenge","<- RECV_CHALLENGE peer cannot "
+		      "handle binary float encoding");
+	goto error;
+    }
 
     if (getpeername(fd, (struct sockaddr *) &sin, &sin_len) < 0) {
 	EI_TRACE_ERR0("recv_challenge","<- RECV_CHALLENGE can't get peername");
diff --git a/lib/erl_interface/src/connect/ei_connect_int.h b/lib/erl_interface/src/connect/ei_connect_int.h
index 9926f799df..83ee18256f 100644
--- a/lib/erl_interface/src/connect/ei_connect_int.h
+++ b/lib/erl_interface/src/connect/ei_connect_int.h
@@ -101,6 +101,7 @@ extern int h_errno;
 #define DFLAG_FUN_TAGS            16
 #define DFLAG_NEW_FUN_TAGS        0x80
 #define DFLAG_EXTENDED_PIDS_PORTS 0x100
+#define DFLAG_NEW_FLOATS          0x800
 
 ei_cnode   *ei_fd_to_cnode(int fd);
 int         ei_distversion(int fd);
diff --git a/lib/erl_interface/src/decode/decode_double.c b/lib/erl_interface/src/decode/decode_double.c
index 66dbe474ec..2462032eaa 100644
--- a/lib/erl_interface/src/decode/decode_double.c
+++ b/lib/erl_interface/src/decode/decode_double.c
@@ -26,14 +26,22 @@ int ei_decode_double(const char *buf, int *index, double *p)
 {
   const char *s = buf + *index;
   const char *s0 = s;
-  double f;
+  FloatExt f;
 
-  if (get8(s) != ERL_FLOAT_EXT) return -1;
-  
-  if (sscanf(s, "%lf", &f) != 1) return -1;
+  switch (get8(s)) {
+    case ERL_FLOAT_EXT:
+      if (sscanf(s, "%lf", &f.d) != 1) return -1;
+      s += 31;
+      break;
+    case NEW_FLOAT_EXT:
+      /* IEEE 754 format */
+      f.val = get64be(s);
+      break;
+    default:
+      return -1;
+  }
 
-  s += 31;
-  if (p) *p = f;
+  if (p) *p = f.d;
   *index += s-s0; 
   return 0; 
 }
diff --git a/lib/erl_interface/src/decode/decode_skip.c b/lib/erl_interface/src/decode/decode_skip.c
index 316b5bee98..efe323d0d5 100644
--- a/lib/erl_interface/src/decode/decode_skip.c
+++ b/lib/erl_interface/src/decode/decode_skip.c
@@ -77,6 +77,7 @@ int ei_skip_term(const char* buf, int* index)
 	if (ei_decode_big(buf, index, NULL) < 0) return -1;
 	break;
     case ERL_FLOAT_EXT:
+    case NEW_FLOAT_EXT:
 	if (ei_decode_double(buf, index, NULL) < 0) return -1;
 	break;
     case ERL_FUN_EXT:
diff --git a/lib/erl_interface/src/encode/encode_double.c b/lib/erl_interface/src/encode/encode_double.c
index 53f3d52ba6..1c8cb286f5 100644
--- a/lib/erl_interface/src/encode/encode_double.c
+++ b/lib/erl_interface/src/encode/encode_double.c
@@ -27,13 +27,13 @@ int ei_encode_double(char *buf, int *index, double p)
   char *s = buf + *index;
   char *s0 = s;
 
-  if (!buf) s ++;
+  if (!buf)
+    s += 9;
   else {
-    put8(s,ERL_FLOAT_EXT);
-    memset(s, 0, 31);
-    sprintf(s, "%.20e", p);
+    /* IEEE 754 format */
+    put8(s, NEW_FLOAT_EXT);
+    put64be(s, ((FloatExt*)&p)->val);
   }
-  s += 31;
 
   *index += s-s0; 
 
diff --git a/lib/erl_interface/src/legacy/decode_term.c b/lib/erl_interface/src/legacy/decode_term.c
index ef29d6f57d..ddd1682f95 100644
--- a/lib/erl_interface/src/legacy/decode_term.c
+++ b/lib/erl_interface/src/legacy/decode_term.c
@@ -59,6 +59,7 @@ int ei_decode_term(const char *buf, int *index, void *t)
       return ei_decode_long(buf,index,NULL);
 
     case ERL_FLOAT_EXT:  
+    case NEW_FLOAT_EXT:
       return ei_decode_double(buf,index,NULL);
 
     case ERL_ATOM_EXT:
diff --git a/lib/erl_interface/src/legacy/erl_marshal.c b/lib/erl_interface/src/legacy/erl_marshal.c
index 4b5f28178f..ad0cc5672c 100644
--- a/lib/erl_interface/src/legacy/erl_marshal.c
+++ b/lib/erl_interface/src/legacy/erl_marshal.c
@@ -102,6 +102,7 @@ void erl_init_marshal(void)
     cmp_array[ERL_SMALL_INTEGER_EXT] = 1;
     cmp_array[ERL_INTEGER_EXT]       = 1;
     cmp_array[ERL_FLOAT_EXT]         = 1;
+    cmp_array[NEW_FLOAT_EXT]         = 1;
     cmp_array[ERL_SMALL_BIG_EXT]     = 1;
     cmp_array[ERL_LARGE_BIG_EXT]     = 1;
     cmp_array[ERL_ATOM_EXT]          = 2;
@@ -124,6 +125,7 @@ void erl_init_marshal(void)
     cmp_num_class[ERL_SMALL_INTEGER_EXT] = SMALL;
     cmp_num_class[ERL_INTEGER_EXT]       = SMALL;
     cmp_num_class[ERL_FLOAT_EXT]         = FLOAT;
+    cmp_num_class[NEW_FLOAT_EXT]         = FLOAT;
     cmp_num_class[ERL_SMALL_BIG_EXT]     = BIG;
     cmp_num_class[ERL_LARGE_BIG_EXT]     = BIG;
     init_cmp_num_class_p = 0;
@@ -1008,10 +1010,13 @@ static ETERM *erl_decode_it(unsigned char **ext)
 	return ep;
 
     case ERL_FLOAT_EXT:
+    case NEW_FLOAT_EXT:
 	ERL_TYPE(ep) = ERL_FLOAT;
-	if (sscanf((char *) *ext, "%lf", &ff) != 1)
+	cp = (char *) *ext;
+	i = -1;
+	if (ei_decode_double(cp, &i, &ff) == -1)
 	    goto failure;
-	*ext += 31;
+	*ext += i;
 	ep->uval.fval.f = ff;
 	return ep;
 
@@ -1176,6 +1181,7 @@ unsigned char erl_ext_type(unsigned char *ext)
     case ERL_LARGE_TUPLE_EXT:
 	return ERL_TUPLE;
     case ERL_FLOAT_EXT:
+    case NEW_FLOAT_EXT:
 	return ERL_FLOAT;
     case ERL_BINARY_EXT:
 	return ERL_BINARY;
@@ -1218,6 +1224,7 @@ int erl_ext_size(unsigned char *t)
     case ERL_BINARY_EXT:
     case ERL_STRING_EXT:
     case ERL_FLOAT_EXT:
+    case NEW_FLOAT_EXT:
     case ERL_SMALL_BIG_EXT:
     case ERL_LARGE_BIG_EXT:
 	return 0;
@@ -1332,6 +1339,9 @@ static int jump(unsigned char **ext)
     case ERL_FLOAT_EXT:
 	*ext += 31;
 	break;
+    case NEW_FLOAT_EXT:
+	*ext += 8;
+	break;
     case ERL_BINARY_EXT:
 	i = (**ext << 24) | ((*ext)[1] << 16) |((*ext)[2] << 8) | (*ext)[3];
 	*ext += 4+i;
@@ -1696,12 +1706,15 @@ static int cmp_exe2(unsigned char **e1, unsigned char **e2)
       }
       return 0;
     case ERL_FLOAT_EXT:
-      if (sscanf((char *) *e1, "%lf", &ff1) != 1)
-	return -1;
-      *e1 += 31;
-      if (sscanf((char *) *e2, "%lf", &ff2) != 1)
-	return -1;
-      *e2 += 31;
+    case NEW_FLOAT_EXT:
+      i = -1;
+      if (ei_decode_double((char *) *e1, &i, &ff1) != 0)
+        return -1;
+      *e1 += i;
+      j = -1;
+      if (ei_decode_double((char *) *e2, &j, &ff2) != 0)
+        return -1;
+      *e2 += j;
       return cmp_floats(ff1,ff2);
 
     case ERL_BINARY_EXT:
diff --git a/lib/erl_interface/src/misc/ei_decode_term.c b/lib/erl_interface/src/misc/ei_decode_term.c
index 7b95ff232f..32aec988e1 100644
--- a/lib/erl_interface/src/misc/ei_decode_term.c
+++ b/lib/erl_interface/src/misc/ei_decode_term.c
@@ -25,9 +25,9 @@
 #include "ei_decode_term.h"
 #include "putget.h"
 
-/* Returns 1 if term is decoded, 0 if term is OK, but not decoded here
-   and -1 if something is wrong.
-   ONLY changes index if term is decoded (return value 1)! */
+/* Returns 0 on successful encoding, -1 on error, and 1 if the term seems
+   alright, but does not fit in the term structure. If it returns 0, the
+   index will be incremented, and the term contains the decoded term. */
 
 int ei_decode_ei_term(const char* buf, int* index, ei_term* term)
 {
@@ -46,11 +46,8 @@ int ei_decode_ei_term(const char* buf, int* index, ei_term* term)
 	term->value.i_val = get32be(s);
 	break;
     case ERL_FLOAT_EXT:
-    	if (s[30]) return -1;
-	if (sscanf(s, "%lf", &f) != 1) return -1;
-	s += 31;
-	term->value.d_val = f;
-	break;
+    case NEW_FLOAT_EXT:
+        return ei_decode_double(buf, index, &term->value.d_val);
     case ERL_ATOM_EXT:
 	len = get16be(s);
 	memcpy(term->value.atom_name, s, len); 
diff --git a/lib/erl_interface/src/misc/ei_printterm.c b/lib/erl_interface/src/misc/ei_printterm.c
index 8d0eef5e79..00f98ba0a1 100644
--- a/lib/erl_interface/src/misc/ei_printterm.c
+++ b/lib/erl_interface/src/misc/ei_printterm.c
@@ -272,6 +272,7 @@ static int print_term(FILE* fp, ei_x_buff* x,
         break;
 
     case ERL_FLOAT_EXT:
+    case NEW_FLOAT_EXT:
 	if (ei_decode_double(buf, index, &d) < 0) goto err;
 	ch_written += xprintf(fp, x, "%f", d);
 	break;
diff --git a/lib/erl_interface/src/misc/putget.h b/lib/erl_interface/src/misc/putget.h
index 98d9ebb64c..f0fbd9a211 100644
--- a/lib/erl_interface/src/misc/putget.h
+++ b/lib/erl_interface/src/misc/putget.h
@@ -54,6 +54,18 @@
   (s) += 4; \
 } while (0)
 
+#define put64be(s,n) do {  \
+  (s)[0] = ((n) >>  56) & 0xff; \
+  (s)[1] = ((n) >>  48) & 0xff; \
+  (s)[2] = ((n) >>  40) & 0xff; \
+  (s)[3] = ((n) >>  32) & 0xff; \
+  (s)[4] = ((n) >>  24) & 0xff; \
+  (s)[5] = ((n) >>  16) & 0xff; \
+  (s)[6] = ((n) >>  8)  & 0xff; \
+  (s)[7] = (n) & 0xff; \
+  (s) += 8; \
+} while (0)
+
 #define get8(s) \
      ((s) += 1, \
       ((unsigned char *)(s))[-1] & 0xff)
@@ -82,4 +94,20 @@
        (((unsigned char *)(s))[-2] << 8) | \
        ((unsigned char *)(s))[-1]))
      
+#define get64be(s) \
+     ((s) += 8, \
+      (((EI_ULONGLONG)((unsigned char *)(s))[-8] << 56) | \
+       ((EI_ULONGLONG)((unsigned char *)(s))[-7] << 48) | \
+       ((EI_ULONGLONG)((unsigned char *)(s))[-6] << 40) | \
+       ((EI_ULONGLONG)((unsigned char *)(s))[-5] << 32) | \
+       ((EI_ULONGLONG)((unsigned char *)(s))[-4] << 24) | \
+       ((EI_ULONGLONG)((unsigned char *)(s))[-3] << 16) | \
+       ((EI_ULONGLONG)((unsigned char *)(s))[-2] << 8)  | \
+        (EI_ULONGLONG)((unsigned char *)(s))[-1]))
+
+typedef union float_ext {
+    double d;
+    EI_ULONGLONG val;
+} FloatExt;
+
 #endif /* _PUTGET_H */
diff --git a/lib/erl_interface/src/misc/show_msg.c b/lib/erl_interface/src/misc/show_msg.c
index 25865d6f8e..9ed27f49d5 100644
--- a/lib/erl_interface/src/misc/show_msg.c
+++ b/lib/erl_interface/src/misc/show_msg.c
@@ -400,6 +400,7 @@ static void show_term(const char *termbuf, int *index, FILE *stream)
 	break;
 
     case ERL_FLOAT_EXT:
+    case NEW_FLOAT_EXT:
 	ei_decode_double(termbuf,index,&fnum);
 	fprintf(stream,"%f",fnum);
 	break;
diff --git a/lib/erl_interface/test/ei_decode_SUITE.erl b/lib/erl_interface/test/ei_decode_SUITE.erl
index ea528728ab..0fd805255f 100644
--- a/lib/erl_interface/test/ei_decode_SUITE.erl
+++ b/lib/erl_interface/test/ei_decode_SUITE.erl
@@ -181,22 +181,9 @@ test_ei_decode_misc(suite) -> [];
 test_ei_decode_misc(Config) when is_list(Config) ->
     ?line P = runner:start(?test_ei_decode_misc),
 
-%    ?line <<131>>  = get_binaries(P),
-
-%    ?line {term,F} = get_term(P),
-%    ?line match_float(F, 0.0),
-%    ?line {term,F} = get_term(P),
-%    ?line match_float(F, 0.0),
-
-%    ?line {term,F} = get_term(P),
-%    ?line true = match_float(F, -1.0),
-%    ?line {term,F} = get_term(P),
-%    ?line true = match_float(F, -1.0),
-
-%    ?line {term,F} = get_term(P),
-%    ?line true = match_float(F, 1.0),
-%    ?line {term,F} = get_term(P),
-%    ?line true = match_float(F, 1.0),
+    ?line send_term_as_binary(P,0.0),
+    ?line send_term_as_binary(P,-1.0),
+    ?line send_term_as_binary(P,1.0),
 
     ?line send_term_as_binary(P,false),
     ?line send_term_as_binary(P,true),
diff --git a/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c b/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c
index d81ea88437..38811c3c77 100644
--- a/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c
+++ b/lib/erl_interface/test/ei_decode_SUITE_data/ei_decode_test.c
@@ -515,11 +515,10 @@ TESTCASE(test_ei_decode_misc)
 /*
     EI_DECODE_0(decode_version);
 */
-/*
-    EI_DECODE_2(decode_double, 0.0);
-    EI_DECODE_2(decode_double, -1.0);
-    EI_DECODE_2(decode_double, 1.0);
-*/
+    EI_DECODE_2(decode_double, 32, double, 0.0);
+    EI_DECODE_2(decode_double, 32, double, -1.0);
+    EI_DECODE_2(decode_double, 32, double, 1.0);
+
     EI_DECODE_2(decode_boolean, 8, int, 0);
     EI_DECODE_2(decode_boolean, 7, int, 1);
 
diff --git a/lib/erl_interface/test/ei_encode_SUITE.erl b/lib/erl_interface/test/ei_encode_SUITE.erl
index fb790eb7c3..dd6f37e453 100644
--- a/lib/erl_interface/test/ei_encode_SUITE.erl
+++ b/lib/erl_interface/test/ei_encode_SUITE.erl
@@ -181,20 +181,14 @@ test_ei_encode_misc(Config) when is_list(Config) ->
 
     ?line <<131>>  = get_binaries(P),
 
-%    ?line {term,F} = get_term(P),
-%    ?line match_float(F, 0.0),
-%    ?line {term,F} = get_term(P),
-%    ?line match_float(F, 0.0),
+    ?line {<<70,_:8/binary>>,F0} = get_buf_and_term(P),
+    ?line true = match_float(F0, 0.0),
 
-%    ?line {term,F} = get_term(P),
-%    ?line true = match_float(F, -1.0),
-%    ?line {term,F} = get_term(P),
-%    ?line true = match_float(F, -1.0),
+    ?line {<<70,_:8/binary>>,Fn1} = get_buf_and_term(P),
+    ?line true = match_float(Fn1, -1.0),
 
-%    ?line {term,F} = get_term(P),
-%    ?line true = match_float(F, 1.0),
-%    ?line {term,F} = get_term(P),
-%    ?line true = match_float(F, 1.0),
+    ?line {<<70,_:8/binary>>,Fp1} = get_buf_and_term(P),
+    ?line true = match_float(Fp1, 1.0),
 
     ?line {<<100,0,5,"false">>,false}  = get_buf_and_term(P),
     ?line {<<100,0,4,"true">> ,true}   = get_buf_and_term(P),
@@ -310,6 +304,8 @@ get_term(P) ->
  
 %%
 
+match_float(F, Match) when is_float(F), is_float(Match), F == Match ->
+    true;
 match_float(F, Match) when is_float(F), F > Match*0.99, F < Match*1.01 ->
     true.
     
diff --git a/lib/erl_interface/test/ei_encode_SUITE_data/ei_encode_test.c b/lib/erl_interface/test/ei_encode_SUITE_data/ei_encode_test.c
index f8de0b7878..2d326757a1 100644
--- a/lib/erl_interface/test/ei_encode_SUITE_data/ei_encode_test.c
+++ b/lib/erl_interface/test/ei_encode_SUITE_data/ei_encode_test.c
@@ -350,13 +350,13 @@ TESTCASE(test_ei_encode_char)
 TESTCASE(test_ei_encode_misc)
 {
     EI_ENCODE_0(encode_version);
-/*
+
     EI_ENCODE_1(encode_double, 0.0);
 
     EI_ENCODE_1(encode_double, -1.0);
 
     EI_ENCODE_1(encode_double, 1.0);
-*/
+
     EI_ENCODE_1(encode_boolean, 0) /* Only case it should be false */;
 
     EI_ENCODE_1(encode_boolean, 1);