Replace io_list_len() with erts_iolist_size()

The io_list_len() function returns an int, where a negative return
value indicates a type error. One problem is that an int only consists
of 32 bits in a 64-bit emulator. Changing the return type to Sint
will solve that problem, but in the 32-bit emulator, a large iolist
and a iolist with a type error will both return a negative number.
(Noticed by Jon Meredith.)

Another problem is that for iolists whose total size exceed the
word size, the result would be truncated, leading to a subsequent
buffer overflow and emulator crash.

Therefore, introduce the new erts_iolist_size() function which
returns a status indication and writes the result size through
a passed pointer. If the result size does not fit in a word,
return an overflow indication.

9 files changed, 109 insertions, 69 deletions

diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 9486602633..91eb2114b1 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -355,21 +355,24 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1)
 BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg)
 {
     Eterm bin;
-    int i;
+    Uint size;
     int offset;
     byte* bytes;
+
     if (is_nil(arg)) {
 	BIF_RET(new_binary(p,(byte*)"",0));
     }
     if (is_not_list(arg)) {
 	goto error;
     }
-    if ((i = io_list_len(arg)) < 0) {
-	goto error;
+    switch (erts_iolist_size(arg, &size)) {
+    case ERTS_IOLIST_OVERFLOW: BIF_ERROR(p, SYSTEM_LIMIT);
+    case ERTS_IOLIST_TYPE: goto error;
+    default: ;
     }
-    bin = new_binary(p, (byte *)NULL, i);
+    bin = new_binary(p, (byte *)NULL, size);
     bytes = binary_bytes(bin);
-    offset = io_list_to_buf(arg, (char*) bytes, i);
+    offset = io_list_to_buf(arg, (char*) bytes, size);
     ASSERT(offset == 0);
     BIF_RET(bin);
     
diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c
index c9cdcb87a6..9631fb50db 100644
--- a/erts/emulator/beam/erl_bif_ddll.c
+++ b/erts/emulator/beam/erl_bif_ddll.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2006-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2006-2011. All Rights Reserved.
  * 
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -146,7 +146,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 				Eterm name_term, Eterm options)
 {
     char *path = NULL;
-    int path_len;
+    Uint path_len;
     char *name = NULL;
     DE_Handle *dh;
     erts_driver_t *drv;
@@ -221,9 +221,7 @@ BIF_RETTYPE erl_ddll_try_load_3(Process *p, Eterm path_term,
 	goto error;
     }
 
-    path_len = io_list_len(path_term);
-
-    if (path_len <= 0) {
+    if (erts_iolist_size(path_term, &path_len)) {
 	goto error;
     }
     path = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, path_len + 1 /* might need path separator */ + sys_strlen(name) + 1);
@@ -1878,7 +1876,7 @@ static Eterm mkatom(char *str)
 static char *pick_list_or_atom(Eterm name_term)
 { 
     char *name = NULL;
-    int name_len;
+    Uint name_len;
     if (is_atom(name_term)) {
 	Atom *ap = atom_tab(atom_val(name_term));
 	if (ap->len == 0) {
@@ -1890,8 +1888,7 @@ static char *pick_list_or_atom(Eterm name_term)
 	memcpy(name,ap->name,ap->len);
 	name[ap->len] = '\0';
     } else {
-	name_len = io_list_len(name_term);
-	if (name_len <= 0) {
+	if (erts_iolist_size(name_term, &name_len)) {
 	    goto error;
 	}
 	name = erts_alloc(ERTS_ALC_T_DDLL_TMP_BUF, name_len + 1);
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index e50fc18e64..f264bf44df 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1732,14 +1732,14 @@ info_1_tuple(Process* BIF_P,	/* Pointer to current process. */
 #    define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF_XML
 #  endif
 #endif
-	    int buf_size = 8*1024; /* Try with 8KB first */
+	    Uint buf_size = 8*1024; /* Try with 8KB first */
 	    char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size);
 	    int r = io_list_to_buf(*tp, (char*) buf, buf_size - 1);
 	    if (r < 0) {
 		erts_free(ERTS_ALC_T_TMP, (void *) buf);
-		buf_size = io_list_len(*tp);
-		if (buf_size < 0)
+		if (erts_iolist_size(*tp, &buf_size)) {
 		    goto badarg;
+		}
 		buf_size++;
 		buf = erts_alloc(ERTS_ALC_T_TMP, buf_size);
 		r = io_list_to_buf(*tp, (char*) buf, buf_size - 1);
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index fbc92b9730..3fd35dd963 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -996,6 +996,7 @@ static byte* convert_environment(Process* p, Eterm env)
     Eterm* hp;
     Uint heap_size;
     int n;
+    Uint size;
     byte* bytes;
 
     if ((n = list_length(env)) < 0) {
@@ -1039,15 +1040,15 @@ static byte* convert_environment(Process* p, Eterm env)
     if (is_not_nil(env)) {
 	goto done;
     }
-    if ((n = io_list_len(all)) < 0) {
+    if (erts_iolist_size(all, &size)) {
 	goto done;
     }
 
     /*
      * Put the result in a binary (no risk for a memory leak that way).
      */
-    (void) erts_new_heap_binary(p, NULL, n, &bytes);
-    io_list_to_buf(all, (char*)bytes, n);
+    (void) erts_new_heap_binary(p, NULL, size, &bytes);
+    io_list_to_buf(all, (char*)bytes, size);
 
  done:
     erts_free(ERTS_ALC_T_TMP, temp_heap);
diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c
index d4a8a3aaa7..26891c4348 100644
--- a/erts/emulator/beam/erl_bif_re.c
+++ b/erts/emulator/beam/erl_bif_re.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -417,7 +417,7 @@ build_compile_result(Process *p, Eterm error_tag, pcre *result, int errcode, con
 BIF_RETTYPE
 re_compile_2(BIF_ALIST_2)
 {
-    int slen;
+    Uint slen;
     char *expr;
     pcre *result;
     int errcode = 0;
@@ -444,7 +444,7 @@ re_compile_2(BIF_ALIST_2)
 	BIF_TRAP2(ucompile_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2);
     }
 
-    if ((slen = io_list_len(BIF_ARG_1)) < 0) {
+    if (erts_iolist_size(BIF_ARG_1, &slen)) {
         BIF_ERROR(BIF_P,BADARG);
     }
     expr = erts_alloc(ERTS_ALC_T_RE_TMP_BUF, slen + 1);
@@ -795,8 +795,8 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code)
 			memcpy(tmpb,ap->name,ap->len);
 			tmpb[ap->len] = '\0';
 		    } else {
-			int slen = io_list_len(val);
-			if (slen < 0) {
+			Uint slen;
+			if (erts_iolist_size(val, &slen)) {
 			    goto error;
 			}
 			if ((slen + 1) > tmpbsiz) {
@@ -851,7 +851,7 @@ re_run_3(BIF_ALIST_3)
     const pcre *code_tmp;
     RestartContext restart;
     byte *temp_alloc = NULL;
-    int slength;
+    Uint slength;
     int startoffset = 0;
     int options = 0, comp_options = 0;
     int ovsize;
@@ -875,7 +875,7 @@ re_run_3(BIF_ALIST_3)
     if (is_not_tuple(BIF_ARG_2) || (arityval(*tuple_val(BIF_ARG_2)) != 4)) {
 	if (is_binary(BIF_ARG_2) || is_list(BIF_ARG_2) || is_nil(BIF_ARG_2)) { 
 	    /* Compile from textual RE */
-	    int slen;
+	    Uint slen;
 	    char *expr;
 	    pcre *result;
 	    int errcode = 0;
@@ -889,7 +889,7 @@ re_run_3(BIF_ALIST_3)
 		BIF_TRAP3(urun_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
 	    }
 	    
-	    if ((slen = io_list_len(BIF_ARG_2)) < 0) {
+	    if (erts_iolist_size(BIF_ARG_2, &slen)) {
 		BIF_ERROR(BIF_P,BADARG);
 	    }
 	    
@@ -1027,7 +1027,7 @@ re_run_3(BIF_ALIST_3)
 	restart.flags |= RESTART_FLAG_SUBJECT_IN_BINARY;
     } else {
 handle_iolist:
-	if ((slength = io_list_len(BIF_ARG_1)) < 0) {
+	if (erts_iolist_size(BIF_ARG_1, &slength)) {
 	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.ovector);
 	    erts_free(ERTS_ALC_T_RE_SUBJECT, restart.code);
 	    if (restart.ret_info != NULL) {
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 8b48444904..68421b4387 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -472,7 +472,7 @@ static void tmp_alloc_dtor(struct enif_tmp_obj_t* obj)
 int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
 {
     struct enif_tmp_obj_t* tobj;
-    int sz;
+    Uint sz;
     if (is_binary(term)) {
 	return enif_inspect_binary(env,term,bin);
     }
@@ -483,7 +483,7 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin)
 	bin->ref_bin = NULL;
 	return 1;
     }
-    if ((sz = io_list_len(term)) < 0) {
+    if (erts_iolist_size(term, &sz)) {
 	return 0;
     }
     
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 96da894d90..7d5b1853e6 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1653,10 +1653,14 @@ struct Sint_buf {
 };	
 char* Sint_to_buf(Sint, struct Sint_buf*);
 
+#define ERTS_IOLIST_OK 0
+#define ERTS_IOLIST_OVERFLOW 1
+#define ERTS_IOLIST_TYPE 2
+
 Eterm buf_to_intlist(Eterm**, char*, int, Eterm); /* most callers pass plain char*'s */
 int io_list_to_buf(Eterm, char*, int);
 int io_list_to_buf2(Eterm, char*, int);
-int io_list_len(Eterm);
+int erts_iolist_size(Eterm, Uint *);
 int is_string(Eterm);
 void erl_at_exit(void (*) (void*), void*);
 Eterm collect_memory(Process *);
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index bf49417c3f..d9df90fe7d 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -961,6 +961,7 @@ do {									\
     if (thing_subtag(*binary_val(_real)) == REFC_BINARY_SUBTAG &&	\
 	_bitoffs == 0) {						\
 	b_size += _size;						\
+        if (b_size < _size) goto L_overflow_error;			\
 	in_clist = 0;							\
 	v_size++;							\
         if (_size >= ERL_SMALL_IO_BIN_LIMIT) {				\
@@ -975,6 +976,7 @@ do {									\
         }								\
     } else {								\
 	c_size += _size;						\
+        if (c_size < _size) goto L_overflow_error;			\
 	if (!in_clist) {						\
 	    in_clist = 1;						\
 	    v_size++;							\
@@ -989,28 +991,30 @@ do {									\
 
 
 /* 
-** Size of a io list in bytes
-** return -1 if error
-** returns:            - Total size of io list
-**           vsize     - SysIOVec size needed for a writev
-**           csize     - Number of bytes not in binary (in the common binary)
-**           pvsize    - SysIOVec size needed if packing small binaries
-**           pcsize    - Number of bytes in the common binary if packing
-*/
+ * Returns 0 if successful and a non-zero value otherwise.
+ *
+ * Return values through pointers:
+ *    *vsize      - SysIOVec size needed for a writev
+ *    *csize      - Number of bytes not in binary (in the common binary)
+ *    *pvsize     - SysIOVec size needed if packing small binaries
+ *    *pcsize     - Number of bytes in the common binary if packing
+ *    *total_size - Total size of iolist in bytes
+ */
 
 static int 
-io_list_vec_len(Eterm obj, int* vsize, int* csize, 
-		int * pvsize, int * pcsize)
+io_list_vec_len(Eterm obj, Uint* vsize, Uint* csize,
+		Uint* pvsize, Uint* pcsize, Uint* total_size)
 {
     DECLARE_ESTACK(s);
     Eterm* objp;
-    int v_size = 0;
-    int c_size = 0;
-    int b_size = 0;
-    int in_clist = 0;
-    int p_v_size = 0;
-    int p_c_size = 0;
-    int p_in_clist = 0;
+    Uint v_size = 0;
+    Uint c_size = 0;
+    Uint b_size = 0;
+    Uint in_clist = 0;
+    Uint p_v_size = 0;
+    Uint p_c_size = 0;
+    Uint p_in_clist = 0;
+    Uint total;
 
     goto L_jump_start;  /* avoid a push */
 
@@ -1024,6 +1028,9 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize,
 
 	    if (is_byte(obj)) {
 		c_size++;
+		if (c_size == 0) {
+		    goto L_overflow_error;
+		}
 		if (!in_clist) {
 		    in_clist = 1;
 		    v_size++;
@@ -1063,16 +1070,23 @@ io_list_vec_len(Eterm obj, int* vsize, int* csize,
 	}
     }
 
+    total = c_size + b_size;
+    if (total < c_size) {
+	goto L_overflow_error;
+    }
+    *total_size = total;
+
     DESTROY_ESTACK(s);
     *vsize = v_size;
     *csize = c_size;
     *pvsize = p_v_size;
     *pcsize = p_c_size;
-    return c_size + b_size;
+    return 0;
 
  L_type_error:
+ L_overflow_error:
     DESTROY_ESTACK(s);
-    return -1;
+    return 1;
 }
 
 /* write data to a port */
@@ -1080,7 +1094,7 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 {
     char *buf;
     erts_driver_t *drv = p->drv_ptr;
-    int size;
+    Uint size;
     int fpe_was_unmasked;
     
     ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p));
@@ -1088,10 +1102,10 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 
     p->caller = caller_id;
     if (drv->outputv != NULL) {
-	int vsize;
-	int csize;
-	int pvsize;
-	int pcsize;
+	Uint vsize;
+	Uint csize;
+	Uint pvsize;
+	Uint pcsize;
 	int blimit;
 	SysIOVec iv[SMALL_WRITE_VEC];
 	ErlDrvBinary* bv[SMALL_WRITE_VEC];
@@ -1100,8 +1114,8 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 	ErlDrvBinary* cbin;
 	ErlIOVec ev;
 
-	if ((size = io_list_vec_len(list, &vsize, &csize, 
-				    &pvsize, &pcsize)) < 0) {
+	if (io_list_vec_len(list, &vsize, &csize,
+			    &pvsize, &pcsize, &size)) {
 	    goto bad_value;
 	}
 	/* To pack or not to pack (small binaries) ...? */
@@ -1176,7 +1190,7 @@ int erts_write_to_port(Eterm caller_id, Port *p, Eterm list)
 	else {
 	    ASSERT(r == -1); /* Overflow */
 	    erts_free(ERTS_ALC_T_TMP, buf);
-	    if ((size = io_list_len(list)) < 0) {
+	    if (erts_iolist_size(list, &size)) {
 		goto bad_value;
 	    }
 
@@ -2140,7 +2154,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
     byte* to_port = NULL;	/* Buffer to write to port. */
 				/* Initialization is for shutting up
 				   warning about use before set. */
-    int to_len = 0;		/* Length of buffer. */
+    Uint to_len = 0;		/* Length of buffer. */
     int must_free = 0;		/* True if the buffer should be freed. */
     char port_result[ERL_ONHEAP_BIN_LIMIT];  /* Default buffer for result from port. */
     char* port_resp;		/* Pointer to result buffer. */
@@ -2185,7 +2199,7 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist)
 	} else {
 	    ASSERT(r == -1);	/* Overflow */
 	    erts_free(ERTS_ALC_T_TMP, (void *) to_port);
-	    if ((to_len = io_list_len(iolist)) < 0) { /* Type error */
+	    if (erts_iolist_size(iolist, &to_len)) { /* Type error */
 		return THE_NON_VALUE;
 	    }
 	    must_free = 1;
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index f531d1430b..afea51c44b 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3021,13 +3021,25 @@ int io_list_to_buf(Eterm obj, char* buf, int len)
     return -1;
 }
 
-int io_list_len(Eterm obj)
+/*
+ * Return 0 if successful, and non-zero if unsuccessful.
+ */
+int erts_iolist_size(Eterm obj, Uint* sizep)
 {
     Eterm* objp;
-    Sint len = 0;
+    Uint size = 0;
     DECLARE_ESTACK(s);
     goto L_again;
 
+#define SAFE_ADD(Var, Val)			\
+    do {					\
+        Uint valvar = (Val);			\
+	Var += valvar;				\
+	if (Var < valvar) {			\
+	    goto L_overflow_error;		\
+	}					\
+    } while (0)
+
     while (!ESTACK_ISEMPTY(s)) {
 	obj = ESTACK_POP(s);
     L_again:
@@ -3037,9 +3049,12 @@ int io_list_len(Eterm obj)
 	    /* Head */
 	    obj = CAR(objp);
 	    if (is_byte(obj)) {
-		len++;
+		size++;
+		if (size == 0) {
+		    goto L_overflow_error;
+		}
 	    } else if (is_binary(obj) && binary_bitsize(obj) == 0) {
-		len += binary_size(obj);
+		SAFE_ADD(size, binary_size(obj));
 	    } else if (is_list(obj)) {
 		ESTACK_PUSH(s, CDR(objp));
 		goto L_iter_list; /* on head */
@@ -3051,23 +3066,29 @@ int io_list_len(Eterm obj)
 	    if (is_list(obj))
 		goto L_iter_list; /* on tail */
 	    else if (is_binary(obj) && binary_bitsize(obj) == 0) {
-		len += binary_size(obj);
+		SAFE_ADD(size, binary_size(obj));
 	    } else if (is_not_nil(obj)) {
 		goto L_type_error;
 	    }
 	} else if (is_binary(obj) && binary_bitsize(obj) == 0) { /* Tail was binary */
-	    len += binary_size(obj);
+	    SAFE_ADD(size, binary_size(obj));
 	} else if (is_not_nil(obj)) {
 	    goto L_type_error;
 	}
     }
+#undef SAFE_ADD
 
     DESTROY_ESTACK(s);
-    return len;
+    *sizep = size;
+    return ERTS_IOLIST_OK;
+
+ L_overflow_error:
+    DESTROY_ESTACK(s);
+    return ERTS_IOLIST_OVERFLOW;
 
  L_type_error:
     DESTROY_ESTACK(s);
-    return -1;
+    return ERTS_IOLIST_TYPE;
 }
 
 /* return 0 if item is not a non-empty flat list of bytes */