The R13B03 release.OTP_R13B03

1 files changed, 1815 insertions, 0 deletions

diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
new file mode 100644
index 0000000000..ab5811c70f
--- /dev/null
+++ b/erts/emulator/beam/erl_unicode.c
@@ -0,0 +1,1815 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2008-2009. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+#include "sys.h"
+#include "erl_vm.h"
+#include "global.h"
+#include "erl_process.h"
+#include "error.h"
+#include "bif.h"
+#include "erl_binary.h"
+#include "big.h"
+
+#include "erl_unicode.h"
+
+typedef struct _restart_context {
+    byte *bytes;
+    Uint num_processed_bytes;
+    Uint num_bytes_to_process;
+    Uint num_resulting_chars;
+    int state;
+} RestartContext;
+
+
+#define LOOP_FACTOR 10
+#define LOOP_FACTOR_SIMPLE 50 /* When just counting */
+
+static Uint max_loop_limit;
+
+static BIF_RETTYPE utf8_to_list(BIF_ALIST_1);
+static BIF_RETTYPE finalize_list_to_list(Process *p, 
+					 byte *bytes,
+					 Eterm rest,
+					 Uint num_processed_bytes,
+					 Uint num_bytes_to_process, 
+					 Uint num_resulting_chars, 
+					 int state, int left,
+					 Eterm tail);
+static int analyze_utf8(byte *source, Uint size, 
+			byte **err_pos, Uint *num_chars, int *left);
+#define UTF8_OK 0
+#define UTF8_INCOMPLETE 1
+#define UTF8_ERROR 2
+#define UTF8_ANALYZE_MORE 3
+
+static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3);
+static BIF_RETTYPE characters_to_list_trap_1(BIF_ALIST_3);
+static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3);
+
+static BIF_RETTYPE characters_to_list_trap_3(BIF_ALIST_3);
+static BIF_RETTYPE characters_to_list_trap_4(BIF_ALIST_1);
+
+static Export characters_to_utf8_trap_exp;
+static Export characters_to_list_trap_1_exp;
+static Export characters_to_list_trap_2_exp;
+
+static Export characters_to_list_trap_3_exp;
+static Export characters_to_list_trap_4_exp;
+
+static Export *c_to_b_int_trap_exportp = NULL;
+static Export *c_to_l_int_trap_exportp = NULL;
+
+void erts_init_unicode(void)
+{
+    max_loop_limit = CONTEXT_REDS * LOOP_FACTOR;
+    /* Non visual BIFs to trap to. */
+    memset(&characters_to_utf8_trap_exp, 0, sizeof(Export));
+    characters_to_utf8_trap_exp.address = 
+	&characters_to_utf8_trap_exp.code[3];
+    characters_to_utf8_trap_exp.code[0] = am_erlang;
+    characters_to_utf8_trap_exp.code[1] = 
+	am_atom_put("characters_to_utf8_trap",23);
+    characters_to_utf8_trap_exp.code[2] = 3;
+    characters_to_utf8_trap_exp.code[3] =
+	(Eterm) em_apply_bif;
+    characters_to_utf8_trap_exp.code[4] = 
+	(Eterm) &characters_to_utf8_trap;
+
+    memset(&characters_to_list_trap_1_exp, 0, sizeof(Export));
+    characters_to_list_trap_1_exp.address = 
+	&characters_to_list_trap_1_exp.code[3];
+    characters_to_list_trap_1_exp.code[0] = am_erlang;
+    characters_to_list_trap_1_exp.code[1] = 
+	am_atom_put("characters_to_list_trap_1",25);
+    characters_to_list_trap_1_exp.code[2] = 3;
+    characters_to_list_trap_1_exp.code[3] =
+	(Eterm) em_apply_bif;
+    characters_to_list_trap_1_exp.code[4] = 
+	(Eterm) &characters_to_list_trap_1;
+
+    memset(&characters_to_list_trap_2_exp, 0, sizeof(Export));
+    characters_to_list_trap_2_exp.address = 
+	&characters_to_list_trap_2_exp.code[3];
+    characters_to_list_trap_2_exp.code[0] = am_erlang;
+    characters_to_list_trap_2_exp.code[1] = 
+	am_atom_put("characters_to_list_trap_2",25);
+    characters_to_list_trap_2_exp.code[2] = 3;
+    characters_to_list_trap_2_exp.code[3] =
+	(Eterm) em_apply_bif;
+    characters_to_list_trap_2_exp.code[4] = 
+	(Eterm) &characters_to_list_trap_2;
+
+
+    memset(&characters_to_list_trap_3_exp, 0, sizeof(Export));
+    characters_to_list_trap_3_exp.address = 
+	&characters_to_list_trap_3_exp.code[3];
+    characters_to_list_trap_3_exp.code[0] = am_erlang;
+    characters_to_list_trap_3_exp.code[1] = 
+	am_atom_put("characters_to_list_trap_3",25);
+    characters_to_list_trap_3_exp.code[2] = 3;
+    characters_to_list_trap_3_exp.code[3] =
+	(Eterm) em_apply_bif;
+    characters_to_list_trap_3_exp.code[4] = 
+	(Eterm) &characters_to_list_trap_3;
+
+    memset(&characters_to_list_trap_4_exp, 0, sizeof(Export));
+    characters_to_list_trap_4_exp.address = 
+	&characters_to_list_trap_4_exp.code[3];
+    characters_to_list_trap_4_exp.code[0] = am_erlang;
+    characters_to_list_trap_4_exp.code[1] = 
+	am_atom_put("characters_to_list_trap_4",25);
+    characters_to_list_trap_4_exp.code[2] = 1;
+    characters_to_list_trap_4_exp.code[3] =
+	(Eterm) em_apply_bif;
+    characters_to_list_trap_4_exp.code[4] = 
+	(Eterm) &characters_to_list_trap_4;
+
+    c_to_b_int_trap_exportp =  erts_export_put(am_unicode,am_characters_to_binary_int,2);
+    c_to_l_int_trap_exportp =  erts_export_put(am_unicode,am_characters_to_list_int,2);
+    
+
+}
+
+
+static ERTS_INLINE void *alloc_restart(size_t size)
+{
+    return erts_alloc(ERTS_ALC_T_UNICODE_BUFFER,size);
+}
+
+static ERTS_INLINE void free_restart(void *ptr)
+{
+    erts_free(ERTS_ALC_T_UNICODE_BUFFER, ptr);
+}
+
+static void cleanup_restart_context(RestartContext *rc)
+{
+    if (rc->bytes != NULL) {
+	free_restart(rc->bytes);
+	rc->bytes = NULL;
+    }
+}
+
+static void cleanup_restart_context_bin(Binary *bp)
+{
+    RestartContext *rc = ERTS_MAGIC_BIN_DATA(bp);
+    cleanup_restart_context(rc);
+}
+
+static RestartContext *get_rc_from_bin(Eterm bin)
+{
+    Binary *mbp;
+    ASSERT(ERTS_TERM_IS_MAGIC_BINARY(bin));
+
+    mbp = ((ProcBin *) binary_val(bin))->val;
+
+    ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(mbp)
+	   == cleanup_restart_context_bin);
+    return (RestartContext *) ERTS_MAGIC_BIN_DATA(mbp);    
+}
+
+static Eterm make_magic_bin_for_restart(Process *p, RestartContext *rc)
+{
+    Binary *mbp = erts_create_magic_binary(sizeof(RestartContext),
+					   cleanup_restart_context_bin);
+    RestartContext *restartp = ERTS_MAGIC_BIN_DATA(mbp);
+    Eterm *hp;
+    memcpy(restartp,rc,sizeof(RestartContext));
+    hp = HAlloc(p, PROC_BIN_SIZE);
+    return erts_mk_magic_binary_term(&hp, &MSO(p), mbp);
+}
+
+	
+Sint erts_unicode_set_loop_limit(Sint limit) 
+{
+    Sint save = (Sint) max_loop_limit;
+    if (limit <= 0) {
+	max_loop_limit = CONTEXT_REDS * LOOP_FACTOR;
+    } else {
+	max_loop_limit = (Uint) limit;
+    }
+    return save;
+}
+
+static ERTS_INLINE int allowed_iterations(Process *p)
+{
+    int tmp = ERTS_BIF_REDS_LEFT(p) * LOOP_FACTOR;
+    int tmp2 = max_loop_limit;
+    if (tmp2 < tmp)
+	return tmp2;
+    else
+	return tmp;
+}
+static ERTS_INLINE int cost_to_proc(Process *p, int cost)
+{
+    int x = (cost / LOOP_FACTOR);
+    BUMP_REDS(p,x);
+    return x;
+}
+static ERTS_INLINE int simple_loops_to_common(int cost)
+{
+    int factor = (LOOP_FACTOR_SIMPLE / LOOP_FACTOR);
+    return (cost / factor);
+}
+
+static Sint aligned_binary_size(Eterm binary)
+{
+    unsigned char *bytes;
+    Uint bitoffs;
+    Uint bitsize;
+    
+    ERTS_GET_BINARY_BYTES(binary, bytes, bitoffs, bitsize);
+    if (bitsize != 0) {
+	return (Sint) -1;
+    }
+    return binary_size(binary);
+}
+
+static Sint latin1_binary_need(Eterm binary)
+{
+    unsigned char *bytes;
+    byte *temp_alloc = NULL;
+    Uint bitoffs;
+    Uint bitsize;
+    Uint size;
+    Sint need = 0;
+    Sint i;
+    
+    ERTS_GET_BINARY_BYTES(binary, bytes, bitoffs, bitsize);
+    if (bitsize != 0) {
+	return (Sint) -1;
+    }
+    if (bitoffs != 0) {
+	bytes = erts_get_aligned_binary_bytes(binary, &temp_alloc);
+	/* The call to erts_get_aligned_binary_bytes cannot fail as 
+	   we'we already checked bitsize and that this is a binary */
+    }
+    size = binary_size(binary);
+    for(i = 0; i < size; ++i) {
+	if (bytes[i] & ((byte) 0x80)) {
+	    need += 2;
+	} else {
+	    need += 1;
+	}
+    }
+    erts_free_aligned_binary_bytes(temp_alloc);
+    return need;
+}
+
+static int utf8_len(byte first) 
+{
+    if ((first & ((byte) 0x80)) == 0) {
+	return 1;
+    } else if ((first & ((byte) 0xE0)) == 0xC0) {
+	return 2;
+    } else if ((first & ((byte) 0xF0)) == 0xE0) {
+	return 3;
+    } else if ((first & ((byte) 0xF8)) == 0xF0) {
+	return 4;
+    } 
+    return -1;
+}
+
+static int copy_utf8_bin(byte *target, byte *source, Uint size, 
+			 byte *leftover, int *num_leftovers, 
+			 byte **err_pos, Uint *characters) {
+    int copied = 0;
+    if (leftover != NULL && *num_leftovers) {
+	int need = utf8_len(leftover[0]);
+	int from_source = need - (*num_leftovers);
+	int c;
+	byte *tmp_err_pos = NULL;
+	ASSERT(need > 0);
+	ASSERT(from_source > 0);
+	if (size < from_source) {
+	    memcpy(leftover + (*num_leftovers), source, size);
+	    *num_leftovers += size;
+	    return 0;
+	}
+	/* leftover has room for four bytes (see bif) */
+	memcpy(leftover + (*num_leftovers),source,from_source);
+	c = copy_utf8_bin(target, leftover, need, NULL, NULL, &tmp_err_pos, characters);
+	if (tmp_err_pos != 0) {
+	    *err_pos = source;
+	    return 0;
+	}
+	copied += c;
+	*num_leftovers = 0;
+	size -= from_source;
+	target += c;
+	source += from_source;
+    }
+    while (size) {
+	if (((*source) & ((byte) 0x80)) == 0) {
+	    *(target++) = *(source++);
+	    --size; ++copied;
+	} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
+	    if (leftover && size < 2) {
+		*leftover = *source;
+		*num_leftovers = 1;
+		break;
+	    }
+	    if (size < 2 || ((source[1] & ((byte) 0xC0)) != 0x80) ||
+		((*source) < 0xC2) /* overlong */) {
+		*err_pos = source;
+		return copied;
+	    }
+	    *(target++) = *(source++);
+	    *(target++) = *(source++);
+	    size -= 2; copied += 2;
+	} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
+	    if (leftover && size < 3) {
+		memcpy(leftover, source, (int) size);
+		*num_leftovers = (int) size;
+		break;
+	    }
+	    if (size < 3 || ((source[1] & ((byte) 0xC0)) != 0x80) ||
+		((source[2] & ((byte) 0xC0)) != 0x80) ||
+		(((*source) == 0xE0) && (source[1] < 0xA0)) /* overlong */ ) {
+		*err_pos = source;
+		return copied;
+	    }
+	    if ((((*source) & ((byte) 0xF)) == 0xD) && 
+		((source[1] & 0x20) != 0)) {
+		*err_pos = source;
+		return copied;
+	    }
+
+	    if (((*source) == 0xEF) && (source[1] == 0xBF) &&
+		((source[2] == 0xBE) || (source[2] == 0xBF))) {
+		*err_pos = source;
+		return copied;
+	    }
+		
+	    *(target++) = *(source++);
+	    *(target++) = *(source++);
+	    *(target++) = *(source++);
+	    size -= 3; copied += 3;
+	} else if (((*source) & ((byte) 0xF8)) == 0xF0) {
+	    if (leftover && size < 4) {
+		memcpy(leftover, source, (int) size);
+		*num_leftovers = (int) size;
+		break;
+	    }
+	    if (size < 4 || ((source[1] & ((byte) 0xC0)) != 0x80) ||
+		((source[2] & ((byte) 0xC0)) != 0x80) ||
+		((source[3] & ((byte) 0xC0)) != 0x80) ||
+		(((*source) == 0xF0) && (source[1] < 0x90)) /* overlong */) {
+		*err_pos = source;
+		return copied;
+	    }
+	    if ((((*source) & ((byte)0x7)) > 0x4U) ||
+		((((*source) & ((byte)0x7)) == 0x4U) && 
+		 ((source[1] & ((byte)0x3F)) > 0xFU))) {
+		*err_pos = source;
+		return copied;
+	    }
+	    *(target++) = *(source++);
+	    *(target++) = *(source++);
+	    *(target++) = *(source++);
+	    *(target++) = *(source++);
+	    size -= 4; copied +=4;
+	} else {
+	    *err_pos = source;
+	    return copied;
+	}
+	++(*characters);
+    }
+    return copied;
+}
+	    
+	    
+    
+static Sint utf8_need(Eterm ioterm, int latin1, Uint *costp) 
+{
+    Eterm *objp;
+    Eterm obj;
+    DECLARE_ESTACK(stack);
+    Sint need = 0;
+    Uint cost = 0;
+
+    if (is_nil(ioterm)) {
+	DESTROY_ESTACK(stack);
+	*costp = 0;
+	return need;
+    }
+    if(is_binary(ioterm)) {
+	DESTROY_ESTACK(stack);
+	if (latin1) {
+	    Sint x = latin1_binary_need(ioterm);
+	    *costp = x;
+	    return x;
+	} else {
+	    *costp = 1;
+	    return aligned_binary_size(ioterm);
+	}
+    }
+    
+    if (!is_list(ioterm)) {
+	DESTROY_ESTACK(stack);
+	*costp = 0;
+	return (Sint) -1;
+    }
+    /* OK a list, needs to be processed in order, handling each flat list-level
+       as they occur, just like io_list_to_binary would */
+    ESTACK_PUSH(stack,ioterm);
+    while (!ESTACK_ISEMPTY(stack)) {
+	ioterm = ESTACK_POP(stack);
+	if (is_nil(ioterm)) {
+	    /* ignore empty lists */
+	    continue;
+	}
+	if(is_list(ioterm)) {
+L_Again:   /* Restart with sublist, old listend was pushed on stack */
+	    objp = list_val(ioterm);
+	    obj = CAR(objp);
+	    for(;;) { /* loop over one flat list of bytes and binaries
+		         until sublist or list end is encountered */
+		if (is_small(obj)) { /* Always small */
+		    for(;;) {
+			Uint x = unsigned_val(obj);
+			if (x < 0x80)
+			    need +=1;
+			else if (x < 0x800)
+			    need += 2;
+			else if (x < 0x10000) 
+			    need += 3;
+			else 
+			    need += 4; 
+			/* everything else will give badarg later 
+			   in the process, so we dont check */
+			++cost;
+			ioterm = CDR(objp);
+			if (!is_list(ioterm)) {
+			    break;
+			}
+			objp = list_val(ioterm);
+			obj = CAR(objp);
+			if (!is_byte(obj))
+			    break;
+		    }
+		} else if (is_nil(obj)) {
+		    ioterm = CDR(objp);
+		    if (!is_list(ioterm)) {
+			break;
+		    }
+		    objp = list_val(ioterm);
+		    obj = CAR(objp);
+		} else if (is_list(obj)) {
+		    /* push rest of list for later processing, start 
+		       again with sublist */
+		    ESTACK_PUSH(stack,CDR(objp));
+		    ioterm = obj;
+		    goto L_Again;
+		} else if (is_binary(obj)) {
+		    Sint x;
+
+		    if (latin1) { 
+			x = latin1_binary_need(obj);
+			if (x < 0) {
+			    DESTROY_ESTACK(stack);
+			    *costp = cost;
+			    return x;
+			} 
+			cost += x;
+		    } else {
+			x = aligned_binary_size(obj);
+			if (x < 0) {
+			    DESTROY_ESTACK(stack);
+			    *costp = cost;
+			    return x;
+			} 
+			++cost;
+		    }
+		    need += x;
+		    ioterm = CDR(objp);
+		    if (is_list(ioterm)) {
+			/* objp and obj need to be updated if 
+			   loop is to continue */
+			objp = list_val(ioterm);
+			obj = CAR(objp);
+		    }
+		} else {
+		    DESTROY_ESTACK(stack);
+		    *costp = cost;
+		    return ((Sint) -1);
+		} 
+		if (is_nil(ioterm) || !is_list(ioterm)) {
+		    break;
+		}
+	    } /* for(;;) */
+	} /* is_list(ioterm) */
+	
+	if (!is_list(ioterm) && !is_nil(ioterm)) {
+	    /* inproper list end */
+	    if (is_binary(ioterm)) {
+		Sint x; 
+		if (latin1) {
+		    x = latin1_binary_need(ioterm);
+		    if (x < 0) {
+			DESTROY_ESTACK(stack);
+			*costp = cost;
+			return x;
+		    } 
+		    cost += x;
+		} else {
+		    x = aligned_binary_size(ioterm);
+		    if (x < 0) {
+			DESTROY_ESTACK(stack);
+			*costp = cost;
+			return x;
+		    } 
+		    ++cost;
+		}
+		need += x;
+	    } else {
+		DESTROY_ESTACK(stack);
+		*costp = cost;
+		return ((Sint) -1);
+	    }
+	}
+    } /* while  not estack empty */
+    DESTROY_ESTACK(stack);
+    *costp = cost;
+    return need;
+}
+    
+    
+static Eterm do_build_utf8(Process *p, Eterm ioterm, int *left, int latin1,
+			   byte *target, int *pos, Uint *characters, int *err, 
+			   byte *leftover, int *num_leftovers)
+{
+    int c;
+    Eterm *objp;
+    Eterm obj;
+    DECLARE_ESTACK(stack);
+
+    *err = 0;
+    if ((*left) <= 0 || is_nil(ioterm)) {
+	DESTROY_ESTACK(stack);
+	return ioterm;
+    }
+    if(is_binary(ioterm)) {
+	Uint bitoffs;
+	Uint bitsize;
+	Uint size;
+	Uint i;
+	Eterm res_term = NIL;
+	unsigned char *bytes;
+	byte *temp_alloc = NULL;
+	Uint orig_size;
+	
+	ERTS_GET_BINARY_BYTES(ioterm, bytes, bitoffs, bitsize);
+	if (bitsize != 0) {
+	    *err = 1;
+	    DESTROY_ESTACK(stack);
+	    return ioterm;
+	}
+	if (bitoffs != 0) {
+	    bytes = erts_get_aligned_binary_bytes(ioterm, &temp_alloc);
+	    /* The call to erts_get_aligned_binary_bytes cannot fail as 
+	       we'we already checked bitsize and that this is a binary */
+	}
+
+	orig_size = size = binary_size(ioterm);
+
+	/* This is done to avoid splitting binaries in two 
+	   and then create an unnecessary rest that eventually gives an error.
+	   For cases where errors are not returned this is unnecessary */
+	if (!latin1) { 
+	    /* Find a valid character boundary */
+	    while (size > (*left) && 
+		   (((byte) bytes[(*left)]) & ((byte) 0xC0)) == ((byte) 0x80)) {
+		++(*left);
+	    }
+	}
+
+	if (size > (*left)) {
+	    Eterm *hp;
+	    ErlSubBin *sb;
+	    Eterm orig;
+	    Uint offset;
+	    /* Split the binary in two parts, of which we 
+	       only process the first */
+	    hp = HAlloc(p, ERL_SUB_BIN_SIZE);
+	    sb = (ErlSubBin *) hp;
+	    ERTS_GET_REAL_BIN(ioterm, orig, offset, bitoffs, bitsize);
+	    sb->thing_word = HEADER_SUB_BIN;
+	    sb->size = size - (*left);
+	    sb->offs = offset + (*left);
+	    sb->orig = orig;
+	    sb->bitoffs = bitoffs;
+	    sb->bitsize = bitsize;
+	    sb->is_writable = 0;
+	    res_term = make_binary(sb);
+	    size = (*left);
+	}
+
+	if (!latin1) {
+	    int num;
+	    byte *err_pos = NULL;
+	    num = copy_utf8_bin(target + (*pos), bytes, 
+				size, leftover, num_leftovers,&err_pos,characters);
+	    *pos += num;
+	    if (err_pos != NULL) {
+		int rest_bin_offset;
+		int rest_bin_size;
+		Eterm *hp;
+		ErlSubBin *sb;
+		Eterm orig;
+		Uint offset;
+
+		*err = 1;
+		/* we have no real stack, just build a list of the binaries
+		   we have not decoded... */
+		DESTROY_ESTACK(stack);
+
+		rest_bin_offset = (err_pos - bytes);
+		rest_bin_size = orig_size - rest_bin_offset;
+		
+		hp = HAlloc(p, ERL_SUB_BIN_SIZE);
+		sb = (ErlSubBin *) hp;
+		ERTS_GET_REAL_BIN(ioterm, orig, offset, bitoffs, bitsize);
+		sb->thing_word = HEADER_SUB_BIN;
+		sb->size = rest_bin_size;
+		sb->offs = offset + rest_bin_offset;
+		sb->orig = orig;
+		sb->bitoffs = bitoffs;
+		sb->bitsize = bitsize;
+		sb->is_writable = 0;
+		res_term = make_binary(sb);
+		erts_free_aligned_binary_bytes(temp_alloc);
+		return res_term;
+	    }
+	} else {
+	    i = 0;
+	    while(i < size) {
+		if (bytes[i] < 0x80) {
+		    target[(*pos)++] = bytes[i++];
+		} else {
+		    target[(*pos)++] = ((bytes[i] >> 6) | ((byte) 0xC0));
+		    target[(*pos)++] = ((bytes[i] & 0x3F) | ((byte) 0x80));
+		    ++i;
+		}
+		++(*characters);
+	    }
+	}
+	*left -= size;
+	DESTROY_ESTACK(stack);
+	erts_free_aligned_binary_bytes(temp_alloc);
+	return res_term;
+    }
+	
+    if (!is_list(ioterm)) {
+	*err = 1;
+	goto done;
+    }
+
+    /* OK a list, needs to be processed in order, handling each flat list-level
+       as they occur, just like io_list_to_binary would */
+    ESTACK_PUSH(stack,ioterm);
+    while (!ESTACK_ISEMPTY(stack) && (*left)) {
+	ioterm = ESTACK_POP(stack);
+	if (is_nil(ioterm)) {
+	    /* ignore empty lists */
+	    continue;
+	}
+	if(is_list(ioterm)) {
+L_Again:   /* Restart with sublist, old listend was pushed on stack */
+	    objp = list_val(ioterm);
+	    obj = CAR(objp);
+	    for(;;) { /* loop over one flat list of bytes and binaries
+		         until sublist or list end is encountered */
+		if (is_small(obj)) { /* Always small in unicode*/
+		    if (*num_leftovers) {
+			/* Have rest from previous bin and this is an integer, not allowed */
+			*err = 1;
+			goto done;
+		    }
+		    for(;;) {
+			Uint x = unsigned_val(obj);
+			if (latin1 && x > 255) {
+			    *err = 1;
+			    goto done;
+			}
+			if (x < 0x80) {
+			    target[(*pos)++] = (byte) x;
+			}
+			else if (x < 0x800) {
+			    target[(*pos)++] = (((byte) (x >> 6)) | 
+						 ((byte) 0xC0));
+			    target[(*pos)++] = (((byte) (x & 0x3F)) | 
+						((byte) 0x80));
+			} else if (x < 0x10000) {
+			    if ((x >= 0xD800 && x <= 0xDFFF) ||
+				(x == 0xFFFE) ||
+				(x == 0xFFFF)) { /* Invalid unicode range */
+				*err = 1;
+				goto done;
+			    }
+			    target[(*pos)++] = (((byte) (x >> 12)) | 
+						((byte) 0xE0));
+			    target[(*pos)++] = ((((byte) (x >> 6)) & 0x3F)  | 
+						((byte) 0x80));
+			    target[(*pos)++] = (((byte) (x & 0x3F)) | 
+						((byte) 0x80));
+			} else if (x < 0x110000) { /* Standard imposed max */
+			    target[(*pos)++] = (((byte) (x >> 18)) | 
+						((byte) 0xF0));
+			    target[(*pos)++] = ((((byte) (x >> 12)) & 0x3F)  | 
+						((byte) 0x80));
+			    target[(*pos)++] = ((((byte) (x >> 6)) & 0x3F)  | 
+						((byte) 0x80));
+			    target[(*pos)++] = (((byte) (x & 0x3F)) | 
+						((byte) 0x80));
+			} else {
+				*err = 1;
+				goto done;
+			}
+			++(*characters);
+			--(*left);
+			ioterm = CDR(objp);
+			if (!is_list(ioterm) || !(*left)) {
+			    break;
+			}
+			objp = list_val(ioterm);
+			obj = CAR(objp);
+			if (!is_small(obj))
+			    break;
+		    }
+		} else if (is_nil(obj)) {
+		    ioterm = CDR(objp);
+		    if (!is_list(ioterm)) {
+			break;
+		    }
+		    objp = list_val(ioterm);
+		    obj = CAR(objp);
+		} else if (is_list(obj)) {
+		    /* push rest of list for later processing, start 
+		       again with sublist */
+		    ESTACK_PUSH(stack,CDR(objp));
+		    ioterm = obj;
+		    goto L_Again;
+		} else if (is_binary(obj)) {
+		    Eterm rest_term;
+		    rest_term = do_build_utf8(p,obj,left,latin1,target,pos, characters, err, 
+					      leftover, num_leftovers);
+		    if ((*err) != 0) {
+			Eterm *hp;
+			hp = HAlloc(p, 2);
+			obj = CDR(objp);
+			ioterm = CONS(hp, rest_term, obj);
+			//(*left) = 0;
+			goto done;
+		    }
+		    if (rest_term != NIL) {
+			Eterm *hp;
+			hp = HAlloc(p, 2);
+			obj = CDR(objp);
+			ioterm = CONS(hp, rest_term, obj);
+			(*left) = 0;
+			break;
+		    }
+		    ioterm = CDR(objp);
+		    if (is_list(ioterm)) {
+			/* objp and obj need to be updated if 
+			   loop is to continue */
+			objp = list_val(ioterm);
+			obj = CAR(objp);
+		    }
+		} else {
+		    *err = 1;
+		    goto done;
+		} 
+		if (!(*left) || is_nil(ioterm) || !is_list(ioterm)) {
+		    break;
+		}
+	    } /* for(;;) */
+	} /* is_list(ioterm) */
+
+	if ((*left) && !is_list(ioterm) && !is_nil(ioterm)) {
+	    /* inproper list end */
+	    if (is_binary(ioterm)) {
+		ioterm = do_build_utf8(p,ioterm,left,latin1,target,pos,characters,err,leftover,num_leftovers);
+		if ((*err) != 0) {
+		    goto done;
+		}
+	    } else {
+		*err = 1;
+		goto done;
+	    }
+	}
+    } /* while left and not estack empty */
+ done:
+    c = ESTACK_COUNT(stack);
+    if (c > 0) {
+	Eterm *hp = HAlloc(p,2*c);
+	while(!ESTACK_ISEMPTY(stack)) {
+	    Eterm st = ESTACK_POP(stack);
+	    ioterm = CONS(hp, ioterm, st);
+	    hp += 2;
+	}
+    }
+    DESTROY_ESTACK(stack);
+    return ioterm;
+
+}
+
+static int check_leftovers(byte *source, int size) 
+{
+    if (((*source) & ((byte) 0xE0)) == 0xC0) {
+	return 0;
+    } else if (((*source) & ((byte) 0xF0)) == 0xE0) {
+	if (size < 2 || 
+	    (size < 3 && ((source[1] & ((byte) 0xC0)) == 0x80))) { 
+	    return 0;
+	}
+    } else if (((*source) & ((byte) 0xF8)) == 0xF0) {
+	if (size < 2 ||
+	    (size < 3 && ((source[1] & ((byte) 0xC0)) == 0x80)) ||
+	    (size < 4 && 
+	     ((source[1] & ((byte) 0xC0)) == 0x80) &&
+	     ((source[2] & ((byte) 0xC0)) == 0x80))) {
+	    return 0;
+	}
+    }
+    return -1;
+}
+	
+	 
+
+static BIF_RETTYPE build_utf8_return(Process *p,Eterm bin,int pos,
+			       Eterm rest_term,int err,
+			       byte *leftover,int num_leftovers,Eterm latin1)
+{
+    Eterm *hp;
+    Eterm ret;
+
+    binary_size(bin) = pos;
+    if (err) {
+	if (num_leftovers > 0) {
+	    Eterm leftover_bin = new_binary(p, leftover, num_leftovers);
+	    hp = HAlloc(p,8);
+	    rest_term = CONS(hp,rest_term,NIL);
+	    hp += 2;
+	    rest_term = CONS(hp,leftover_bin,rest_term);
+	    hp += 2;
+	} else {
+	   hp = HAlloc(p,4);
+	} 
+	ret = TUPLE3(hp,am_error,bin,rest_term);
+    } else if (rest_term == NIL && num_leftovers != 0) {
+	Eterm leftover_bin = new_binary(p, leftover, num_leftovers);
+	if (check_leftovers(leftover,num_leftovers) != 0) {
+	    hp = HAlloc(p,4);
+	    ret = TUPLE3(hp,am_error,bin,leftover_bin);
+	} else {
+	    hp = HAlloc(p,4);
+	    ret = TUPLE3(hp,am_incomplete,bin,leftover_bin);
+	}
+    } else { /* All OK */	    
+	if (rest_term != NIL) { /* Trap */
+	    if (num_leftovers > 0) {
+		Eterm rest_bin = new_binary(p, leftover, num_leftovers);
+		hp = HAlloc(p,2);
+		rest_term = CONS(hp,rest_bin,rest_term);
+	    }
+	    BUMP_ALL_REDS(p);
+	    BIF_TRAP3(&characters_to_utf8_trap_exp, p, bin, rest_term, latin1);
+	} else { /* Success */
+	    /*hp = HAlloc(p,5);
+	      ret = TUPLE4(hp,bin,rest_term,make_small(pos),make_small(err));*/
+	    ret = bin;
+	}
+    }
+    BIF_RET(ret);
+}
+
+
+static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3)
+{
+    Eterm *real_bin;
+    Sint need;
+    byte* bytes;
+    Eterm rest_term;
+    int left, sleft;
+    int pos;
+    int err;
+    byte leftover[4]; /* used for temp buffer too, 
+			 otherwise 3 bytes would have been enough */
+    int num_leftovers = 0;
+    int latin1 = 0;
+    Uint characters = 0;
+    
+    /*erts_printf("Trap %T!\r\n",BIF_ARG_2);*/
+    ASSERT(is_binary(BIF_ARG_1));
+    real_bin = binary_val(BIF_ARG_1);
+    ASSERT(*real_bin == HEADER_PROC_BIN);
+    need = ((ProcBin *) real_bin)->val->orig_size;
+    pos = (int) binary_size(BIF_ARG_1);
+    bytes = binary_bytes(BIF_ARG_1);
+    sleft = left = allowed_iterations(BIF_P);
+    err = 0;
+    if (BIF_ARG_3 == am_latin1) {
+	latin1 = 1;
+    } 
+    rest_term = do_build_utf8(BIF_P, BIF_ARG_2, &left, latin1,
+			      bytes, &pos, &characters, &err, leftover, &num_leftovers); 
+    cost_to_proc(BIF_P, sleft - left);
+    return build_utf8_return(BIF_P,BIF_ARG_1,pos,rest_term,err,
+			      leftover,num_leftovers,BIF_ARG_3);
+}
+
+BIF_RETTYPE unicode_bin_is_7bit_1(BIF_ALIST_1)
+{
+    Sint need;
+    if(!is_binary(BIF_ARG_1)) {
+	BIF_RET(am_false);
+    }
+    need = latin1_binary_need(BIF_ARG_1);
+    if(need >= 0 && aligned_binary_size(BIF_ARG_1) == need) {
+	BIF_RET(am_true);
+    }
+    BIF_RET(am_false);
+}
+
+static int is_valid_utf8(Eterm orig_bin)
+{
+    Uint bitoffs;
+    Uint bitsize;
+    Uint size;
+    byte *temp_alloc = NULL;
+    byte *endpos;
+    Uint numchar;
+    byte *bytes;
+    int ret;
+
+    ERTS_GET_BINARY_BYTES(orig_bin, bytes, bitoffs, bitsize);
+    if (bitsize != 0) {
+	return 0;
+    }
+    if (bitoffs != 0) {
+	bytes = erts_get_aligned_binary_bytes(orig_bin, &temp_alloc);
+    }
+    size = binary_size(orig_bin);
+    ret = analyze_utf8(bytes,
+		       size,
+		       &endpos,&numchar,NULL);
+    erts_free_aligned_binary_bytes(temp_alloc);
+    return (ret == UTF8_OK);
+}
+
+BIF_RETTYPE unicode_characters_to_binary_2(BIF_ALIST_2)
+{
+    Sint need;
+    Uint characters;
+    int latin1;
+    Eterm bin;
+    byte *bytes;
+    int pos;
+    int err;
+    int left, sleft;
+    Eterm rest_term, subject;
+    byte leftover[4]; /* used for temp buffer too, o
+			 therwise 3 bytes would have been enough */
+    int num_leftovers = 0;
+    Uint cost_of_utf8_need;
+
+
+    if (BIF_ARG_2 == am_latin1) {
+	latin1 = 1;
+    } else if (BIF_ARG_2 == am_unicode || BIF_ARG_2 == am_utf8) {
+	latin1 = 0;
+    } else {
+	BIF_TRAP2(c_to_b_int_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2);
+    }	
+    if (is_list(BIF_ARG_1) && is_binary(CAR(list_val(BIF_ARG_1))) && 
+	is_nil(CDR(list_val(BIF_ARG_1)))) {
+	subject = CAR(list_val(BIF_ARG_1));
+    } else {
+	subject = BIF_ARG_1;
+    }
+
+    need = utf8_need(subject,latin1,&cost_of_utf8_need);
+    if (need < 0) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    if (is_binary(subject) && need >= 0 && aligned_binary_size(subject) == need
+	&& (latin1 || is_valid_utf8(subject))) {
+	cost_to_proc(BIF_P, simple_loops_to_common(cost_of_utf8_need)); 
+	    BIF_RET(subject);
+    }
+	
+
+    bin = erts_new_mso_binary(BIF_P, (byte *)NULL, need);
+    bytes = binary_bytes(bin);
+    cost_to_proc(BIF_P, simple_loops_to_common(cost_of_utf8_need)); 
+    left = allowed_iterations(BIF_P) - 
+	simple_loops_to_common(cost_of_utf8_need);
+    if (left <= 0) {
+	/* simplified - let everything be setup by setting left to 1 */
+	left = 1;
+    }
+    sleft = left;
+    pos = 0;
+    err = 0;
+
+
+    rest_term = do_build_utf8(BIF_P, subject, &left, latin1,
+			      bytes, &pos, &characters, &err, leftover, &num_leftovers); 
+#ifdef HARDDEBUG
+    if (left == 0) {
+	Eterm bin;
+	if (is_binary(subject)) {
+	    bin = subject;
+	} else if(is_list(subject) && is_binary(CAR(list_val(subject)))) {
+	    bin = CAR(list_val(subject));
+	} else {
+	    bin = NIL;
+	}
+	if (is_binary(bin)) {
+	    byte *t = NULL;
+	    Uint sz = binary_size(bin);
+	    byte *by = erts_get_aligned_binary_bytes(bin,&t);
+	    int i;
+	    erts_printf("<<");
+	    for (i = 0;i < sz; ++i) {
+		erts_printf((i == sz -1) ? "0x%X" : "0x%X, ", (unsigned) by[i]);
+	    }
+	    erts_printf(">>: ");
+	    erts_free_aligned_binary_bytes(t);
+	}
+	erts_printf("%d - %d = %d\n",sleft,left,sleft - left);
+    }
+#endif
+    cost_to_proc(BIF_P, sleft - left); 
+    return build_utf8_return(BIF_P,bin,pos,rest_term,err,
+			     leftover,num_leftovers,BIF_ARG_2);
+}
+
+static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint characters,
+				     Eterm rest_term, int err,
+				     byte *leftover, int num_leftovers,
+				     Eterm latin1, int left)
+{
+    Eterm *hp;
+    
+    if (left <= 0) {
+	left = 1;
+    }
+    
+    if (err) {
+	if (num_leftovers > 0) {
+	    Eterm leftover_bin = new_binary(p, leftover, num_leftovers);
+	    hp = HAlloc(p,4);
+	    rest_term = CONS(hp,rest_term,NIL);
+	    hp += 2;
+	    rest_term = CONS(hp,leftover_bin,rest_term);
+	}
+	BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, UTF8_ERROR, left, NIL));
+    } else if (rest_term == NIL && num_leftovers != 0) {
+	Eterm leftover_bin = new_binary(p, leftover, num_leftovers);
+	if (check_leftovers(leftover,num_leftovers) != 0) {
+	    BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_ERROR, 
+					  left, NIL));
+	} else {
+	    BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_INCOMPLETE, 
+					  left, NIL));
+	}
+    } else { /* All OK */	    
+	if (rest_term != NIL) { /* Trap */
+	    RestartContext rc;
+	    if (num_leftovers > 0) {
+		Eterm rest_bin = new_binary(p, leftover, num_leftovers);
+		hp = HAlloc(p,2);
+		rest_term = CONS(hp,rest_bin,rest_term);
+	    }
+	    BUMP_ALL_REDS(p);
+	    rc.bytes = bytes;
+	    rc.num_processed_bytes = 0; /* not used */
+	    rc.num_bytes_to_process = pos;
+	    rc.num_resulting_chars = characters;
+	    rc.state = UTF8_OK; /* not used */
+	    BIF_TRAP3(&characters_to_list_trap_1_exp, p, make_magic_bin_for_restart(p,&rc), 
+		      rest_term, latin1);
+	} else { /* Success */
+	    BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, UTF8_OK, left, NIL));
+	}
+    }
+}
+
+static BIF_RETTYPE characters_to_list_trap_1(BIF_ALIST_3)
+{
+    RestartContext *rc;
+    byte* bytes;
+    int pos;
+    Uint characters;
+    int err;
+    Eterm rest_term;
+    int left, sleft;
+
+    int latin1 = 0;
+    byte leftover[4]; /* used for temp buffer too, 
+			 otherwise 3 bytes would have been enough */
+    int num_leftovers = 0;
+    
+
+    rc = get_rc_from_bin(BIF_ARG_1);
+
+    bytes = rc->bytes;
+    rc->bytes = NULL; /* to avoid free due to later GC */
+    pos = rc->num_bytes_to_process;
+    characters = rc->num_resulting_chars;
+
+    sleft = left = allowed_iterations(BIF_P);
+    err = 0;
+    if (BIF_ARG_3 == am_latin1) {
+	latin1 = 1;
+    } 
+    rest_term = do_build_utf8(BIF_P, BIF_ARG_2, &left, latin1,
+			      bytes, &pos, &characters, &err, leftover, &num_leftovers); 
+    cost_to_proc(BIF_P, sleft - left);
+    return build_list_return(BIF_P,bytes,pos,characters,rest_term,err,
+			      leftover,num_leftovers,BIF_ARG_3,left);
+}
+
+BIF_RETTYPE unicode_characters_to_list_2(BIF_ALIST_2)
+{
+    Sint need;
+    int latin1;
+    Uint characters = 0;
+    byte *bytes;
+    int pos;
+    int err;
+    int left, sleft;
+    Eterm rest_term;
+    byte leftover[4]; /* used for temp buffer too, o
+			 therwise 3 bytes would have been enough */
+    int num_leftovers = 0;
+    Uint cost_of_utf8_need;
+
+    if (BIF_ARG_2 == am_latin1) {
+	latin1 = 1;
+    } else if (BIF_ARG_2 == am_unicode || BIF_ARG_2 == am_utf8) {
+	latin1 = 0;
+    } else {
+	BIF_TRAP2(c_to_l_int_trap_exportp, BIF_P, BIF_ARG_1, BIF_ARG_2);
+    }	
+    if (is_binary(BIF_ARG_1) && !latin1) { /* Optimized behaviour for this case */
+	    return utf8_to_list(BIF_P,BIF_ARG_1);
+    }
+    need = utf8_need(BIF_ARG_1,latin1,&cost_of_utf8_need);
+    if (need < 0) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    bytes = alloc_restart(need);
+    cost_to_proc(BIF_P, simple_loops_to_common(cost_of_utf8_need)); 
+    left = allowed_iterations(BIF_P) - 
+	simple_loops_to_common(cost_of_utf8_need);
+    if (left <= 0) {
+	/* simplified - let everything be setup by setting left to 1 */
+	left = 1;
+    }
+    sleft = left;
+    pos = 0;
+    err = 0;
+
+
+    rest_term = do_build_utf8(BIF_P, BIF_ARG_1, &left, latin1,
+			      bytes, &pos, &characters, &err, leftover, &num_leftovers); 
+    cost_to_proc(BIF_P, sleft - left); 
+    return build_list_return(BIF_P,bytes,pos,characters,rest_term,err,
+			     leftover,num_leftovers,BIF_ARG_2,left);
+}
+
+
+/*
+ * When input to characters_to_list is a plain binary and the format is 'unicode', we do
+ * a faster analyze and size count with this function.
+ */
+static int analyze_utf8(byte *source, Uint size, 
+			byte **err_pos, Uint *num_chars, int *left)
+{
+    *err_pos = source;
+    *num_chars = 0;
+    while (size) {
+	if (((*source) & ((byte) 0x80)) == 0) {
+	    source++;
+	    --size; 
+	} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
+	    if (size < 2) {
+		return UTF8_INCOMPLETE;
+	    }
+	    if (((source[1] & ((byte) 0xC0)) != 0x80) ||
+		((*source) < 0xC2) /* overlong */) {
+		return UTF8_ERROR;
+	    }
+	    source += 2;
+	    size -= 2;
+	} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
+	    if (size < 3) {
+		return UTF8_INCOMPLETE;
+	    }
+	    if (((source[1] & ((byte) 0xC0)) != 0x80) ||
+		((source[2] & ((byte) 0xC0)) != 0x80) ||
+		(((*source) == 0xE0) && (source[1] < 0xA0)) /* overlong */ ) {
+		return UTF8_ERROR;
+	    }
+	    if ((((*source) & ((byte) 0xF)) == 0xD) && 
+		((source[1] & 0x20) != 0)) {
+		return UTF8_ERROR;
+	    }
+	    if (((*source) == 0xEF) && (source[1] == 0xBF) &&
+		((source[2] == 0xBE) || (source[2] == 0xBF))) {
+		return UTF8_ERROR;
+	    }
+	    source += 3;
+	    size -= 3;
+	} else if (((*source) & ((byte) 0xF8)) == 0xF0) {
+	    if (size < 4) {
+		return UTF8_INCOMPLETE;
+	    }
+	    if (((source[1] & ((byte) 0xC0)) != 0x80) ||
+		((source[2] & ((byte) 0xC0)) != 0x80) ||
+		((source[3] & ((byte) 0xC0)) != 0x80) ||
+		(((*source) == 0xF0) && (source[1] < 0x90)) /* overlong */) {
+		return UTF8_ERROR;
+	    }
+	    if ((((*source) & ((byte)0x7)) > 0x4U) ||
+		((((*source) & ((byte)0x7)) == 0x4U) && 
+		 ((source[1] & ((byte)0x3F)) > 0xFU))) {
+		return UTF8_ERROR;
+	    }
+	    source += 4;
+	    size -= 4; 
+	} else {
+	    return UTF8_ERROR;
+	}
+	++(*num_chars);
+	*err_pos = source;
+	if (left && --(*left) <= 0) {
+	    return UTF8_ANALYZE_MORE;
+	}
+    }
+    return UTF8_OK;
+}
+
+/*
+ * No errors should be able to occur - no overlongs, no malformed, no nothing
+ */    
+static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, 
+			     Uint left,
+			     Uint *num_built, Uint *num_eaten, Eterm tail)
+{
+    Eterm *hp;
+    Eterm ret;
+    byte *source, *ssource;
+    Uint unipoint;
+
+    ASSERT(num > 0);
+    if (left < num) {
+	if (left > 0)
+	    num = left;
+	else
+	    num = 1;
+    }
+    
+    *num_built = num; /* Always */
+
+    hp = HAlloc(p,num * 2);
+    ret = tail;
+    source = bytes + sz;
+    ssource = source;
+    while(--source >= bytes) {
+	if (((*source) & ((byte) 0x80)) == 0) {
+	    unipoint = (Uint) *source;
+	} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
+	    unipoint = 
+		(((Uint) ((*source) & ((byte) 0x1F))) << 6) |
+		((Uint) (source[1] & ((byte) 0x3F))); 	
+	} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
+	    unipoint = 
+		(((Uint) ((*source) & ((byte) 0xF))) << 12) |
+		(((Uint) (source[1] & ((byte) 0x3F))) << 6) |
+		((Uint) (source[2] & ((byte) 0x3F))); 	 	
+	} else if (((*source) & ((byte) 0xF8)) == 0xF0) {
+	    unipoint = 
+		(((Uint) ((*source) & ((byte) 0x7))) << 18) |
+		(((Uint) (source[1] & ((byte) 0x3F))) << 12) |
+		(((Uint) (source[2] & ((byte) 0x3F))) << 6) |
+		((Uint) (source[3] & ((byte) 0x3F))); 	 	
+	} else {
+	    /* ignore 2#10XXXXXX */
+	    continue;
+	}
+	ret = CONS(hp,make_small(unipoint),ret);
+	hp += 2;
+	if (--num <= 0) {
+	    break;
+	}
+    }
+    *num_eaten = (ssource - source);
+    return ret;
+}
+
+/*
+ * The last step of characters_to_list, build a list from the buffer 'bytes' (created in the same way
+ * as for characters_to_utf8). All sizes are known in advance and most data will be held in a 
+ * "magic binary" during trapping.
+ */
+static BIF_RETTYPE finalize_list_to_list(Process *p, 
+					 byte *bytes,
+					 Eterm rest,
+					 Uint num_processed_bytes,
+					 Uint num_bytes_to_process, 
+					 Uint num_resulting_chars, 
+					 int state, int left,
+					 Eterm tail) 
+{
+    Uint num_built; /* characters */
+    Uint num_eaten; /* bytes */
+    Eterm *hp;
+    Eterm converted,ret;
+
+    if (!num_bytes_to_process) {
+	converted = tail;
+    } else {
+	num_built = 0;
+	num_eaten = 0;
+	converted = do_utf8_to_list(p, num_resulting_chars,
+				    bytes, num_bytes_to_process,
+				    left, &num_built, &num_eaten, tail);
+	cost_to_proc(p,num_built);
+	
+	if (num_built != num_resulting_chars) { /* work left to do */
+	    RestartContext rc;
+
+	    rc.num_resulting_chars = num_resulting_chars - num_built;
+	    rc.num_bytes_to_process = num_bytes_to_process - num_eaten;
+	    rc.num_processed_bytes = num_processed_bytes + num_eaten;
+	    rc.state = state;
+	    rc.bytes = bytes;
+	    BUMP_ALL_REDS(p);
+	    BIF_TRAP3(&characters_to_list_trap_2_exp, p, 
+		       make_magic_bin_for_restart(p, &rc), rest, converted); 
+	}
+    }
+
+    /* 
+     * OK, no more trapping, let's get rid of the temporary array...
+     */
+
+    free_restart(bytes);
+    if (state == UTF8_INCOMPLETE) {
+	hp = HAlloc(p,4);
+	ret = TUPLE3(hp,am_incomplete,converted,rest);
+    } else if (state == UTF8_ERROR) {
+	hp = HAlloc(p,4);
+	ret = TUPLE3(hp,am_error,converted,rest);
+    } else {
+	ret = converted;
+    }
+
+    BIF_RET(ret);
+}
+ 
+static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3)
+{
+    RestartContext *rc;
+    byte *bytes;
+
+    rc = get_rc_from_bin(BIF_ARG_1);
+
+    bytes = rc->bytes;
+    rc->bytes = NULL; /* Don't want this freed just yet... */
+    return finalize_list_to_list(BIF_P, bytes, BIF_ARG_2, rc->num_processed_bytes,
+				 rc->num_bytes_to_process, rc->num_resulting_chars,
+				 rc->state, allowed_iterations(BIF_P), BIF_ARG_3);
+}
+
+
+/*
+ * Hooks into the process of decoding a binary depending on state.
+ * If last_state is UTF8_ANALYZE_MORE, num_bytes_to_process 
+ * and num_resulting_chars will grow
+ * until we're done analyzing the binary. Then we'll eat 
+ * the bytes to process, lowering num_bytes_to_process and num_resulting_chars,
+ * while increasing num_processed_bytes until we're done. the state 
+ * indicates how to return (error, incomplete or ok) in this stage.
+ * note that num_processed_bytes and num_bytes_to_process will make up the 
+ * length of the binary part to process, not necessarily the length of the 
+ * whole binary (if there are errors or an incomplete tail).
+ *
+ * Analyzing happens from the beginning of the binary towards the end,
+ * while result is built from the end of the analyzed/accepted part 
+ * towards the beginning.
+ *
+ * Note that this routine is *only* called when original input was a plain utf8 binary,
+ * otherwise the rest and the sizes are known in advance, so finalize_list_to_list is 
+ * used to build the resulting list (no analyzing needed).
+ */
+static BIF_RETTYPE do_bif_utf8_to_list(Process *p, 
+				       Eterm orig_bin,
+				       Uint num_processed_bytes,
+				       Uint num_bytes_to_process, 
+				       Uint num_resulting_chars, 
+				       int state,
+				       Eterm tail) 
+{
+    int left;
+    Uint bitoffs;
+    Uint bitsize;
+    Uint size;
+    byte *bytes;
+    Eterm converted = NIL;
+    Eterm rest = NIL;
+    Eterm *hp;
+    Eterm ret;
+    byte *temp_alloc = NULL;
+    byte *endpos;
+    Uint numchar;
+
+    Uint b_sz; /* size of the non analyzed tail */
+    Uint num_built; /* characters */
+    Uint num_eaten; /* bytes */
+
+    ERTS_GET_BINARY_BYTES(orig_bin, bytes, bitoffs, bitsize);
+    if (bitsize != 0) {
+	converted = NIL;
+	rest = orig_bin;
+	goto error_return;
+    }
+    if (bitoffs != 0) {
+	bytes = erts_get_aligned_binary_bytes(orig_bin, &temp_alloc);
+    }
+    
+    size = binary_size(orig_bin);
+
+    left = allowed_iterations(p);
+    
+    if (state == UTF8_ANALYZE_MORE) {
+	state = analyze_utf8(bytes + num_bytes_to_process,
+			     size - num_bytes_to_process,
+			     &endpos,&numchar,&left);
+	cost_to_proc(p,numchar);
+	num_resulting_chars += numchar;
+	num_bytes_to_process = endpos - bytes;
+	if (state == UTF8_ANALYZE_MORE) {
+	    Eterm epos = erts_make_integer(num_bytes_to_process,p);
+	    Eterm enumchar = erts_make_integer(num_resulting_chars,p);
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    BUMP_ALL_REDS(p);
+	    BIF_TRAP3(&characters_to_list_trap_3_exp, p, orig_bin, epos, 
+		      enumchar);
+	}
+    } 
+
+    /* 
+     * If we're here, we have everything analyzed and are instead building 
+     */
+
+
+    if (!num_bytes_to_process) {
+	converted = tail;
+    } else {
+	num_built = 0;
+	num_eaten = 0;
+	converted = do_utf8_to_list(p, num_resulting_chars,
+				    bytes, num_bytes_to_process,
+				    left, &num_built, &num_eaten, tail);
+	cost_to_proc(p,num_built);
+	
+	if (num_built != num_resulting_chars) { /* work left to do */
+	    Eterm newnum_resulting_chars = 
+		erts_make_integer(num_resulting_chars - num_built,p);
+	    Eterm newnum_bytes_to_process = 
+		erts_make_integer(num_bytes_to_process - num_eaten,p);
+	    Eterm newnum_processed_bytes = 
+		erts_make_integer(num_processed_bytes + num_eaten,p);
+	    Eterm traptuple;
+	    hp = HAlloc(p,7);
+	    traptuple = TUPLE6(hp,orig_bin,newnum_processed_bytes,
+			       newnum_bytes_to_process, 
+			       newnum_resulting_chars,
+			       make_small(state),
+			       converted);
+	    BUMP_ALL_REDS(p);
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    BIF_TRAP1(&characters_to_list_trap_4_exp,p,traptuple);
+	}
+    }
+
+    /* 
+     * OK, no more trapping, let's build rest binary if there should
+     * be one. 
+     */
+
+    b_sz = size - (num_bytes_to_process + num_processed_bytes);
+
+    if (b_sz) {
+	ErlSubBin *sb;
+	Eterm orig;
+	Uint offset;
+	ASSERT(state != UTF8_OK);
+	hp = HAlloc(p, ERL_SUB_BIN_SIZE);
+	sb = (ErlSubBin *) hp;
+	ERTS_GET_REAL_BIN(orig_bin, orig, offset, bitoffs, bitsize);
+	sb->thing_word = HEADER_SUB_BIN;
+	sb->size = b_sz;
+	sb->offs = num_bytes_to_process + num_processed_bytes;
+	sb->orig = orig;
+	sb->bitoffs = bitoffs;
+	sb->bitsize = bitsize;
+	sb->is_writable = 0;
+	rest = make_binary(sb);
+    } 
+
+    /* Done */
+
+    if (state == UTF8_INCOMPLETE) {
+	if (check_leftovers(bytes + num_bytes_to_process + num_processed_bytes,
+			    b_sz) != 0) {
+	    goto error_return;
+	}
+	hp = HAlloc(p,4);
+	ret = TUPLE3(hp,am_incomplete,converted,rest);
+    } else if (state == UTF8_ERROR) {
+ error_return:
+	hp = HAlloc(p,4);
+	ret = TUPLE3(hp,am_error,converted,rest);
+    } else {
+	ret = converted;
+    }
+
+    erts_free_aligned_binary_bytes(temp_alloc);
+    BIF_RET(ret);
+}
+
+
+/* 
+ * This is called when there's still analyzing left to do,
+ * we only reach this if original input was a binary.
+ */
+
+static BIF_RETTYPE characters_to_list_trap_3(BIF_ALIST_3)
+{
+    Uint num_bytes_to_process;
+    Uint num_resulting_chars;
+
+    term_to_Uint(BIF_ARG_2, &num_bytes_to_process); /* The number of already
+						       analyzed and accepted 
+						       bytes */
+    term_to_Uint(BIF_ARG_3, &num_resulting_chars); /* The number of chars
+						      procuced by the
+						      already analyzed
+						      part of the binary */
+
+    /*erts_printf("Trap: %T, %T, %T\n",BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);*/
+
+    return do_bif_utf8_to_list(BIF_P, 
+			       BIF_ARG_1, /* the binary */ 
+			       0U, /* nothing processed yet */
+			       num_bytes_to_process, 
+			       num_resulting_chars,
+			       UTF8_ANALYZE_MORE, /* always this state here */
+			       NIL); /* Nothing built -> no tail yet */
+	
+}
+
+/*
+ * This is called when analyzing is done and we are trapped during building,
+ * we only reach this if original input was a binary.
+ */
+static BIF_RETTYPE characters_to_list_trap_4(BIF_ALIST_1)
+{
+    Uint num_processed_bytes;
+    Uint num_bytes_to_process;
+    Uint num_resulting_chars;
+    Eterm orig_bin, tail;
+    int last_state;
+    Eterm *tplp = tuple_val(BIF_ARG_1);
+
+    orig_bin = tplp[1];
+    term_to_Uint(tplp[2], &num_processed_bytes);
+    term_to_Uint(tplp[3], &num_bytes_to_process);
+    term_to_Uint(tplp[4], &num_resulting_chars);
+    last_state = (int) signed_val(tplp[5]);
+    tail = tplp[6];
+
+    /*erts_printf("Trap: {%T, %lu, %lu, %lu, %d, %T}\n",
+      orig_bin, num_processed_bytes, num_bytes_to_process, 
+      num_resulting_chars, last_state, tail);*/
+
+    return do_bif_utf8_to_list(BIF_P, 
+			       orig_bin, /* The whole binary */
+			       num_processed_bytes,  /* Number of bytes 
+							already processed */
+			       num_bytes_to_process, /* Bytes left to proc. */
+			       num_resulting_chars,  /* Num chars left to 
+							build */
+			       last_state,           /* The current state 
+							(never ANALYZE_MORE)*/
+			       tail);                /* The already built 
+							tail */  
+	
+}
+/*
+ * This is only used when characters are a plain unicode (utf8) binary.
+ * Instead of building an utf8 buffer, we analyze the binary given and use that.
+ */
+
+static BIF_RETTYPE utf8_to_list(BIF_ALIST_1)
+{
+    if (!is_binary(BIF_ARG_1) || aligned_binary_size(BIF_ARG_1) < 0) {
+	BIF_ERROR(BIF_P,BADARG);
+    }
+    return do_bif_utf8_to_list(BIF_P, BIF_ARG_1, 0U, 0U, 0U, 
+			       UTF8_ANALYZE_MORE,NIL);
+}
+
+
+BIF_RETTYPE atom_to_binary_2(BIF_ALIST_2)
+{
+    Atom* ap;
+
+    if (is_not_atom(BIF_ARG_1)) {
+	goto error;
+    }
+
+    ap = atom_tab(atom_val(BIF_ARG_1));
+
+    if (BIF_ARG_2 == am_latin1) {
+	BIF_RET(new_binary(BIF_P, ap->name, ap->len));
+    } else if (BIF_ARG_2 == am_utf8 || BIF_ARG_2 == am_unicode) {
+	int bin_size = 0;
+	int i;
+	Eterm bin_term;
+	byte* bin_p;
+
+	for (i = 0; i < ap->len; i++) {
+	    bin_size += (ap->name[i] >= 0x80) ? 2 : 1;
+	}
+	if (bin_size == ap->len) {
+	    BIF_RET(new_binary(BIF_P, ap->name, ap->len));
+	}
+	bin_term = new_binary(BIF_P, 0, bin_size);
+	bin_p = binary_bytes(bin_term);
+	for (i = 0; i < ap->len; i++) {
+	    byte b = ap->name[i];
+	    if (b < 0x80) {
+		*bin_p++ = b;
+	    } else {
+		*bin_p++ = 0xC0 | (b >> 6);
+		*bin_p++ = 0x80 | (b & 0x3F);
+	    }
+	}
+	BIF_RET(bin_term);
+    } else {
+    error:
+	BIF_ERROR(BIF_P, BADARG);
+    }
+}
+
+static BIF_RETTYPE
+binary_to_atom(Process* p, Eterm bin, Eterm enc, int must_exist)
+{
+    byte* bytes;
+    byte *temp_alloc = NULL;
+    Uint bin_size;
+
+    if ((bytes = erts_get_aligned_binary_bytes(bin, &temp_alloc)) == 0) {
+	BIF_ERROR(p, BADARG);
+    }
+    bin_size = binary_size(bin);
+    if (enc == am_latin1) {
+	Eterm a;
+	if (bin_size > MAX_ATOM_LENGTH) {
+	system_limit:
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    BIF_ERROR(p, SYSTEM_LIMIT);
+	}
+	if (!must_exist) {
+	    a = am_atom_put((char *)bytes, bin_size);
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    BIF_RET(a);
+	} else if (erts_atom_get((char *)bytes, bin_size, &a)) {
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    BIF_RET(a);
+	} else {
+	    goto badarg;
+	}
+    } else if (enc == am_utf8 || enc == am_unicode) {
+	char *buf;
+	char *dst;
+	int i;
+	int num_chars;
+	Eterm res;
+
+	if (bin_size > 2*MAX_ATOM_LENGTH) {
+	    byte* err_pos;
+	    Uint n;
+	    int reds_left = bin_size+1; /* Number of reductions left. */
+
+	    if (analyze_utf8(bytes, bin_size, &err_pos,
+			     &n, &reds_left) == UTF8_OK) {
+		/* 
+		 * Correct UTF-8 encoding, but too many characters to
+		 * fit in an atom.
+		 */
+		goto system_limit;
+	    } else {
+		/*
+		 * Something wrong in the UTF-8 encoding or Unicode code
+		 * points > 255.
+		 */
+		goto badarg;
+	    }
+	}
+
+	/*
+	 * Allocate a temporary buffer the same size as the binary,
+	 * so that we don't need an extra overflow test.
+	 */
+	buf = (char *) erts_alloc(ERTS_ALC_T_TMP, bin_size);
+	dst = buf;
+	for (i = 0; i < bin_size; i++) {
+	    int c = bytes[i];
+	    if (c < 0x80) {
+		*dst++ = c;
+	    } else if (i < bin_size-1) {
+		int c2;
+		if ((c & 0xE0) != 0xC0) {
+		    goto free_badarg;
+		}
+		i++;
+		c = (c & 0x3F) << 6;
+		c2 = bytes[i];
+		if ((c2 & 0xC0) != 0x80) {
+		    goto free_badarg;
+		}
+		c = c | (c2 & 0x3F);
+		if (0x80 <= c && c < 256) {
+		    *dst++ = c;
+		} else {
+		    goto free_badarg;
+		}
+	    } else {
+	    free_badarg:
+		erts_free(ERTS_ALC_T_TMP, (void *) buf);
+		goto badarg;
+	    }
+	}
+	num_chars = dst - buf;
+	if (num_chars > MAX_ATOM_LENGTH) {
+	    erts_free(ERTS_ALC_T_TMP, (void *) buf);
+	    goto system_limit;
+	}
+	if (!must_exist) {
+	    res = am_atom_put(buf, num_chars);
+	    erts_free(ERTS_ALC_T_TMP, (void *) buf);
+	    erts_free_aligned_binary_bytes(temp_alloc);
+	    BIF_RET(res);
+	} else {
+	    int exists = erts_atom_get(buf, num_chars, &res);
+	    erts_free(ERTS_ALC_T_TMP, (void *) buf);
+	    if (exists) {
+		erts_free_aligned_binary_bytes(temp_alloc);
+		BIF_RET(res);
+	    } else {
+		goto badarg;
+	    }
+	}
+    } else {
+    badarg:
+	erts_free_aligned_binary_bytes(temp_alloc);
+	BIF_ERROR(p, BADARG);
+    }
+}
+
+BIF_RETTYPE binary_to_atom_2(BIF_ALIST_2)
+{
+    return binary_to_atom(BIF_P, BIF_ARG_1, BIF_ARG_2, 0);
+}
+
+BIF_RETTYPE binary_to_existing_atom_2(BIF_ALIST_2)
+{
+    return binary_to_atom(BIF_P, BIF_ARG_1, BIF_ARG_2, 1);
+}