From 685d009efcfd7521e9c918a14b58eac19755299d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 21 Dec 2012 15:50:21 +0100
Subject: erl_interface: Enable decode of unicode atoms

No API changes or additions. Just the ability for erl_interface to decode
unicode atoms and convert them into latin1 strings to preserve backward
compatibility for the existing API.
---
 lib/erl_interface/include/ei.h                |   2 +
 lib/erl_interface/src/decode/decode_atom.c    |  68 +++++++--
 lib/erl_interface/src/decode/decode_boolean.c |  32 ++---
 lib/erl_interface/src/decode/decode_pid.c     |  13 +-
 lib/erl_interface/src/decode/decode_port.c    |  12 +-
 lib/erl_interface/src/decode/decode_ref.c     |  26 +---
 lib/erl_interface/src/legacy/erl_marshal.c    | 200 +++++++++++++-------------
 lib/erl_interface/src/misc/ei_decode_term.c   |  37 +----
 lib/erl_interface/src/misc/ei_printterm.c     |   2 +
 lib/erl_interface/src/misc/get_type.c         |   8 +-
 lib/erl_interface/src/misc/putget.h           |   3 +
 11 files changed, 202 insertions(+), 201 deletions(-)

(limited to 'lib/erl_interface')

diff --git a/lib/erl_interface/include/ei.h b/lib/erl_interface/include/ei.h
index ae815b414a..8f07b24852 100644
--- a/lib/erl_interface/include/ei.h
+++ b/lib/erl_interface/include/ei.h
@@ -115,6 +115,8 @@
 #define ERL_FLOAT_EXT         'c'
 #define NEW_FLOAT_EXT         'F'
 #define ERL_ATOM_EXT          'd'
+#define ERL_SMALL_ATOM_EXT    's'
+#define ERL_UNICODE_ATOM_EXT  'v'
 #define ERL_REFERENCE_EXT     'e'
 #define ERL_NEW_REFERENCE_EXT 'r'
 #define ERL_PORT_EXT          'f'
diff --git a/lib/erl_interface/src/decode/decode_atom.c b/lib/erl_interface/src/decode/decode_atom.c
index c2e6a0426e..84edf1766a 100644
--- a/lib/erl_interface/src/decode/decode_atom.c
+++ b/lib/erl_interface/src/decode/decode_atom.c
@@ -21,24 +21,76 @@
 #include "eiext.h"
 #include "putget.h"
 
+static int utf8_to_latin1(char* dest, const char* source, unsigned len);
+
 int ei_decode_atom(const char *buf, int *index, char *p)
 {
   const char *s = buf + *index;
   const char *s0 = s;
   int len;
 
-  if (get8(s) != ERL_ATOM_EXT) return -1;
+  switch (get8(s)) {
+  case ERL_ATOM_EXT:
+      len = get16be(s);
+      if (len > MAXATOMLEN) return -1;
+      if (p) {
+	  memmove(p,s,len); 
+	  p[len] = (char)0;
+      }
+      break;
+
+  case ERL_SMALL_ATOM_EXT:
+      len = get8(s);
+      if (p) {
+	  memmove(p,s,len); 
+	  p[len] = (char)0;
+      }
+      break;
+
+  case ERL_UNICODE_ATOM_EXT:
+      len = get16be(s);
 
-  len = get16be(s);
+      if (len > 2*MAXATOMLEN) return -1;
 
-  if (len > MAXATOMLEN) return -1;
+      if (p && utf8_to_latin1(p, s, len) < 0) return -1;
+      break;
 
-  if (p) {
-    memmove(p,s,len); 
-    p[len] = (char)0;
+  default:
+      return -1;
   }
+
   s += len;
   *index += s-s0;
-  
-  return 0; 
+  return 0;
+}
+
+int ei_internal_get_atom(const char** bufp, char* p)
+{
+    int ix = 0;
+    if (ei_decode_atom(*bufp, &ix, p) < 0) return -1;
+    *bufp += ix;
+    return 0;
+}
+
+static int utf8_to_latin1(char* dest, const char* source, unsigned slen)
+{
+    const char* dest_end = dest + MAXATOMLEN - 1;
+
+    while (slen > 0 && dest < dest_end) {
+	if ((source[0] & 0x80) == 0) {
+	    *dest++ = *source++;
+	    --slen;
+	}
+	else if (slen > 1 &&
+		 (source[0] & 0xFE) == 0xC2 &&
+		 (source[1] & 0xC0) == 0x80) {
+	    *dest++ = (char) ((source[0] << 6) | (source[1] & 0x3F));
+	    source += 2;
+	    slen -= 2;
+	}
+	else return -1;
+    }
+    *dest = 0;
+    return 0;
 }
+
diff --git a/lib/erl_interface/src/decode/decode_boolean.c b/lib/erl_interface/src/decode/decode_boolean.c
index 9fd09c63f1..0a7a06f1d4 100644
--- a/lib/erl_interface/src/decode/decode_boolean.c
+++ b/lib/erl_interface/src/decode/decode_boolean.c
@@ -26,32 +26,20 @@ int ei_decode_boolean(const char *buf, int *index, int *p)
 {
   const char *s = buf + *index;
   const char *s0 = s;
-  int len;
+  char tbuf[MAXATOMLEN+1];
   int t;
 
-  if (get8(s) != ERL_ATOM_EXT) return -1;
+  if (get_atom(&s, tbuf) < 0) return -1;
 
-  len = get16be(s);
-
-  switch (len) {
-  case 4:
-    /* typecast makes ansi happy */
-    if (strncmp((char*)s,"true",4)) return -1;
-    t = 1;
-    break;
-
-  case 5:
-    if (strncmp((char*)s,"false",5)) return -1;
-    t = 0;
-    break;
-    
-  default:
-    return -1;
-  }
-  
-  s += len;
+  if (memcmp(tbuf, "true", 5) == 0)
+      t = 1;
+  else if (memcmp(tbuf, "false", 6) == 0)
+      t = 0;
+  else
+      return -1;
+      
   if (p) *p = t;
   *index += s-s0;
-
   return 0; 
 }
+
diff --git a/lib/erl_interface/src/decode/decode_pid.c b/lib/erl_interface/src/decode/decode_pid.c
index 9ed1c36db6..a762ae499e 100644
--- a/lib/erl_interface/src/decode/decode_pid.c
+++ b/lib/erl_interface/src/decode/decode_pid.c
@@ -21,6 +21,7 @@
 #include "eiext.h"
 #include "putget.h"
 
+
 int ei_decode_pid(const char *buf, int *index, erlang_pid *p)
 {
   const char *s = buf + *index;
@@ -30,17 +31,7 @@ int ei_decode_pid(const char *buf, int *index, erlang_pid *p)
   if (get8(s) != ERL_PID_EXT) return -1;
 
   /* first the nodename */
-  if (get8(s) != ERL_ATOM_EXT) return -1;
-
-  len = get16be(s);
-
-  if (len > MAXATOMLEN) return -1;
-  
-  if (p) {
-    memmove(p->node, s, len);
-    p->node[len] = (char)0;
-  }
-  s += len;
+  if (get_atom(&s, p->node) < 0) return -1;
   
   /* now the numbers: num (4), serial (4), creation (1) */
   if (p) {
diff --git a/lib/erl_interface/src/decode/decode_port.c b/lib/erl_interface/src/decode/decode_port.c
index 28abed801a..6eb2bc9197 100644
--- a/lib/erl_interface/src/decode/decode_port.c
+++ b/lib/erl_interface/src/decode/decode_port.c
@@ -30,17 +30,7 @@ int ei_decode_port(const char *buf, int *index, erlang_port *p)
   if (get8(s) != ERL_PORT_EXT) return -1;
 
   /* first the nodename */
-  if (get8(s) != ERL_ATOM_EXT) return -1;
-
-  len = get16be(s);
-
-  if (len > MAXATOMLEN) return -1;
-
-  if (p) {
-    memmove(p->node, s, len);
-    p->node[len] = (char)0;
-  }
-  s += len;
+  if (get_atom(&s, p->node) < 0) return -1;
   
   /* now the numbers: num (4), creation (1) */
   if (p) {
diff --git a/lib/erl_interface/src/decode/decode_ref.c b/lib/erl_interface/src/decode/decode_ref.c
index 7b15808bc5..df3c30777b 100644
--- a/lib/erl_interface/src/decode/decode_ref.c
+++ b/lib/erl_interface/src/decode/decode_ref.c
@@ -21,6 +21,7 @@
 #include "eiext.h"
 #include "putget.h"
 
+
 int ei_decode_ref(const char *buf, int *index, erlang_ref *p)
 {
   const char *s = buf + *index;
@@ -30,18 +31,8 @@ int ei_decode_ref(const char *buf, int *index, erlang_ref *p)
   switch (get8(s)) {
     case ERL_REFERENCE_EXT:
 
-      /* first the nodename */
-      if (get8(s) != ERL_ATOM_EXT) return -1;
-
-      len = get16be(s);
-
-      if (len > MAXATOMLEN) return -1;
-
-      if (p) {
-	  memmove(p->node, s, len);
-	  p->node[len] = (char)0;
-      }
-      s += len;
+      /* nodename */
+      if (get_atom(&s, p->node) < 0) return -1;
   
       /* now the numbers: num (4), creation (1) */
       if (p) {
@@ -62,15 +53,7 @@ int ei_decode_ref(const char *buf, int *index, erlang_ref *p)
       if (p) p->len = count;
 
       /* then the nodename */
-      if (get8(s) != ERL_ATOM_EXT) return -1;
-      len = get16be(s);
-      if (len > MAXATOMLEN) return -1;
-
-      if (p) {
-	  memmove(p->node, s, len);
-	  p->node[len] = (char)0;
-      }
-      s += len;
+      if (get_atom(&s, p->node) < 0) return -1;
 
       /* creation */
       if (p) {
@@ -95,3 +78,4 @@ int ei_decode_ref(const char *buf, int *index, erlang_ref *p)
       return -1;
   }
 }
+
diff --git a/lib/erl_interface/src/legacy/erl_marshal.c b/lib/erl_interface/src/legacy/erl_marshal.c
index dad715c762..775d7e82ca 100644
--- a/lib/erl_interface/src/legacy/erl_marshal.c
+++ b/lib/erl_interface/src/legacy/erl_marshal.c
@@ -51,7 +51,13 @@ static void erl_long_to_fp(long l, unsigned *d);
 
 #define CMP_ARRAY_SIZE 256
 /* FIXME problem for threaded ? */
-static char cmp_array[CMP_ARRAY_SIZE]; 
+
+static enum
+{
+    ERL_NUM_CMP=1, ERL_ATOM_CMP, ERL_REF_CMP, ERL_FUN_CMP, ERL_PORT_CMP,
+    ERL_PID_CMP, ERL_TUPLE_CMP, ERL_NIL_CMP, ERL_LIST_CMP, ERL_BIN_CMP
+}cmp_array[CMP_ARRAY_SIZE];
+
 static int init_cmp_array_p=1; /* initialize array, the first time */
 
 #if defined(VXWORKS) && CPU == PPC860
@@ -69,10 +75,8 @@ static int init_cmp_array_p=1; /* initialize array, the first time */
 static int cmp_floats(double f1, double f2);
 static INLINE double to_float(long l);
 
-#define ERL_NUM_CMP 1
-#define ERL_REF_CMP 3
-
 #define IS_ERL_NUM(t) (cmp_array[t]==ERL_NUM_CMP)
+#define IS_ERL_ATOM(t) (cmp_array[t]==ERL_ATOM_CMP)
 
 #define CMP_NUM_CLASS_SIZE 256
 static unsigned char cmp_num_class[CMP_NUM_CLASS_SIZE]; 
@@ -100,25 +104,27 @@ void erl_init_marshal(void)
 {
   if (init_cmp_array_p) {
     memset(cmp_array, 0, CMP_ARRAY_SIZE);
-    cmp_array[ERL_SMALL_INTEGER_EXT] = 1;
-    cmp_array[ERL_INTEGER_EXT]       = 1;
-    cmp_array[ERL_FLOAT_EXT]         = 1;
-    cmp_array[NEW_FLOAT_EXT]         = 1;
-    cmp_array[ERL_SMALL_BIG_EXT]     = 1;
-    cmp_array[ERL_LARGE_BIG_EXT]     = 1;
-    cmp_array[ERL_ATOM_EXT]          = 2;
-    cmp_array[ERL_REFERENCE_EXT]     = 3;
-    cmp_array[ERL_NEW_REFERENCE_EXT] = 3;
-    cmp_array[ERL_FUN_EXT]           = 4;
-    cmp_array[ERL_NEW_FUN_EXT]       = 4;
-    cmp_array[ERL_PORT_EXT]          = 5;
-    cmp_array[ERL_PID_EXT]           = 6;
-    cmp_array[ERL_SMALL_TUPLE_EXT]   = 7;
-    cmp_array[ERL_LARGE_TUPLE_EXT]   = 7;
-    cmp_array[ERL_NIL_EXT]           = 8;
-    cmp_array[ERL_STRING_EXT]        = 9;
-    cmp_array[ERL_LIST_EXT]          = 9;
-    cmp_array[ERL_BINARY_EXT]        = 10;
+    cmp_array[ERL_SMALL_INTEGER_EXT] = ERL_NUM_CMP;
+    cmp_array[ERL_INTEGER_EXT]       = ERL_NUM_CMP;
+    cmp_array[ERL_FLOAT_EXT]         = ERL_NUM_CMP;
+    cmp_array[NEW_FLOAT_EXT]         = ERL_NUM_CMP;
+    cmp_array[ERL_SMALL_BIG_EXT]     = ERL_NUM_CMP;
+    cmp_array[ERL_LARGE_BIG_EXT]     = ERL_NUM_CMP;
+    cmp_array[ERL_ATOM_EXT]          = ERL_ATOM_CMP;
+    cmp_array[ERL_SMALL_ATOM_EXT]    = ERL_ATOM_CMP;
+    cmp_array[ERL_UNICODE_ATOM_EXT]  = ERL_ATOM_CMP;
+    cmp_array[ERL_REFERENCE_EXT]     = ERL_REF_CMP;
+    cmp_array[ERL_NEW_REFERENCE_EXT] = ERL_REF_CMP;
+    cmp_array[ERL_FUN_EXT]           = ERL_FUN_CMP;
+    cmp_array[ERL_NEW_FUN_EXT]       = ERL_FUN_CMP;
+    cmp_array[ERL_PORT_EXT]          = ERL_PORT_CMP;
+    cmp_array[ERL_PID_EXT]           = ERL_PID_CMP;
+    cmp_array[ERL_SMALL_TUPLE_EXT]   = ERL_TUPLE_CMP;
+    cmp_array[ERL_LARGE_TUPLE_EXT]   = ERL_TUPLE_CMP;
+    cmp_array[ERL_NIL_EXT]           = ERL_NIL_CMP;
+    cmp_array[ERL_STRING_EXT]        = ERL_LIST_CMP;
+    cmp_array[ERL_LIST_EXT]          = ERL_LIST_CMP;
+    cmp_array[ERL_BINARY_EXT]        = ERL_BIN_CMP;
     init_cmp_array_p = 0;
   }
   if (init_cmp_num_class_p) {
@@ -644,31 +650,14 @@ int erl_encode_buf(ETERM *ep, unsigned char **ext)
 
 } /* erl_encode_buf */
 
-/*
- * A nice macro to make it look cleaner in the 
- * cases of PID's,PORT's and REF's below. 
- * It reads the NODE name from a buffer.
- */
-#define READ_THE_NODE(ext,cp,len,i) \
-/* eat first atom, repr. the node */ \
-if (**ext != ERL_ATOM_EXT) \
-  return (ETERM *) NULL; \
-*ext += 1; \
-i = (**ext << 8) | (*ext)[1]; \
-cp = (char *) *(ext) + 2; \
-*ext += (i + 2); \
-len = i
-
-#define STATIC_NODE_BUF_SZ 30
-
-#define SET_NODE(node,node_buf,cp,len) \
-if (len >= STATIC_NODE_BUF_SZ) node = erl_malloc(len+1); \
-else node = node_buf; \
-memcpy(node, cp, len); \
-node[len] = '\0'
-
-#define RESET_NODE(node,len) \
-if (len >= STATIC_NODE_BUF_SZ) free(node)
+
+static int read_atom(unsigned char** ext, char* dst)
+{
+    int offs = 0;
+    int ret = ei_decode_atom((char*)*ext, &offs, dst);
+    *ext += offs;
+    return ret;
+}
 
 /*
  * The actual DECODE engine.
@@ -676,16 +665,17 @@ if (len >= STATIC_NODE_BUF_SZ) free(node)
  */
 static ETERM *erl_decode_it(unsigned char **ext)
 {
+    char atom_buf[MAXATOMLEN+1];
     char *cp;
     ETERM *ep,*tp,*np;
     unsigned int u,sign;
-    int i,j,len,arity;
+    int i,j,arity;
     double ff;
     
     /* Assume we are going to decode an integer */
     ep = erl_alloc_eterm(ERL_INTEGER);
     ERL_COUNT(ep) = 1;
-    
+
     switch (*(*ext)++) 
     {
     case ERL_INTEGER_EXT:
@@ -774,27 +764,27 @@ static ETERM *erl_decode_it(unsigned char **ext)
 	return ep;
       
     case ERL_ATOM_EXT:
+    case ERL_SMALL_ATOM_EXT:
+    case ERL_UNICODE_ATOM_EXT:
 	ERL_TYPE(ep) = ERL_ATOM;
-	i = (**ext << 8) | (*ext)[1];
-	cp = (char *) *(ext) + 2;
-	*ext += (i + 2);
+	--(*ext);
+	if (read_atom(ext, atom_buf) < 0) return NULL;
+
+	i = strlen(atom_buf);
 	ep->uval.aval.len = i;
 	ep->uval.aval.a = (char *) erl_malloc(i+1);
-	memcpy(ep->uval.aval.a, cp, i);
-	ep->uval.aval.a[i]='\0';
+	memcpy(ep->uval.aval.a, atom_buf, i+1);
 	return ep;
       
     case ERL_PID_EXT:
 	erl_free_term(ep);
 	{			/* Why not use the constructors? */
-	    char *node;
-	    char node_buf[STATIC_NODE_BUF_SZ];
+	    char* node = atom_buf;
 	    unsigned int number, serial;
 	    unsigned char creation;
 	    ETERM *eterm_p;
 
-	    READ_THE_NODE(ext,cp,len,i);
-	    SET_NODE(node,node_buf,cp,len);
+	    if (read_atom(ext, node) < 0) return NULL;
 
 	    /* get the integers */
 #if 0
@@ -816,20 +806,17 @@ static ETERM *erl_decode_it(unsigned char **ext)
 #endif
 	    creation =  *(*ext)++; 
 	    eterm_p = erl_mk_pid(node, number, serial, creation);
-	    RESET_NODE(node,len);
 	    return eterm_p;
 	}
     case ERL_REFERENCE_EXT:
 	erl_free_term(ep);
 	{
-	    char *node;
-	    char node_buf[STATIC_NODE_BUF_SZ];
+	    char* node = atom_buf;
 	    unsigned int number;
 	    unsigned char creation;
 	    ETERM *eterm_p;
 
-	    READ_THE_NODE(ext,cp,len,i);
-	    SET_NODE(node,node_buf,cp,len);
+	    if (read_atom(ext, node) < 0) return NULL;
 
 	    /* get the integers */
 #if 0
@@ -841,15 +828,13 @@ static ETERM *erl_decode_it(unsigned char **ext)
 #endif
 	    creation =  *(*ext)++; 
 	    eterm_p = erl_mk_ref(node, number, creation);
-	    RESET_NODE(node,len);
 	    return eterm_p;
 	}
 
     case ERL_NEW_REFERENCE_EXT: 
 	erl_free_term(ep);
 	{
-	    char *node;
-	    char node_buf[STATIC_NODE_BUF_SZ];
+	    char* node = atom_buf;
 	    size_t cnt, i;
 	    unsigned int n[3];
 	    unsigned char creation;
@@ -862,8 +847,7 @@ static ETERM *erl_decode_it(unsigned char **ext)
 	    *ext += 2;
 #endif
 
-	    READ_THE_NODE(ext,cp,len,i);
-	    SET_NODE(node,node_buf,cp,len);
+	    if (read_atom(ext, node) < 0) return NULL;
 
 	    /* get the integers */
 	    creation =  *(*ext)++; 
@@ -878,21 +862,18 @@ static ETERM *erl_decode_it(unsigned char **ext)
 #endif
 	    }
 	    eterm_p = __erl_mk_reference(node, cnt, n, creation);
-	    RESET_NODE(node,len);
 	    return eterm_p;
 	}
 
     case ERL_PORT_EXT:
 	erl_free_term(ep);
 	{
-	    char *node;
-	    char node_buf[STATIC_NODE_BUF_SZ];
+	    char* node = atom_buf;
 	    unsigned int number;
 	    unsigned char creation;
 	    ETERM *eterm_p;
 
-	    READ_THE_NODE(ext,cp,len,i);
-	    SET_NODE(node,node_buf,cp,len);
+	    if (read_atom(ext, node) < 0) return NULL;
 
 	    /* get the integers */
 #if 0
@@ -904,7 +885,6 @@ static ETERM *erl_decode_it(unsigned char **ext)
 #endif
 	    creation =  *(*ext)++; 
 	    eterm_p = erl_mk_port(node, number, creation);
-	    RESET_NODE(node,len);
 	    return eterm_p;
 	}
 
@@ -1140,6 +1120,8 @@ unsigned char erl_ext_type(unsigned char *ext)
     case ERL_INTEGER_EXT:
 	return ERL_INTEGER;
     case ERL_ATOM_EXT:
+    case ERL_SMALL_ATOM_EXT:
+    case ERL_UNICODE_ATOM_EXT:
 	return ERL_ATOM;
     case ERL_PID_EXT:
 	return ERL_PID;
@@ -1191,6 +1173,8 @@ int erl_ext_size(unsigned char *t)
     case ERL_SMALL_INTEGER_EXT:
     case ERL_INTEGER_EXT:
     case ERL_ATOM_EXT:
+    case ERL_SMALL_ATOM_EXT:
+    case ERL_UNICODE_ATOM_EXT:
     case ERL_PID_EXT:
     case ERL_PORT_EXT:
     case ERL_REFERENCE_EXT:
@@ -1229,15 +1213,31 @@ int erl_ext_size(unsigned char *t)
 
 } /* ext_size */
 
-/*
- * A nice macro that eats up the atom pointed to.
- */
-#define JUMP_ATOM(ext,i) \
-if (**ext != ERL_ATOM_EXT) \
-  return 0; \
-*ext += 1; \
-i = (**ext << 8) | (*ext)[1]; \
-*ext += (i + 2)
+
+static int jump_atom(unsigned char** ext)
+{
+    unsigned char* e = *ext;
+    int len;
+
+    switch (*e++) {
+    case ERL_ATOM_EXT:
+    case ERL_UNICODE_ATOM_EXT:
+	len = (e[0] << 8) | e[1];
+	e += (len + 2);
+	break;
+
+    case ERL_SMALL_ATOM_EXT:
+	len = e[0];
+	e += (len + 1);
+	break;
+
+    default:
+	return 0;
+    }
+    *ext = e;
+    return 1;
+}
+
 
 /*
  * MOVE the POINTER PAST the ENCODED ETERM we
@@ -1259,25 +1259,26 @@ static int jump(unsigned char **ext)
 	*ext += 1;
 	break;
     case ERL_ATOM_EXT:
-	i = (**ext << 8) | (*ext)[1];
-	*ext += (i + 2);
+    case ERL_SMALL_ATOM_EXT:
+    case ERL_UNICODE_ATOM_EXT:
+	jump_atom(ext);
 	break;
     case ERL_PID_EXT:
 	/* eat first atom */
-	JUMP_ATOM(ext,i);
+	if (!jump_atom(ext)) return 0;
 	*ext += 9;		/* Two int's and the creation field */
 	break;
     case ERL_REFERENCE_EXT:
     case ERL_PORT_EXT:
 	/* first field is an atom */
-	JUMP_ATOM(ext,i);
+	if (!jump_atom(ext)) return 0;
 	*ext += 5;		/* One int and the creation field */
 	break;
     case ERL_NEW_REFERENCE_EXT:
 	n = (**ext << 8) | (*ext)[1];
 	*ext += 2;
 	/* first field is an atom */
-	JUMP_ATOM(ext,i);
+	if (!jump_atom(ext)) return 0;
 	*ext += 4*n+1;
 	break;
     case ERL_NIL_EXT:
@@ -1437,9 +1438,8 @@ do {								\
 
 #define CMP_EXT_SKIP_ATOM(EP)					\
 do {								\
-    if ((EP)[0] != ERL_ATOM_EXT)				\
+    if (!jump_atom(&(EP)))					\
 	return CMP_EXT_ERROR_CODE;				\
-    (EP) += 3 + ((EP)[1] << 8 | (EP)[2]);			\
 } while (0)
 
 /* 
@@ -1569,6 +1569,7 @@ static int cmp_exe2(unsigned char **e1, unsigned char **e2)
   }
 
   *e2 += 1;
+  i = j = 0;
   switch (*(*e1)++) 
     {
     case ERL_SMALL_INTEGER_EXT:
@@ -1589,11 +1590,16 @@ static int cmp_exe2(unsigned char **e1, unsigned char **e2)
       *e1 += 4; *e2 += 4;
       return ret;
     case ERL_ATOM_EXT:
-      i = (**e1 << 8) | (*e1)[1];
-      j = (**e2 << 8) | (*e2)[1];
-      ret = cmpbytes(*e1 +2, i, *e2 +2, j);
-      *e1 += (i + 2);
-      *e2 += (j + 2);
+    case ERL_UNICODE_ATOM_EXT:
+      i = (**e1) << 8; (*e1)++;
+      j = (**e2) << 8; (*e2)++;
+      /*fall through*/
+    case ERL_SMALL_ATOM_EXT:
+      i |= (**e1); (*e1)++;
+      j |= (**e2); (*e2)++;
+      ret = cmpbytes(*e1, i, *e2, j);
+      *e1 += i;
+      *e2 += j;
       return ret;
     case ERL_PID_EXT: {
       unsigned char *n1 = *e1;
@@ -1622,7 +1628,7 @@ static int cmp_exe2(unsigned char **e1, unsigned char **e2)
     }
     case ERL_PORT_EXT:
       /* First compare node names ... */
-      if (**e1 != ERL_ATOM_EXT || **e2 != ERL_ATOM_EXT)
+      if (!IS_ERL_ATOM(**e1) || !IS_ERL_ATOM(**e2))
 	  return CMP_EXT_ERROR_CODE;
       ret = cmp_exe2(e1, e2);
       *e1 += 5; *e2 += 5;
diff --git a/lib/erl_interface/src/misc/ei_decode_term.c b/lib/erl_interface/src/misc/ei_decode_term.c
index 0b82ef0e35..6773f90bfc 100644
--- a/lib/erl_interface/src/misc/ei_decode_term.c
+++ b/lib/erl_interface/src/misc/ei_decode_term.c
@@ -48,20 +48,12 @@ int ei_decode_ei_term(const char* buf, int* index, ei_term* term)
     case NEW_FLOAT_EXT:
         return ei_decode_double(buf, index, &term->value.d_val);
     case ERL_ATOM_EXT:
-	len = get16be(s);
-	if (len > MAXATOMLEN) return -1;
-	memcpy(term->value.atom_name, s, len); 
-	term->value.atom_name[len] = '\0';
-	s += len;
-	break;
+    case ERL_SMALL_ATOM_EXT:
+    case ERL_UNICODE_ATOM_EXT:
+	return ei_decode_atom(buf, index, term->value.atom_name);
     case ERL_REFERENCE_EXT:
 	/* first the nodename */
-	if (get8(s) != ERL_ATOM_EXT) return -1;
-	len = get16be(s);
-	if (len > MAXATOMLEN) return -1;
-	memcpy(term->value.ref.node, s, len);
-	term->value.ref.node[len] = '\0';
-	s += len;
+	if (get_atom(&s, term->value.ref.node) < 0) return -1;
         /* now the numbers: num (4), creation (1) */
 	term->value.ref.n[0] = get32be(s);
 	term->value.ref.len = 1;
@@ -71,12 +63,7 @@ int ei_decode_ei_term(const char* buf, int* index, ei_term* term)
 	/* first the integer count */
 	term->value.ref.len = get16be(s);
 	/* then the nodename */
-	if (get8(s) != ERL_ATOM_EXT) return -1;
-	len = get16be(s);
-	if (len > MAXATOMLEN) return -1;
-	memcpy(term->value.ref.node, s, len);
-	term->value.ref.node[len] = '\0';
-	s += len;
+	if (get_atom(&s, term->value.ref.node) < 0) return -1;
 	/* creation */
 	term->value.ref.creation = get8(s) & 0x03;
 	/* finally the id integers */
@@ -88,22 +75,12 @@ int ei_decode_ei_term(const char* buf, int* index, ei_term* term)
 	}
 	break;
     case ERL_PORT_EXT:
-	if (get8(s) != ERL_ATOM_EXT) return -1;
-	len = get16be(s);
-	if (len > MAXATOMLEN) return -1;
-	memcpy(term->value.port.node, s, len);
-	term->value.port.node[len] = '\0';
+	if (get_atom(&s, term->value.port.node) < 0) return -1;
 	term->value.port.id = get32be(s) & 0x0fffffff; /* 28 bits */;
 	term->value.port.creation = get8(s) & 0x03;
 	break;
     case ERL_PID_EXT:
-	if (get8(s) != ERL_ATOM_EXT) return -1;
-	/* name first */
-	len = get16be(s); 
-	if (len > MAXATOMLEN) return -1;
-	memcpy(term->value.pid.node, s, len);
-	term->value.pid.node[len] = '\0';
-	s += len;
+	if (get_atom(&s, term->value.pid.node) < 0) return -1;
 	/* now the numbers: num (4), serial (4), creation (1) */
 	term->value.pid.num = get32be(s) & 0x7fff; /* 15 bits */
 	term->value.pid.serial = get32be(s) & 0x1fff; /* 13 bits */
diff --git a/lib/erl_interface/src/misc/ei_printterm.c b/lib/erl_interface/src/misc/ei_printterm.c
index 5fc6b3542c..620c6e72e2 100644
--- a/lib/erl_interface/src/misc/ei_printterm.c
+++ b/lib/erl_interface/src/misc/ei_printterm.c
@@ -133,6 +133,8 @@ static int print_term(FILE* fp, ei_x_buff* x,
     ei_get_type_internal(buf, index, &ty, &n);
     switch (ty) {
     case ERL_ATOM_EXT:
+    case ERL_SMALL_ATOM_EXT:
+    case ERL_UNICODE_ATOM_EXT:
 	if (ei_decode_atom(buf, index, a) < 0)
 	   goto err;
 	doquote = !islower((int)a[0]);
diff --git a/lib/erl_interface/src/misc/get_type.c b/lib/erl_interface/src/misc/get_type.c
index 2a680d0f94..c9a040fbbd 100644
--- a/lib/erl_interface/src/misc/get_type.c
+++ b/lib/erl_interface/src/misc/get_type.c
@@ -55,10 +55,12 @@ int ei_get_type(const char *buf, const int *index, int *type, int *len)
 	break;
 
     case ERL_SMALL_TUPLE_EXT:
+    case ERL_SMALL_ATOM_EXT:
 	*len  = get8(s);
 	break;
     
     case ERL_ATOM_EXT:
+    case ERL_UNICODE_ATOM_EXT:
     case ERL_STRING_EXT:
 	*len  = get16be(s);
 	break;
@@ -114,10 +116,14 @@ int ei_get_type_internal(const char *buf, const int *index,
   *type = get8(s);
   
   switch (*type) {
+  case ERL_SMALL_ATOM_EXT:
+    *type = ERL_ATOM_EXT;
   case ERL_SMALL_TUPLE_EXT:
     *len = get8(s);
     break;
-    
+
+  case ERL_UNICODE_ATOM_EXT:
+    *type = ERL_ATOM_EXT;
   case ERL_ATOM_EXT:
   case ERL_STRING_EXT:
     *len = get16be(s);
diff --git a/lib/erl_interface/src/misc/putget.h b/lib/erl_interface/src/misc/putget.h
index 7a43de324b..8b0d4d3404 100644
--- a/lib/erl_interface/src/misc/putget.h
+++ b/lib/erl_interface/src/misc/putget.h
@@ -105,6 +105,9 @@
        ((EI_ULONGLONG)((unsigned char *)(s))[-2] << 8)  | \
         (EI_ULONGLONG)((unsigned char *)(s))[-1]))
 
+int ei_internal_get_atom(const char** bufp, char* p);
+#define get_atom ei_internal_get_atom
+
 typedef union float_ext {
     double d;
     EI_ULONGLONG val;
-- 
cgit v1.2.3


From b553664f54034e8c04ae6f9cc44f16b7f516518b Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 11 Jan 2013 17:27:29 +0100
Subject: erl_interface: utf8 atoms continued

---
 lib/erl_interface/doc/src/ei.xml                   |  59 +++-
 lib/erl_interface/doc/src/erl_eterm.xml            |  18 +-
 lib/erl_interface/include/ei.h                     |  35 +-
 lib/erl_interface/include/erl_interface.h          |  40 ++-
 lib/erl_interface/src/connect/ei_connect.c         |   5 +-
 lib/erl_interface/src/connect/ei_connect_int.h     |   2 +
 lib/erl_interface/src/connect/eirecv.c             |  12 +-
 lib/erl_interface/src/decode/decode_atom.c         | 185 +++++++---
 lib/erl_interface/src/decode/decode_boolean.c      |   8 +-
 lib/erl_interface/src/decode/decode_fun.c          |   6 +-
 lib/erl_interface/src/decode/decode_pid.c          |   3 +-
 lib/erl_interface/src/decode/decode_port.c         |   3 +-
 lib/erl_interface/src/decode/decode_ref.c          |   6 +-
 lib/erl_interface/src/encode/encode_atom.c         | 100 +++++-
 lib/erl_interface/src/encode/encode_fun.c          |   4 +-
 lib/erl_interface/src/encode/encode_pid.c          |  24 +-
 lib/erl_interface/src/encode/encode_port.c         |  23 +-
 lib/erl_interface/src/encode/encode_ref.c          |  24 +-
 lib/erl_interface/src/legacy/erl_connect.c         |  16 +-
 lib/erl_interface/src/legacy/erl_eterm.c           | 158 +++++++--
 lib/erl_interface/src/legacy/erl_eterm.h           |   4 +-
 lib/erl_interface/src/legacy/erl_format.c          |  20 +-
 lib/erl_interface/src/legacy/erl_malloc.c          |  20 +-
 lib/erl_interface/src/legacy/erl_marshal.c         | 259 ++++++++------
 lib/erl_interface/src/legacy/global_whereis.c      |  11 +-
 lib/erl_interface/src/misc/ei_decode_term.c        |  13 +-
 lib/erl_interface/src/misc/ei_printterm.c          |   5 +-
 lib/erl_interface/src/misc/ei_x_encode.c           |  21 +-
 lib/erl_interface/src/misc/get_type.c              |  77 +---
 lib/erl_interface/src/misc/putget.h                |   6 +-
 lib/erl_interface/src/misc/show_msg.c              |  12 +-
 lib/erl_interface/test/ei_decode_encode_SUITE.erl  |  66 +++-
 .../ei_decode_encode_test.c                        | 386 ++++++++++++---------
 33 files changed, 1059 insertions(+), 572 deletions(-)

(limited to 'lib/erl_interface')

diff --git a/lib/erl_interface/doc/src/ei.xml b/lib/erl_interface/doc/src/ei.xml
index 539e16d837..0b0b1eeb79 100644
--- a/lib/erl_interface/doc/src/ei.xml
+++ b/lib/erl_interface/doc/src/ei.xml
@@ -82,6 +82,22 @@
       function returns the size required (note that for strings an
       extra byte is needed for the 0 string terminator).</p>
   </description>
+  <section>
+    <title>DATA TYPES</title>
+
+    <taglist>
+      <tag><marker id="erlang_char_encoding"/>enum erlang_char_encoding</tag>
+       <item>
+	 <p/>
+	 <code type="none">
+enum erlang_char_encoding {
+    ERLANG_ASCII, ERLANG_LATIN1, ERLANG_UTF8, ERLANG_WHATEVER
+};
+</code>
+         <p>The character encoding used for atoms.</p>
+      </item>
+    </taglist>
+  </section>
   <funcs>
     <func>
       <name><ret>void</ret><nametext>ei_set_compat_rel(release_number)</nametext></name>
@@ -225,11 +241,31 @@
       <fsummary>Encode an atom</fsummary>
       <desc>
         <p>Encodes an atom in the binary format. The <c><![CDATA[p]]></c> parameter
-          is the name of the atom. Only upto <c><![CDATA[MAXATOMLEN]]></c> bytes
+          is the name of the atom in latin1 encoding. Only upto <c>MAXATOMLEN-1</c> bytes
           are encoded. The name should be zero-terminated, except for
           the <c><![CDATA[ei_x_encode_atom_len()]]></c> function.</p>
       </desc>
     </func>
+    <func>
+      <name><ret>int</ret><nametext>ei_encode_atom_as(char *buf, int *index, const char *p, enum erlang_char_encoding from_enc, enum erlang_char_encoding to_enc)</nametext></name>
+      <name><ret>int</ret><nametext>ei_encode_atom_len_as(char *buf, int *index, const char *p, int len, enum erlang_char_encoding from_enc, enum erlang_char_encoding to_enc)</nametext></name>
+      <name><ret>int</ret><nametext>ei_x_encode_atom_as(ei_x_buff* x, const char *p, enum erlang_char_encoding from_enc, enum erlang_char_encoding to_enc)</nametext></name>
+      <name><ret>int</ret><nametext>ei_x_encode_atom_len_as(ei_x_buff* x, const char *p, int len, enum erlang_char_encoding from_enc, enum erlang_char_encoding to_enc)</nametext></name>
+      <fsummary>Encode an atom</fsummary>
+      <desc>
+        <p>Encodes an atom in the binary format with character encoding
+	<c><seealso marker="#erlang_char_encoding">to_enc</seealso></c> (latin1 or utf8).
+	The <c>p</c> parameter is the name of the atom with character encoding
+	<c><seealso marker="#erlang_char_encoding">from_enc</seealso></c>.
+	The name must either be zero-terminated or a function variant with a <c>len</c>
+	parameter must be used.</p>
+	<p>The encoding will fail if the atom is too long or if it can not be represented
+	with character encoding <c>to_enc</c>.</p>
+	<p>These functions were introduced in R16 release of Erlang/OTP as part of a first step
+	to support UTF8 atoms. Atoms encoded with <c>ERLANG_UTF8</c>
+	can not be decoded by earlier releases than R16.</p>
+      </desc>
+    </func>
     <func>
       <name><ret>int</ret><nametext>ei_encode_binary(char *buf, int *index, const void *p, long len)</nametext></name>
       <name><ret>int</ret><nametext>ei_x_encode_binary(ei_x_buff* x, const void *p, long len)</nametext></name>
@@ -490,10 +526,29 @@ ei_x_encode_empty_list(&amp;x);
       <fsummary>Decode an atom</fsummary>
       <desc>
         <p>This function decodes an atom from the binary format.  The
-          name of the atom is placed at <c><![CDATA[p]]></c>. There can be at most
+	null terminated name of the atom is placed at <c><![CDATA[p]]></c>. There can be at most
           <c><![CDATA[MAXATOMLEN]]></c> bytes placed in the buffer.</p>
       </desc>
     </func>
+    <func>
+      <name><ret>int</ret><nametext>ei_decode_atom_as(const char *buf, int *index, char *p, int plen, enum erlang_char_encoding want, enum erlang_char_encoding* was, enum erlang_char_encoding* result)</nametext></name>
+      <fsummary>Decode an atom</fsummary>
+      <desc>
+        <p>This function decodes an atom from the binary format. The
+	null terminated name of the atom is placed in buffer at <c>p</c> of length
+	<c>plen</c> bytes.</p>
+	<p>The wanted string encoding is specified by <c><seealso marker="#erlang_char_encoding">
+	want</seealso></c>. The original encoding used in the
+	binary format (latin1 or utf8) can be obtained from <c>*was</c>. The actual encoding of the resulting string
+	(7-bit ascii, latin1 or utf8) can be obtained from <c>*result</c>. Both <c>was</c> and <c>result</c> can be <c>NULL</c>.
+	<c>*result</c> may differ from <c>want</c> if <c>want</c> is <c>ERLANG_WHATEVER</c> or if
+	<c>*result</c> turn out to be pure 7-bit ascii (compatible with both latin1 and utf8).</p>
+	<p>This function fails if the atom is too long for the buffer
+	or if it can not be represented with encoding <c>want</c>.</p>
+	<p>This functions was introduced in R16 release of Erlang/OTP as part of a first step
+	to support UTF8 atoms.</p>
+      </desc>
+    </func>
     <func>
       <name><ret>int</ret><nametext>ei_decode_binary(const char *buf, int *index, void *p, long *len)</nametext></name>
       <fsummary>Decode a binary</fsummary>
diff --git a/lib/erl_interface/doc/src/erl_eterm.xml b/lib/erl_interface/doc/src/erl_eterm.xml
index f403618c59..c7840d7813 100644
--- a/lib/erl_interface/doc/src/erl_eterm.xml
+++ b/lib/erl_interface/doc/src/erl_eterm.xml
@@ -77,10 +77,12 @@
       </p>
     <taglist>
       <tag><c><![CDATA[char *ERL_ATOM_PTR(t)]]></c></tag>
+      <tag><c><![CDATA[char *ERL_ATOM_PTR_UTF8(t)]]></c></tag>
       <item>A string representing atom <c><![CDATA[t]]></c>.
       </item>
       <tag><c><![CDATA[int ERL_ATOM_SIZE(t)]]></c></tag>
-      <item>The length (in characters) of atom t.</item>
+      <tag><c><![CDATA[int ERL_ATOM_SIZE_UTF8(t)]]></c></tag>
+      <item>The length (in bytes) of atom t.</item>
       <tag><c><![CDATA[void *ERL_BIN_PTR(t)]]></c></tag>
       <item>A pointer to the contents of <c><![CDATA[t]]></c></item>
       <tag><c><![CDATA[int ERL_BIN_SIZE(t)]]></c></tag>
@@ -92,6 +94,7 @@
       <tag><c><![CDATA[double ERL_FLOAT_VALUE(t)]]></c></tag>
       <item>The floating point value of <c><![CDATA[t]]></c>.</item>
       <tag><c><![CDATA[ETERM *ERL_PID_NODE(t)]]></c></tag>
+      <tag><c><![CDATA[ETERM *ERL_PID_NODE_UTF8(t)]]></c></tag>
       <item>The Node in pid <c><![CDATA[t]]></c>.</item>
       <tag><c><![CDATA[int ERL_PID_NUMBER(t)]]></c></tag>
       <item>The sequence number in pid <c><![CDATA[t]]></c>.</item>
@@ -104,6 +107,7 @@
       <tag><c><![CDATA[int ERL_PORT_CREATION(t)]]></c></tag>
       <item>The creation number in port <c><![CDATA[t]]></c>.</item>
       <tag><c><![CDATA[ETERM *ERL_PORT_NODE(t)]]></c></tag>
+      <tag><c><![CDATA[ETERM *ERL_PORT_NODE_UTF8(t)]]></c></tag>
       <item>The node in port <c><![CDATA[t]]></c>.</item>
       <tag><c><![CDATA[int ERL_REF_NUMBER(t)]]></c></tag>
       <item>The first part of the reference number in ref <c><![CDATA[t]]></c>. Use
@@ -296,7 +300,7 @@ iohead ::= Binary
       <name><ret>ETERM *</ret><nametext>erl_mk_atom(string)</nametext></name>
       <fsummary>Creates an atom</fsummary>
       <type>
-        <v>char *string;</v>
+        <v>const char *string;</v>
       </type>
       <desc>
         <p>Creates an atom.</p>
@@ -305,10 +309,12 @@ iohead ::= Binary
         <p>Returns an Erlang term containing an atom. Note that it is
           the callers responsibility to make sure that <c><![CDATA[string]]></c>
           contains a valid name for an atom.</p>
-        <p><c><![CDATA[ERL_ATOM_PTR(atom)]]></c> can be used to retrieve the 
-          atom name (as a string). Note that the string is not
-          0-terminated in the atom. <c><![CDATA[ERL_ATOM_SIZE(atom)]]></c>returns
-          the length of the atom name.</p>
+        <p><c><![CDATA[ERL_ATOM_PTR(atom)]]></c> and <c><![CDATA[ERL_ATOM_PTR_UTF8(atom)]]></c>
+	can be used to retrieve the atom name (as a null terminated string). <c><![CDATA[ERL_ATOM_SIZE(atom)]]></c>
+	and <c><![CDATA[ERL_ATOM_SIZE_UTF8(atom)]]></c> returns the length of the atom name.</p>
+	<note><p>Note that the UTF8 variants were introduced in Erlang/OTP releases R16
+	and the string returned by <c>ERL_ATOM_PTR(atom)</c> was not null terminated on older releases.</p>
+	</note>
       </desc>
     </func>
     <func>
diff --git a/lib/erl_interface/include/ei.h b/lib/erl_interface/include/ei.h
index 8f07b24852..20e575f64d 100644
--- a/lib/erl_interface/include/ei.h
+++ b/lib/erl_interface/include/ei.h
@@ -116,7 +116,8 @@
 #define NEW_FLOAT_EXT         'F'
 #define ERL_ATOM_EXT          'd'
 #define ERL_SMALL_ATOM_EXT    's'
-#define ERL_UNICODE_ATOM_EXT  'v'
+#define ERL_ATOM_UTF8_EXT     'v'
+#define ERL_SMALL_ATOM_UTF8_EXT 'w'
 #define ERL_REFERENCE_EXT     'e'
 #define ERL_NEW_REFERENCE_EXT 'r'
 #define ERL_PORT_EXT          'f'
@@ -185,12 +186,16 @@ extern volatile int __erl_errno;
 #define EI_MAXHOSTNAMELEN 64
 #define EI_MAXALIVELEN 63
 #define EI_MAX_COOKIE_SIZE 512
-#define MAXATOMLEN 255
+#define MAXATOMLEN (255 + 1)
+#define MAXATOMLEN_UTF8 (255*4 + 1)
 #define MAXNODELEN EI_MAXALIVELEN+1+EI_MAXHOSTNAMELEN
 
+enum erlang_char_encoding { ERLANG_ASCII, ERLANG_LATIN1, ERLANG_UTF8, ERLANG_WHATEVER };
+
 /* a pid */
 typedef struct {
-  char node[MAXATOMLEN+1];
+  char node[MAXATOMLEN_UTF8];
+  enum erlang_char_encoding node_org_enc;
   unsigned int num;
   unsigned int serial;
   unsigned int creation;
@@ -198,14 +203,16 @@ typedef struct {
 
 /* a port */
 typedef struct {
-  char node[MAXATOMLEN+1];
+  char node[MAXATOMLEN_UTF8];
+  enum erlang_char_encoding node_org_enc;
   unsigned int id;
   unsigned int creation;
 } erlang_port;
 
 /* a ref */
 typedef struct {
-  char node[MAXATOMLEN+1];
+  char node[MAXATOMLEN_UTF8];
+  enum erlang_char_encoding node_org_enc;  
   int len;
   unsigned int n[3];
   unsigned int creation;
@@ -225,15 +232,16 @@ typedef struct {
   long msgtype;
   erlang_pid from;
   erlang_pid to;
-  char toname[MAXATOMLEN+1];
-  char cookie[MAXATOMLEN+1];
+  char toname[MAXATOMLEN_UTF8];
+  char cookie[MAXATOMLEN_UTF8];
   erlang_trace token;
 } erlang_msg;
 
 /* a fun */
 typedef struct {
     long arity;
-    char module[MAXATOMLEN+1];
+    char module[MAXATOMLEN_UTF8];
+    enum erlang_char_encoding module_org_enc;
     char md5[16];
     long index;
     long old_index;
@@ -258,7 +266,7 @@ typedef struct {
     union {
 	long i_val;
 	double d_val;
-	char atom_name[MAXATOMLEN+1];
+	char atom_name[MAXATOMLEN_UTF8];
 	erlang_pid pid;
 	erlang_port port;
 	erlang_ref ref;
@@ -427,9 +435,17 @@ int ei_encode_string_len(char *buf, int *index, const char *p, int len);
 int ei_x_encode_string(ei_x_buff* x, const char* s);
 int ei_x_encode_string_len(ei_x_buff* x, const char* s, int len);
 int ei_encode_atom(char *buf, int *index, const char *p);
+int ei_encode_atom_as(char *buf, int *index, const char *p,
+		    enum erlang_char_encoding from, enum erlang_char_encoding to);
 int ei_encode_atom_len(char *buf, int *index, const char *p, int len);
+int ei_encode_atom_len_as(char *buf, int *index, const char *p, int len,
+			enum erlang_char_encoding from, enum erlang_char_encoding to);
 int ei_x_encode_atom(ei_x_buff* x, const char* s);
+int ei_x_encode_atom_as(ei_x_buff* x, const char* s,
+		      enum erlang_char_encoding from, enum erlang_char_encoding to);
 int ei_x_encode_atom_len(ei_x_buff* x, const char* s, int len);
+int ei_x_encode_atom_len_as(ei_x_buff* x, const char* s, int len,
+			  enum erlang_char_encoding from, enum erlang_char_encoding to);
 int ei_encode_binary(char *buf, int *index, const void *p, long len);
 int ei_x_encode_binary(ei_x_buff* x, const void* s, int len);
 int ei_encode_pid(char *buf, int *index, const erlang_pid *p);
@@ -479,6 +495,7 @@ int ei_decode_boolean(const char *buf, int *index, int *p);
 int ei_decode_char(const char *buf, int *index, char *p);
 int ei_decode_string(const char *buf, int *index, char *p);
 int ei_decode_atom(const char *buf, int *index, char *p);
+int ei_decode_atom_as(const char *buf, int *index, char *p, int destlen, enum erlang_char_encoding want, enum erlang_char_encoding* was, enum erlang_char_encoding* result);
 int ei_decode_binary(const char *buf, int *index, void *p, long *len);
 int ei_decode_fun(const char* buf, int* index, erlang_fun* p);
 void free_fun(erlang_fun* f);
diff --git a/lib/erl_interface/include/erl_interface.h b/lib/erl_interface/include/erl_interface.h
index 1c4a94700d..98acc0d71d 100644
--- a/lib/erl_interface/include/erl_interface.h
+++ b/lib/erl_interface/include/erl_interface.h
@@ -95,19 +95,24 @@
 
 #define ERL_FLOAT_VALUE(x) ((x)->uval.fval.f)
 
-#define ERL_ATOM_PTR(x) ((x)->uval.aval.a)
-#define ERL_ATOM_SIZE(x) ((x)->uval.aval.len)
+#define ERL_ATOM_PTR(x) erl_atom_ptr_latin1((Erl_Atom_data*) &(x)->uval.aval.d)
+#define ERL_ATOM_PTR_UTF8(x) erl_atom_ptr_utf8((Erl_Atom_data*) &(x)->uval.aval.d)
+#define ERL_ATOM_SIZE(x) erl_atom_size_latin1((Erl_Atom_data*) &(x)->uval.aval.d)
+#define ERL_ATOM_SIZE_UTF8(x) erl_atom_size_utf8((Erl_Atom_data*) &(x)->uval.aval.d)
 
-#define ERL_PID_NODE(x) ((x)->uval.pidval.node)
+#define ERL_PID_NODE(x) erl_atom_ptr_latin1((Erl_Atom_data*) &(x)->uval.pidval.node)
+#define ERL_PID_NODE_UTF8(x) erl_atom_ptr_utf8((Erl_Atom_data*) &(x)->uval.pidval.node)
 #define ERL_PID_NUMBER(x) ((x)->uval.pidval.number)
 #define ERL_PID_SERIAL(x) ((x)->uval.pidval.serial)
 #define ERL_PID_CREATION(x) ((x)->uval.pidval.creation)
 
-#define ERL_PORT_NODE(x) ((x)->uval.portval.node)
+#define ERL_PORT_NODE(x) erl_atom_ptr_latin1((Erl_Atom_data*) &(x)->uval.portval.node)
+#define ERL_PORT_NODE_UTF8(x) erl_atom_ptr_utf8((Erl_Atom_data*) &(x)->uval.portval.node)
 #define ERL_PORT_NUMBER(x) ((x)->uval.portval.number)
 #define ERL_PORT_CREATION(x) ((x)->uval.portval.creation)
 
-#define ERL_REF_NODE(x) ((x)->uval.refval.node)
+#define ERL_REF_NODE(x) erl_atom_ptr_latin1((Erl_Atom_data*) &(x)->uval.refval.node)
+#define ERL_REF_NODE_UTF8(x) erl_atom_ptr_utf8((Erl_Atom_data*) &(x)->uval.refval.node)
 #define ERL_REF_NUMBER(x) ((x)->uval.refval.n[0])
 #define ERL_REF_NUMBERS(x) ((x)->uval.refval.n)
 #define ERL_REF_LEN(x) ((x)->uval.refval.len)
@@ -182,15 +187,27 @@ typedef struct {
   double f;
 } Erl_Float;
 
+typedef struct {
+  char *utf8;
+  int lenU;
+  char *latin1;
+  int lenL;
+} Erl_Atom_data;
+
+char* erl_atom_ptr_latin1(Erl_Atom_data*);
+char* erl_atom_ptr_utf8(Erl_Atom_data*);
+int erl_atom_size_latin1(Erl_Atom_data*);
+int erl_atom_size_utf8(Erl_Atom_data*);
+char* erl_atom_init_latin1(Erl_Atom_data*, const char*);
+
 typedef struct {
   Erl_Header h;
-  int len;
-  char *a;
+  Erl_Atom_data d;
 } Erl_Atom;
 
 typedef struct {
   Erl_Header h;
-  char * node;
+  Erl_Atom_data node;
   unsigned int number;
   unsigned int serial;
   unsigned char creation;
@@ -198,14 +215,14 @@ typedef struct {
 
 typedef struct {    
   Erl_Header h;
-  char * node;
+  Erl_Atom_data node;
   unsigned int number;
   unsigned char creation;
 } Erl_Port;
 
 typedef struct {
   Erl_Header h;
-  char * node;
+  Erl_Atom_data node;
   int len;
   unsigned int n[3];
   unsigned char creation;
@@ -289,7 +306,7 @@ typedef struct _eterm {
 } ETERM;
 
 
-#define MAXREGLEN 255  /* max length of registered (atom) name */
+#define MAXREGLEN (255*4)  /* max length of registered (atom) name */
 
 typedef struct {
   int type;   /* one of the message type constants in eiext.h */
@@ -409,6 +426,7 @@ unsigned char erl_ext_type(unsigned char*); /* Note: returned 'char' before R9C
 unsigned char *erl_peek_ext(unsigned char*,int);
 int    erl_term_len(ETERM*);
 
+int cmp_latin1_vs_utf8(const char* sL, int lenL, const char* sU, int lenU);
 
 /* -------------------------------------------------------------------- */
 /*                      Wrappers around ei functions                    */
diff --git a/lib/erl_interface/src/connect/ei_connect.c b/lib/erl_interface/src/connect/ei_connect.c
index 34362b4b9f..a17257795e 100644
--- a/lib/erl_interface/src/connect/ei_connect.c
+++ b/lib/erl_interface/src/connect/ei_connect.c
@@ -459,6 +459,7 @@ int ei_connect_xinit(ei_cnode* ec, const char *thishostname,
 /*    memmove(&ec->this_ipaddr, thisipaddr, sizeof(ec->this_ipaddr)); */
     
     strcpy(ec->self.node,thisnodename);
+    ec->self.node_org_enc = ERLANG_LATIN1;
     ec->self.num = 0;
     ec->self.serial = 0;
     ec->self.creation = creation;
@@ -1332,7 +1333,9 @@ static int send_name_or_challenge(int fd, char *nodename,
 		| DFLAG_EXTENDED_PIDS_PORTS
 		| DFLAG_FUN_TAGS
 		| DFLAG_NEW_FUN_TAGS
-                | DFLAG_NEW_FLOATS));
+                | DFLAG_NEW_FLOATS
+		| DFLAG_SMALL_ATOM_TAGS
+		| DFLAG_UTF8_ATOMS));
     if (f_chall)
 	put32be(s, challenge);
     memcpy(s, nodename, strlen(nodename));
diff --git a/lib/erl_interface/src/connect/ei_connect_int.h b/lib/erl_interface/src/connect/ei_connect_int.h
index 3c42b49b82..81c384e38d 100644
--- a/lib/erl_interface/src/connect/ei_connect_int.h
+++ b/lib/erl_interface/src/connect/ei_connect_int.h
@@ -102,6 +102,8 @@ extern int h_errno;
 #define DFLAG_NEW_FUN_TAGS        0x80
 #define DFLAG_EXTENDED_PIDS_PORTS 0x100
 #define DFLAG_NEW_FLOATS          0x800
+#define DFLAG_SMALL_ATOM_TAGS     0x4000
+#define DFLAG_UTF8_ATOMS          0x10000
 
 ei_cnode   *ei_fd_to_cnode(int fd);
 int         ei_distversion(int fd);
diff --git a/lib/erl_interface/src/connect/eirecv.c b/lib/erl_interface/src/connect/eirecv.c
index 86852f947d..075f78e3d2 100644
--- a/lib/erl_interface/src/connect/eirecv.c
+++ b/lib/erl_interface/src/connect/eirecv.c
@@ -108,7 +108,7 @@ ei_recv_internal (int fd,
   switch (msg->msgtype) {
   case ERL_SEND:          /* { SEND, Cookie, ToPid } */
     if (ei_tracelevel >= 4) show_this_msg = 1;
-    if (ei_decode_atom(header,&index,msg->cookie) 
+    if (ei_decode_atom_as(header,&index,msg->cookie,sizeof(msg->cookie),ERLANG_UTF8,NULL,NULL) 
 	|| ei_decode_pid(header,&index,&msg->to))
     {
 	erl_errno = EIO;
@@ -120,8 +120,8 @@ ei_recv_internal (int fd,
   case ERL_REG_SEND:     /* { REG_SEND, From, Cookie, ToName } */
     if (ei_tracelevel >= 4) show_this_msg = 1;
     if (ei_decode_pid(header,&index,&msg->from) 
-	|| ei_decode_atom(header,&index,msg->cookie) 
-	|| ei_decode_atom(header,&index,msg->toname))
+	|| ei_decode_atom_as(header,&index,msg->cookie,sizeof(msg->cookie),ERLANG_UTF8,NULL,NULL) 
+	|| ei_decode_atom_as(header,&index,msg->toname,sizeof(msg->toname),ERLANG_UTF8,NULL,NULL))
     {
 	erl_errno = EIO;
 	return -1;
@@ -157,7 +157,7 @@ ei_recv_internal (int fd,
     
   case ERL_SEND_TT:      /* { SEND_TT, Cookie, ToPid, TraceToken } */
     if (ei_tracelevel >= 4) show_this_msg = 1;
-    if (ei_decode_atom(header,&index,msg->cookie) 
+    if (ei_decode_atom_as(header,&index,msg->cookie,sizeof(msg->cookie),ERLANG_UTF8,NULL,NULL) 
 	|| ei_decode_pid(header,&index,&msg->to)
 	|| ei_decode_trace(header,&index,&msg->token))
     {
@@ -171,8 +171,8 @@ ei_recv_internal (int fd,
   case ERL_REG_SEND_TT: /* { REG_SEND_TT, From, Cookie, ToName, TraceToken } */
     if (ei_tracelevel >= 4) show_this_msg = 1;
     if (ei_decode_pid(header,&index,&msg->from) 
-	|| ei_decode_atom(header,&index,msg->cookie) 
-	|| ei_decode_atom(header,&index,msg->toname)
+	|| ei_decode_atom_as(header,&index,msg->cookie,sizeof(msg->cookie),ERLANG_UTF8,NULL,NULL) 
+	|| ei_decode_atom_as(header,&index,msg->toname,sizeof(msg->toname),ERLANG_UTF8,NULL,NULL)
 	|| ei_decode_trace(header,&index,&msg->token))
     {
 	erl_errno = EIO;
diff --git a/lib/erl_interface/src/decode/decode_atom.c b/lib/erl_interface/src/decode/decode_atom.c
index 84edf1766a..2ada418243 100644
--- a/lib/erl_interface/src/decode/decode_atom.c
+++ b/lib/erl_interface/src/decode/decode_atom.c
@@ -21,76 +21,155 @@
 #include "eiext.h"
 #include "putget.h"
 
-static int utf8_to_latin1(char* dest, const char* source, unsigned len);
 
 int ei_decode_atom(const char *buf, int *index, char *p)
 {
-  const char *s = buf + *index;
-  const char *s0 = s;
-  int len;
-
-  switch (get8(s)) {
-  case ERL_ATOM_EXT:
-      len = get16be(s);
-      if (len > MAXATOMLEN) return -1;
-      if (p) {
-	  memmove(p,s,len); 
-	  p[len] = (char)0;
-      }
-      break;
-
-  case ERL_SMALL_ATOM_EXT:
-      len = get8(s);
-      if (p) {
-	  memmove(p,s,len); 
-	  p[len] = (char)0;
-      }
-      break;
-
-  case ERL_UNICODE_ATOM_EXT:
-      len = get16be(s);
-
-      if (len > 2*MAXATOMLEN) return -1;
-
-      if (p && utf8_to_latin1(p, s, len) < 0) return -1;
-      break;
-
-  default:
-      return -1;
-  }
-
-  s += len;
-  *index += s-s0;
-  return 0;
+    return ei_decode_atom_as(buf, index, p, MAXATOMLEN, ERLANG_LATIN1, NULL, NULL);
 }
 
-int ei_internal_get_atom(const char** bufp, char* p)
+int ei_decode_atom_as(const char *buf, int *index, char* p, int destlen,
+		      enum erlang_char_encoding want_enc,
+		      enum erlang_char_encoding* was_encp,
+		      enum erlang_char_encoding* res_encp)
 {
-    int ix = 0;
-    if (ei_decode_atom(*bufp, &ix, p) < 0) return -1;
-    *bufp += ix;
+    const char *s = buf + *index;
+    const char *s0 = s;
+    int len;
+    enum erlang_char_encoding got_enc;
+    
+    switch (get8(s)) {
+    case ERL_ATOM_EXT:
+	len = get16be(s);
+	got_enc = ERLANG_LATIN1;
+	break;
+    case ERL_SMALL_ATOM_EXT:
+	len = get8(s);
+	got_enc = ERLANG_LATIN1;
+	break;
+    case ERL_ATOM_UTF8_EXT:
+	len = get16be(s);
+	got_enc = ERLANG_UTF8;
+	break;
+    case ERL_SMALL_ATOM_UTF8_EXT:
+	len = get8(s);
+	got_enc = ERLANG_UTF8;
+	break;
+    default:
+	return -1;
+    }
+    
+    if (want_enc == got_enc || want_enc == ERLANG_WHATEVER || want_enc == ERLANG_ASCII) {
+	int i, found_non_ascii = 0;
+	if (len >= destlen)
+	    return -1;
+	for (i=0; i<len; i++) {
+	    if (s[i] & 0x80) found_non_ascii = 1;
+	    if (p) p[i] = s[i];
+	}
+	if (p) p[len] = 0;
+	if (want_enc == ERLANG_ASCII && found_non_ascii) {
+	    return -1;
+	}
+	if (res_encp) {
+	    *res_encp = found_non_ascii ? got_enc : ERLANG_ASCII;
+	}
+    }
+    else {
+	int plen = (got_enc == ERLANG_LATIN1) ?
+	    utf8_to_latin1(p, s, len, destlen-1, res_encp) :
+	    latin1_to_utf8(p, s, len, destlen-1, res_encp);
+	if (plen < 0) return -1;
+	if (p) p[plen] = 0;
+    }
+    if (was_encp) {
+	*was_encp = got_enc;
+    }
+    
+    s += len;
+    *index += s-s0;
     return 0;
-}
+}	
+
 
-static int utf8_to_latin1(char* dest, const char* source, unsigned slen)
+int utf8_to_latin1(char* dst, const char* src, int slen, int destlen,
+		   enum erlang_char_encoding* res_encp)
 {
-    const char* dest_end = dest + MAXATOMLEN - 1;
+    const char* const dst_start = dst;
+    const char* const dst_end = dst + destlen;
+    int found_non_ascii = 0;
 
-    while (slen > 0 && dest < dest_end) {
-	if ((source[0] & 0x80) == 0) {
-	    *dest++ = *source++;
+    while (slen > 0) {
+	if (dst >= dst_end) return -1;
+	if ((src[0] & 0x80) == 0) {
+	    if (dst_start) {
+		*dst = *src;
+	    }
+	    ++dst;
+	    ++src;
 	    --slen;
 	}
 	else if (slen > 1 &&
-		 (source[0] & 0xFE) == 0xC2 &&
-		 (source[1] & 0xC0) == 0x80) {
-	    *dest++ = (char) ((source[0] << 6) | (source[1] & 0x3F));
-	    source += 2;
+		 (src[0] & 0xFE) == 0xC2 &&
+		 (src[1] & 0xC0) == 0x80) {
+	    if (dst_start) {
+		*dst = (char) ((src[0] << 6) | (src[1] & 0x3F));
+	    }
+	    ++dst;
+	    src += 2;
 	    slen -= 2;
+	    found_non_ascii = 1;
 	}
 	else return -1;
     }
-    *dest = 0;
+    if (res_encp) {
+	*res_encp = found_non_ascii ? ERLANG_LATIN1 : ERLANG_ASCII;
+    }
+    return dst - dst_start;
+}
+
+int latin1_to_utf8(char* dst, const char* src, int slen, int destlen,
+		   enum erlang_char_encoding* res_encp)
+{
+    const char* const src_end = src + slen;
+    const char* const dst_start = dst;
+    const char* const dst_end = dst + destlen;
+    int found_non_ascii = 0;
+
+    while (src < src_end) {
+	if (dst >= dst_end) return -1;
+	if ((src[0] & 0x80) == 0) {
+	    if (dst_start) {
+		*dst = *src;
+	    }
+	    ++dst;
+	}
+	else {
+	    if (dst_start) {
+		unsigned char ch = *src;
+		dst[0] = 0xC0 | (ch >> 6);
+		dst[1] = 0x80 | (ch & 0x3F);
+	    }
+	    dst += 2;
+	    found_non_ascii = 1;
+	}
+	++src;
+    }
+    if (res_encp) {
+	*res_encp = found_non_ascii ? ERLANG_UTF8 : ERLANG_ASCII;
+    }
+    return dst - dst_start;
+}
+
+
+
+int ei_internal_get_atom(const char** bufp, char* p,
+			 enum erlang_char_encoding* was_encp)
+{
+    int ix = 0;
+    if (ei_decode_atom_as(*bufp, &ix, p, MAXATOMLEN_UTF8, ERLANG_UTF8, was_encp, NULL) < 0)
+	return -1;
+    *bufp += ix;
     return 0;
 }
 
+
diff --git a/lib/erl_interface/src/decode/decode_boolean.c b/lib/erl_interface/src/decode/decode_boolean.c
index 0a7a06f1d4..f20690249b 100644
--- a/lib/erl_interface/src/decode/decode_boolean.c
+++ b/lib/erl_interface/src/decode/decode_boolean.c
@@ -24,12 +24,11 @@
 /* c non-zero -> erlang "true" atom, otherwise "false" */
 int ei_decode_boolean(const char *buf, int *index, int *p)
 {
-  const char *s = buf + *index;
-  const char *s0 = s;
-  char tbuf[MAXATOMLEN+1];
+  char tbuf[6];
   int t;
 
-  if (get_atom(&s, tbuf) < 0) return -1;
+  if (ei_decode_atom_as(buf, index, tbuf, sizeof(tbuf), ERLANG_ASCII, NULL, NULL) < 0)
+      return -1;
 
   if (memcmp(tbuf, "true", 5) == 0)
       t = 1;
@@ -39,7 +38,6 @@ int ei_decode_boolean(const char *buf, int *index, int *p)
       return -1;
       
   if (p) *p = t;
-  *index += s-s0;
   return 0; 
 }
 
diff --git a/lib/erl_interface/src/decode/decode_fun.c b/lib/erl_interface/src/decode/decode_fun.c
index 64fb9e86d8..7bbef5db44 100644
--- a/lib/erl_interface/src/decode/decode_fun.c
+++ b/lib/erl_interface/src/decode/decode_fun.c
@@ -42,7 +42,8 @@ int ei_decode_fun(const char *buf, int *index, erlang_fun *p)
 	if (ei_decode_pid(s, &ix, (p == NULL ? (erlang_pid*)NULL : &p->pid)) < 0)
 	    return -1;
 	/* then the module (atom) */
-	if (ei_decode_atom(s, &ix, (p == NULL ? (char*)NULL : p->module)) < 0)
+	if (ei_decode_atom_as(s, &ix, (p == NULL ? (char*)NULL : p->module),
+			      MAXATOMLEN_UTF8, ERLANG_UTF8, &p->module_org_enc, NULL) < 0)
 	    return -1;
 	/* then the index */
 	if (ei_decode_long(s, &ix, (p == NULL ? (long*)NULL : &p->index)) < 0)
@@ -84,7 +85,8 @@ int ei_decode_fun(const char *buf, int *index, erlang_fun *p)
 	if (p != NULL) p->n_free_vars = i;
 	/* then the module (atom) */
 	ix = 0;
-	if (ei_decode_atom(s, &ix, (p == NULL ? (char*)NULL : p->module)) < 0)
+	if (ei_decode_atom_as(s, &ix, (p == NULL ? (char*)NULL : p->module),
+			      MAXATOMLEN_UTF8, ERLANG_UTF8, &p->module_org_enc, NULL) < 0)
 	    return -1;
 	/* then the old_index */
 	if (ei_decode_long(s, &ix, (p == NULL ? (long*)NULL : &p->old_index)) < 0)
diff --git a/lib/erl_interface/src/decode/decode_pid.c b/lib/erl_interface/src/decode/decode_pid.c
index a762ae499e..e79952195d 100644
--- a/lib/erl_interface/src/decode/decode_pid.c
+++ b/lib/erl_interface/src/decode/decode_pid.c
@@ -26,12 +26,11 @@ int ei_decode_pid(const char *buf, int *index, erlang_pid *p)
 {
   const char *s = buf + *index;
   const char *s0 = s;
-  int len;
   
   if (get8(s) != ERL_PID_EXT) return -1;
 
   /* first the nodename */
-  if (get_atom(&s, p->node) < 0) return -1;
+  if (get_atom(&s, p->node, &p->node_org_enc) < 0) return -1;
   
   /* now the numbers: num (4), serial (4), creation (1) */
   if (p) {
diff --git a/lib/erl_interface/src/decode/decode_port.c b/lib/erl_interface/src/decode/decode_port.c
index 6eb2bc9197..5fd96b51a4 100644
--- a/lib/erl_interface/src/decode/decode_port.c
+++ b/lib/erl_interface/src/decode/decode_port.c
@@ -25,12 +25,11 @@ int ei_decode_port(const char *buf, int *index, erlang_port *p)
 {
   const char *s = buf + *index;
   const char *s0 = s;
-  int len;
   
   if (get8(s) != ERL_PORT_EXT) return -1;
 
   /* first the nodename */
-  if (get_atom(&s, p->node) < 0) return -1;
+  if (get_atom(&s, p->node, &p->node_org_enc) < 0) return -1;
   
   /* now the numbers: num (4), creation (1) */
   if (p) {
diff --git a/lib/erl_interface/src/decode/decode_ref.c b/lib/erl_interface/src/decode/decode_ref.c
index df3c30777b..7294e5d239 100644
--- a/lib/erl_interface/src/decode/decode_ref.c
+++ b/lib/erl_interface/src/decode/decode_ref.c
@@ -26,13 +26,13 @@ int ei_decode_ref(const char *buf, int *index, erlang_ref *p)
 {
   const char *s = buf + *index;
   const char *s0 = s;
-  int count, len, i;
+  int count, i;
   
   switch (get8(s)) {
     case ERL_REFERENCE_EXT:
 
       /* nodename */
-      if (get_atom(&s, p->node) < 0) return -1;
+      if (get_atom(&s, p->node, &p->node_org_enc) < 0) return -1;
   
       /* now the numbers: num (4), creation (1) */
       if (p) {
@@ -53,7 +53,7 @@ int ei_decode_ref(const char *buf, int *index, erlang_ref *p)
       if (p) p->len = count;
 
       /* then the nodename */
-      if (get_atom(&s, p->node) < 0) return -1;
+      if (get_atom(&s, p->node, &p->node_org_enc) < 0) return -1;
 
       /* creation */
       if (p) {
diff --git a/lib/erl_interface/src/encode/encode_atom.c b/lib/erl_interface/src/encode/encode_atom.c
index 6f41f045e0..a3d7c4c759 100644
--- a/lib/erl_interface/src/encode/encode_atom.c
+++ b/lib/erl_interface/src/encode/encode_atom.c
@@ -26,25 +26,95 @@ int ei_encode_atom(char *buf, int *index, const char *p)
 {
     size_t len = strlen(p);
 
-    if (len >= INT_MAX) return -1;
-    return ei_encode_atom_len(buf, index, p, len);
+    if (len >= MAXATOMLEN)
+	len = MAXATOMLEN - 1;
+    return ei_encode_atom_len_as(buf, index, p, len, ERLANG_LATIN1, ERLANG_LATIN1);
 }
 
 int ei_encode_atom_len(char *buf, int *index, const char *p, int len)
+{
+    /* This function is documented to truncate at MAXATOMLEN (256) */ 
+    if (len >= MAXATOMLEN)
+	len = MAXATOMLEN - 1;
+    return ei_encode_atom_len_as(buf, index, p, len, ERLANG_LATIN1, ERLANG_LATIN1);
+}
+
+int ei_encode_atom_as(char *buf, int *index, const char *p,
+		      enum erlang_char_encoding from_enc,
+		      enum erlang_char_encoding to_enc)
+{	
+    return ei_encode_atom_len_as(buf, index, p, strlen(p), from_enc, to_enc);
+}
+
+int ei_encode_atom_len_as(char *buf, int *index, const char *p, int len,
+			  enum erlang_char_encoding from_enc,
+			  enum erlang_char_encoding to_enc)
 {
   char *s = buf + *index;
   char *s0 = s;
+  int offs;
 
-  /* This function is documented to truncate at MAXATOMLEN (256) */ 
-  if (len > MAXATOMLEN)
-    len = MAXATOMLEN;
+  if (from_enc == ERLANG_LATIN1 && len >= MAXATOMLEN) {
+      return -1;
+  }
 
-  if (!buf) s += 3;
-  else {
-    put8(s,ERL_ATOM_EXT);
-    put16be(s,len);
+  switch(to_enc) {
+  case ERLANG_LATIN1:
+      if (buf) {
+	  put8(s,ERL_ATOM_EXT);
+	  switch (from_enc) {
+	  case ERLANG_UTF8:
+	      len = utf8_to_latin1(s+2, p, len, MAXATOMLEN-1, NULL);
+	      if (len < 0) return -1;
+	      break;
+	  case ERLANG_ASCII:
+	  case ERLANG_LATIN1:
+	      memcpy(s+2, p, len);
+	      break;
+	  default:
+	      return -1;
+	  }
+	  put16be(s,len);
+      }
+      else {
+	  s += 3;
+	  if (from_enc == ERLANG_UTF8) {
+	      len = utf8_to_latin1(NULL, p, len, MAXATOMLEN-1, NULL);
+	      if (len < 0) return -1;
+	  }
+      }
+      break;
+      
+  case ERLANG_UTF8:
+      offs =  1 + 1;
+      switch (from_enc) {
+      case ERLANG_LATIN1:
+	  if (len >= 256/2) offs++;
+	  len = latin1_to_utf8((buf ? s+offs : NULL), p, len, MAXATOMLEN_UTF8-1, NULL);
+	  break;
+      case ERLANG_ASCII:
+      case ERLANG_UTF8:
+	  if (len >= 256) offs++;
+	  if (buf) memcpy(s+offs, p, len);
+	  break;
+      default:
+	  return -1;
+      }
+      if (buf) {
+	  if (offs == 2) {
+	      put8(s, ERL_SMALL_ATOM_UTF8_EXT);
+	      put8(s, len);
+	  }
+	  else {
+	      put8(s, ERL_ATOM_UTF8_EXT);
+	      put16be(s, len);
+	  }
+      }
+      else s+= offs;
+      break;
 
-    memmove(s,p,len); /* unterminated string */
+  default:
+      return -1;
   }
   s += len;
 
@@ -53,3 +123,13 @@ int ei_encode_atom_len(char *buf, int *index, const char *p, int len)
   return 0; 
 }
 
+int
+ei_internal_put_atom(char** bufp, const char* p, int slen,
+		     enum erlang_char_encoding to_enc)
+{
+    int ix = 0;
+    if (ei_encode_atom_len_as(*bufp, &ix, p, slen, ERLANG_UTF8, to_enc) < 0)
+	return -1;
+    *bufp += ix;
+    return 0;
+}
diff --git a/lib/erl_interface/src/encode/encode_fun.c b/lib/erl_interface/src/encode/encode_fun.c
index 54ee2083d6..4daee32648 100644
--- a/lib/erl_interface/src/encode/encode_fun.c
+++ b/lib/erl_interface/src/encode/encode_fun.c
@@ -35,7 +35,7 @@ int ei_encode_fun(char *buf, int *index, const erlang_fun *p)
 	ix += sizeof(char) + 4;
 	if (ei_encode_pid(buf, &ix, &p->pid) < 0)
 	    return -1;
-	if (ei_encode_atom(buf, &ix, p->module) < 0)
+	if (ei_encode_atom_as(buf, &ix, p->module, ERLANG_UTF8, p->module_org_enc) < 0)
 	    return -1;
 	if (ei_encode_long(buf, &ix, p->index) < 0)
 	    return -1;
@@ -60,7 +60,7 @@ int ei_encode_fun(char *buf, int *index, const erlang_fun *p)
 	} else
 	    size_p = NULL;
 	ix += 1 + 4 + 1 + sizeof(p->md5) + 4 + 4;
-	if (ei_encode_atom(buf, &ix, p->module) < 0)
+	if (ei_encode_atom_as(buf, &ix, p->module, ERLANG_UTF8, p->module_org_enc) < 0)
 	    return -1;
 	if (ei_encode_long(buf, &ix, p->old_index) < 0)
 	    return -1;
diff --git a/lib/erl_interface/src/encode/encode_pid.c b/lib/erl_interface/src/encode/encode_pid.c
index ee7f235c17..0cf3ef4efb 100644
--- a/lib/erl_interface/src/encode/encode_pid.c
+++ b/lib/erl_interface/src/encode/encode_pid.c
@@ -24,29 +24,23 @@
 int ei_encode_pid(char *buf, int *index, const erlang_pid *p)
 {
   char *s = buf + *index;
-  char *s0 = s;
-  int len = strlen(p->node);
-  
-  if (!buf) s += 13 + len;
-  else {
-    put8(s,ERL_PID_EXT);
 
-    /* first the nodename */
-    put8(s,ERL_ATOM_EXT);
+  ++(*index); /* skip ERL_PID_EXT */
+  if (ei_encode_atom_len_as(buf, index, p->node, strlen(p->node), ERLANG_UTF8, p->node_org_enc) < 0)
+      return -1;
+
+  if (buf) {
+    put8(s,ERL_PID_EXT);
 
-    put16be(s,len);
-  
-    memmove(s, p->node, len);
-    s += len;
+    s = buf + *index;
 
     /* now the integers */
     put32be(s,p->num & 0x7fff); /* 15 bits */
     put32be(s,p->serial & 0x1fff); /* 13 bits */
     put8(s,(p->creation & 0x03)); /* 2 bits */
   }
-  
-  *index += s-s0;
-  
+
+  *index += 4 + 4 + 1;  
   return 0;
 }
 
diff --git a/lib/erl_interface/src/encode/encode_port.c b/lib/erl_interface/src/encode/encode_port.c
index fbbb33182e..2bf9e26d78 100644
--- a/lib/erl_interface/src/encode/encode_port.c
+++ b/lib/erl_interface/src/encode/encode_port.c
@@ -24,28 +24,23 @@
 int ei_encode_port(char *buf, int *index, const erlang_port *p)
 {
   char *s = buf + *index;
-  char *s0 = s;
-  int len = strlen(p->node);
-  
-  if (!buf) s += 9 + len;
-  else {
-    put8(s,ERL_PORT_EXT);
 
-    /* first the nodename */
-    put8(s,ERL_ATOM_EXT);
+  ++(*index); /* skip ERL_PORT_EXT */
+  if (ei_encode_atom_len_as(buf, index, p->node, strlen(p->node), ERLANG_UTF8,
+			    p->node_org_enc) < 0) {
+      return -1;
+  }
+  if (buf) {
+    put8(s,ERL_PORT_EXT);
 
-    put16be(s,len);
-  
-    memmove(s, p->node, len);
-    s += len;
+    s = buf + *index;
 
     /* now the integers */
     put32be(s,p->id & 0x0fffffff /* 28 bits */);
     put8(s,(p->creation & 0x03));
   }
   
-  *index += s-s0;
-  
+  *index += 4 + 1;
   return 0;
 }
 
diff --git a/lib/erl_interface/src/encode/encode_ref.c b/lib/erl_interface/src/encode/encode_ref.c
index 292b452864..e8b3173315 100644
--- a/lib/erl_interface/src/encode/encode_ref.c
+++ b/lib/erl_interface/src/encode/encode_ref.c
@@ -24,36 +24,32 @@
 int ei_encode_ref(char *buf, int *index, const erlang_ref *p)
 {
   char *s = buf + *index;
-  char *s0 = s;
-  int len = strlen(p->node);
   int i;
 
+  (*index) += 1 + 2; /* skip to node atom */
+  if (ei_encode_atom_len_as(buf, index, p->node, strlen(p->node), ERLANG_UTF8,
+			    p->node_org_enc) < 0) {
+      return -1;
+  }
+
   /* Always encode as an extended reference; all participating parties
      are now expected to be able to decode extended references. */
-      if (!buf) s += 1 + 2 + (3+len) + p->len*4 + 1;
-      else {
+  if (buf) {
 	  put8(s,ERL_NEW_REFERENCE_EXT);
 
 	  /* first, number of integers */
 	  put16be(s, p->len);
 
 	  /* then the nodename */
-	  put8(s,ERL_ATOM_EXT);
-
-	  put16be(s,len);
-  
-	  memmove(s, p->node, len);
-	  s += len;
+	  s = buf + *index;
 
 	  /* now the integers */
 	  put8(s,(p->creation & 0x03));
 	  for (i = 0; i < p->len; i++)
 	      put32be(s,p->n[i]);
-
-      }
-  
-  *index += s-s0;
+  }
   
+  *index += p->len*4 + 1;  
   return 0;
 }
 
diff --git a/lib/erl_interface/src/legacy/erl_connect.c b/lib/erl_interface/src/legacy/erl_connect.c
index 41d4fa3138..be83fa8469 100644
--- a/lib/erl_interface/src/legacy/erl_connect.c
+++ b/lib/erl_interface/src/legacy/erl_connect.c
@@ -247,9 +247,15 @@ int erl_send(int fd, ETERM *to ,ETERM *msg)
 	erl_errno = EINVAL;
 	return -1;
     }
-    
-    strncpy(topid.node, (char *)ERL_PID_NODE(to), sizeof(topid.node));
-    topid.node[sizeof(topid.node)-1] = '\0';
+
+    if (to->uval.pidval.node.latin1) {
+	strcpy(topid.node, to->uval.pidval.node.latin1);
+	topid.node_org_enc = ERLANG_LATIN1;
+    }
+    else {
+	strcpy(topid.node, to->uval.pidval.node.utf8);
+	topid.node_org_enc = ERLANG_UTF8;
+    }    
     topid.num = ERL_PID_NUMBER(to);
     topid.serial = ERL_PID_SERIAL(to);
     topid.creation = ERL_PID_CREATION(to);
@@ -263,7 +269,7 @@ static int erl_do_receive_msg(int fd, ei_x_buff* x, ErlMessage* emsg)
     erlang_msg msg;
 
     int r;
-    msg.from.node[0] = msg.to.node[0] = '\0';
+    msg.from.node[0] = msg.to.node[0] = msg.toname[0] = '\0';
     r = ei_do_receive_msg(fd, 0, &msg, x, 0);
 
     if (r == ERL_MSG) {
@@ -299,7 +305,7 @@ static int erl_do_receive_msg(int fd, ei_x_buff* x, ErlMessage* emsg)
 	emsg->to = erl_mk_pid(msg.to.node, msg.to.num, msg.to.serial, msg.to.creation);
     else
 	emsg->to = NULL;
-    memcpy(emsg->to_name, msg.toname, MAXATOMLEN+1);
+    strcpy(emsg->to_name, msg.toname);
     return r;
 }
 
diff --git a/lib/erl_interface/src/legacy/erl_eterm.c b/lib/erl_interface/src/legacy/erl_eterm.c
index 8d559f0f55..aa0fd5ddcf 100644
--- a/lib/erl_interface/src/legacy/erl_eterm.c
+++ b/lib/erl_interface/src/legacy/erl_eterm.c
@@ -36,6 +36,7 @@
 #include "erl_error.h"
 #include "erl_internal.h"
 #include "ei_internal.h"
+#include "putget.h"
 
 #define ERL_IS_BYTE(x) (ERL_IS_INTEGER(x) && (ERL_INT_VALUE(x) & ~0xFF) == 0)
 
@@ -142,9 +143,7 @@ ETERM *erl_mk_atom (const char *s)
 
   ep = erl_alloc_eterm(ERL_ATOM);
   ERL_COUNT(ep) = 1;
-  ERL_ATOM_SIZE(ep) = strlen(s);
-  if ((ERL_ATOM_PTR(ep) = strsave(s)) == NULL)
-  {
+  if (erl_atom_init_latin1(&ep->uval.aval.d, s) == NULL) {
       erl_free_term(ep);
       erl_errno = ENOMEM;
       return NULL;
@@ -152,6 +151,65 @@ ETERM *erl_mk_atom (const char *s)
   return ep;
 } 
 
+char* erl_atom_ptr_latin1(Erl_Atom_data* a)
+{
+    if (a->latin1 == NULL) {
+	enum erlang_char_encoding enc;
+	a->lenL = utf8_to_latin1(NULL, a->utf8, a->lenU, a->lenU, &enc);
+	if (a->lenL < 0) {
+	    a->lenL = 0;
+	    return NULL;
+	}
+	if (enc == ERLANG_ASCII) {
+	    a->latin1 = a->utf8; 
+	}
+	else {
+	    a->latin1 = malloc(a->lenL+1);
+	    utf8_to_latin1(a->latin1, a->utf8, a->lenU, a->lenL, NULL);
+	    a->latin1[a->lenL] = '\0';
+	}
+    }
+    return a->latin1;
+}
+
+char* erl_atom_ptr_utf8(Erl_Atom_data* a)
+{
+    if (a->utf8 == NULL) {
+	int dlen = a->lenL * 2; /* over estimation */
+	a->utf8 = malloc(dlen + 1); 
+	a->lenU = latin1_to_utf8(a->utf8, a->latin1, a->lenL, dlen, NULL);
+	a->utf8[a->lenU] = '\0';
+    }
+    return a->utf8;
+
+}
+int erl_atom_size_latin1(Erl_Atom_data* a)
+{
+    if (a->latin1 == NULL) {
+	erl_atom_ptr_latin1(a);
+    }
+    return a->lenL;
+}
+int erl_atom_size_utf8(Erl_Atom_data* a)
+{
+    if (a->utf8 == NULL) {
+	erl_atom_ptr_utf8(a);
+    }
+    return a->lenU;
+}
+char* erl_atom_init_latin1(Erl_Atom_data* a, const char* s)
+{
+    a->lenL = strlen(s);
+    if ((a->latin1 = strsave(s)) == NULL)
+    {
+	return NULL;
+    }
+    a->utf8 = NULL;
+    a->lenU = 0;
+    return a->latin1;
+}
+
+
 /*
  * Given a string as input, creates a list.
  */
@@ -208,12 +266,19 @@ ETERM *erl_mk_pid(const char *node,
 
     ep = erl_alloc_eterm(ERL_PID);
     ERL_COUNT(ep) = 1;
-    if ((ERL_PID_NODE(ep) = strsave(node)) == NULL)
+    if (erl_atom_init_latin1(&ep->uval.pidval.node, node) == NULL)
     {	     
 	erl_free_term(ep);
 	erl_errno = ENOMEM;
 	return NULL;
     }
+    erl_mk_pid_helper(ep, number, serial, creation);
+    return ep;
+}
+
+void erl_mk_pid_helper(ETERM *ep, unsigned int number, 
+		       unsigned int serial, unsigned char creation)
+{
     ERL_PID_NUMBER(ep)   = number & 0x7fff; /* 15 bits */
     if (ei_internal_use_r9_pids_ports()) {
 	ERL_PID_SERIAL(ep)   = serial & 0x07;  /* 3 bits */
@@ -222,7 +287,6 @@ ETERM *erl_mk_pid(const char *node,
 	ERL_PID_SERIAL(ep)   = serial & 0x1fff;  /* 13 bits */
     }
     ERL_PID_CREATION(ep) = creation & 0x03; /* 2 bits */
-    return ep;
 }
 
 /*
@@ -239,12 +303,18 @@ ETERM *erl_mk_port(const char *node,
 
     ep = erl_alloc_eterm(ERL_PORT);
     ERL_COUNT(ep) = 1;
-    if ((ERL_PORT_NODE(ep) = strsave(node)) == NULL)
+    if (erl_atom_init_latin1(&ep->uval.portval.node, node) == NULL)
     {	     
 	erl_free_term(ep);
 	erl_errno = ENOMEM;
 	return NULL;
     }
+    erl_mk_port_helper(ep, number, creation);
+    return ep;
+}
+
+void erl_mk_port_helper(ETERM* ep, unsigned number, unsigned char creation)
+{
     if (ei_internal_use_r9_pids_ports()) {
 	ERL_PORT_NUMBER(ep)   = number & 0x3ffff; /* 18 bits */
     }
@@ -252,29 +322,29 @@ ETERM *erl_mk_port(const char *node,
 	ERL_PORT_NUMBER(ep)   = number & 0x0fffffff; /* 18 bits */
     }
     ERL_PORT_CREATION(ep) = creation & 0x03; /* 2 bits */
-    return ep;
 }
 
 /*
  * Create any kind of reference.
  */
-ETERM *__erl_mk_reference (const char *node,
+ETERM *__erl_mk_reference (ETERM* t,
+			   const char *node,
 			   size_t len,
 			   unsigned int n[],
 			   unsigned char creation)
 {
-    ETERM * t;
-
-    if (node == NULL) return NULL;
-
-    t = erl_alloc_eterm(ERL_REF);
-    ERL_COUNT(t) = 1;
-
-    if ((ERL_REF_NODE(t) = strsave(node)) == NULL)
-    {	     
-	erl_free_term(t);
-	erl_errno = ENOMEM;
-	return NULL;
+    if (t == NULL) {
+	if (node == NULL) return NULL;
+    
+	t = erl_alloc_eterm(ERL_REF);
+	ERL_COUNT(t) = 1;
+    
+	if (erl_atom_init_latin1(&t->uval.refval.node, node) == NULL)
+	{	     
+	    erl_free_term(t);
+	    erl_errno = ENOMEM;
+	    return NULL;
+	}
     }
     ERL_REF_LEN(t) = len;
     ERL_REF_NUMBERS(t)[0]   = n[0] & 0x3ffff; /* 18 bits */
@@ -294,7 +364,7 @@ ETERM *erl_mk_ref (const char *node,
 {
     unsigned int n[3] = {0, 0, 0};
     n[0] = number;
-    return __erl_mk_reference(node, 1, n, creation);
+    return __erl_mk_reference(NULL, node, 1, n, creation);
 }
 
 /*
@@ -307,7 +377,7 @@ erl_mk_long_ref (const char *node,
 {
     unsigned int n[3] = {0, 0, 0};
     n[0] = n3; n[1] = n2; n[2] = n1;
-    return __erl_mk_reference(node, 3, n, creation);
+    return __erl_mk_reference(NULL, node, 3, n, creation);
 }
 
 /*
@@ -758,6 +828,28 @@ int erl_iolist_length (const ETERM* term)
 	return -1;
 }
 
+static int erl_atom_copy(Erl_Atom_data* dst, const Erl_Atom_data* src)
+{
+    if (src->latin1 == src->utf8) {
+	dst->latin1 = dst->utf8 = strsave(src->latin1);
+	dst->lenL = dst->lenU = strlen(src->latin1);
+    }
+    else if (src->latin1) {
+	dst->latin1 = strsave(src->latin1);
+	dst->lenL = strlen(src->latin1);
+	dst->utf8 = NULL;
+	dst->lenU = 0;
+    }
+    else {
+	dst->utf8 = strsave(src->utf8);
+	dst->lenU = strlen(src->utf8);
+	dst->latin1 = NULL;
+	dst->lenL = 0;
+    }
+    return (dst->latin1 != NULL || dst->utf8 == NULL);
+}
+
+
 /*
  * Return a brand NEW COPY of an ETERM.
  */
@@ -796,9 +888,7 @@ ETERM *erl_copy_term(const ETERM *ep)
 	ERL_FLOAT_VALUE(cp) = ERL_FLOAT_VALUE(ep);
 	break;
     case ERL_ATOM:
-	ERL_ATOM_SIZE(cp) = ERL_ATOM_SIZE(ep);
-	ERL_ATOM_PTR(cp) = strsave(ERL_ATOM_PTR(ep));
-	if (ERL_ATOM_PTR(cp) == NULL)
+	if (!erl_atom_copy(&cp->uval.aval.d, &ep->uval.aval.d))
 	{
 	    erl_free_term(cp);
 	    erl_errno = ENOMEM;
@@ -810,17 +900,17 @@ ETERM *erl_copy_term(const ETERM *ep)
            name and plug in. Somewhat ugly (also done with port and
            ref below). */
 	memcpy(&cp->uval.pidval, &ep->uval.pidval, sizeof(Erl_Pid));
-	ERL_PID_NODE(cp) = strsave(ERL_PID_NODE(ep));
+	erl_atom_copy(&cp->uval.pidval.node, &ep->uval.pidval.node);
 	ERL_COUNT(cp) = 1;
 	break;
     case ERL_PORT:
 	memcpy(&cp->uval.portval, &ep->uval.portval, sizeof(Erl_Port));
-	ERL_PORT_NODE(cp) = strsave(ERL_PORT_NODE(ep));
+	erl_atom_copy(&cp->uval.portval.node, &ep->uval.portval.node);
 	ERL_COUNT(cp) = 1;
 	break;
     case ERL_REF:
 	memcpy(&cp->uval.refval, &ep->uval.refval, sizeof(Erl_Ref));
-	ERL_REF_NODE(cp) = strsave(ERL_REF_NODE(ep));
+	erl_atom_copy(&cp->uval.refval.node, &ep->uval.refval.node);
 	ERL_COUNT(cp) = 1;
 	break;
     case ERL_LIST:
@@ -883,29 +973,29 @@ int erl_print_term(FILE *fp, const ETERM *ep)
     j = i = doquote = 0;
     switch(ERL_TYPE(ep)) 
     {
-    case ERL_ATOM:
+    case ERL_ATOM: {
+	char* adata = ERL_ATOM_PTR(ep);
 	/* FIXME: what if some weird locale is in use? */
-	if (!islower((int)ERL_ATOM_PTR(ep)[0]))
+	if (!islower(adata[0]))
 	    doquote = 1;
 
 	for (i = 0; !doquote && i < ERL_ATOM_SIZE(ep); i++) 
 	{
-	    doquote = !(isalnum((int)ERL_ATOM_PTR(ep)[i]) 
-			|| (ERL_ATOM_PTR(ep)[i] == '_'));
+	    doquote = !(isalnum(adata[i]) || (adata[i] == '_'));
 	}
 
 	if (doquote) {
 	    putc('\'', fp);
 	    ch_written++; 
 	}
-	fputs(ERL_ATOM_PTR(ep), fp);
+	fputs(adata, fp);
 	ch_written += ERL_ATOM_SIZE(ep);	
 	if (doquote) {
 	    putc('\'', fp);
 	    ch_written++;
 	}
 	break;
-
+    }
     case ERL_VARIABLE:
 	if (!isupper((int)ERL_VAR_NAME(ep)[0])) {
 	    doquote = 1;
diff --git a/lib/erl_interface/src/legacy/erl_eterm.h b/lib/erl_interface/src/legacy/erl_eterm.h
index 41b008f04f..2e8129d9cd 100644
--- a/lib/erl_interface/src/legacy/erl_eterm.h
+++ b/lib/erl_interface/src/legacy/erl_eterm.h
@@ -55,7 +55,9 @@ typedef struct _heapmark {
 } Erl_HeapMark;
 
 
-ETERM * __erl_mk_reference(const char *, size_t, unsigned int n[], unsigned char);
+void erl_mk_port_helper(ETERM* ep, unsigned number, unsigned char creation);
+void erl_mk_pid_helper(ETERM*, unsigned,unsigned, unsigned char);
+ETERM * __erl_mk_reference(ETERM*, const char *, size_t, unsigned int n[], unsigned char);
 int erl_current_fix_desc(void);
 
 #endif /* _ERL_ETERM_H */
diff --git a/lib/erl_interface/src/legacy/erl_format.c b/lib/erl_interface/src/legacy/erl_format.c
index dc85806c36..533241e396 100644
--- a/lib/erl_interface/src/legacy/erl_format.c
+++ b/lib/erl_interface/src/legacy/erl_format.c
@@ -574,10 +574,22 @@ static int ematch(ETERM *p, ETERM *t)
 
   switch (type_p) {
 
-  case ERL_ATOM:
-    return p->uval.aval.len == t->uval.aval.len &&
-      memcmp(p->uval.aval.a, t->uval.aval.a, p->uval.aval.len) == 0;
-	
+  case ERL_ATOM: {
+      Erl_Atom_data* pa = &p->uval.aval.d;
+      Erl_Atom_data* ta = &t->uval.aval.d;
+      if (pa->utf8 && ta->utf8) {
+	  return pa->lenU == ta->lenU && memcmp(pa->utf8, ta->utf8, pa->lenU)==0; 
+      }
+      else if (pa->latin1 && ta->latin1) {
+	  return pa->lenL == ta->lenL && memcmp(pa->latin1, ta->latin1, pa->lenL)==0; 
+      }
+      else if (pa->latin1) {
+	  return cmp_latin1_vs_utf8(pa->latin1, pa->lenL, ta->utf8, ta->lenU)==0;
+      }
+      else {
+	  return cmp_latin1_vs_utf8(ta->latin1, ta->lenL, pa->utf8, pa->lenU)==0;
+      }
+  }
   case ERL_VARIABLE:
     if (strcmp(p->uval.vval.name, "_") == 0) /* anon. variable */
       return ERL_TRUE;
diff --git a/lib/erl_interface/src/legacy/erl_malloc.c b/lib/erl_interface/src/legacy/erl_malloc.c
index f51a6c69b3..d09239e02d 100644
--- a/lib/erl_interface/src/legacy/erl_malloc.c
+++ b/lib/erl_interface/src/legacy/erl_malloc.c
@@ -112,6 +112,18 @@ do {						\
     (ptr) = NULL;				\
 } while (0)
 
+static void erl_atom_free(Erl_Atom_data* p)
+{
+    erl_free(p->latin1);
+    if (p->utf8 != p->latin1) {
+	erl_free(p->utf8);
+    }
+    p->latin1 = NULL;
+    p->utf8 = NULL;
+    p->lenL = 0;
+    p->lenU = 0;
+}
+
 static void _erl_free_term (ETERM *ep, int external, int compound)
 {
 restart:
@@ -122,7 +134,7 @@ restart:
 	switch(ERL_TYPE(ep)) 
 	    {
 	    case ERL_ATOM:
-		FREE_AND_CLEAR(ERL_ATOM_PTR(ep));
+		erl_atom_free(&ep->uval.aval.d);
 		break;
 	    case ERL_VARIABLE:
 		FREE_AND_CLEAR(ERL_VAR_NAME(ep));
@@ -161,13 +173,13 @@ restart:
 		FREE_AND_CLEAR(ERL_BIN_PTR(ep));
 		break;
 	    case ERL_PID:
-		FREE_AND_CLEAR(ERL_PID_NODE(ep));
+		erl_atom_free(&ep->uval.pidval.node);
 		break;
 	    case ERL_PORT:
-		FREE_AND_CLEAR(ERL_PORT_NODE(ep));
+		erl_atom_free(&ep->uval.portval.node);
 		break;
 	    case ERL_REF:
-		FREE_AND_CLEAR(ERL_REF_NODE(ep));
+		erl_atom_free(&ep->uval.refval.node);
 		break;
 	    case ERL_EMPTY_LIST:
 	    case ERL_INTEGER:
diff --git a/lib/erl_interface/src/legacy/erl_marshal.c b/lib/erl_interface/src/legacy/erl_marshal.c
index 775d7e82ca..884e9d421b 100644
--- a/lib/erl_interface/src/legacy/erl_marshal.c
+++ b/lib/erl_interface/src/legacy/erl_marshal.c
@@ -44,6 +44,9 @@ int erl_fp_compare(unsigned *a, unsigned *b);
 static void erl_long_to_fp(long l, unsigned *d);
 #endif
 
+static int cmpbytes(unsigned char* s1,int l1,unsigned char* s2,int l2);
+static int cmpatoms(unsigned char* s1, int l1, unsigned char tag1, unsigned char* s2, int l2, unsigned char tag2);
+
 /* Used when comparing two encoded byte arrays */
 /* this global data is ok (from threading point of view) since it is
  * initialized once and never changed
@@ -111,8 +114,9 @@ void erl_init_marshal(void)
     cmp_array[ERL_SMALL_BIG_EXT]     = ERL_NUM_CMP;
     cmp_array[ERL_LARGE_BIG_EXT]     = ERL_NUM_CMP;
     cmp_array[ERL_ATOM_EXT]          = ERL_ATOM_CMP;
+    cmp_array[ERL_ATOM_UTF8_EXT]     = ERL_ATOM_CMP;
     cmp_array[ERL_SMALL_ATOM_EXT]    = ERL_ATOM_CMP;
-    cmp_array[ERL_UNICODE_ATOM_EXT]  = ERL_ATOM_CMP;
+    cmp_array[ERL_SMALL_ATOM_UTF8_EXT] = ERL_ATOM_CMP;
     cmp_array[ERL_REFERENCE_EXT]     = ERL_REF_CMP;
     cmp_array[ERL_NEW_REFERENCE_EXT] = ERL_REF_CMP;
     cmp_array[ERL_FUN_EXT]           = ERL_FUN_CMP;
@@ -162,6 +166,21 @@ static int erl_length_x(const ETERM *ep) {
  *==============================================================
  */
 
+static void encode_atom(Erl_Atom_data* a, unsigned char **ext)
+{
+    int ix = 0;
+    if (a->latin1) {
+	ei_encode_atom_len_as((char*)*ext, &ix, a->latin1, a->lenL,
+			      ERLANG_LATIN1, ERLANG_LATIN1);
+    }
+    else if (ei_encode_atom_len_as((char*)*ext, &ix, a->utf8, a->lenU,
+				   ERLANG_UTF8, ERLANG_LATIN1) < 0) {
+	ei_encode_atom_len_as((char*)*ext, &ix, a->utf8, a->lenU,
+			      ERLANG_UTF8, ERLANG_UTF8);
+    }
+    *ext += ix;
+}
+
 /* 
  * The actual ENCODE engine.
  * Returns 0 on success, otherwise 1.
@@ -176,12 +195,7 @@ int erl_encode_it(ETERM *ep, unsigned char **ext, int dist)
     switch(ERL_TYPE(ep)) 
     {
     case ERL_ATOM:
-	i =  ep->uval.aval.len;
-	*(*ext)++ = ERL_ATOM_EXT;
-	*(*ext)++ = (i >>8) &0xff;
-	*(*ext)++ = i &0xff;
-	memcpy((void *) *ext, (const void *) ep->uval.aval.a, i);
-	*ext += i;
+	encode_atom(&ep->uval.aval.d, ext);
 	return 0;
 
     case ERL_INTEGER:
@@ -292,12 +306,7 @@ int erl_encode_it(ETERM *ep, unsigned char **ext, int dist)
     case ERL_PID:
 	*(*ext)++ = ERL_PID_EXT;    
 	/* First poke in node as an atom */    
-	i = strlen((char *)ERL_PID_NODE(ep));
-	*(*ext)++ = ERL_ATOM_EXT;
-	*(*ext)++ = (i >>8) &0xff;
-	*(*ext)++ = i &0xff;
-	memcpy(*ext, ERL_PID_NODE(ep), i);
-	*ext += i;
+	encode_atom(&ep->uval.pidval.node, ext);
 	/* And then fill in the integer fields */
 	i = ERL_PID_NUMBER(ep);
 	*(*ext)++ = (i >> 24) &0xff;
@@ -325,11 +334,8 @@ int erl_encode_it(ETERM *ep, unsigned char **ext, int dist)
 	    *(*ext)++ = (len >> 8) &0xff;
 	    *(*ext)++ = len &0xff;
 
-	    *(*ext)++ = ERL_ATOM_EXT;
-	    *(*ext)++ = (i >> 8) &0xff;
-	    *(*ext)++ = i &0xff;
-	    memcpy(*ext, ERL_REF_NODE(ep), i);
-	    *ext += i;
+	    encode_atom(&ep->uval.refval.node, ext);
+
 	    *(*ext)++ = ERL_REF_CREATION(ep);
 	    /* Then the integer fields */
 	    for (j = 0; j < ERL_REF_LEN(ep); j++) {
@@ -344,12 +350,7 @@ int erl_encode_it(ETERM *ep, unsigned char **ext, int dist)
     case ERL_PORT:
 	*(*ext)++ = ERL_PORT_EXT;
 	/* First poke in node as an atom */
-	i = strlen((char *)ERL_PORT_NODE(ep));
-	*(*ext)++ = ERL_ATOM_EXT;
-	*(*ext)++ = (i >>8) &0xff;
-	*(*ext)++ = i &0xff;
-	memcpy(*ext, ERL_PORT_NODE(ep), i);
-	*ext += i;
+	encode_atom(&ep->uval.portval.node, ext);
 	/* Then the integer fields */
 	i = ERL_PORT_NUMBER(ep);
 	*(*ext)++ = (i >> 24) &0xff;
@@ -500,6 +501,16 @@ int erl_term_len(ETERM *ep)
   return 1+erl_term_len_helper(ep, 4);
 }
 
+static int atom_len_helper(Erl_Atom_data* a)
+{
+    if (erl_atom_ptr_latin1(a)) {
+	return 1 + 2 + a->lenL; /* ERL_ATOM_EXT */
+    }
+    else {
+	return 1 + 1 + (a->lenU > 255) + a->lenU;
+    }
+}
+
 static int erl_term_len_helper(ETERM *ep, int dist)
 {
   int len = 0;
@@ -511,8 +522,7 @@ static int erl_term_len_helper(ETERM *ep, int dist)
   if (ep) {
     switch (ERL_TYPE(ep)) {
     case ERL_ATOM:
-      i = ep->uval.aval.len;
-      len = i + 3;
+      len = atom_len_helper(&ep->uval.aval.d);
       break;
 
     case ERL_INTEGER:
@@ -544,20 +554,15 @@ static int erl_term_len_helper(ETERM *ep, int dist)
       break;
 
     case ERL_PID:
-      /* 1 + N + 4 + 4 + 1 where N = 3 + strlen */
-      i = strlen((char *)ERL_PID_NODE(ep));
-      len = 13 + i;
+      len = 1 + atom_len_helper(&ep->uval.pidval.node) + 4 + 4 + 1;
       break;
 
     case ERL_REF:
-      i = strlen((char *)ERL_REF_NODE(ep));
-      len = 1 + 2 + (i+3) + 1 + ERL_REF_LEN(ep) * 4;
+      len = 1 + 2 + atom_len_helper(&ep->uval.refval.node) + 1 + ERL_REF_LEN(ep) * 4;
       break;
 
     case ERL_PORT:
-      /* 1 + N + 4 + 1 where N = 3 + strlen */
-      i = strlen((char *)ERL_PORT_NODE(ep));
-      len = 9 + i;
+      len = 1 + atom_len_helper(&ep->uval.portval.node) + 4 + 1;
       break;
 
     case ERL_EMPTY_LIST:
@@ -651,11 +656,33 @@ int erl_encode_buf(ETERM *ep, unsigned char **ext)
 } /* erl_encode_buf */
 
 
-static int read_atom(unsigned char** ext, char* dst)
+static int read_atom(unsigned char** ext, Erl_Atom_data* a)
 {
+    char buf[MAXATOMLEN_UTF8];
     int offs = 0;
-    int ret = ei_decode_atom((char*)*ext, &offs, dst);
+    enum erlang_char_encoding enc;
+    int ret = ei_decode_atom_as((char*)*ext, &offs, buf, MAXATOMLEN_UTF8,
+				ERLANG_WHATEVER, NULL, &enc);
     *ext += offs;
+
+    if (ret == 0) {
+	int i = strlen(buf);
+	char* clone = erl_malloc(i+1);
+	memcpy(clone, buf, i+1);
+
+	a->latin1 = NULL; 	    
+	a->lenL = 0;
+	a->utf8 = NULL;
+	a->lenU = 0;
+	if (enc == ERLANG_LATIN1 || enc == ERLANG_ASCII) {
+	    a->latin1 = clone; 	    
+	    a->lenL = i;
+	}
+	if (enc == ERLANG_UTF8 || enc == ERLANG_ASCII) {
+	    a->utf8 = clone;
+	    a->lenU = i;
+	}
+    }
     return ret;
 }
 
@@ -665,7 +692,6 @@ static int read_atom(unsigned char** ext, char* dst)
  */
 static ETERM *erl_decode_it(unsigned char **ext)
 {
-    char atom_buf[MAXATOMLEN+1];
     char *cp;
     ETERM *ep,*tp,*np;
     unsigned int u,sign;
@@ -765,127 +791,89 @@ static ETERM *erl_decode_it(unsigned char **ext)
       
     case ERL_ATOM_EXT:
     case ERL_SMALL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
+    case ERL_ATOM_UTF8_EXT:
+    case ERL_SMALL_ATOM_UTF8_EXT:
+	
 	ERL_TYPE(ep) = ERL_ATOM;
 	--(*ext);
-	if (read_atom(ext, atom_buf) < 0) return NULL;
-
-	i = strlen(atom_buf);
-	ep->uval.aval.len = i;
-	ep->uval.aval.a = (char *) erl_malloc(i+1);
-	memcpy(ep->uval.aval.a, atom_buf, i+1);
+	if (read_atom(ext, &ep->uval.aval.d) < 0) return NULL;
 	return ep;
       
     case ERL_PID_EXT:
-	erl_free_term(ep);
-	{			/* Why not use the constructors? */
-	    char* node = atom_buf;
+	{
 	    unsigned int number, serial;
 	    unsigned char creation;
-	    ETERM *eterm_p;
 
-	    if (read_atom(ext, node) < 0) return NULL;
+	    ERL_TYPE(ep) = ERL_PID;
+	    if (read_atom(ext, &ep->uval.pidval.node) < 0) return NULL;
 
 	    /* get the integers */
-#if 0
-	    /* FIXME: Remove code or whatever....
-               Ints on the wire are big-endian (== network byte order)
-               so use ntoh[sl]. (But some are little-endian! Arrrgh!)
-               Also, the libc authors can be expected to optimize them
-               heavily. However, the marshalling makes no guarantees
-               about alignments -- so it won't work at all. */
-	    number = ntohl(*((unsigned int *)*ext)++);
-	    serial = ntohl(*((unsigned int *)*ext)++);
-#else
 	    number = ((*ext)[0] << 24) | ((*ext)[1]) << 16 | 
 		((*ext)[2]) << 8 | ((*ext)[3]);	
 	    *ext += 4;
 	    serial = ((*ext)[0] << 24) | ((*ext)[1]) << 16 | 
 		((*ext)[2]) << 8 | ((*ext)[3]);	
 	    *ext += 4;
-#endif
 	    creation =  *(*ext)++; 
-	    eterm_p = erl_mk_pid(node, number, serial, creation);
-	    return eterm_p;
+	    erl_mk_pid_helper(ep, number, serial, creation);
+	    return ep;
 	}
     case ERL_REFERENCE_EXT:
-	erl_free_term(ep);
 	{
-	    char* node = atom_buf;
-	    unsigned int number;
+	    unsigned int n[3] = {0, 0, 0};
 	    unsigned char creation;
-	    ETERM *eterm_p;
 
-	    if (read_atom(ext, node) < 0) return NULL;
+	    ERL_TYPE(ep) = ERL_REF;
+	    if (read_atom(ext, &ep->uval.refval.node) < 0) return NULL;
 
 	    /* get the integers */
-#if 0
-	    number = ntohl(*((unsigned int *)*ext)++);
-#else
-	    number = ((*ext)[0] << 24) | ((*ext)[1]) << 16 | 
+	    n[0] = ((*ext)[0] << 24) | ((*ext)[1]) << 16 | 
 		((*ext)[2]) << 8 | ((*ext)[3]);	
 	    *ext += 4;
-#endif
 	    creation =  *(*ext)++; 
-	    eterm_p = erl_mk_ref(node, number, creation);
-	    return eterm_p;
+	    __erl_mk_reference(ep, NULL, 1, n, creation);
+	    return ep;
 	}
 
     case ERL_NEW_REFERENCE_EXT: 
-	erl_free_term(ep);
 	{
-	    char* node = atom_buf;
 	    size_t cnt, i;
 	    unsigned int n[3];
 	    unsigned char creation;
-	    ETERM *eterm_p;
 
-#if 0
-	    cnt = ntohs(*((unsigned short *)*ext)++);
-#else
+	    ERL_TYPE(ep) = ERL_REF;
 	    cnt = ((*ext)[0] << 8) | (*ext)[1];
 	    *ext += 2;
-#endif
 
-	    if (read_atom(ext, node) < 0) return NULL;
+	    if (read_atom(ext, &ep->uval.refval.node) < 0) return NULL;
 
 	    /* get the integers */
 	    creation =  *(*ext)++; 
 	    for(i = 0; i < cnt; i++)
 	    {
-#if 0
-		n[i] = ntohl(*((unsigned int *)*ext)++);
-#else
 		n[i] = ((*ext)[0] << 24) | ((*ext)[1]) << 16 | 
 		    ((*ext)[2]) << 8 | ((*ext)[3]);	
 		*ext += 4;
-#endif
 	    }
-	    eterm_p = __erl_mk_reference(node, cnt, n, creation);
-	    return eterm_p;
+	    __erl_mk_reference(ep, NULL, cnt, n, creation);
+	    return ep;
 	}
 
     case ERL_PORT_EXT:
-	erl_free_term(ep);
 	{
-	    char* node = atom_buf;
 	    unsigned int number;
 	    unsigned char creation;
-	    ETERM *eterm_p;
 
-	    if (read_atom(ext, node) < 0) return NULL;
+	    ERL_TYPE(ep) = ERL_PORT;
+	    if (read_atom(ext, &ep->uval.portval.node) < 0) return NULL;
 
 	    /* get the integers */
-#if 0
-	    number = ntohl(*((unsigned int *)*ext)++);
-#else
 	    number = ((*ext)[0] << 24) | ((*ext)[1]) << 16 | 
 		((*ext)[2]) << 8 | ((*ext)[3]);	
 	    *ext += 4;
-#endif
 	    creation =  *(*ext)++; 
-	    eterm_p = erl_mk_port(node, number, creation);
-	    return eterm_p;
+	    erl_mk_port_helper(ep, number, creation);
+	    return ep;
 	}
 
     case ERL_NIL_EXT:
@@ -1120,8 +1108,9 @@ unsigned char erl_ext_type(unsigned char *ext)
     case ERL_INTEGER_EXT:
 	return ERL_INTEGER;
     case ERL_ATOM_EXT:
+    case ERL_ATOM_UTF8_EXT:
     case ERL_SMALL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
+    case ERL_SMALL_ATOM_UTF8_EXT:
 	return ERL_ATOM;
     case ERL_PID_EXT:
 	return ERL_PID;
@@ -1173,8 +1162,9 @@ int erl_ext_size(unsigned char *t)
     case ERL_SMALL_INTEGER_EXT:
     case ERL_INTEGER_EXT:
     case ERL_ATOM_EXT:
+    case ERL_ATOM_UTF8_EXT:
     case ERL_SMALL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
+    case ERL_SMALL_ATOM_UTF8_EXT:
     case ERL_PID_EXT:
     case ERL_PORT_EXT:
     case ERL_REFERENCE_EXT:
@@ -1221,12 +1211,13 @@ static int jump_atom(unsigned char** ext)
 
     switch (*e++) {
     case ERL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
+    case ERL_ATOM_UTF8_EXT:
 	len = (e[0] << 8) | e[1];
 	e += (len + 2);
 	break;
 
     case ERL_SMALL_ATOM_EXT:
+    case ERL_SMALL_ATOM_UTF8_EXT:
 	len = e[0];
 	e += (len + 1);
 	break;
@@ -1259,8 +1250,9 @@ static int jump(unsigned char **ext)
 	*ext += 1;
 	break;
     case ERL_ATOM_EXT:
+    case ERL_ATOM_UTF8_EXT:
     case ERL_SMALL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
+    case ERL_SMALL_ATOM_UTF8_EXT:
 	jump_atom(ext);
 	break;
     case ERL_PID_EXT:
@@ -1426,6 +1418,58 @@ static int cmpbytes(unsigned char* s1,int l1,unsigned char* s2,int l2)
 
 } /* cmpbytes */
 
+#define tag2enc(T) ((T)==ERL_ATOM_EXT || (T)==ERL_SMALL_ATOM_EXT ? ERLANG_LATIN1 : ERLANG_UTF8)
+
+static int cmpatoms(unsigned char* s1, int l1, unsigned char tag1,	 
+                    unsigned char* s2, int l2, unsigned char tag2)
+{
+    enum erlang_char_encoding enc1 = tag2enc(tag1);
+    enum erlang_char_encoding enc2 = tag2enc(tag2);
+
+    if (enc1 == enc2) {
+	return cmpbytes(s1, l1,s2,l2);
+    }
+    
+    if (enc1 == ERLANG_LATIN1) {
+	return cmp_latin1_vs_utf8((char*)s1, l1, (char*)s2, l2);
+    }
+    else {
+	return -cmp_latin1_vs_utf8((char*)s2, l2, (char*)s1, l1);
+    }
+}
+
+int cmp_latin1_vs_utf8(const char* strL, int lenL, const char* strU, int lenU)
+{
+    unsigned char* sL = (unsigned char*)strL;
+    unsigned char* sU = (unsigned char*)strU;
+    unsigned char* sL_end = sL + lenL;
+    unsigned char* sU_end = sU + lenU;
+
+    while(sL < sL_end && sU < sU_end) {
+	unsigned char UasL;
+	if (*sL >= 0x80) {
+	    if (*sU < 0xC4 && (sU+1) < sU_end) {
+		UasL = ((sU[0] & 0x3) << 6) | (sU[1] & 0x3F);
+	    }
+	    else return -1;
+	}
+	else {
+	    UasL = *sU;
+	}
+	if (*sL < UasL) return -1;
+	if (*sL > UasL) return 1;
+
+	sL++;
+	if (*sU < 0x80) sU++;
+	else if (*sU < 0xE0) sU += 2;
+	else if (*sU < 0xF0) sU += 3;
+	else /*if (*sU < 0xF8)*/ sU += 4;
+    }
+
+    return (sU >= sU_end) - (sL >= sL_end); /* -1, 0 or 1 */
+}
+
+
 #define CMP_EXT_ERROR_CODE 4711
 
 #define CMP_EXT_INT32_BE(AP, BP)				\
@@ -1561,6 +1605,7 @@ static int cmp_exe2(unsigned char **e1, unsigned char **e2)
   int min,  ret,i,j,k;
   double ff1, ff2;
   unsigned char *tmp1, *tmp2;
+  unsigned char tag1, tag2;
 
   if ( ((*e1)[0] == ERL_STRING_EXT) && ((*e2)[0] == ERL_LIST_EXT) ) {
     return cmp_string_list(e1, e2);
@@ -1568,9 +1613,10 @@ static int cmp_exe2(unsigned char **e1, unsigned char **e2)
     return -cmp_string_list(e2, e1);
   }
 
-  *e2 += 1;
+  tag1 = *(*e1)++;
+  tag2 = *(*e2)++;
   i = j = 0;
-  switch (*(*e1)++) 
+  switch (tag1) 
     {
     case ERL_SMALL_INTEGER_EXT:
       if (**e1 < **e2) ret = -1;
@@ -1590,14 +1636,15 @@ static int cmp_exe2(unsigned char **e1, unsigned char **e2)
       *e1 += 4; *e2 += 4;
       return ret;
     case ERL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
+    case ERL_ATOM_UTF8_EXT:
       i = (**e1) << 8; (*e1)++;
       j = (**e2) << 8; (*e2)++;
       /*fall through*/
     case ERL_SMALL_ATOM_EXT:
+    case ERL_SMALL_ATOM_UTF8_EXT:
       i |= (**e1); (*e1)++;
       j |= (**e2); (*e2)++;
-      ret = cmpbytes(*e1, i, *e2, j);
+      ret = cmpatoms(*e1, i, tag1, *e2, j, tag2);
       *e1 += i;
       *e2 += j;
       return ret;
diff --git a/lib/erl_interface/src/legacy/global_whereis.c b/lib/erl_interface/src/legacy/global_whereis.c
index 2afb193504..e6c556d907 100644
--- a/lib/erl_interface/src/legacy/global_whereis.c
+++ b/lib/erl_interface/src/legacy/global_whereis.c
@@ -85,7 +85,16 @@ ETERM *erl_global_whereis(int fd, const char *name, char *node)
   opid = erl_decode((unsigned char*)buf);
 
   /* extract the nodename for the caller */
-  if (node) strcpy(node,epid.node);
+  if (node) {
+      char* node_str = ERL_PID_NODE(opid);
+      if (node_str) {
+	  strcpy(node, node_str);
+      }
+      else {
+	  erl_free_term(opid);
+	  return NULL;
+      }
+  }
 
   return opid;
 }
diff --git a/lib/erl_interface/src/misc/ei_decode_term.c b/lib/erl_interface/src/misc/ei_decode_term.c
index 6773f90bfc..65afee89cc 100644
--- a/lib/erl_interface/src/misc/ei_decode_term.c
+++ b/lib/erl_interface/src/misc/ei_decode_term.c
@@ -32,7 +32,7 @@
 int ei_decode_ei_term(const char* buf, int* index, ei_term* term)
 {
     const char* s = buf + *index, * s0 = s;
-    int len, i, n, sign;
+    int i, n, sign;
     char c;
 
     if (term == NULL) return -1;
@@ -48,12 +48,13 @@ int ei_decode_ei_term(const char* buf, int* index, ei_term* term)
     case NEW_FLOAT_EXT:
         return ei_decode_double(buf, index, &term->value.d_val);
     case ERL_ATOM_EXT:
+    case ERL_ATOM_UTF8_EXT:
     case ERL_SMALL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
+    case ERL_SMALL_ATOM_UTF8_EXT:
 	return ei_decode_atom(buf, index, term->value.atom_name);
     case ERL_REFERENCE_EXT:
 	/* first the nodename */
-	if (get_atom(&s, term->value.ref.node) < 0) return -1;
+	if (get_atom(&s, term->value.ref.node, &term->value.ref.node_org_enc) < 0) return -1;
         /* now the numbers: num (4), creation (1) */
 	term->value.ref.n[0] = get32be(s);
 	term->value.ref.len = 1;
@@ -63,7 +64,7 @@ int ei_decode_ei_term(const char* buf, int* index, ei_term* term)
 	/* first the integer count */
 	term->value.ref.len = get16be(s);
 	/* then the nodename */
-	if (get_atom(&s, term->value.ref.node) < 0) return -1;
+	if (get_atom(&s, term->value.ref.node, &term->value.ref.node_org_enc) < 0) return -1;
 	/* creation */
 	term->value.ref.creation = get8(s) & 0x03;
 	/* finally the id integers */
@@ -75,12 +76,12 @@ int ei_decode_ei_term(const char* buf, int* index, ei_term* term)
 	}
 	break;
     case ERL_PORT_EXT:
-	if (get_atom(&s, term->value.port.node) < 0) return -1;
+	if (get_atom(&s, term->value.port.node, &term->value.port.node_org_enc) < 0) return -1;
 	term->value.port.id = get32be(s) & 0x0fffffff; /* 28 bits */;
 	term->value.port.creation = get8(s) & 0x03;
 	break;
     case ERL_PID_EXT:
-	if (get_atom(&s, term->value.pid.node) < 0) return -1;
+	if (get_atom(&s, term->value.pid.node, &term->value.port.node_org_enc) < 0) return -1;
 	/* now the numbers: num (4), serial (4), creation (1) */
 	term->value.pid.num = get32be(s) & 0x7fff; /* 15 bits */
 	term->value.pid.serial = get32be(s) & 0x1fff; /* 13 bits */
diff --git a/lib/erl_interface/src/misc/ei_printterm.c b/lib/erl_interface/src/misc/ei_printterm.c
index 620c6e72e2..91fe73e68c 100644
--- a/lib/erl_interface/src/misc/ei_printterm.c
+++ b/lib/erl_interface/src/misc/ei_printterm.c
@@ -132,9 +132,10 @@ static int print_term(FILE* fp, ei_x_buff* x,
     doquote = 0;
     ei_get_type_internal(buf, index, &ty, &n);
     switch (ty) {
-    case ERL_ATOM_EXT:
+    case ERL_ATOM_EXT:   
+    case ERL_ATOM_UTF8_EXT:
     case ERL_SMALL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
+    case ERL_SMALL_ATOM_UTF8_EXT:
 	if (ei_decode_atom(buf, index, a) < 0)
 	   goto err;
 	doquote = !islower((int)a[0]);
diff --git a/lib/erl_interface/src/misc/ei_x_encode.c b/lib/erl_interface/src/misc/ei_x_encode.c
index fa1e26ccbb..44dcff7664 100644
--- a/lib/erl_interface/src/misc/ei_x_encode.c
+++ b/lib/erl_interface/src/misc/ei_x_encode.c
@@ -197,18 +197,33 @@ int ei_x_encode_tuple_header(ei_x_buff* x, long n)
 
 int ei_x_encode_atom(ei_x_buff* x, const char* s)
 {
-    return ei_x_encode_atom_len(x, s, strlen(s));
+    return ei_x_encode_atom_len_as(x, s, strlen(s), ERLANG_LATIN1, ERLANG_LATIN1);
 }
 
 int ei_x_encode_atom_len(ei_x_buff* x, const char* s, int len)
+{
+    return ei_x_encode_atom_len_as(x, s, len, ERLANG_LATIN1, ERLANG_LATIN1);
+}
+
+int ei_x_encode_atom_as(ei_x_buff* x, const char* s,
+			enum erlang_char_encoding from_enc,
+			enum erlang_char_encoding to_enc)
+{
+    return ei_x_encode_atom_len_as(x, s, strlen(s), from_enc, to_enc);
+}
+
+int ei_x_encode_atom_len_as(ei_x_buff* x, const char* s, int len,
+			    enum erlang_char_encoding from_enc,
+			    enum erlang_char_encoding to_enc)
 {
     int i = x->index;
-    ei_encode_atom_len(NULL, &i, s, len);
+    ei_encode_atom_len_as(NULL, &i, s, len, from_enc, to_enc);
     if (!x_fix_buff(x, i))
 	return -1;
-    return ei_encode_atom_len(x->buff, &x->index, s, len);
+    return ei_encode_atom_len_as(x->buff, &x->index, s, len, from_enc, to_enc);
 }
 
+
 int ei_x_encode_pid(ei_x_buff* x, const erlang_pid* pid)
 {
     int i = x->index;
diff --git a/lib/erl_interface/src/misc/get_type.c b/lib/erl_interface/src/misc/get_type.c
index c9a040fbbd..54465196b0 100644
--- a/lib/erl_interface/src/misc/get_type.c
+++ b/lib/erl_interface/src/misc/get_type.c
@@ -33,80 +33,6 @@ int ei_get_type(const char *buf, const int *index, int *type, int *len)
     return ei_get_type_internal(buf, index, type, len);
 }
 
-#if 0
-int ei_get_type(const char *buf, const int *index, int *type, int *len)
-{
-    const char *s = buf + *index;
-    int itype = get8(s);	/* Internal type */
-
-    *len = 0;
-  
-    switch (*type) {
-
-    case ERL_SMALL_INTEGER_EXT:
-    case ERL_INTEGER_EXT:
-    case ERL_SMALL_BIG_EXT:
-    case ERL_LARGE_BIG_EXT:
-	*type = EI_TYPE_INTEGER;
-	break;
-
-    case ERL_FLOAT_EXT:
-	*type = EI_TYPE_FLOAT;
-	break;
-
-    case ERL_SMALL_TUPLE_EXT:
-    case ERL_SMALL_ATOM_EXT:
-	*len  = get8(s);
-	break;
-    
-    case ERL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
-    case ERL_STRING_EXT:
-	*len  = get16be(s);
-	break;
-    
-    case ERL_LARGE_TUPLE_EXT:
-    case ERL_LIST_EXT:
-    case ERL_BINARY_EXT:
-	*len  = get32be(s);
-	break;
-    
-    case ERL_SMALL_BIG_EXT:
-	*len  = (get8(s)+1)/2; /* big arity */
-	break;
-
-    case ERL_LARGE_BIG_EXT:
-	*len  = (get32be(s)+1)/2; /* big arity */
-	break;
-
-    case ERL_BINARY_EXT:
-	*type = EI_TYPE_BINARY;
-	break;
-
-    case ERL_PID_EXT:
-	*type = EI_TYPE_PID;
-	break;
-
-    case ERL_PORT_EXT:
-	*type = EI_TYPE_PORT;
-	break;
-
-    case ERL_REFERENCE_EXT:
-    case ERL_NEW_REFERENCE_EXT:
-	*type = EI_TYPE_REF;
-	break;
-
-    default:
-	break;
-    }
-
-    /* leave index unchanged */
-    return 0;
-}
-#endif
-
-
-/* Old definition of function above */
    
 int ei_get_type_internal(const char *buf, const int *index,
 			 int *type, int *len)
@@ -117,12 +43,13 @@ int ei_get_type_internal(const char *buf, const int *index,
   
   switch (*type) {
   case ERL_SMALL_ATOM_EXT:
+  case ERL_SMALL_ATOM_UTF8_EXT:
     *type = ERL_ATOM_EXT;
   case ERL_SMALL_TUPLE_EXT:
     *len = get8(s);
     break;
 
-  case ERL_UNICODE_ATOM_EXT:
+  case ERL_ATOM_UTF8_EXT:
     *type = ERL_ATOM_EXT;
   case ERL_ATOM_EXT:
   case ERL_STRING_EXT:
diff --git a/lib/erl_interface/src/misc/putget.h b/lib/erl_interface/src/misc/putget.h
index 8b0d4d3404..77ae168f8c 100644
--- a/lib/erl_interface/src/misc/putget.h
+++ b/lib/erl_interface/src/misc/putget.h
@@ -105,8 +105,12 @@
        ((EI_ULONGLONG)((unsigned char *)(s))[-2] << 8)  | \
         (EI_ULONGLONG)((unsigned char *)(s))[-1]))
 
-int ei_internal_get_atom(const char** bufp, char* p);
+int utf8_to_latin1(char* dst, const char* src, int slen, int destlen, enum erlang_char_encoding* res_encp);
+int latin1_to_utf8(char* dst, const char* src, int slen, int destlen, enum erlang_char_encoding* res_encp);
+int ei_internal_get_atom(const char** bufp, char* p, enum erlang_char_encoding*);
+int ei_internal_put_atom(char** bufp, const char* p, int slen, enum erlang_char_encoding);
 #define get_atom ei_internal_get_atom
+#define put_atom ei_internal_put_atom
 
 typedef union float_ext {
     double d;
diff --git a/lib/erl_interface/src/misc/show_msg.c b/lib/erl_interface/src/misc/show_msg.c
index 194296798b..ca46b15aff 100644
--- a/lib/erl_interface/src/misc/show_msg.c
+++ b/lib/erl_interface/src/misc/show_msg.c
@@ -132,13 +132,13 @@ int ei_show_sendmsg(FILE *stream, const char *header, const char *msgbuf)
 
     switch (msg.msgtype) {
     case ERL_SEND:
-	if (ei_decode_atom(header,&index,msg.cookie) 
+	if (ei_decode_atom_as(header,&index,msg.cookie,sizeof(msg.cookie),ERLANG_UTF8,NULL,NULL) 
 	    || ei_decode_pid(header,&index,&msg.to)) return -1;
 	mbuf = msgbuf;
 	break;
 
     case ERL_SEND_TT:
-	if (ei_decode_atom(header,&index,msg.cookie) 
+	if (ei_decode_atom_as(header,&index,msg.cookie,sizeof(msg.cookie),ERLANG_UTF8,NULL,NULL) 
 	    || ei_decode_pid(header,&index,&msg.to)
 	    || ei_decode_trace(header,&index,&msg.token)) return -1;
 	mbuf = msgbuf;
@@ -146,15 +146,15 @@ int ei_show_sendmsg(FILE *stream, const char *header, const char *msgbuf)
     
     case ERL_REG_SEND:
 	if (ei_decode_pid(header,&index,&msg.from) 
-	    || ei_decode_atom(header,&index,msg.cookie) 
-	    || ei_decode_atom(header,&index,msg.toname)) return -1;
+	    || ei_decode_atom_as(header,&index,msg.cookie,sizeof(msg.cookie),ERLANG_UTF8,NULL,NULL) 
+	    || ei_decode_atom_as(header,&index,msg.toname,sizeof(msg.toname),ERLANG_UTF8,NULL,NULL)) return -1;
 	mbuf = msgbuf;
 	break;
     
     case ERL_REG_SEND_TT:
 	if (ei_decode_pid(header,&index,&msg.from) 
-	    || ei_decode_atom(header,&index,msg.cookie) 
-	    || ei_decode_atom(header,&index,msg.toname)
+	    || ei_decode_atom_as(header,&index,msg.cookie,sizeof(msg.cookie),ERLANG_UTF8,NULL,NULL) 
+	    || ei_decode_atom_as(header,&index,msg.toname,sizeof(msg.toname),ERLANG_UTF8,NULL,NULL)
 	    || ei_decode_trace(header,&index,&msg.token)) return -1;
 	mbuf = msgbuf;
 	break;
diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE.erl b/lib/erl_interface/test/ei_decode_encode_SUITE.erl
index 85cb62239b..e8ae7a6f81 100644
--- a/lib/erl_interface/test/ei_decode_encode_SUITE.erl
+++ b/lib/erl_interface/test/ei_decode_encode_SUITE.erl
@@ -118,6 +118,9 @@ test_ei_decode_encode(Config) when is_list(Config) ->
     ?line send_rec(P, OXPort),
     ?line send_rec(P, OXRef),
 
+    %% Unicode atoms
+    [send_rec(P, Atom) || Atom <- unicode_atom_data()],
+
     ?line runner:recv_eot(P),
     ok.
 
@@ -127,7 +130,7 @@ test_ei_decode_encode(Config) when is_list(Config) ->
 % We read two packets for each test, the ei_decode_encode and ei_x_decode_encode  version....
 
 send_rec(P, Term) when is_port(P) ->
-    ?t:format("Testing: ~p~n", [Term]),
+    %%?t:format("Testing: ~p~n", [Term]),
     P ! {self(), {command, term_to_binary(Term)}},
     {_B,Term} = get_buf_and_term(P).
     
@@ -146,7 +149,7 @@ get_buf_and_term(P) ->
 	_ ->
 	    B1 = list_to_binary([131,B]),	% No magic, add
 	    T = binary_to_term(B1),
-	    io:format("~w\n~w\n(got no magic)\n",[B,T]),
+	    %io:format("~w\n~w\n(got no magic)\n",[B,T]),
 	    {B,T}
     end.
     
@@ -160,7 +163,7 @@ get_binary(P) ->
     case runner:get_term(P) of
 	{bytes,L} ->
 	    B = list_to_binary(L),
-	    io:format("~w\n",[L]),
+	    %%io:format("~w\n",[L]),
 % For strange reasons <<131>> show up as <>....
 %	    io:format("~w\n",[B]),
 	    B;
@@ -305,3 +308,60 @@ mk_ref({NodeName, Creation}, Numbers) when is_list(NodeName),
 	    exit({unexpected_binary_to_term_result, Other})
     end.
 
+
+
+unicode_atom_data() ->
+    [uc_atup(lists:seq(16#1f600, 16#1f600+254)),
+     uc_atup(lists:seq(16#1f600, 16#1f600+63)),
+     uc_atup(lists:seq(1, 255)),
+     uc_atup(lists:seq(100, 163)),
+     uc_atup(lists:seq(200, 354)),
+     uc_atup(lists:seq(200, 263)),
+     uc_atup(lists:seq(2000, 2254)),
+     uc_atup(lists:seq(2000, 2063)),
+     uc_atup(lists:seq(65500, 65754)),
+     uc_atup(lists:seq(65500, 65563))
+     | lists:map(fun (N) ->
+			 uc_atup(lists:seq(64000+N, 64254+N))
+		 end,
+		 lists:seq(1, 2000))].
+
+uc_atup(ATxt) ->
+    string_to_atom(ATxt).
+
+string_to_atom(String) ->
+    Utf8List = string_to_utf8_list(String),
+    Len = length(Utf8List),
+    TagLen = case Len < 256 of
+		 true -> [119, Len];
+		 false -> [118, Len bsr 8, Len band 16#ff]
+	     end,
+    binary_to_term(list_to_binary([131, TagLen, Utf8List])).
+
+string_to_utf8_list([]) ->
+    [];
+string_to_utf8_list([CP|CPs]) when is_integer(CP),
+				   0 =< CP,
+				   CP =< 16#7F ->
+    [CP | string_to_utf8_list(CPs)];
+string_to_utf8_list([CP|CPs]) when is_integer(CP),
+				   16#80 =< CP,
+				   CP =< 16#7FF ->
+    [16#C0 bor (CP bsr 6),
+     16#80 bor (16#3F band CP)
+     | string_to_utf8_list(CPs)];
+string_to_utf8_list([CP|CPs]) when is_integer(CP),
+				   16#800 =< CP,
+				   CP =< 16#FFFF ->
+    [16#E0 bor (CP bsr 12),
+     16#80 bor (16#3F band (CP bsr 6)),
+     16#80 bor (16#3F band CP)
+     | string_to_utf8_list(CPs)];
+string_to_utf8_list([CP|CPs]) when is_integer(CP),
+				   16#10000 =< CP,
+				   CP =< 16#10FFFF ->
+    [16#F0 bor (CP bsr 18),
+     16#80 bor (16#3F band (CP bsr 12)),
+     16#80 bor (16#3F band (CP bsr 6)),
+     16#80 bor (16#3F band CP)
+     | string_to_utf8_list(CPs)].
diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c b/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c
index 406f02ecfb..194ce9057b 100644
--- a/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c
+++ b/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c
@@ -29,171 +29,222 @@
  * Author:  kent@erix.ericsson.se
  */
 
-#define EI_DECODE_ENCODE(FUNC,TYPE) \
-  { \
-    char *buf; \
-    char buf2[1024]; \
-    TYPE p; \
-    int size1 = 0; \
-    int size2 = 0; \
-    int size3 = 0; \
-    int err; \
-    ei_x_buff arg; \
-\
-    message("ei_decode_" #FUNC ", arg is type " #TYPE); \
-    buf = read_packet(NULL); \
-    err = ei_decode_ ## FUNC(buf+1, &size1, &p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("decode returned non zero but not -1"); \
-      } else { \
-	fail("decode returned non zero"); \
-      } \
-      return; \
-    } \
-    if (size1 < 1) { \
-      fail("size is < 1"); \
-      return; \
-    } \
-\
-    message("ei_encode_" #FUNC " buf is NULL, arg is type " #TYPE); \
-    err = ei_encode_ ## FUNC(NULL, &size2, &p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("size calculation returned non zero but not -1"); \
-	return; \
-      } else { \
-	fail("size calculation returned non zero"); \
-	return; \
-      } \
-    } \
-    if (size1 != size2) { \
-      message("size1 = %d, size2 = %d\n",size1,size2); \
-      fail("decode and encode size differs when buf is NULL"); \
-      return; \
-    } \
-    message("ei_encode_" #FUNC ", arg is type " #TYPE); \
-    err = ei_encode_ ## FUNC(buf2, &size3, &p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("returned non zero but not -1"); \
-      } else { \
-	fail("returned non zero"); \
-      } \
-      return; \
-    } \
-    if (size1 != size3) { \
-      message("size1 = %d, size2 = %d\n",size1,size3); \
-      fail("decode and encode size differs"); \
-      return; \
-    } \
-    send_buffer(buf2, size1); \
-\
-    message("ei_x_encode_" #FUNC ", arg is type " #TYPE); \
-    ei_x_new(&arg); \
-    err = ei_x_encode_ ## FUNC(&arg, &p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("returned non zero but not -1"); \
-      } else { \
-	fail("returned non zero"); \
-      } \
-      ei_x_free(&arg); \
-      return; \
-    } \
-    if (arg.index < 1) { \
-      fail("size is < 1"); \
-      ei_x_free(&arg); \
-      return; \
-    } \
-    send_buffer(arg.buff, arg.index); \
-    ei_x_free(&arg); \
-  }
-
-#define EI_DECODE_ENCODE_BIG(FUNC,TYPE) \
-  { \
-    char *buf; \
-    char buf2[2048]; \
-    TYPE *p; \
-    int size1 = 0; \
-    int size2 = 0; \
-    int size3 = 0; \
-    int err, index = 0, len, type; \
-    ei_x_buff arg; \
-\
-    message("ei_decode_" #FUNC ", arg is type " #TYPE); \
-    buf = read_packet(NULL); \
-    ei_get_type(buf+1, &index, &type, &len); \
-    p = ei_alloc_big(len); \
-    err = ei_decode_ ## FUNC(buf+1, &size1, p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("decode returned non zero but not -1"); \
-      } else { \
-	fail("decode returned non zero"); \
-      } \
-      return; \
-    } \
-    if (size1 < 1) { \
-      fail("size is < 1"); \
-      return; \
-    } \
-\
-    message("ei_encode_" #FUNC " buf is NULL, arg is type " #TYPE); \
-    err = ei_encode_ ## FUNC(NULL, &size2, p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("size calculation returned non zero but not -1"); \
-	return; \
-      } else { \
-	fail("size calculation returned non zero"); \
-	return; \
-      } \
-    } \
-    if (size1 != size2) { \
-      message("size1 = %d, size2 = %d\n",size1,size2); \
-      fail("decode and encode size differs when buf is NULL"); \
-      return; \
-    } \
-    message("ei_encode_" #FUNC ", arg is type " #TYPE); \
-    err = ei_encode_ ## FUNC(buf2, &size3, p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("returned non zero but not -1"); \
-      } else { \
-	fail("returned non zero"); \
-      } \
-      return; \
-    } \
-    if (size1 != size3) { \
-      message("size1 = %d, size2 = %d\n",size1,size3); \
-      fail("decode and encode size differs"); \
-      return; \
-    } \
-    send_buffer(buf2, size1); \
-\
-    message("ei_x_encode_" #FUNC ", arg is type " #TYPE); \
-    ei_x_new(&arg); \
-    err = ei_x_encode_ ## FUNC(&arg, p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("returned non zero but not -1"); \
-      } else { \
-	fail("returned non zero"); \
-      } \
-      ei_x_free(&arg); \
-      return; \
-    } \
-    if (arg.index < 1) { \
-      fail("size is < 1"); \
-      ei_x_free(&arg); \
-      return; \
-    } \
-    send_buffer(arg.buff, arg.index); \
-    ei_x_free(&arg); \
-    ei_free_big(p); \
-  }
+/*#define MESSAGE(FMT,A1,A2) message(FMT,A1,A2)*/
+#define MESSAGE(FMT,A1,A2)
 
+typedef int decodeFT(const char *buf, int *index, void*);
+typedef int encodeFT(char *buf, int *index, void*);
+typedef int x_encodeFT(ei_x_buff*, void*);
+
+struct Type {
+    char* name;
+    char* type;
+    decodeFT* ei_decode_fp;
+    encodeFT* ei_encode_fp;
+    x_encodeFT* ei_x_encode_fp;
+};
+
+typedef struct
+{
+    char name[MAXATOMLEN_UTF8];
+    enum erlang_char_encoding enc;
+}my_atom;
+
+int ei_decode_my_atom(const char *buf, int *index, my_atom* a)
+{
+    return ei_decode_atom_as(buf, index, a->name, sizeof(a->name), ERLANG_UTF8, &a->enc, NULL);
+}
+int ei_encode_my_atom(char *buf, int *index, my_atom* a)
+{
+    return ei_encode_atom_as(buf, index, a->name, ERLANG_UTF8, a->enc);
+}
+int ei_x_encode_my_atom(ei_x_buff* x, my_atom* a)
+{
+    return ei_x_encode_atom_as(x, a->name, ERLANG_UTF8, a->enc);
+}
+
+void decode_encode(struct Type* t, void* obj)
+{
+    char *buf;
+    char buf2[1024];
+    int size1 = 0;
+    int size2 = 0;
+    int size3 = 0;
+    int err;
+    ei_x_buff arg;
+    
+    MESSAGE("ei_decode_%s, arg is type %s", t->name, t->type);
+    buf = read_packet(NULL);
+    err = t->ei_decode_fp(buf+1, &size1, obj);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("decode returned non zero but not -1");
+	} else {
+	    fail("decode returned non zero");
+	}
+	return;
+    }
+    if (size1 < 1) {
+	fail("size is < 1");
+	return;
+    }
+
+    MESSAGE("ei_encode_%s buf is NULL, arg is type %s", t->name, t->type);
+    err = t->ei_encode_fp(NULL, &size2, obj);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("size calculation returned non zero but not -1");
+	    return;
+	} else {
+	    fail("size calculation returned non zero");
+	    return;
+	}
+    }
+    if (size1 != size2) {
+	MESSAGE("size1 = %d, size2 = %d\n",size1,size2);
+	fail("decode and encode size differs when buf is NULL");
+	return;
+    }
+    MESSAGE("ei_encode_%s, arg is type %s", t->name, t->type);
+    err = t->ei_encode_fp(buf2, &size3, obj);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("returned non zero but not -1");
+	} else {
+	    fail("returned non zero");
+	}
+	return;
+    }
+    if (size1 != size3) {
+	MESSAGE("size1 = %d, size2 = %d\n",size1,size3);
+	fail("decode and encode size differs");
+	return;
+    }
+    send_buffer(buf2, size1);
+
+    MESSAGE("ei_x_encode_%s, arg is type %s", t->name, t->type);
+    ei_x_new(&arg);
+    err = t->ei_x_encode_fp(&arg, obj);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("returned non zero but not -1");
+	} else {
+	    fail("returned non zero");
+	}
+	ei_x_free(&arg);
+	return;
+    }
+    if (arg.index < 1) {
+	fail("size is < 1");
+	ei_x_free(&arg);
+	return;
+    }
+    send_buffer(arg.buff, arg.index);
+    ei_x_free(&arg);
+}
+
+
+#define EI_DECODE_ENCODE(TYPE, ERLANG_TYPE) {			\
+	struct Type type_struct = {#TYPE, #ERLANG_TYPE,		\
+				   (decodeFT*)ei_decode_##TYPE,		\
+				   (encodeFT*)ei_encode_##TYPE,		\
+				   (x_encodeFT*)ei_x_encode_##TYPE };	\
+	ERLANG_TYPE type_obj;						\
+	decode_encode(&type_struct, &type_obj);				\
+    }
+
+
+void decode_encode_big(struct Type* t)
+{
+    char *buf;
+    char buf2[2048];
+    void *p; /* (TYPE*) */
+    int size1 = 0;
+    int size2 = 0;
+    int size3 = 0;
+    int err, index = 0, len, type;
+    ei_x_buff arg;
+
+    MESSAGE("ei_decode_%s, arg is type %s", t->name, t->type);
+    buf = read_packet(NULL);
+    ei_get_type(buf+1, &index, &type, &len);
+    p = ei_alloc_big(len);
+    err = t->ei_decode_fp(buf+1, &size1, p);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("decode returned non zero but not -1");
+	} else {
+	    fail("decode returned non zero");
+	}
+	return;
+    }
+    if (size1 < 1) {
+	fail("size is < 1");
+	return;
+    }
+
+    MESSAGE("ei_encode_%s buf is NULL, arg is type %s", t->name, t->type);
+    err = t->ei_encode_fp(NULL, &size2, p);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("size calculation returned non zero but not -1");
+	    return;
+	} else {
+	    fail("size calculation returned non zero");
+	    return;
+	}
+    }
+    if (size1 != size2) {
+	MESSAGE("size1 = %d, size2 = %d\n",size1,size2);
+	fail("decode and encode size differs when buf is NULL");
+	return;
+    }
+    MESSAGE("ei_encode_%s, arg is type %s", t->name, t->type);
+    err = t->ei_encode_fp(buf2, &size3, p);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("returned non zero but not -1");
+	} else {
+	    fail("returned non zero");
+	}
+	return;
+    }
+    if (size1 != size3) {
+	MESSAGE("size1 = %d, size2 = %d\n",size1,size3);
+	fail("decode and encode size differs");
+	return;
+    }
+    send_buffer(buf2, size1);
+
+    MESSAGE("ei_x_encode_%s, arg is type %s", t->name, t->type);
+    ei_x_new(&arg);
+    err = t->ei_x_encode_fp(&arg, p);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("returned non zero but not -1");
+	} else {
+	    fail("returned non zero");
+	}
+	ei_x_free(&arg);
+	return;
+    }
+    if (arg.index < 1) {
+	fail("size is < 1");
+	ei_x_free(&arg);
+	return;
+    }
+    send_buffer(arg.buff, arg.index);
+    ei_x_free(&arg);
+    ei_free_big(p);
+}
+
+#define EI_DECODE_ENCODE_BIG(TYPE, ERLANG_TYPE) {					\
+	struct Type type_struct = {#TYPE, #ERLANG_TYPE,		\
+				   (decodeFT*)ei_decode_##TYPE,	\
+				   (encodeFT*)ei_encode_##TYPE,		\
+				   (x_encodeFT*)ei_x_encode_##TYPE };	\
+	decode_encode_big(&type_struct);				\
+    }
 
 
 
@@ -201,6 +252,8 @@
 
 TESTCASE(test_ei_decode_encode)
 {
+    int i;
+
     EI_DECODE_ENCODE(fun  , erlang_fun);
     EI_DECODE_ENCODE(pid  , erlang_pid);
     EI_DECODE_ENCODE(port , erlang_port);
@@ -223,6 +276,11 @@ TESTCASE(test_ei_decode_encode)
     EI_DECODE_ENCODE(port , erlang_port);
     EI_DECODE_ENCODE(ref  , erlang_ref);
 
+    /* Unicode atoms */
+    for (i=0; i<2010; i++) {
+	EI_DECODE_ENCODE(my_atom , my_atom);
+    }
+
     report(1);
 }
 
-- 
cgit v1.2.3


From 1f4765cca4874fa92fcfad888fbe6d5f2fbf74d1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 22 Jan 2013 19:25:36 +0100
Subject: erl_interface: even more utf8 atom stuff

---
 lib/erl_interface/doc/src/ei.xml                   | 12 +++--
 lib/erl_interface/src/connect/ei_connect.c         |  2 +-
 lib/erl_interface/src/encode/encode_atom.c         | 59 +++++++++++++++++++-
 lib/erl_interface/src/legacy/erl_connect.c         |  2 +-
 lib/erl_interface/src/misc/ei_format.c             |  4 +-
 lib/erl_interface/src/misc/ei_printterm.c          |  2 +-
 lib/erl_interface/src/misc/show_msg.c              |  2 +-
 lib/erl_interface/src/prog/ei_fake_prog.c          |  6 +++
 lib/erl_interface/test/ei_decode_encode_SUITE.erl  | 63 +++++++++++-----------
 .../ei_decode_encode_test.c                        | 16 ++++--
 10 files changed, 122 insertions(+), 46 deletions(-)

(limited to 'lib/erl_interface')

diff --git a/lib/erl_interface/doc/src/ei.xml b/lib/erl_interface/doc/src/ei.xml
index 0b0b1eeb79..e9c7c644b5 100644
--- a/lib/erl_interface/doc/src/ei.xml
+++ b/lib/erl_interface/doc/src/ei.xml
@@ -94,7 +94,11 @@ enum erlang_char_encoding {
     ERLANG_ASCII, ERLANG_LATIN1, ERLANG_UTF8, ERLANG_WHATEVER
 };
 </code>
-         <p>The character encoding used for atoms.</p>
+         <p>The character encoding used for atoms. <c>ERLANG_ASCII</c> represents 7-bit ASCII.
+	 Latin1 and UTF8 are different extensions of 7-bit ASCII. All 7-bit ASCII characters
+	 are valid Latin1 and UTF8 characters. ASCII and Latin1 both represent each character
+	 by one byte. A UTF8 character can consist of one to four bytes. <c>ERLANG_WHATEVER</c>
+	 is not an encoding but rather used as a wildcard.</p>
       </item>
     </taglist>
   </section>
@@ -256,11 +260,11 @@ enum erlang_char_encoding {
         <p>Encodes an atom in the binary format with character encoding
 	<c><seealso marker="#erlang_char_encoding">to_enc</seealso></c> (latin1 or utf8).
 	The <c>p</c> parameter is the name of the atom with character encoding
-	<c><seealso marker="#erlang_char_encoding">from_enc</seealso></c>.
+	<c><seealso marker="#erlang_char_encoding">from_enc</seealso></c>  (ascii, latin1 or utf8).
 	The name must either be zero-terminated or a function variant with a <c>len</c>
 	parameter must be used.</p>
-	<p>The encoding will fail if the atom is too long or if it can not be represented
-	with character encoding <c>to_enc</c>.</p>
+	<p>The encoding will fail if <c>p</c> is not a valid string in encoding <c>from_enc</c>,
+	if the string is too long or if it can not be represented with character encoding <c>to_enc</c>.</p>
 	<p>These functions were introduced in R16 release of Erlang/OTP as part of a first step
 	to support UTF8 atoms. Atoms encoded with <c>ERLANG_UTF8</c>
 	can not be decoded by earlier releases than R16.</p>
diff --git a/lib/erl_interface/src/connect/ei_connect.c b/lib/erl_interface/src/connect/ei_connect.c
index a17257795e..4421bbb7fe 100644
--- a/lib/erl_interface/src/connect/ei_connect.c
+++ b/lib/erl_interface/src/connect/ei_connect.c
@@ -1071,7 +1071,7 @@ int ei_rpc(ei_cnode* ec, int fd, char *mod, char *fun,
     int i, index;
     ei_term t;
     erlang_msg msg;
-    char rex[MAXATOMLEN+1];
+    char rex[MAXATOMLEN];
 
     if (ei_rpc_to(ec, fd, mod, fun, inbuf, inbuflen) < 0) {
 	return -1;
diff --git a/lib/erl_interface/src/encode/encode_atom.c b/lib/erl_interface/src/encode/encode_atom.c
index a3d7c4c759..8bbe962396 100644
--- a/lib/erl_interface/src/encode/encode_atom.c
+++ b/lib/erl_interface/src/encode/encode_atom.c
@@ -22,6 +22,11 @@
 #include "eiext.h"
 #include "putget.h"
 
+
+static int copy_ascii_atom(char* dst, const char* src, int slen);
+static int copy_utf8_atom(char* dst, const char* src, int slen);
+
+
 int ei_encode_atom(char *buf, int *index, const char *p)
 {
     size_t len = strlen(p);
@@ -54,7 +59,8 @@ int ei_encode_atom_len_as(char *buf, int *index, const char *p, int len,
   char *s0 = s;
   int offs;
 
-  if (from_enc == ERLANG_LATIN1 && len >= MAXATOMLEN) {
+  if (len >= MAXATOMLEN && (from_enc == ERLANG_LATIN1 ||
+			    from_enc == ERLANG_ASCII)) {
       return -1;
   }
 
@@ -68,6 +74,8 @@ int ei_encode_atom_len_as(char *buf, int *index, const char *p, int len,
 	      if (len < 0) return -1;
 	      break;
 	  case ERLANG_ASCII:
+	      if (copy_ascii_atom(s+2, p, len) < 0) return -1;
+	      break;
 	  case ERLANG_LATIN1:
 	      memcpy(s+2, p, len);
 	      break;
@@ -93,9 +101,11 @@ int ei_encode_atom_len_as(char *buf, int *index, const char *p, int len,
 	  len = latin1_to_utf8((buf ? s+offs : NULL), p, len, MAXATOMLEN_UTF8-1, NULL);
 	  break;
       case ERLANG_ASCII:
+	  if (buf && copy_ascii_atom(s+offs, p, len) < 0) return -1;
+	  break;
       case ERLANG_UTF8:
 	  if (len >= 256) offs++;
-	  if (buf) memcpy(s+offs, p, len);
+	  if (buf && copy_utf8_atom(s+offs, p, len) < 0) return -1;
 	  break;
       default:
 	  return -1;
@@ -133,3 +143,48 @@ ei_internal_put_atom(char** bufp, const char* p, int slen,
     *bufp += ix;
     return 0;
 }
+
+
+int copy_ascii_atom(char* dst, const char* src, int slen)
+{
+    while (slen > 0) {
+	if ((src[0] & 0x80) != 0) return -1;
+	*dst++ = *src++;
+	slen--;
+    }
+    return 0;
+}
+
+int copy_utf8_atom(char* dst, const char* src, int slen)
+{
+    int num_chars = 0;
+
+    while (slen > 0) {
+	if (++num_chars >= MAXATOMLEN) return -1;
+	if ((src[0] & 0x80) != 0) {
+	    if ((src[0] & 0xE0) == 0xC0) {
+		if (slen < 2 || (src[1] & 0xC0) != 0x80) return -1;
+		*dst++ = *src++;
+		slen--;
+	    }
+	    else if ((src[0] & 0xF0) == 0xE0) {
+		if (slen < 3 || (src[1] & 0xC0) != 0x80 || (src[2] & 0xC0) != 0x80) return -1;
+		*dst++ = *src++;
+		*dst++ = *src++;
+		slen -= 2;
+	    }
+	    else if ((src[0] & 0xF8) == 0xF0) {
+		if (slen < 4 || (src[1] & 0xC0) != 0x80 || (src[2] & 0xC0) != 0x80 || (src[3] & 0xC0) != 0x80) return -1;
+		*dst++ = *src++;
+		*dst++ = *src++;
+		*dst++ = *src++;
+		slen -= 3;
+	    }
+	    else return -1;
+	}
+	*dst++ = *src++;
+	slen--;
+    }
+    return 0;
+}
+
diff --git a/lib/erl_interface/src/legacy/erl_connect.c b/lib/erl_interface/src/legacy/erl_connect.c
index be83fa8469..f82704ea8b 100644
--- a/lib/erl_interface/src/legacy/erl_connect.c
+++ b/lib/erl_interface/src/legacy/erl_connect.c
@@ -125,7 +125,7 @@ static ei_cnode erl_if_ec;
 
 int erl_connect_init(int this_node_number, char *cookie, short creation)
 {
-    char nn[MAXATOMLEN+1];
+    char nn[MAXATOMLEN];
 
     sprintf(nn, "c%d", this_node_number);
 
diff --git a/lib/erl_interface/src/misc/ei_format.c b/lib/erl_interface/src/misc/ei_format.c
index 281a192535..b5f11e618e 100644
--- a/lib/erl_interface/src/misc/ei_format.c
+++ b/lib/erl_interface/src/misc/ei_format.c
@@ -139,8 +139,8 @@ static int patom(const char** fmt, ei_x_buff* x)
     --(*fmt);
     len = *fmt - start;
     /* FIXME why truncate atom name and not fail?! */
-    if (len > MAXATOMLEN)
-	len = MAXATOMLEN;
+    if (len >= MAXATOMLEN)
+	len = MAXATOMLEN-1;
     return ei_x_encode_atom_len(x, start, len);
 }
 
diff --git a/lib/erl_interface/src/misc/ei_printterm.c b/lib/erl_interface/src/misc/ei_printterm.c
index 91fe73e68c..f3003a6172 100644
--- a/lib/erl_interface/src/misc/ei_printterm.c
+++ b/lib/erl_interface/src/misc/ei_printterm.c
@@ -115,7 +115,7 @@ static int print_term(FILE* fp, ei_x_buff* x,
 			       const char* buf, int* index)
 {
     int i, doquote, n, m, ty, r;
-    char a[MAXATOMLEN+1], *p;
+    char a[MAXATOMLEN], *p;
     int ch_written = 0;		/* counter of written chars */
     erlang_pid pid;
     erlang_port port;
diff --git a/lib/erl_interface/src/misc/show_msg.c b/lib/erl_interface/src/misc/show_msg.c
index ca46b15aff..33b09643ca 100644
--- a/lib/erl_interface/src/misc/show_msg.c
+++ b/lib/erl_interface/src/misc/show_msg.c
@@ -457,7 +457,7 @@ static void show_term(const char *termbuf, int *index, FILE *stream)
 	break;
     
     case ERL_FUN_EXT: {
-	char atom[MAXATOMLEN+1];
+	char atom[MAXATOMLEN];
 	long idx;
 	long uniq;
 	const char* s = termbuf + *index, * s0 = s;
diff --git a/lib/erl_interface/src/prog/ei_fake_prog.c b/lib/erl_interface/src/prog/ei_fake_prog.c
index 68eb537211..34101a2851 100644
--- a/lib/erl_interface/src/prog/ei_fake_prog.c
+++ b/lib/erl_interface/src/prog/ei_fake_prog.c
@@ -96,6 +96,7 @@ int main(void)
   EI_ULONGLONG *ulonglongp = (EI_ULONGLONG*)NULL;
   EI_ULONGLONG ulonglongx = 0;
 #endif
+  enum erlang_char_encoding enc;
 
   intx = erl_errno;
 
@@ -148,9 +149,13 @@ int main(void)
   ei_x_encode_string(&eix, charp);
   ei_x_encode_string_len(&eix, charp, intx);
   ei_encode_atom(charp, intp, charp);
+  ei_encode_atom_as(charp, intp, charp, ERLANG_LATIN1, ERLANG_UTF8);
   ei_encode_atom_len(charp, intp, charp, intx);
+  ei_encode_atom_len_as(charp, intp, charp, intx, ERLANG_ASCII, ERLANG_LATIN1);
   ei_x_encode_atom(&eix, charp);
+  ei_x_encode_atom_as(&eix, charp, ERLANG_LATIN1, ERLANG_UTF8);
   ei_x_encode_atom_len(&eix, charp, intx);
+  ei_x_encode_atom_len_as(&eix, charp, intx, ERLANG_LATIN1, ERLANG_UTF8);
   ei_encode_binary(charp, intp, (void *)0, longx);
   ei_x_encode_binary(&eix, (void*)0, intx);
   ei_encode_pid(charp, intp, &epid);
@@ -181,6 +186,7 @@ int main(void)
   ei_decode_char(charp, intp, charp);
   ei_decode_string(charp, intp, charp);
   ei_decode_atom(charp, intp, charp);
+  ei_decode_atom_as(charp, intp, charp, MAXATOMLEN_UTF8, ERLANG_WHATEVER, &enc, &enc);
   ei_decode_binary(charp, intp, (void *)0, longp);
   ei_decode_fun(charp, intp, &efun);
   free_fun(&efun);
diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE.erl b/lib/erl_interface/test/ei_decode_encode_SUITE.erl
index e8ae7a6f81..0c98b494ec 100644
--- a/lib/erl_interface/test/ei_decode_encode_SUITE.erl
+++ b/lib/erl_interface/test/ei_decode_encode_SUITE.erl
@@ -119,8 +119,12 @@ test_ei_decode_encode(Config) when is_list(Config) ->
     ?line send_rec(P, OXRef),
 
     %% Unicode atoms
-    [send_rec(P, Atom) || Atom <- unicode_atom_data()],
-
+    [begin send_rec(P, Atom),
+	   send_rec(P, mk_pid({Atom,1}, 23434, 3434)),
+	   send_rec(P, mk_port({Atom,1}, 2343434)),
+	   send_rec(P, mk_ref({Atom,1}, [262143, 8723648, 24097245])),
+	   void
+     end || Atom <- unicode_atom_data()],
     ?line runner:recv_eot(P),
     ok.
 
@@ -229,38 +233,36 @@ uint8(Uint) ->
 
 
 mk_pid({NodeName, Creation}, Number, Serial) when is_atom(NodeName) ->
-    mk_pid({atom_to_list(NodeName), Creation}, Number, Serial);
-mk_pid({NodeName, Creation}, Number, Serial) ->
+    <<?VERSION_MAGIC, NodeNameExt/binary>> = term_to_binary(NodeName),
+    mk_pid({NodeNameExt, Creation}, Number, Serial);
+mk_pid({NodeNameExt, Creation}, Number, Serial) ->
     case catch binary_to_term(list_to_binary([?VERSION_MAGIC,
 					?PID_EXT,
-					?ATOM_EXT,
-					uint16_be(length(NodeName)),
-					NodeName,
+					NodeNameExt,
 					uint32_be(Number),
 					uint32_be(Serial),
 					uint8(Creation)])) of
 	Pid when is_pid(Pid) ->
 	    Pid;
 	{'EXIT', {badarg, _}} ->
-	    exit({badarg, mk_pid, [{NodeName, Creation}, Number, Serial]});
+	    exit({badarg, mk_pid, [{NodeNameExt, Creation}, Number, Serial]});
 	Other ->
 	    exit({unexpected_binary_to_term_result, Other})
     end.
 
 mk_port({NodeName, Creation}, Number) when is_atom(NodeName) ->
-    mk_port({atom_to_list(NodeName), Creation}, Number);
-mk_port({NodeName, Creation}, Number) ->
+    <<?VERSION_MAGIC, NodeNameExt/binary>> = term_to_binary(NodeName),
+    mk_port({NodeNameExt, Creation}, Number);
+mk_port({NodeNameExt, Creation}, Number) ->
     case catch binary_to_term(list_to_binary([?VERSION_MAGIC,
 					      ?PORT_EXT,
-					      ?ATOM_EXT,
-					      uint16_be(length(NodeName)),
-					      NodeName,
+					      NodeNameExt,
 					      uint32_be(Number),
 					      uint8(Creation)])) of
 	Port when is_port(Port) ->
 	    Port;
 	{'EXIT', {badarg, _}} ->
-	    exit({badarg, mk_port, [{NodeName, Creation}, Number]});
+	    exit({badarg, mk_port, [{NodeNameExt, Creation}, Number]});
 	Other ->
 	    exit({unexpected_binary_to_term_result, Other})
     end.
@@ -268,33 +270,30 @@ mk_port({NodeName, Creation}, Number) ->
 mk_ref({NodeName, Creation}, Numbers) when is_atom(NodeName),
 					   is_integer(Creation),
 					   is_list(Numbers) ->
-    mk_ref({atom_to_list(NodeName), Creation}, Numbers);
-mk_ref({NodeName, Creation}, [Number]) when is_list(NodeName),
-					    is_integer(Creation),
-					    is_integer(Number) ->
+    <<?VERSION_MAGIC, NodeNameExt/binary>> = term_to_binary(NodeName),
+    mk_ref({NodeNameExt, Creation}, Numbers);
+mk_ref({NodeNameExt, Creation}, [Number]) when is_binary(NodeNameExt),
+					       is_integer(Creation),
+					       is_integer(Number) ->
     case catch binary_to_term(list_to_binary([?VERSION_MAGIC,
 					      ?REFERENCE_EXT,
-					      ?ATOM_EXT,
-					      uint16_be(length(NodeName)),
-					      NodeName,
+					      NodeNameExt,
 					      uint32_be(Number),
 					      uint8(Creation)])) of
 	Ref when is_reference(Ref) ->
 	    Ref;
 	{'EXIT', {badarg, _}} ->
-	    exit({badarg, mk_ref, [{NodeName, Creation}, [Number]]});
+	    exit({badarg, mk_ref, [{NodeNameExt, Creation}, [Number]]});
 	Other ->
 	    exit({unexpected_binary_to_term_result, Other})
     end;
-mk_ref({NodeName, Creation}, Numbers) when is_list(NodeName),
-					   is_integer(Creation),
-					   is_list(Numbers) ->
+mk_ref({NodeNameExt, Creation}, Numbers) when is_binary(NodeNameExt),
+					      is_integer(Creation),
+					      is_list(Numbers) ->
     case catch binary_to_term(list_to_binary([?VERSION_MAGIC,
 					      ?NEW_REFERENCE_EXT,
 					      uint16_be(length(Numbers)),
-					      ?ATOM_EXT,
-					      uint16_be(length(NodeName)),
-					      NodeName,
+					      NodeNameExt,
 					      uint8(Creation),
 					      lists:map(fun (N) ->
 								uint32_be(N)
@@ -303,7 +302,7 @@ mk_ref({NodeName, Creation}, Numbers) when is_list(NodeName),
 	Ref when is_reference(Ref) ->
 	    Ref;
 	{'EXIT', {badarg, _}} ->
-	    exit({badarg, mk_ref, [{NodeName, Creation}, Numbers]});
+	    exit({badarg, mk_ref, [{NodeNameExt, Creation}, Numbers]});
 	Other ->
 	    exit({unexpected_binary_to_term_result, Other})
     end.
@@ -322,9 +321,11 @@ unicode_atom_data() ->
      uc_atup(lists:seq(65500, 65754)),
      uc_atup(lists:seq(65500, 65563))
      | lists:map(fun (N) ->
-			 uc_atup(lists:seq(64000+N, 64254+N))
+			 Pow2 = (1 bsl N),
+			 uc_atup(lists:seq(Pow2 - 127, Pow2 + 127))
 		 end,
-		 lists:seq(1, 2000))].
+		 lists:seq(7, 20))
+    ].
 
 uc_atup(ATxt) ->
     string_to_atom(ATxt).
diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c b/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c
index 194ce9057b..e57663f984 100644
--- a/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c
+++ b/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c
@@ -63,10 +63,12 @@ int ei_x_encode_my_atom(ei_x_buff* x, my_atom* a)
     return ei_x_encode_atom_as(x, a->name, ERLANG_UTF8, a->enc);
 }
 
+#define BUFSZ 2000
+
 void decode_encode(struct Type* t, void* obj)
 {
     char *buf;
-    char buf2[1024];
+    char buf2[BUFSZ];
     int size1 = 0;
     int size2 = 0;
     int size3 = 0;
@@ -89,6 +91,11 @@ void decode_encode(struct Type* t, void* obj)
 	return;
     }
 
+    if (size1 > BUFSZ) {
+	fail("size is > BUFSZ");
+	return;
+    }
+
     MESSAGE("ei_encode_%s buf is NULL, arg is type %s", t->name, t->type);
     err = t->ei_encode_fp(NULL, &size2, obj);
     if (err != 0) {
@@ -277,8 +284,11 @@ TESTCASE(test_ei_decode_encode)
     EI_DECODE_ENCODE(ref  , erlang_ref);
 
     /* Unicode atoms */
-    for (i=0; i<2010; i++) {
-	EI_DECODE_ENCODE(my_atom , my_atom);
+    for (i=0; i<24; i++) {
+	EI_DECODE_ENCODE(my_atom, my_atom);
+	EI_DECODE_ENCODE(pid, erlang_pid);
+	EI_DECODE_ENCODE(port, erlang_port);
+	EI_DECODE_ENCODE(ref, erlang_ref);
     }
 
     report(1);
-- 
cgit v1.2.3


From c596e17cf3d69cf5e10d28ee2a8ee35162786da1 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Wed, 23 Jan 2013 16:04:38 +0100
Subject: erl_interface: Changed erlang_char_encoding interface

to allow bitwise-or'd combinations.
---
 lib/erl_interface/doc/src/ei.xml           | 13 +++++++------
 lib/erl_interface/include/ei.h             |  7 ++++++-
 lib/erl_interface/src/decode/decode_atom.c |  2 +-
 lib/erl_interface/src/encode/encode_atom.c |  3 +--
 lib/erl_interface/src/legacy/erl_marshal.c |  6 +++---
 5 files changed, 18 insertions(+), 13 deletions(-)

(limited to 'lib/erl_interface')

diff --git a/lib/erl_interface/doc/src/ei.xml b/lib/erl_interface/doc/src/ei.xml
index e9c7c644b5..117c787da6 100644
--- a/lib/erl_interface/doc/src/ei.xml
+++ b/lib/erl_interface/doc/src/ei.xml
@@ -91,14 +91,13 @@
 	 <p/>
 	 <code type="none">
 enum erlang_char_encoding {
-    ERLANG_ASCII, ERLANG_LATIN1, ERLANG_UTF8, ERLANG_WHATEVER
+    ERLANG_ASCII, ERLANG_LATIN1, ERLANG_UTF8
 };
 </code>
          <p>The character encoding used for atoms. <c>ERLANG_ASCII</c> represents 7-bit ASCII.
 	 Latin1 and UTF8 are different extensions of 7-bit ASCII. All 7-bit ASCII characters
 	 are valid Latin1 and UTF8 characters. ASCII and Latin1 both represent each character
-	 by one byte. A UTF8 character can consist of one to four bytes. <c>ERLANG_WHATEVER</c>
-	 is not an encoding but rather used as a wildcard.</p>
+	 by one byte. A UTF8 character can consist of one to four bytes.</p>
       </item>
     </taglist>
   </section>
@@ -545,11 +544,13 @@ ei_x_encode_empty_list(&amp;x);
 	want</seealso></c>. The original encoding used in the
 	binary format (latin1 or utf8) can be obtained from <c>*was</c>. The actual encoding of the resulting string
 	(7-bit ascii, latin1 or utf8) can be obtained from <c>*result</c>. Both <c>was</c> and <c>result</c> can be <c>NULL</c>.
-	<c>*result</c> may differ from <c>want</c> if <c>want</c> is <c>ERLANG_WHATEVER</c> or if
-	<c>*result</c> turn out to be pure 7-bit ascii (compatible with both latin1 and utf8).</p>
+	
+	<c>*result</c> may differ from <c>want</c> if <c>want</c> is a bitwise-or'd combination like
+	<c>ERLANG_LATIN1|ERLANG_UTF8</c> or if <c>*result</c> turn out to be pure 7-bit ascii
+	(compatible with both latin1 and utf8).</p>
 	<p>This function fails if the atom is too long for the buffer
 	or if it can not be represented with encoding <c>want</c>.</p>
-	<p>This functions was introduced in R16 release of Erlang/OTP as part of a first step
+	<p>This function was introduced in R16 release of Erlang/OTP as part of a first step
 	to support UTF8 atoms.</p>
       </desc>
     </func>
diff --git a/lib/erl_interface/include/ei.h b/lib/erl_interface/include/ei.h
index 20e575f64d..2278a28adb 100644
--- a/lib/erl_interface/include/ei.h
+++ b/lib/erl_interface/include/ei.h
@@ -190,7 +190,12 @@ extern volatile int __erl_errno;
 #define MAXATOMLEN_UTF8 (255*4 + 1)
 #define MAXNODELEN EI_MAXALIVELEN+1+EI_MAXHOSTNAMELEN
 
-enum erlang_char_encoding { ERLANG_ASCII, ERLANG_LATIN1, ERLANG_UTF8, ERLANG_WHATEVER };
+enum erlang_char_encoding { 
+    ERLANG_ASCII = 1,
+    ERLANG_LATIN1 = 2,
+    ERLANG_UTF8 = 4,
+    ERLANG_ANY = ERLANG_ASCII|ERLANG_LATIN1|ERLANG_UTF8
+};
 
 /* a pid */
 typedef struct {
diff --git a/lib/erl_interface/src/decode/decode_atom.c b/lib/erl_interface/src/decode/decode_atom.c
index 2ada418243..556c400cb3 100644
--- a/lib/erl_interface/src/decode/decode_atom.c
+++ b/lib/erl_interface/src/decode/decode_atom.c
@@ -58,7 +58,7 @@ int ei_decode_atom_as(const char *buf, int *index, char* p, int destlen,
 	return -1;
     }
     
-    if (want_enc == got_enc || want_enc == ERLANG_WHATEVER || want_enc == ERLANG_ASCII) {
+    if ((want_enc & got_enc) || want_enc == ERLANG_ASCII) {
 	int i, found_non_ascii = 0;
 	if (len >= destlen)
 	    return -1;
diff --git a/lib/erl_interface/src/encode/encode_atom.c b/lib/erl_interface/src/encode/encode_atom.c
index 8bbe962396..044f17cb60 100644
--- a/lib/erl_interface/src/encode/encode_atom.c
+++ b/lib/erl_interface/src/encode/encode_atom.c
@@ -59,8 +59,7 @@ int ei_encode_atom_len_as(char *buf, int *index, const char *p, int len,
   char *s0 = s;
   int offs;
 
-  if (len >= MAXATOMLEN && (from_enc == ERLANG_LATIN1 ||
-			    from_enc == ERLANG_ASCII)) {
+  if (len >= MAXATOMLEN && (from_enc & (ERLANG_LATIN1|ERLANG_ASCII))) {
       return -1;
   }
 
diff --git a/lib/erl_interface/src/legacy/erl_marshal.c b/lib/erl_interface/src/legacy/erl_marshal.c
index 884e9d421b..4c45cebb02 100644
--- a/lib/erl_interface/src/legacy/erl_marshal.c
+++ b/lib/erl_interface/src/legacy/erl_marshal.c
@@ -662,7 +662,7 @@ static int read_atom(unsigned char** ext, Erl_Atom_data* a)
     int offs = 0;
     enum erlang_char_encoding enc;
     int ret = ei_decode_atom_as((char*)*ext, &offs, buf, MAXATOMLEN_UTF8,
-				ERLANG_WHATEVER, NULL, &enc);
+				ERLANG_LATIN1|ERLANG_UTF8, NULL, &enc);
     *ext += offs;
 
     if (ret == 0) {
@@ -674,11 +674,11 @@ static int read_atom(unsigned char** ext, Erl_Atom_data* a)
 	a->lenL = 0;
 	a->utf8 = NULL;
 	a->lenU = 0;
-	if (enc == ERLANG_LATIN1 || enc == ERLANG_ASCII) {
+	if (enc & (ERLANG_LATIN1 | ERLANG_ASCII)) {
 	    a->latin1 = clone; 	    
 	    a->lenL = i;
 	}
-	if (enc == ERLANG_UTF8 || enc == ERLANG_ASCII) {
+	if (enc & (ERLANG_UTF8 | ERLANG_ASCII)) {
 	    a->utf8 = clone;
 	    a->lenU = i;
 	}
-- 
cgit v1.2.3


From d6e3e256b850050b7a86323b2948009d5fcc30a9 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang-solutions.com>
Date: Wed, 23 Jan 2013 10:17:29 +0100
Subject: erl_interface: Fix bug when transcoding atoms from and to UTF8

---
 lib/erl_interface/src/decode/decode_atom.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/erl_interface')

diff --git a/lib/erl_interface/src/decode/decode_atom.c b/lib/erl_interface/src/decode/decode_atom.c
index 556c400cb3..9779ad3f35 100644
--- a/lib/erl_interface/src/decode/decode_atom.c
+++ b/lib/erl_interface/src/decode/decode_atom.c
@@ -76,8 +76,8 @@ int ei_decode_atom_as(const char *buf, int *index, char* p, int destlen,
     }
     else {
 	int plen = (got_enc == ERLANG_LATIN1) ?
-	    utf8_to_latin1(p, s, len, destlen-1, res_encp) :
-	    latin1_to_utf8(p, s, len, destlen-1, res_encp);
+	  latin1_to_utf8(p, s, len, destlen-1, res_encp) :
+	  utf8_to_latin1(p, s, len, destlen-1, res_encp);
 	if (plen < 0) return -1;
 	if (p) p[plen] = 0;
     }
-- 
cgit v1.2.3