erl_interface: utf8 atoms continued

7 files changed, 333 insertions, 155 deletions

diff --git a/lib/erl_interface/src/legacy/erl_connect.c b/lib/erl_interface/src/legacy/erl_connect.c
index 41d4fa3138..be83fa8469 100644
--- a/lib/erl_interface/src/legacy/erl_connect.c
+++ b/lib/erl_interface/src/legacy/erl_connect.c
@@ -247,9 +247,15 @@ int erl_send(int fd, ETERM *to ,ETERM *msg)
 	erl_errno = EINVAL;
 	return -1;
     }
-    
-    strncpy(topid.node, (char *)ERL_PID_NODE(to), sizeof(topid.node));
-    topid.node[sizeof(topid.node)-1] = '\0';
+
+    if (to->uval.pidval.node.latin1) {
+	strcpy(topid.node, to->uval.pidval.node.latin1);
+	topid.node_org_enc = ERLANG_LATIN1;
+    }
+    else {
+	strcpy(topid.node, to->uval.pidval.node.utf8);
+	topid.node_org_enc = ERLANG_UTF8;
+    }    
     topid.num = ERL_PID_NUMBER(to);
     topid.serial = ERL_PID_SERIAL(to);
     topid.creation = ERL_PID_CREATION(to);
@@ -263,7 +269,7 @@ static int erl_do_receive_msg(int fd, ei_x_buff* x, ErlMessage* emsg)
     erlang_msg msg;
 
     int r;
-    msg.from.node[0] = msg.to.node[0] = '\0';
+    msg.from.node[0] = msg.to.node[0] = msg.toname[0] = '\0';
     r = ei_do_receive_msg(fd, 0, &msg, x, 0);
 
     if (r == ERL_MSG) {
@@ -299,7 +305,7 @@ static int erl_do_receive_msg(int fd, ei_x_buff* x, ErlMessage* emsg)
 	emsg->to = erl_mk_pid(msg.to.node, msg.to.num, msg.to.serial, msg.to.creation);
     else
 	emsg->to = NULL;
-    memcpy(emsg->to_name, msg.toname, MAXATOMLEN+1);
+    strcpy(emsg->to_name, msg.toname);
     return r;
 }
 
diff --git a/lib/erl_interface/src/legacy/erl_eterm.c b/lib/erl_interface/src/legacy/erl_eterm.c
index 8d559f0f55..aa0fd5ddcf 100644
--- a/lib/erl_interface/src/legacy/erl_eterm.c
+++ b/lib/erl_interface/src/legacy/erl_eterm.c
@@ -36,6 +36,7 @@
 #include "erl_error.h"
 #include "erl_internal.h"
 #include "ei_internal.h"
+#include "putget.h"
 
 #define ERL_IS_BYTE(x) (ERL_IS_INTEGER(x) && (ERL_INT_VALUE(x) & ~0xFF) == 0)
 
@@ -142,9 +143,7 @@ ETERM *erl_mk_atom (const char *s)
 
   ep = erl_alloc_eterm(ERL_ATOM);
   ERL_COUNT(ep) = 1;
-  ERL_ATOM_SIZE(ep) = strlen(s);
-  if ((ERL_ATOM_PTR(ep) = strsave(s)) == NULL)
-  {
+  if (erl_atom_init_latin1(&ep->uval.aval.d, s) == NULL) {
       erl_free_term(ep);
       erl_errno = ENOMEM;
       return NULL;
@@ -152,6 +151,65 @@ ETERM *erl_mk_atom (const char *s)
   return ep;
 } 
 
+char* erl_atom_ptr_latin1(Erl_Atom_data* a)
+{
+    if (a->latin1 == NULL) {
+	enum erlang_char_encoding enc;
+	a->lenL = utf8_to_latin1(NULL, a->utf8, a->lenU, a->lenU, &enc);
+	if (a->lenL < 0) {
+	    a->lenL = 0;
+	    return NULL;
+	}
+	if (enc == ERLANG_ASCII) {
+	    a->latin1 = a->utf8; 
+	}
+	else {
+	    a->latin1 = malloc(a->lenL+1);
+	    utf8_to_latin1(a->latin1, a->utf8, a->lenU, a->lenL, NULL);
+	    a->latin1[a->lenL] = '\0';
+	}
+    }
+    return a->latin1;
+}
+
+char* erl_atom_ptr_utf8(Erl_Atom_data* a)
+{
+    if (a->utf8 == NULL) {
+	int dlen = a->lenL * 2; /* over estimation */
+	a->utf8 = malloc(dlen + 1); 
+	a->lenU = latin1_to_utf8(a->utf8, a->latin1, a->lenL, dlen, NULL);
+	a->utf8[a->lenU] = '\0';
+    }
+    return a->utf8;
+
+}
+int erl_atom_size_latin1(Erl_Atom_data* a)
+{
+    if (a->latin1 == NULL) {
+	erl_atom_ptr_latin1(a);
+    }
+    return a->lenL;
+}
+int erl_atom_size_utf8(Erl_Atom_data* a)
+{
+    if (a->utf8 == NULL) {
+	erl_atom_ptr_utf8(a);
+    }
+    return a->lenU;
+}
+char* erl_atom_init_latin1(Erl_Atom_data* a, const char* s)
+{
+    a->lenL = strlen(s);
+    if ((a->latin1 = strsave(s)) == NULL)
+    {
+	return NULL;
+    }
+    a->utf8 = NULL;
+    a->lenU = 0;
+    return a->latin1;
+}
+
+
 /*
  * Given a string as input, creates a list.
  */
@@ -208,12 +266,19 @@ ETERM *erl_mk_pid(const char *node,
 
     ep = erl_alloc_eterm(ERL_PID);
     ERL_COUNT(ep) = 1;
-    if ((ERL_PID_NODE(ep) = strsave(node)) == NULL)
+    if (erl_atom_init_latin1(&ep->uval.pidval.node, node) == NULL)
     {	     
 	erl_free_term(ep);
 	erl_errno = ENOMEM;
 	return NULL;
     }
+    erl_mk_pid_helper(ep, number, serial, creation);
+    return ep;
+}
+
+void erl_mk_pid_helper(ETERM *ep, unsigned int number, 
+		       unsigned int serial, unsigned char creation)
+{
     ERL_PID_NUMBER(ep)   = number & 0x7fff; /* 15 bits */
     if (ei_internal_use_r9_pids_ports()) {
 	ERL_PID_SERIAL(ep)   = serial & 0x07;  /* 3 bits */
@@ -222,7 +287,6 @@ ETERM *erl_mk_pid(const char *node,
 	ERL_PID_SERIAL(ep)   = serial & 0x1fff;  /* 13 bits */
     }
     ERL_PID_CREATION(ep) = creation & 0x03; /* 2 bits */
-    return ep;
 }
 
 /*
@@ -239,12 +303,18 @@ ETERM *erl_mk_port(const char *node,
 
     ep = erl_alloc_eterm(ERL_PORT);
     ERL_COUNT(ep) = 1;
-    if ((ERL_PORT_NODE(ep) = strsave(node)) == NULL)
+    if (erl_atom_init_latin1(&ep->uval.portval.node, node) == NULL)
     {	     
 	erl_free_term(ep);
 	erl_errno = ENOMEM;
 	return NULL;
     }
+    erl_mk_port_helper(ep, number, creation);
+    return ep;
+}
+
+void erl_mk_port_helper(ETERM* ep, unsigned number, unsigned char creation)
+{
     if (ei_internal_use_r9_pids_ports()) {
 	ERL_PORT_NUMBER(ep)   = number & 0x3ffff; /* 18 bits */
     }
@@ -252,29 +322,29 @@ ETERM *erl_mk_port(const char *node,
 	ERL_PORT_NUMBER(ep)   = number & 0x0fffffff; /* 18 bits */
     }
     ERL_PORT_CREATION(ep) = creation & 0x03; /* 2 bits */
-    return ep;
 }
 
 /*
  * Create any kind of reference.
  */
-ETERM *__erl_mk_reference (const char *node,
+ETERM *__erl_mk_reference (ETERM* t,
+			   const char *node,
 			   size_t len,
 			   unsigned int n[],
 			   unsigned char creation)
 {
-    ETERM * t;
-
-    if (node == NULL) return NULL;
-
-    t = erl_alloc_eterm(ERL_REF);
-    ERL_COUNT(t) = 1;
-
-    if ((ERL_REF_NODE(t) = strsave(node)) == NULL)
-    {	     
-	erl_free_term(t);
-	erl_errno = ENOMEM;
-	return NULL;
+    if (t == NULL) {
+	if (node == NULL) return NULL;
+    
+	t = erl_alloc_eterm(ERL_REF);
+	ERL_COUNT(t) = 1;
+    
+	if (erl_atom_init_latin1(&t->uval.refval.node, node) == NULL)
+	{	     
+	    erl_free_term(t);
+	    erl_errno = ENOMEM;
+	    return NULL;
+	}
     }
     ERL_REF_LEN(t) = len;
     ERL_REF_NUMBERS(t)[0]   = n[0] & 0x3ffff; /* 18 bits */
@@ -294,7 +364,7 @@ ETERM *erl_mk_ref (const char *node,
 {
     unsigned int n[3] = {0, 0, 0};
     n[0] = number;
-    return __erl_mk_reference(node, 1, n, creation);
+    return __erl_mk_reference(NULL, node, 1, n, creation);
 }
 
 /*
@@ -307,7 +377,7 @@ erl_mk_long_ref (const char *node,
 {
     unsigned int n[3] = {0, 0, 0};
     n[0] = n3; n[1] = n2; n[2] = n1;
-    return __erl_mk_reference(node, 3, n, creation);
+    return __erl_mk_reference(NULL, node, 3, n, creation);
 }
 
 /*
@@ -758,6 +828,28 @@ int erl_iolist_length (const ETERM* term)
 	return -1;
 }
 
+static int erl_atom_copy(Erl_Atom_data* dst, const Erl_Atom_data* src)
+{
+    if (src->latin1 == src->utf8) {
+	dst->latin1 = dst->utf8 = strsave(src->latin1);
+	dst->lenL = dst->lenU = strlen(src->latin1);
+    }
+    else if (src->latin1) {
+	dst->latin1 = strsave(src->latin1);
+	dst->lenL = strlen(src->latin1);
+	dst->utf8 = NULL;
+	dst->lenU = 0;
+    }
+    else {
+	dst->utf8 = strsave(src->utf8);
+	dst->lenU = strlen(src->utf8);
+	dst->latin1 = NULL;
+	dst->lenL = 0;
+    }
+    return (dst->latin1 != NULL || dst->utf8 == NULL);
+}
+
+
 /*
  * Return a brand NEW COPY of an ETERM.
  */
@@ -796,9 +888,7 @@ ETERM *erl_copy_term(const ETERM *ep)
 	ERL_FLOAT_VALUE(cp) = ERL_FLOAT_VALUE(ep);
 	break;
     case ERL_ATOM:
-	ERL_ATOM_SIZE(cp) = ERL_ATOM_SIZE(ep);
-	ERL_ATOM_PTR(cp) = strsave(ERL_ATOM_PTR(ep));
-	if (ERL_ATOM_PTR(cp) == NULL)
+	if (!erl_atom_copy(&cp->uval.aval.d, &ep->uval.aval.d))
 	{
 	    erl_free_term(cp);
 	    erl_errno = ENOMEM;
@@ -810,17 +900,17 @@ ETERM *erl_copy_term(const ETERM *ep)
            name and plug in. Somewhat ugly (also done with port and
            ref below). */
 	memcpy(&cp->uval.pidval, &ep->uval.pidval, sizeof(Erl_Pid));
-	ERL_PID_NODE(cp) = strsave(ERL_PID_NODE(ep));
+	erl_atom_copy(&cp->uval.pidval.node, &ep->uval.pidval.node);
 	ERL_COUNT(cp) = 1;
 	break;
     case ERL_PORT:
 	memcpy(&cp->uval.portval, &ep->uval.portval, sizeof(Erl_Port));
-	ERL_PORT_NODE(cp) = strsave(ERL_PORT_NODE(ep));
+	erl_atom_copy(&cp->uval.portval.node, &ep->uval.portval.node);
 	ERL_COUNT(cp) = 1;
 	break;
     case ERL_REF:
 	memcpy(&cp->uval.refval, &ep->uval.refval, sizeof(Erl_Ref));
-	ERL_REF_NODE(cp) = strsave(ERL_REF_NODE(ep));
+	erl_atom_copy(&cp->uval.refval.node, &ep->uval.refval.node);
 	ERL_COUNT(cp) = 1;
 	break;
     case ERL_LIST:
@@ -883,29 +973,29 @@ int erl_print_term(FILE *fp, const ETERM *ep)
     j = i = doquote = 0;
     switch(ERL_TYPE(ep)) 
     {
-    case ERL_ATOM:
+    case ERL_ATOM: {
+	char* adata = ERL_ATOM_PTR(ep);
 	/* FIXME: what if some weird locale is in use? */
-	if (!islower((int)ERL_ATOM_PTR(ep)[0]))
+	if (!islower(adata[0]))
 	    doquote = 1;
 
 	for (i = 0; !doquote && i < ERL_ATOM_SIZE(ep); i++) 
 	{
-	    doquote = !(isalnum((int)ERL_ATOM_PTR(ep)[i]) 
-			|| (ERL_ATOM_PTR(ep)[i] == '_'));
+	    doquote = !(isalnum(adata[i]) || (adata[i] == '_'));
 	}
 
 	if (doquote) {
 	    putc('\'', fp);
 	    ch_written++; 
 	}
-	fputs(ERL_ATOM_PTR(ep), fp);
+	fputs(adata, fp);
 	ch_written += ERL_ATOM_SIZE(ep);	
 	if (doquote) {
 	    putc('\'', fp);
 	    ch_written++;
 	}
 	break;
-
+    }
     case ERL_VARIABLE:
 	if (!isupper((int)ERL_VAR_NAME(ep)[0])) {
 	    doquote = 1;
diff --git a/lib/erl_interface/src/legacy/erl_eterm.h b/lib/erl_interface/src/legacy/erl_eterm.h
index 41b008f04f..2e8129d9cd 100644
--- a/lib/erl_interface/src/legacy/erl_eterm.h
+++ b/lib/erl_interface/src/legacy/erl_eterm.h
@@ -55,7 +55,9 @@ typedef struct _heapmark {
 } Erl_HeapMark;
 
 
-ETERM * __erl_mk_reference(const char *, size_t, unsigned int n[], unsigned char);
+void erl_mk_port_helper(ETERM* ep, unsigned number, unsigned char creation);
+void erl_mk_pid_helper(ETERM*, unsigned,unsigned, unsigned char);
+ETERM * __erl_mk_reference(ETERM*, const char *, size_t, unsigned int n[], unsigned char);
 int erl_current_fix_desc(void);
 
 #endif /* _ERL_ETERM_H */
diff --git a/lib/erl_interface/src/legacy/erl_format.c b/lib/erl_interface/src/legacy/erl_format.c
index dc85806c36..533241e396 100644
--- a/lib/erl_interface/src/legacy/erl_format.c
+++ b/lib/erl_interface/src/legacy/erl_format.c
@@ -574,10 +574,22 @@ static int ematch(ETERM *p, ETERM *t)
 
   switch (type_p) {
 
-  case ERL_ATOM:
-    return p->uval.aval.len == t->uval.aval.len &&
-      memcmp(p->uval.aval.a, t->uval.aval.a, p->uval.aval.len) == 0;
-	
+  case ERL_ATOM: {
+      Erl_Atom_data* pa = &p->uval.aval.d;
+      Erl_Atom_data* ta = &t->uval.aval.d;
+      if (pa->utf8 && ta->utf8) {
+	  return pa->lenU == ta->lenU && memcmp(pa->utf8, ta->utf8, pa->lenU)==0; 
+      }
+      else if (pa->latin1 && ta->latin1) {
+	  return pa->lenL == ta->lenL && memcmp(pa->latin1, ta->latin1, pa->lenL)==0; 
+      }
+      else if (pa->latin1) {
+	  return cmp_latin1_vs_utf8(pa->latin1, pa->lenL, ta->utf8, ta->lenU)==0;
+      }
+      else {
+	  return cmp_latin1_vs_utf8(ta->latin1, ta->lenL, pa->utf8, pa->lenU)==0;
+      }
+  }
   case ERL_VARIABLE:
     if (strcmp(p->uval.vval.name, "_") == 0) /* anon. variable */
       return ERL_TRUE;
diff --git a/lib/erl_interface/src/legacy/erl_malloc.c b/lib/erl_interface/src/legacy/erl_malloc.c
index f51a6c69b3..d09239e02d 100644
--- a/lib/erl_interface/src/legacy/erl_malloc.c
+++ b/lib/erl_interface/src/legacy/erl_malloc.c
@@ -112,6 +112,18 @@ do {						\
     (ptr) = NULL;				\
 } while (0)
 
+static void erl_atom_free(Erl_Atom_data* p)
+{
+    erl_free(p->latin1);
+    if (p->utf8 != p->latin1) {
+	erl_free(p->utf8);
+    }
+    p->latin1 = NULL;
+    p->utf8 = NULL;
+    p->lenL = 0;
+    p->lenU = 0;
+}
+
 static void _erl_free_term (ETERM *ep, int external, int compound)
 {
 restart:
@@ -122,7 +134,7 @@ restart:
 	switch(ERL_TYPE(ep)) 
 	    {
 	    case ERL_ATOM:
-		FREE_AND_CLEAR(ERL_ATOM_PTR(ep));
+		erl_atom_free(&ep->uval.aval.d);
 		break;
 	    case ERL_VARIABLE:
 		FREE_AND_CLEAR(ERL_VAR_NAME(ep));
@@ -161,13 +173,13 @@ restart:
 		FREE_AND_CLEAR(ERL_BIN_PTR(ep));
 		break;
 	    case ERL_PID:
-		FREE_AND_CLEAR(ERL_PID_NODE(ep));
+		erl_atom_free(&ep->uval.pidval.node);
 		break;
 	    case ERL_PORT:
-		FREE_AND_CLEAR(ERL_PORT_NODE(ep));
+		erl_atom_free(&ep->uval.portval.node);
 		break;
 	    case ERL_REF:
-		FREE_AND_CLEAR(ERL_REF_NODE(ep));
+		erl_atom_free(&ep->uval.refval.node);
 		break;
 	    case ERL_EMPTY_LIST:
 	    case ERL_INTEGER:
diff --git a/lib/erl_interface/src/legacy/erl_marshal.c b/lib/erl_interface/src/legacy/erl_marshal.c
index 775d7e82ca..884e9d421b 100644
--- a/lib/erl_interface/src/legacy/erl_marshal.c
+++ b/lib/erl_interface/src/legacy/erl_marshal.c
@@ -44,6 +44,9 @@ int erl_fp_compare(unsigned *a, unsigned *b);
 static void erl_long_to_fp(long l, unsigned *d);
 #endif
 
+static int cmpbytes(unsigned char* s1,int l1,unsigned char* s2,int l2);
+static int cmpatoms(unsigned char* s1, int l1, unsigned char tag1, unsigned char* s2, int l2, unsigned char tag2);
+
 /* Used when comparing two encoded byte arrays */
 /* this global data is ok (from threading point of view) since it is
  * initialized once and never changed
@@ -111,8 +114,9 @@ void erl_init_marshal(void)
     cmp_array[ERL_SMALL_BIG_EXT]     = ERL_NUM_CMP;
     cmp_array[ERL_LARGE_BIG_EXT]     = ERL_NUM_CMP;
     cmp_array[ERL_ATOM_EXT]          = ERL_ATOM_CMP;
+    cmp_array[ERL_ATOM_UTF8_EXT]     = ERL_ATOM_CMP;
     cmp_array[ERL_SMALL_ATOM_EXT]    = ERL_ATOM_CMP;
-    cmp_array[ERL_UNICODE_ATOM_EXT]  = ERL_ATOM_CMP;
+    cmp_array[ERL_SMALL_ATOM_UTF8_EXT] = ERL_ATOM_CMP;
     cmp_array[ERL_REFERENCE_EXT]     = ERL_REF_CMP;
     cmp_array[ERL_NEW_REFERENCE_EXT] = ERL_REF_CMP;
     cmp_array[ERL_FUN_EXT]           = ERL_FUN_CMP;
@@ -162,6 +166,21 @@ static int erl_length_x(const ETERM *ep) {
  *==============================================================
  */
 
+static void encode_atom(Erl_Atom_data* a, unsigned char **ext)
+{
+    int ix = 0;
+    if (a->latin1) {
+	ei_encode_atom_len_as((char*)*ext, &ix, a->latin1, a->lenL,
+			      ERLANG_LATIN1, ERLANG_LATIN1);
+    }
+    else if (ei_encode_atom_len_as((char*)*ext, &ix, a->utf8, a->lenU,
+				   ERLANG_UTF8, ERLANG_LATIN1) < 0) {
+	ei_encode_atom_len_as((char*)*ext, &ix, a->utf8, a->lenU,
+			      ERLANG_UTF8, ERLANG_UTF8);
+    }
+    *ext += ix;
+}
+
 /* 
  * The actual ENCODE engine.
  * Returns 0 on success, otherwise 1.
@@ -176,12 +195,7 @@ int erl_encode_it(ETERM *ep, unsigned char **ext, int dist)
     switch(ERL_TYPE(ep)) 
     {
     case ERL_ATOM:
-	i =  ep->uval.aval.len;
-	*(*ext)++ = ERL_ATOM_EXT;
-	*(*ext)++ = (i >>8) &0xff;
-	*(*ext)++ = i &0xff;
-	memcpy((void *) *ext, (const void *) ep->uval.aval.a, i);
-	*ext += i;
+	encode_atom(&ep->uval.aval.d, ext);
 	return 0;
 
     case ERL_INTEGER:
@@ -292,12 +306,7 @@ int erl_encode_it(ETERM *ep, unsigned char **ext, int dist)
     case ERL_PID:
 	*(*ext)++ = ERL_PID_EXT;    
 	/* First poke in node as an atom */    
-	i = strlen((char *)ERL_PID_NODE(ep));
-	*(*ext)++ = ERL_ATOM_EXT;
-	*(*ext)++ = (i >>8) &0xff;
-	*(*ext)++ = i &0xff;
-	memcpy(*ext, ERL_PID_NODE(ep), i);
-	*ext += i;
+	encode_atom(&ep->uval.pidval.node, ext);
 	/* And then fill in the integer fields */
 	i = ERL_PID_NUMBER(ep);
 	*(*ext)++ = (i >> 24) &0xff;
@@ -325,11 +334,8 @@ int erl_encode_it(ETERM *ep, unsigned char **ext, int dist)
 	    *(*ext)++ = (len >> 8) &0xff;
 	    *(*ext)++ = len &0xff;
 
-	    *(*ext)++ = ERL_ATOM_EXT;
-	    *(*ext)++ = (i >> 8) &0xff;
-	    *(*ext)++ = i &0xff;
-	    memcpy(*ext, ERL_REF_NODE(ep), i);
-	    *ext += i;
+	    encode_atom(&ep->uval.refval.node, ext);
+
 	    *(*ext)++ = ERL_REF_CREATION(ep);
 	    /* Then the integer fields */
 	    for (j = 0; j < ERL_REF_LEN(ep); j++) {
@@ -344,12 +350,7 @@ int erl_encode_it(ETERM *ep, unsigned char **ext, int dist)
     case ERL_PORT:
 	*(*ext)++ = ERL_PORT_EXT;
 	/* First poke in node as an atom */
-	i = strlen((char *)ERL_PORT_NODE(ep));
-	*(*ext)++ = ERL_ATOM_EXT;
-	*(*ext)++ = (i >>8) &0xff;
-	*(*ext)++ = i &0xff;
-	memcpy(*ext, ERL_PORT_NODE(ep), i);
-	*ext += i;
+	encode_atom(&ep->uval.portval.node, ext);
 	/* Then the integer fields */
 	i = ERL_PORT_NUMBER(ep);
 	*(*ext)++ = (i >> 24) &0xff;
@@ -500,6 +501,16 @@ int erl_term_len(ETERM *ep)
   return 1+erl_term_len_helper(ep, 4);
 }
 
+static int atom_len_helper(Erl_Atom_data* a)
+{
+    if (erl_atom_ptr_latin1(a)) {
+	return 1 + 2 + a->lenL; /* ERL_ATOM_EXT */
+    }
+    else {
+	return 1 + 1 + (a->lenU > 255) + a->lenU;
+    }
+}
+
 static int erl_term_len_helper(ETERM *ep, int dist)
 {
   int len = 0;
@@ -511,8 +522,7 @@ static int erl_term_len_helper(ETERM *ep, int dist)
   if (ep) {
     switch (ERL_TYPE(ep)) {
     case ERL_ATOM:
-      i = ep->uval.aval.len;
-      len = i + 3;
+      len = atom_len_helper(&ep->uval.aval.d);
       break;
 
     case ERL_INTEGER:
@@ -544,20 +554,15 @@ static int erl_term_len_helper(ETERM *ep, int dist)
       break;
 
     case ERL_PID:
-      /* 1 + N + 4 + 4 + 1 where N = 3 + strlen */
-      i = strlen((char *)ERL_PID_NODE(ep));
-      len = 13 + i;
+      len = 1 + atom_len_helper(&ep->uval.pidval.node) + 4 + 4 + 1;
       break;
 
     case ERL_REF:
-      i = strlen((char *)ERL_REF_NODE(ep));
-      len = 1 + 2 + (i+3) + 1 + ERL_REF_LEN(ep) * 4;
+      len = 1 + 2 + atom_len_helper(&ep->uval.refval.node) + 1 + ERL_REF_LEN(ep) * 4;
       break;
 
     case ERL_PORT:
-      /* 1 + N + 4 + 1 where N = 3 + strlen */
-      i = strlen((char *)ERL_PORT_NODE(ep));
-      len = 9 + i;
+      len = 1 + atom_len_helper(&ep->uval.portval.node) + 4 + 1;
       break;
 
     case ERL_EMPTY_LIST:
@@ -651,11 +656,33 @@ int erl_encode_buf(ETERM *ep, unsigned char **ext)
 } /* erl_encode_buf */
 
 
-static int read_atom(unsigned char** ext, char* dst)
+static int read_atom(unsigned char** ext, Erl_Atom_data* a)
 {
+    char buf[MAXATOMLEN_UTF8];
     int offs = 0;
-    int ret = ei_decode_atom((char*)*ext, &offs, dst);
+    enum erlang_char_encoding enc;
+    int ret = ei_decode_atom_as((char*)*ext, &offs, buf, MAXATOMLEN_UTF8,
+				ERLANG_WHATEVER, NULL, &enc);
     *ext += offs;
+
+    if (ret == 0) {
+	int i = strlen(buf);
+	char* clone = erl_malloc(i+1);
+	memcpy(clone, buf, i+1);
+
+	a->latin1 = NULL; 	    
+	a->lenL = 0;
+	a->utf8 = NULL;
+	a->lenU = 0;
+	if (enc == ERLANG_LATIN1 || enc == ERLANG_ASCII) {
+	    a->latin1 = clone; 	    
+	    a->lenL = i;
+	}
+	if (enc == ERLANG_UTF8 || enc == ERLANG_ASCII) {
+	    a->utf8 = clone;
+	    a->lenU = i;
+	}
+    }
     return ret;
 }
 
@@ -665,7 +692,6 @@ static int read_atom(unsigned char** ext, char* dst)
  */
 static ETERM *erl_decode_it(unsigned char **ext)
 {
-    char atom_buf[MAXATOMLEN+1];
     char *cp;
     ETERM *ep,*tp,*np;
     unsigned int u,sign;
@@ -765,127 +791,89 @@ static ETERM *erl_decode_it(unsigned char **ext)
       
     case ERL_ATOM_EXT:
     case ERL_SMALL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
+    case ERL_ATOM_UTF8_EXT:
+    case ERL_SMALL_ATOM_UTF8_EXT:
+	
 	ERL_TYPE(ep) = ERL_ATOM;
 	--(*ext);
-	if (read_atom(ext, atom_buf) < 0) return NULL;
-
-	i = strlen(atom_buf);
-	ep->uval.aval.len = i;
-	ep->uval.aval.a = (char *) erl_malloc(i+1);
-	memcpy(ep->uval.aval.a, atom_buf, i+1);
+	if (read_atom(ext, &ep->uval.aval.d) < 0) return NULL;
 	return ep;
       
     case ERL_PID_EXT:
-	erl_free_term(ep);
-	{			/* Why not use the constructors? */
-	    char* node = atom_buf;
+	{
 	    unsigned int number, serial;
 	    unsigned char creation;
-	    ETERM *eterm_p;
 
-	    if (read_atom(ext, node) < 0) return NULL;
+	    ERL_TYPE(ep) = ERL_PID;
+	    if (read_atom(ext, &ep->uval.pidval.node) < 0) return NULL;
 
 	    /* get the integers */
-#if 0
-	    /* FIXME: Remove code or whatever....
-               Ints on the wire are big-endian (== network byte order)
-               so use ntoh[sl]. (But some are little-endian! Arrrgh!)
-               Also, the libc authors can be expected to optimize them
-               heavily. However, the marshalling makes no guarantees
-               about alignments -- so it won't work at all. */
-	    number = ntohl(*((unsigned int *)*ext)++);
-	    serial = ntohl(*((unsigned int *)*ext)++);
-#else
 	    number = ((*ext)[0] << 24) | ((*ext)[1]) << 16 | 
 		((*ext)[2]) << 8 | ((*ext)[3]);	
 	    *ext += 4;
 	    serial = ((*ext)[0] << 24) | ((*ext)[1]) << 16 | 
 		((*ext)[2]) << 8 | ((*ext)[3]);	
 	    *ext += 4;
-#endif
 	    creation =  *(*ext)++; 
-	    eterm_p = erl_mk_pid(node, number, serial, creation);
-	    return eterm_p;
+	    erl_mk_pid_helper(ep, number, serial, creation);
+	    return ep;
 	}
     case ERL_REFERENCE_EXT:
-	erl_free_term(ep);
 	{
-	    char* node = atom_buf;
-	    unsigned int number;
+	    unsigned int n[3] = {0, 0, 0};
 	    unsigned char creation;
-	    ETERM *eterm_p;
 
-	    if (read_atom(ext, node) < 0) return NULL;
+	    ERL_TYPE(ep) = ERL_REF;
+	    if (read_atom(ext, &ep->uval.refval.node) < 0) return NULL;
 
 	    /* get the integers */
-#if 0
-	    number = ntohl(*((unsigned int *)*ext)++);
-#else
-	    number = ((*ext)[0] << 24) | ((*ext)[1]) << 16 | 
+	    n[0] = ((*ext)[0] << 24) | ((*ext)[1]) << 16 | 
 		((*ext)[2]) << 8 | ((*ext)[3]);	
 	    *ext += 4;
-#endif
 	    creation =  *(*ext)++; 
-	    eterm_p = erl_mk_ref(node, number, creation);
-	    return eterm_p;
+	    __erl_mk_reference(ep, NULL, 1, n, creation);
+	    return ep;
 	}
 
     case ERL_NEW_REFERENCE_EXT: 
-	erl_free_term(ep);
 	{
-	    char* node = atom_buf;
 	    size_t cnt, i;
 	    unsigned int n[3];
 	    unsigned char creation;
-	    ETERM *eterm_p;
 
-#if 0
-	    cnt = ntohs(*((unsigned short *)*ext)++);
-#else
+	    ERL_TYPE(ep) = ERL_REF;
 	    cnt = ((*ext)[0] << 8) | (*ext)[1];
 	    *ext += 2;
-#endif
 
-	    if (read_atom(ext, node) < 0) return NULL;
+	    if (read_atom(ext, &ep->uval.refval.node) < 0) return NULL;
 
 	    /* get the integers */
 	    creation =  *(*ext)++; 
 	    for(i = 0; i < cnt; i++)
 	    {
-#if 0
-		n[i] = ntohl(*((unsigned int *)*ext)++);
-#else
 		n[i] = ((*ext)[0] << 24) | ((*ext)[1]) << 16 | 
 		    ((*ext)[2]) << 8 | ((*ext)[3]);	
 		*ext += 4;
-#endif
 	    }
-	    eterm_p = __erl_mk_reference(node, cnt, n, creation);
-	    return eterm_p;
+	    __erl_mk_reference(ep, NULL, cnt, n, creation);
+	    return ep;
 	}
 
     case ERL_PORT_EXT:
-	erl_free_term(ep);
 	{
-	    char* node = atom_buf;
 	    unsigned int number;
 	    unsigned char creation;
-	    ETERM *eterm_p;
 
-	    if (read_atom(ext, node) < 0) return NULL;
+	    ERL_TYPE(ep) = ERL_PORT;
+	    if (read_atom(ext, &ep->uval.portval.node) < 0) return NULL;
 
 	    /* get the integers */
-#if 0
-	    number = ntohl(*((unsigned int *)*ext)++);
-#else
 	    number = ((*ext)[0] << 24) | ((*ext)[1]) << 16 | 
 		((*ext)[2]) << 8 | ((*ext)[3]);	
 	    *ext += 4;
-#endif
 	    creation =  *(*ext)++; 
-	    eterm_p = erl_mk_port(node, number, creation);
-	    return eterm_p;
+	    erl_mk_port_helper(ep, number, creation);
+	    return ep;
 	}
 
     case ERL_NIL_EXT:
@@ -1120,8 +1108,9 @@ unsigned char erl_ext_type(unsigned char *ext)
     case ERL_INTEGER_EXT:
 	return ERL_INTEGER;
     case ERL_ATOM_EXT:
+    case ERL_ATOM_UTF8_EXT:
     case ERL_SMALL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
+    case ERL_SMALL_ATOM_UTF8_EXT:
 	return ERL_ATOM;
     case ERL_PID_EXT:
 	return ERL_PID;
@@ -1173,8 +1162,9 @@ int erl_ext_size(unsigned char *t)
     case ERL_SMALL_INTEGER_EXT:
     case ERL_INTEGER_EXT:
     case ERL_ATOM_EXT:
+    case ERL_ATOM_UTF8_EXT:
     case ERL_SMALL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
+    case ERL_SMALL_ATOM_UTF8_EXT:
     case ERL_PID_EXT:
     case ERL_PORT_EXT:
     case ERL_REFERENCE_EXT:
@@ -1221,12 +1211,13 @@ static int jump_atom(unsigned char** ext)
 
     switch (*e++) {
     case ERL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
+    case ERL_ATOM_UTF8_EXT:
 	len = (e[0] << 8) | e[1];
 	e += (len + 2);
 	break;
 
     case ERL_SMALL_ATOM_EXT:
+    case ERL_SMALL_ATOM_UTF8_EXT:
 	len = e[0];
 	e += (len + 1);
 	break;
@@ -1259,8 +1250,9 @@ static int jump(unsigned char **ext)
 	*ext += 1;
 	break;
     case ERL_ATOM_EXT:
+    case ERL_ATOM_UTF8_EXT:
     case ERL_SMALL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
+    case ERL_SMALL_ATOM_UTF8_EXT:
 	jump_atom(ext);
 	break;
     case ERL_PID_EXT:
@@ -1426,6 +1418,58 @@ static int cmpbytes(unsigned char* s1,int l1,unsigned char* s2,int l2)
 
 } /* cmpbytes */
 
+#define tag2enc(T) ((T)==ERL_ATOM_EXT || (T)==ERL_SMALL_ATOM_EXT ? ERLANG_LATIN1 : ERLANG_UTF8)
+
+static int cmpatoms(unsigned char* s1, int l1, unsigned char tag1,	 
+                    unsigned char* s2, int l2, unsigned char tag2)
+{
+    enum erlang_char_encoding enc1 = tag2enc(tag1);
+    enum erlang_char_encoding enc2 = tag2enc(tag2);
+
+    if (enc1 == enc2) {
+	return cmpbytes(s1, l1,s2,l2);
+    }
+    
+    if (enc1 == ERLANG_LATIN1) {
+	return cmp_latin1_vs_utf8((char*)s1, l1, (char*)s2, l2);
+    }
+    else {
+	return -cmp_latin1_vs_utf8((char*)s2, l2, (char*)s1, l1);
+    }
+}
+
+int cmp_latin1_vs_utf8(const char* strL, int lenL, const char* strU, int lenU)
+{
+    unsigned char* sL = (unsigned char*)strL;
+    unsigned char* sU = (unsigned char*)strU;
+    unsigned char* sL_end = sL + lenL;
+    unsigned char* sU_end = sU + lenU;
+
+    while(sL < sL_end && sU < sU_end) {
+	unsigned char UasL;
+	if (*sL >= 0x80) {
+	    if (*sU < 0xC4 && (sU+1) < sU_end) {
+		UasL = ((sU[0] & 0x3) << 6) | (sU[1] & 0x3F);
+	    }
+	    else return -1;
+	}
+	else {
+	    UasL = *sU;
+	}
+	if (*sL < UasL) return -1;
+	if (*sL > UasL) return 1;
+
+	sL++;
+	if (*sU < 0x80) sU++;
+	else if (*sU < 0xE0) sU += 2;
+	else if (*sU < 0xF0) sU += 3;
+	else /*if (*sU < 0xF8)*/ sU += 4;
+    }
+
+    return (sU >= sU_end) - (sL >= sL_end); /* -1, 0 or 1 */
+}
+
+
 #define CMP_EXT_ERROR_CODE 4711
 
 #define CMP_EXT_INT32_BE(AP, BP)				\
@@ -1561,6 +1605,7 @@ static int cmp_exe2(unsigned char **e1, unsigned char **e2)
   int min,  ret,i,j,k;
   double ff1, ff2;
   unsigned char *tmp1, *tmp2;
+  unsigned char tag1, tag2;
 
   if ( ((*e1)[0] == ERL_STRING_EXT) && ((*e2)[0] == ERL_LIST_EXT) ) {
     return cmp_string_list(e1, e2);
@@ -1568,9 +1613,10 @@ static int cmp_exe2(unsigned char **e1, unsigned char **e2)
     return -cmp_string_list(e2, e1);
   }
 
-  *e2 += 1;
+  tag1 = *(*e1)++;
+  tag2 = *(*e2)++;
   i = j = 0;
-  switch (*(*e1)++) 
+  switch (tag1) 
     {
     case ERL_SMALL_INTEGER_EXT:
       if (**e1 < **e2) ret = -1;
@@ -1590,14 +1636,15 @@ static int cmp_exe2(unsigned char **e1, unsigned char **e2)
       *e1 += 4; *e2 += 4;
       return ret;
     case ERL_ATOM_EXT:
-    case ERL_UNICODE_ATOM_EXT:
+    case ERL_ATOM_UTF8_EXT:
       i = (**e1) << 8; (*e1)++;
       j = (**e2) << 8; (*e2)++;
       /*fall through*/
     case ERL_SMALL_ATOM_EXT:
+    case ERL_SMALL_ATOM_UTF8_EXT:
       i |= (**e1); (*e1)++;
       j |= (**e2); (*e2)++;
-      ret = cmpbytes(*e1, i, *e2, j);
+      ret = cmpatoms(*e1, i, tag1, *e2, j, tag2);
       *e1 += i;
       *e2 += j;
       return ret;
diff --git a/lib/erl_interface/src/legacy/global_whereis.c b/lib/erl_interface/src/legacy/global_whereis.c
index 2afb193504..e6c556d907 100644
--- a/lib/erl_interface/src/legacy/global_whereis.c
+++ b/lib/erl_interface/src/legacy/global_whereis.c
@@ -85,7 +85,16 @@ ETERM *erl_global_whereis(int fd, const char *name, char *node)
   opid = erl_decode((unsigned char*)buf);
 
   /* extract the nodename for the caller */
-  if (node) strcpy(node,epid.node);
+  if (node) {
+      char* node_str = ERL_PID_NODE(opid);
+      if (node_str) {
+	  strcpy(node, node_str);
+      }
+      else {
+	  erl_free_term(opid);
+	  return NULL;
+      }
+  }
 
   return opid;
 }