erl_interface: utf8 atoms continued

2 files changed, 285 insertions, 167 deletions

diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE.erl b/lib/erl_interface/test/ei_decode_encode_SUITE.erl
index 85cb62239b..e8ae7a6f81 100644
--- a/lib/erl_interface/test/ei_decode_encode_SUITE.erl
+++ b/lib/erl_interface/test/ei_decode_encode_SUITE.erl
@@ -118,6 +118,9 @@ test_ei_decode_encode(Config) when is_list(Config) ->
     ?line send_rec(P, OXPort),
     ?line send_rec(P, OXRef),
 
+    %% Unicode atoms
+    [send_rec(P, Atom) || Atom <- unicode_atom_data()],
+
     ?line runner:recv_eot(P),
     ok.
 
@@ -127,7 +130,7 @@ test_ei_decode_encode(Config) when is_list(Config) ->
 % We read two packets for each test, the ei_decode_encode and ei_x_decode_encode  version....
 
 send_rec(P, Term) when is_port(P) ->
-    ?t:format("Testing: ~p~n", [Term]),
+    %%?t:format("Testing: ~p~n", [Term]),
     P ! {self(), {command, term_to_binary(Term)}},
     {_B,Term} = get_buf_and_term(P).
     
@@ -146,7 +149,7 @@ get_buf_and_term(P) ->
 	_ ->
 	    B1 = list_to_binary([131,B]),	% No magic, add
 	    T = binary_to_term(B1),
-	    io:format("~w\n~w\n(got no magic)\n",[B,T]),
+	    %io:format("~w\n~w\n(got no magic)\n",[B,T]),
 	    {B,T}
     end.
     
@@ -160,7 +163,7 @@ get_binary(P) ->
     case runner:get_term(P) of
 	{bytes,L} ->
 	    B = list_to_binary(L),
-	    io:format("~w\n",[L]),
+	    %%io:format("~w\n",[L]),
 % For strange reasons <<131>> show up as <>....
 %	    io:format("~w\n",[B]),
 	    B;
@@ -305,3 +308,60 @@ mk_ref({NodeName, Creation}, Numbers) when is_list(NodeName),
 	    exit({unexpected_binary_to_term_result, Other})
     end.
 
+
+
+unicode_atom_data() ->
+    [uc_atup(lists:seq(16#1f600, 16#1f600+254)),
+     uc_atup(lists:seq(16#1f600, 16#1f600+63)),
+     uc_atup(lists:seq(1, 255)),
+     uc_atup(lists:seq(100, 163)),
+     uc_atup(lists:seq(200, 354)),
+     uc_atup(lists:seq(200, 263)),
+     uc_atup(lists:seq(2000, 2254)),
+     uc_atup(lists:seq(2000, 2063)),
+     uc_atup(lists:seq(65500, 65754)),
+     uc_atup(lists:seq(65500, 65563))
+     | lists:map(fun (N) ->
+			 uc_atup(lists:seq(64000+N, 64254+N))
+		 end,
+		 lists:seq(1, 2000))].
+
+uc_atup(ATxt) ->
+    string_to_atom(ATxt).
+
+string_to_atom(String) ->
+    Utf8List = string_to_utf8_list(String),
+    Len = length(Utf8List),
+    TagLen = case Len < 256 of
+		 true -> [119, Len];
+		 false -> [118, Len bsr 8, Len band 16#ff]
+	     end,
+    binary_to_term(list_to_binary([131, TagLen, Utf8List])).
+
+string_to_utf8_list([]) ->
+    [];
+string_to_utf8_list([CP|CPs]) when is_integer(CP),
+				   0 =< CP,
+				   CP =< 16#7F ->
+    [CP | string_to_utf8_list(CPs)];
+string_to_utf8_list([CP|CPs]) when is_integer(CP),
+				   16#80 =< CP,
+				   CP =< 16#7FF ->
+    [16#C0 bor (CP bsr 6),
+     16#80 bor (16#3F band CP)
+     | string_to_utf8_list(CPs)];
+string_to_utf8_list([CP|CPs]) when is_integer(CP),
+				   16#800 =< CP,
+				   CP =< 16#FFFF ->
+    [16#E0 bor (CP bsr 12),
+     16#80 bor (16#3F band (CP bsr 6)),
+     16#80 bor (16#3F band CP)
+     | string_to_utf8_list(CPs)];
+string_to_utf8_list([CP|CPs]) when is_integer(CP),
+				   16#10000 =< CP,
+				   CP =< 16#10FFFF ->
+    [16#F0 bor (CP bsr 18),
+     16#80 bor (16#3F band (CP bsr 12)),
+     16#80 bor (16#3F band (CP bsr 6)),
+     16#80 bor (16#3F band CP)
+     | string_to_utf8_list(CPs)].
diff --git a/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c b/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c
index 406f02ecfb..194ce9057b 100644
--- a/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c
+++ b/lib/erl_interface/test/ei_decode_encode_SUITE_data/ei_decode_encode_test.c
@@ -29,171 +29,222 @@
  * Author:  [email protected]
  */
 
-#define EI_DECODE_ENCODE(FUNC,TYPE) \
-  { \
-    char *buf; \
-    char buf2[1024]; \
-    TYPE p; \
-    int size1 = 0; \
-    int size2 = 0; \
-    int size3 = 0; \
-    int err; \
-    ei_x_buff arg; \
-\
-    message("ei_decode_" #FUNC ", arg is type " #TYPE); \
-    buf = read_packet(NULL); \
-    err = ei_decode_ ## FUNC(buf+1, &size1, &p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("decode returned non zero but not -1"); \
-      } else { \
-	fail("decode returned non zero"); \
-      } \
-      return; \
-    } \
-    if (size1 < 1) { \
-      fail("size is < 1"); \
-      return; \
-    } \
-\
-    message("ei_encode_" #FUNC " buf is NULL, arg is type " #TYPE); \
-    err = ei_encode_ ## FUNC(NULL, &size2, &p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("size calculation returned non zero but not -1"); \
-	return; \
-      } else { \
-	fail("size calculation returned non zero"); \
-	return; \
-      } \
-    } \
-    if (size1 != size2) { \
-      message("size1 = %d, size2 = %d\n",size1,size2); \
-      fail("decode and encode size differs when buf is NULL"); \
-      return; \
-    } \
-    message("ei_encode_" #FUNC ", arg is type " #TYPE); \
-    err = ei_encode_ ## FUNC(buf2, &size3, &p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("returned non zero but not -1"); \
-      } else { \
-	fail("returned non zero"); \
-      } \
-      return; \
-    } \
-    if (size1 != size3) { \
-      message("size1 = %d, size2 = %d\n",size1,size3); \
-      fail("decode and encode size differs"); \
-      return; \
-    } \
-    send_buffer(buf2, size1); \
-\
-    message("ei_x_encode_" #FUNC ", arg is type " #TYPE); \
-    ei_x_new(&arg); \
-    err = ei_x_encode_ ## FUNC(&arg, &p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("returned non zero but not -1"); \
-      } else { \
-	fail("returned non zero"); \
-      } \
-      ei_x_free(&arg); \
-      return; \
-    } \
-    if (arg.index < 1) { \
-      fail("size is < 1"); \
-      ei_x_free(&arg); \
-      return; \
-    } \
-    send_buffer(arg.buff, arg.index); \
-    ei_x_free(&arg); \
-  }
-
-#define EI_DECODE_ENCODE_BIG(FUNC,TYPE) \
-  { \
-    char *buf; \
-    char buf2[2048]; \
-    TYPE *p; \
-    int size1 = 0; \
-    int size2 = 0; \
-    int size3 = 0; \
-    int err, index = 0, len, type; \
-    ei_x_buff arg; \
-\
-    message("ei_decode_" #FUNC ", arg is type " #TYPE); \
-    buf = read_packet(NULL); \
-    ei_get_type(buf+1, &index, &type, &len); \
-    p = ei_alloc_big(len); \
-    err = ei_decode_ ## FUNC(buf+1, &size1, p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("decode returned non zero but not -1"); \
-      } else { \
-	fail("decode returned non zero"); \
-      } \
-      return; \
-    } \
-    if (size1 < 1) { \
-      fail("size is < 1"); \
-      return; \
-    } \
-\
-    message("ei_encode_" #FUNC " buf is NULL, arg is type " #TYPE); \
-    err = ei_encode_ ## FUNC(NULL, &size2, p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("size calculation returned non zero but not -1"); \
-	return; \
-      } else { \
-	fail("size calculation returned non zero"); \
-	return; \
-      } \
-    } \
-    if (size1 != size2) { \
-      message("size1 = %d, size2 = %d\n",size1,size2); \
-      fail("decode and encode size differs when buf is NULL"); \
-      return; \
-    } \
-    message("ei_encode_" #FUNC ", arg is type " #TYPE); \
-    err = ei_encode_ ## FUNC(buf2, &size3, p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("returned non zero but not -1"); \
-      } else { \
-	fail("returned non zero"); \
-      } \
-      return; \
-    } \
-    if (size1 != size3) { \
-      message("size1 = %d, size2 = %d\n",size1,size3); \
-      fail("decode and encode size differs"); \
-      return; \
-    } \
-    send_buffer(buf2, size1); \
-\
-    message("ei_x_encode_" #FUNC ", arg is type " #TYPE); \
-    ei_x_new(&arg); \
-    err = ei_x_encode_ ## FUNC(&arg, p); \
-    if (err != 0) { \
-      if (err != -1) { \
-	fail("returned non zero but not -1"); \
-      } else { \
-	fail("returned non zero"); \
-      } \
-      ei_x_free(&arg); \
-      return; \
-    } \
-    if (arg.index < 1) { \
-      fail("size is < 1"); \
-      ei_x_free(&arg); \
-      return; \
-    } \
-    send_buffer(arg.buff, arg.index); \
-    ei_x_free(&arg); \
-    ei_free_big(p); \
-  }
+/*#define MESSAGE(FMT,A1,A2) message(FMT,A1,A2)*/
+#define MESSAGE(FMT,A1,A2)
 
+typedef int decodeFT(const char *buf, int *index, void*);
+typedef int encodeFT(char *buf, int *index, void*);
+typedef int x_encodeFT(ei_x_buff*, void*);
+
+struct Type {
+    char* name;
+    char* type;
+    decodeFT* ei_decode_fp;
+    encodeFT* ei_encode_fp;
+    x_encodeFT* ei_x_encode_fp;
+};
+
+typedef struct
+{
+    char name[MAXATOMLEN_UTF8];
+    enum erlang_char_encoding enc;
+}my_atom;
+
+int ei_decode_my_atom(const char *buf, int *index, my_atom* a)
+{
+    return ei_decode_atom_as(buf, index, a->name, sizeof(a->name), ERLANG_UTF8, &a->enc, NULL);
+}
+int ei_encode_my_atom(char *buf, int *index, my_atom* a)
+{
+    return ei_encode_atom_as(buf, index, a->name, ERLANG_UTF8, a->enc);
+}
+int ei_x_encode_my_atom(ei_x_buff* x, my_atom* a)
+{
+    return ei_x_encode_atom_as(x, a->name, ERLANG_UTF8, a->enc);
+}
+
+void decode_encode(struct Type* t, void* obj)
+{
+    char *buf;
+    char buf2[1024];
+    int size1 = 0;
+    int size2 = 0;
+    int size3 = 0;
+    int err;
+    ei_x_buff arg;
+    
+    MESSAGE("ei_decode_%s, arg is type %s", t->name, t->type);
+    buf = read_packet(NULL);
+    err = t->ei_decode_fp(buf+1, &size1, obj);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("decode returned non zero but not -1");
+	} else {
+	    fail("decode returned non zero");
+	}
+	return;
+    }
+    if (size1 < 1) {
+	fail("size is < 1");
+	return;
+    }
+
+    MESSAGE("ei_encode_%s buf is NULL, arg is type %s", t->name, t->type);
+    err = t->ei_encode_fp(NULL, &size2, obj);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("size calculation returned non zero but not -1");
+	    return;
+	} else {
+	    fail("size calculation returned non zero");
+	    return;
+	}
+    }
+    if (size1 != size2) {
+	MESSAGE("size1 = %d, size2 = %d\n",size1,size2);
+	fail("decode and encode size differs when buf is NULL");
+	return;
+    }
+    MESSAGE("ei_encode_%s, arg is type %s", t->name, t->type);
+    err = t->ei_encode_fp(buf2, &size3, obj);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("returned non zero but not -1");
+	} else {
+	    fail("returned non zero");
+	}
+	return;
+    }
+    if (size1 != size3) {
+	MESSAGE("size1 = %d, size2 = %d\n",size1,size3);
+	fail("decode and encode size differs");
+	return;
+    }
+    send_buffer(buf2, size1);
+
+    MESSAGE("ei_x_encode_%s, arg is type %s", t->name, t->type);
+    ei_x_new(&arg);
+    err = t->ei_x_encode_fp(&arg, obj);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("returned non zero but not -1");
+	} else {
+	    fail("returned non zero");
+	}
+	ei_x_free(&arg);
+	return;
+    }
+    if (arg.index < 1) {
+	fail("size is < 1");
+	ei_x_free(&arg);
+	return;
+    }
+    send_buffer(arg.buff, arg.index);
+    ei_x_free(&arg);
+}
+
+
+#define EI_DECODE_ENCODE(TYPE, ERLANG_TYPE) {			\
+	struct Type type_struct = {#TYPE, #ERLANG_TYPE,		\
+				   (decodeFT*)ei_decode_##TYPE,		\
+				   (encodeFT*)ei_encode_##TYPE,		\
+				   (x_encodeFT*)ei_x_encode_##TYPE };	\
+	ERLANG_TYPE type_obj;						\
+	decode_encode(&type_struct, &type_obj);				\
+    }
+
+
+void decode_encode_big(struct Type* t)
+{
+    char *buf;
+    char buf2[2048];
+    void *p; /* (TYPE*) */
+    int size1 = 0;
+    int size2 = 0;
+    int size3 = 0;
+    int err, index = 0, len, type;
+    ei_x_buff arg;
+
+    MESSAGE("ei_decode_%s, arg is type %s", t->name, t->type);
+    buf = read_packet(NULL);
+    ei_get_type(buf+1, &index, &type, &len);
+    p = ei_alloc_big(len);
+    err = t->ei_decode_fp(buf+1, &size1, p);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("decode returned non zero but not -1");
+	} else {
+	    fail("decode returned non zero");
+	}
+	return;
+    }
+    if (size1 < 1) {
+	fail("size is < 1");
+	return;
+    }
+
+    MESSAGE("ei_encode_%s buf is NULL, arg is type %s", t->name, t->type);
+    err = t->ei_encode_fp(NULL, &size2, p);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("size calculation returned non zero but not -1");
+	    return;
+	} else {
+	    fail("size calculation returned non zero");
+	    return;
+	}
+    }
+    if (size1 != size2) {
+	MESSAGE("size1 = %d, size2 = %d\n",size1,size2);
+	fail("decode and encode size differs when buf is NULL");
+	return;
+    }
+    MESSAGE("ei_encode_%s, arg is type %s", t->name, t->type);
+    err = t->ei_encode_fp(buf2, &size3, p);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("returned non zero but not -1");
+	} else {
+	    fail("returned non zero");
+	}
+	return;
+    }
+    if (size1 != size3) {
+	MESSAGE("size1 = %d, size2 = %d\n",size1,size3);
+	fail("decode and encode size differs");
+	return;
+    }
+    send_buffer(buf2, size1);
+
+    MESSAGE("ei_x_encode_%s, arg is type %s", t->name, t->type);
+    ei_x_new(&arg);
+    err = t->ei_x_encode_fp(&arg, p);
+    if (err != 0) {
+	if (err != -1) {
+	    fail("returned non zero but not -1");
+	} else {
+	    fail("returned non zero");
+	}
+	ei_x_free(&arg);
+	return;
+    }
+    if (arg.index < 1) {
+	fail("size is < 1");
+	ei_x_free(&arg);
+	return;
+    }
+    send_buffer(arg.buff, arg.index);
+    ei_x_free(&arg);
+    ei_free_big(p);
+}
+
+#define EI_DECODE_ENCODE_BIG(TYPE, ERLANG_TYPE) {					\
+	struct Type type_struct = {#TYPE, #ERLANG_TYPE,		\
+				   (decodeFT*)ei_decode_##TYPE,	\
+				   (encodeFT*)ei_encode_##TYPE,		\
+				   (x_encodeFT*)ei_x_encode_##TYPE };	\
+	decode_encode_big(&type_struct);				\
+    }
 
 
 
@@ -201,6 +252,8 @@
 
 TESTCASE(test_ei_decode_encode)
 {
+    int i;
+
     EI_DECODE_ENCODE(fun  , erlang_fun);
     EI_DECODE_ENCODE(pid  , erlang_pid);
     EI_DECODE_ENCODE(port , erlang_port);
@@ -223,6 +276,11 @@ TESTCASE(test_ei_decode_encode)
     EI_DECODE_ENCODE(port , erlang_port);
     EI_DECODE_ENCODE(ref  , erlang_ref);
 
+    /* Unicode atoms */
+    for (i=0; i<2010; i++) {
+	EI_DECODE_ENCODE(my_atom , my_atom);
+    }
+
     report(1);
 }