aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
authorPatrik Nyblom <[email protected]>2010-12-03 15:38:14 +0100
committerPatrik Nyblom <[email protected]>2010-12-03 15:38:14 +0100
commit73b53fe24ebc595a81c4ee023bcfef5a71e59db4 (patch)
treeed75b935318dbef275d62ab141a5f59b5b36bce0 /erts/emulator/beam
parent52b0fbbc4daa4e22fcad6fe39dd8f8cec8bb8d1b (diff)
parent159d79b6657e5bb4f3accb6a0d74b74f06da4ba2 (diff)
downloadotp-73b53fe24ebc595a81c4ee023bcfef5a71e59db4.tar.gz
otp-73b53fe24ebc595a81c4ee023bcfef5a71e59db4.tar.bz2
otp-73b53fe24ebc595a81c4ee023bcfef5a71e59db4.zip
Merge branch 'pan/unicode-filenames/OTP-8887' into dev
* pan/unicode-filenames/OTP-8887: (27 commits) Test and correct filelib and filename Add documentation to erlang.xml and slight correction to unicode_usage.xml Add section about Unicode file names to stdlib users guide Correct bug in file_name_SUITE making it fail on Unix instead of Windows7 Add documentation about raw filenames and Unicode file name translation mode Make filelib not crash on re codepoints beyond 255 in re when filename is raw Mend on_load_embedded testcase which did not handle windows links Correct testcase regarding windows versions supporting soft links. Teach filelib to use re in unicode mode when filenames are not raw Treat soft links on Windows correctly in file_name_SUITE Adapt new soft and hard link routines on Windos to Unicode Corrected testcases broken by unicode filenames Update preloaded prim_file Teach prim_file not to accept atoms and not to throw exceptions Adapt inet_drv to Visual Studio 2008 Teach spawn_executable about Unicode Convert filenames read on MacOSX to canonical form Teach file to accept codepoints beyond 255. Add testcases Correct shell utilities to handle unicode and possibly binaries ...
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/bif.tab7
-rw-r--r--erts/emulator/beam/binary.c10
-rw-r--r--erts/emulator/beam/erl_bif_binary.c2
-rw-r--r--erts/emulator/beam/erl_bif_port.c91
-rw-r--r--erts/emulator/beam/erl_binary.h2
-rw-r--r--erts/emulator/beam/erl_init.c23
-rw-r--r--erts/emulator/beam/erl_unicode.c902
-rw-r--r--erts/emulator/beam/erl_unicode_normalize.h1687
-rw-r--r--erts/emulator/beam/global.h13
-rw-r--r--erts/emulator/beam/sys.h16
10 files changed, 2652 insertions, 101 deletions
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 0674aae77f..60b4b1946b 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -795,6 +795,13 @@ bif erlang:nif_error/1
bif erlang:nif_error/2
#
+# Helpers for unicode filenames
+#
+bif prim_file:internal_name2native/1
+bif prim_file:internal_native2name/1
+bif prim_file:internal_normalize_utf8/1
+bif file:native_name_encoding/0
+#
# Obsolete
#
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 8ee8fbcb29..4be869f269 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -217,8 +217,8 @@ erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, ErtsAlcType_t al
return bytes;
}
-static Eterm
-bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs)
+Eterm
+erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs)
{
if (bitoffs == 0) {
while (size) {
@@ -263,7 +263,7 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1)
Eterm* hp = HAlloc(BIF_P, 2 * size);
byte* bytes = binary_bytes(real_bin)+offset;
- BIF_RET(bin_bytes_to_list(NIL, hp, bytes, size, bitoffs));
+ BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs));
}
error:
@@ -295,7 +295,7 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3)
}
i = stop-start+1;
hp = HAlloc(BIF_P, 2*i);
- BIF_RET(bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs));
+ BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs));
error:
BIF_ERROR(BIF_P, BADARG);
@@ -339,7 +339,7 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1)
previous = CONS(hp, make_binary(last), previous);
hp += 2;
}
- BIF_RET(bin_bytes_to_list(previous, hp, bytes, size, bitoffs));
+ BIF_RET(erts_bin_bytes_to_list(previous, hp, bytes, size, bitoffs));
}
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index b6a445c55c..684fa5d12f 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -1477,7 +1477,7 @@ BIF_RETTYPE binary_matches_3(BIF_ALIST_3)
goto badarg;
}
if (hsend == 0) {
- BIF_RET(am_nomatch);
+ BIF_RET(NIL);
}
if (is_tuple(BIF_ARG_2)) {
tp = tuple_val(BIF_ARG_2);
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 378c5e73fd..fbc92b9730 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -610,6 +610,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
int binary_io;
int soft_eof;
Sint linebuf;
+ Eterm edir = NIL;
byte dir[MAXPATHLEN];
/* These are the defaults */
@@ -686,19 +687,10 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
} else if (option == am_arg0) {
char *a0;
- int n;
- if (is_nil(*tp)) {
- n = 0;
- } else if( (n = is_string(*tp)) == 0) {
+
+ if ((a0 = erts_convert_filename_to_native(*tp, ERTS_ALC_T_TMP, 1)) == NULL) {
goto badarg;
}
- a0 = (char *) erts_alloc(ERTS_ALC_T_TMP,
- (n + 1) * sizeof(byte));
- if (intlist_to_buf(*tp, a0, n) != n) {
- erl_exit(1, "%s:%d: Internal error\n",
- __FILE__, __LINE__);
- }
- a0[n] = '\0';
if (opts.argv == NULL) {
opts.argv = erts_alloc(ERTS_ALC_T_TMP,
2 * sizeof(char **));
@@ -711,22 +703,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
opts.argv[0] = a0;
}
} else if (option == am_cd) {
- Eterm iolist;
- DeclareTmpHeap(heap,4,p);
- int r;
-
- UseTmpHeap(4,p);
- heap[0] = *tp;
- heap[1] = make_list(heap+2);
- heap[2] = make_small(0);
- heap[3] = NIL;
- iolist = make_list(heap);
- r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN);
- UnUseTmpHeap(4,p);
- if (r < 0) {
- goto badarg;
- }
- opts.wd = (char *) dir;
+ edir = *tp;
} else {
goto badarg;
}
@@ -838,19 +815,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
goto badarg;
}
name = tp[1];
- if (is_atom(name)) {
- name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP,
- atom_tab(atom_val(name))->len+1);
- sys_memcpy((void *) name_buf,
- (void *) atom_tab(atom_val(name))->name,
- atom_tab(atom_val(name))->len);
- name_buf[atom_tab(atom_val(name))->len] = '\0';
- } else if ((i = is_string(name))) {
- name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1);
- if (intlist_to_buf(name, name_buf, i) != i)
- erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
- name_buf[i] = '\0';
- } else {
+ if ((name_buf = erts_convert_filename_to_native(name,ERTS_ALC_T_TMP,0)) == NULL) {
goto badarg;
}
opts.spawn_type = ERTS_SPAWN_EXECUTABLE;
@@ -892,7 +857,33 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
/* Argument vector only if explicit spawn_executable */
goto badarg;
}
-
+
+ if (edir != NIL) {
+ /* A working directory is expressed differently if spawn_executable, i.e. Unicode is handles
+ for spawn_executable... */
+ if (opts.spawn_type != ERTS_SPAWN_EXECUTABLE) {
+ Eterm iolist;
+ DeclareTmpHeap(heap,4,p);
+ int r;
+
+ UseTmpHeap(4,p);
+ heap[0] = edir;
+ heap[1] = make_list(heap+2);
+ heap[2] = make_small(0);
+ heap[3] = NIL;
+ iolist = make_list(heap);
+ r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN);
+ UnUseTmpHeap(4,p);
+ if (r < 0) {
+ goto badarg;
+ }
+ opts.wd = (char *) dir;
+ } else {
+ if ((opts.wd = erts_convert_filename_to_native(edir,ERTS_ALC_T_TMP,0)) == NULL) {
+ goto badarg;
+ }
+ }
+ }
if (driver != &spawn_driver && opts.exit_status) {
goto badarg;
@@ -941,6 +932,9 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
if (opts.argv) {
free_args(opts.argv);
}
+ if (opts.wd && opts.wd != ((char *)dir)) {
+ erts_free(ERTS_ALC_T_TMP, (void *) opts.wd);
+ }
return port_num;
badarg:
@@ -950,6 +944,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
#undef OPEN_PORT_ERROR
}
+/* Arguments can be given i unicode and as raw binaries, convert filename is used to convert */
static char **convert_args(Eterm l)
{
char **pp;
@@ -966,22 +961,14 @@ static char **convert_args(Eterm l)
pp[i++] = erts_default_arg0;
while (is_list(l)) {
str = CAR(list_val(l));
-
- if (is_nil(str)) {
- n = 0;
- } else if( (n = is_string(str)) == 0) {
- /* Not a string... */
+ if ((b = erts_convert_filename_to_native(str,ERTS_ALC_T_TMP,1)) == NULL) {
int j;
for (j = 1; j < i; ++j)
erts_free(ERTS_ALC_T_TMP, pp[j]);
erts_free(ERTS_ALC_T_TMP, pp);
return NULL;
- }
- b = (char *) erts_alloc(ERTS_ALC_T_TMP, (n + 1) * sizeof(byte));
- pp[i++] = (char *) b;
- if (intlist_to_buf(str, b, n) != n)
- erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
- b[n] = '\0';
+ }
+ pp[i++] = b;
l = CDR(list_val(l));
}
pp[i] = NULL;
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index a569fe2e85..bdf0fe23fc 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -152,6 +152,8 @@ do { \
void erts_init_binary(void);
byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, ErtsAlcType_t, unsigned extra);
+/* Used by unicode module */
+Eterm erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs);
/*
* Common implementation for erlang:list_to_binary/1 and binary:list_to_bin/1
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index a9f4f041ac..464ee750f7 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -251,6 +251,7 @@ erl_init(int ncpu)
erts_init_monitors();
erts_init_gc();
init_time();
+ erts_init_sys_common_misc();
erts_init_process(ncpu);
erts_init_scheduling(use_multi_run_queue,
no_schedulers,
@@ -907,7 +908,27 @@ erl_start(int argc, char **argv)
VERBOSE(DEBUG_SYSTEM,
("using display items %d\n",display_items));
break;
-
+ case 'f':
+ if (!strncmp(argv[i],"-fn",3)) {
+ arg = get_arg(argv[i]+3, argv[i+1], &i);
+ switch (*arg) {
+ case 'u':
+ erts_set_user_requested_filename_encoding(ERL_FILENAME_UTF8);
+ break;
+ case 'l':
+ erts_set_user_requested_filename_encoding(ERL_FILENAME_LATIN1);
+ break;
+ case 'a':
+ erts_set_user_requested_filename_encoding(ERL_FILENAME_UNKNOWN);
+ default:
+ erts_fprintf(stderr, "bad filename encoding %s, can be (l,u or a)\n", arg);
+ erts_usage();
+ }
+ break;
+ } else {
+ erts_fprintf(stderr, "%s unknown flag %s\n", argv[0], argv[i]);
+ erts_usage();
+ }
case 'l':
display_loads++;
break;
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index d01a3661f9..72207df621 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -30,6 +30,8 @@
#include "big.h"
#include "erl_unicode.h"
+#include "erl_unicode_normalize.h"
+
typedef struct _restart_context {
byte *bytes;
@@ -54,13 +56,6 @@ static BIF_RETTYPE finalize_list_to_list(Process *p,
Uint num_resulting_chars,
int state, int left,
Eterm tail);
-static int analyze_utf8(byte *source, Uint size,
- byte **err_pos, Uint *num_chars, int *left);
-#define UTF8_OK 0
-#define UTF8_INCOMPLETE 1
-#define UTF8_ERROR 2
-#define UTF8_ANALYZE_MORE 3
-
static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3);
static BIF_RETTYPE characters_to_list_trap_1(BIF_ALIST_3);
static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3);
@@ -463,7 +458,7 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */
}
objp = list_val(ioterm);
obj = CAR(objp);
- if (!is_byte(obj))
+ if (!is_small(obj))
break;
}
} else if (is_nil(obj)) {
@@ -970,11 +965,11 @@ static int is_valid_utf8(Eterm orig_bin)
bytes = erts_get_aligned_binary_bytes(orig_bin, &temp_alloc);
}
size = binary_size(orig_bin);
- ret = analyze_utf8(bytes,
+ ret = erts_analyze_utf8(bytes,
size,
&endpos,&numchar,NULL);
erts_free_aligned_binary_bytes(temp_alloc);
- return (ret == UTF8_OK);
+ return (ret == ERTS_UTF8_OK);
}
BIF_RETTYPE unicode_characters_to_binary_2(BIF_ALIST_2)
@@ -1084,14 +1079,14 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char
hp += 2;
rest_term = CONS(hp,leftover_bin,rest_term);
}
- BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, UTF8_ERROR, left, NIL));
+ BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, ERTS_UTF8_ERROR, left, NIL));
} else if (rest_term == NIL && num_leftovers != 0) {
Eterm leftover_bin = new_binary(p, leftover, num_leftovers);
if (check_leftovers(leftover,num_leftovers) != 0) {
- BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_ERROR,
+ BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_ERROR,
left, NIL));
} else {
- BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_INCOMPLETE,
+ BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_INCOMPLETE,
left, NIL));
}
} else { /* All OK */
@@ -1107,11 +1102,11 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char
rc.num_processed_bytes = 0; /* not used */
rc.num_bytes_to_process = pos;
rc.num_resulting_chars = characters;
- rc.state = UTF8_OK; /* not used */
+ rc.state = ERTS_UTF8_OK; /* not used */
BIF_TRAP3(&characters_to_list_trap_1_exp, p, make_magic_bin_for_restart(p,&rc),
rest_term, latin1);
} else { /* Success */
- BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, UTF8_OK, left, NIL));
+ BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, ERTS_UTF8_OK, left, NIL));
}
}
}
@@ -1205,7 +1200,7 @@ BIF_RETTYPE unicode_characters_to_list_2(BIF_ALIST_2)
* When input to characters_to_list is a plain binary and the format is 'unicode', we do
* a faster analyze and size count with this function.
*/
-static int analyze_utf8(byte *source, Uint size,
+int erts_analyze_utf8(byte *source, Uint size,
byte **err_pos, Uint *num_chars, int *left)
{
*err_pos = source;
@@ -1216,60 +1211,60 @@ static int analyze_utf8(byte *source, Uint size,
--size;
} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
if (size < 2) {
- return UTF8_INCOMPLETE;
+ return ERTS_UTF8_INCOMPLETE;
}
if (((source[1] & ((byte) 0xC0)) != 0x80) ||
((*source) < 0xC2) /* overlong */) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
source += 2;
size -= 2;
} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
if (size < 3) {
- return UTF8_INCOMPLETE;
+ return ERTS_UTF8_INCOMPLETE;
}
if (((source[1] & ((byte) 0xC0)) != 0x80) ||
((source[2] & ((byte) 0xC0)) != 0x80) ||
(((*source) == 0xE0) && (source[1] < 0xA0)) /* overlong */ ) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
if ((((*source) & ((byte) 0xF)) == 0xD) &&
((source[1] & 0x20) != 0)) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
if (((*source) == 0xEF) && (source[1] == 0xBF) &&
((source[2] == 0xBE) || (source[2] == 0xBF))) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
source += 3;
size -= 3;
} else if (((*source) & ((byte) 0xF8)) == 0xF0) {
if (size < 4) {
- return UTF8_INCOMPLETE;
+ return ERTS_UTF8_INCOMPLETE;
}
if (((source[1] & ((byte) 0xC0)) != 0x80) ||
((source[2] & ((byte) 0xC0)) != 0x80) ||
((source[3] & ((byte) 0xC0)) != 0x80) ||
(((*source) == 0xF0) && (source[1] < 0x90)) /* overlong */) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
if ((((*source) & ((byte)0x7)) > 0x4U) ||
((((*source) & ((byte)0x7)) == 0x4U) &&
((source[1] & ((byte)0x3F)) > 0xFU))) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
source += 4;
size -= 4;
} else {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
++(*num_chars);
*err_pos = source;
if (left && --(*left) <= 0) {
- return UTF8_ANALYZE_MORE;
+ return ERTS_UTF8_ANALYZE_MORE;
}
}
- return UTF8_OK;
+ return ERTS_UTF8_OK;
}
/*
@@ -1304,7 +1299,7 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz,
} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
unipoint =
(((Uint) ((*source) & ((byte) 0x1F))) << 6) |
- ((Uint) (source[1] & ((byte) 0x3F)));
+ ((Uint) (source[1] & ((byte) 0x3F)));
} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
unipoint =
(((Uint) ((*source) & ((byte) 0xF))) << 12) |
@@ -1330,6 +1325,216 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz,
return ret;
}
+static int is_candidate(Uint cp)
+{
+ int index,pos;
+ if (cp < 768) return 0;
+ if (cp > 4023) {
+ if (cp == 12441 || cp == 12442) return 1;
+ return 0;
+ }
+ index = cp / 32 - COMP_CANDIDATE_MAP_OFFSET;
+ pos = cp % 32;
+ return !!(comp_candidate_map[index] & (1UL << pos));
+}
+
+static int hashsearch(int *htab, int htab_size, CompEntry *cv, Uint16 c)
+{
+ int bucket = c % htab_size;
+ while (htab[bucket] != -1 && cv[htab[bucket]].c != c)
+ bucket = (bucket + 1) % htab_size;
+ return htab[bucket];
+}
+
+#define TRANSLATE_NO 0
+#define TRANSLATE_MAYBE -1
+
+/* The s array is reversed */
+static int translate(Uint16 *s, int slen, Uint16 *res)
+{
+ /* Go backwards through buffer and match against tree */
+ int pos = 0;
+ CompEntry *cv = compose_tab;
+ int *hc = hash_compose_tab;
+ int cvs = compose_tab_size;
+ int x;
+ while (pos < slen) {
+ x = hashsearch(hc,cvs*HASH_SIZE_FACTOR,cv,s[pos]);
+ if (x < 0) {
+ return TRANSLATE_NO;
+ }
+ if (cv[x].res) {
+ *res = cv[x].res;
+ return pos;
+ }
+ cvs = cv[x].num_subs;
+ hc = cv[x].hash;
+ cv = cv[x].subs;
+ ++pos;
+ }
+ return TRANSLATE_MAYBE;
+}
+
+static void handle_first_norm(Uint16 *savepoints, int *numpointsp, Uint unipoint)
+{
+ /*erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) *numpointsp);*/
+ *numpointsp = 1;
+ savepoints[0] = (Uint16) unipoint;
+}
+
+static void cleanup_norm(Eterm **hpp, Uint16 *savepoints, int numpoints, Eterm *retp)
+{
+ Eterm *hp = *hpp;
+ int res,i;
+ Uint16 newpoint;
+ Eterm ret = *retp;
+
+ ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+ hp += 2;
+
+ for (i = 1;i < numpoints;) {
+ if(!is_candidate(savepoints[i]) ||
+ ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) {
+ ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+ hp += 2;
+ ++i;
+ } else {
+ ret = CONS(hp,make_small((Uint) newpoint),ret);
+ hp += 2;
+ i += res;
+ }
+ }
+ *retp = ret;
+}
+
+static void handle_potential_norm(Eterm **hpp, Uint16 *savepoints, int *numpointsp, Uint unipoint, Eterm *retp)
+{
+ Eterm *hp = *hpp;
+ int numpoints = *numpointsp;
+ int res,i;
+ Uint16 newpoint;
+ Eterm ret = *retp;
+
+ /* erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) numpoints);*/
+ if ((unipoint >> 16) == 0) { /* otherwise we're done here */
+ savepoints[numpoints++] = (Uint16) unipoint;
+ res = translate(savepoints,numpoints,&newpoint);
+ if (res == TRANSLATE_NO) {
+ ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+ hp += 2;
+ for (i = 1;i < numpoints;) {
+ if(!is_candidate(savepoints[i]) ||
+ ((res = translate(savepoints+i,numpoints - i, &newpoint)) == 0)) {
+ ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+ hp += 2;
+ ++i;
+ } else if (res > 0) {
+ ret = CONS(hp,make_small((Uint) newpoint),ret);
+ hp += 2;
+ i += res;
+ } else { /* res < 0 */
+ /* A "maybe", means we are not done yet */
+ int j = 0;
+ while (i < numpoints) {
+ savepoints[j++] = savepoints[i++];
+ }
+ numpoints = j;
+ goto breakaway;
+ }
+ }
+ numpoints = 0;
+ breakaway:
+ ;
+ } else if (res > 0) {
+ numpoints = 0;
+ ret = CONS(hp,make_small((Uint) newpoint),ret);
+ hp += 2;
+ } /* < 0 means go on */
+ } else {
+ /* Unconditional rollup, this character is larger than 16 bit */
+ ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+ hp += 2;
+
+ for (i = 1;i < numpoints;) {
+ if(!is_candidate(savepoints[i]) ||
+ ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) {
+ ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+ hp += 2;
+ ++i;
+ } else {
+ ret = CONS(hp,make_small((Uint) newpoint),ret);
+ hp += 2;
+ i += res;
+ }
+ }
+ ret = CONS(hp,make_small(unipoint),ret);
+ hp += 2;
+ numpoints = 0;
+ }
+ *hpp = hp;
+ *numpointsp = numpoints;
+ *retp = ret;
+}
+
+static Eterm do_utf8_to_list_normalize(Process *p, Uint num, byte *bytes, Uint sz)
+{
+ Eterm *hp,*hp_end;
+ Eterm ret;
+ byte *source;
+ Uint unipoint;
+ Uint16 savepoints[4];
+ int numpoints = 0;
+
+ ASSERT(num > 0);
+
+ hp = HAlloc(p,num * 2); /* May be to much */
+ hp_end = hp + num * 2;
+ ret = NIL;
+ source = bytes + sz;
+ while(--source >= bytes) {
+ if (((*source) & ((byte) 0x80)) == 0) {
+ unipoint = (Uint) *source;
+ } else if (((*source) & ((byte) 0xE0)) == 0xC0) {
+ unipoint =
+ (((Uint) ((*source) & ((byte) 0x1F))) << 6) |
+ ((Uint) (source[1] & ((byte) 0x3F)));
+ } else if (((*source) & ((byte) 0xF0)) == 0xE0) {
+ unipoint =
+ (((Uint) ((*source) & ((byte) 0xF))) << 12) |
+ (((Uint) (source[1] & ((byte) 0x3F))) << 6) |
+ ((Uint) (source[2] & ((byte) 0x3F)));
+ } else if (((*source) & ((byte) 0xF8)) == 0xF0) {
+ unipoint =
+ (((Uint) ((*source) & ((byte) 0x7))) << 18) |
+ (((Uint) (source[1] & ((byte) 0x3F))) << 12) |
+ (((Uint) (source[2] & ((byte) 0x3F))) << 6) |
+ ((Uint) (source[3] & ((byte) 0x3F)));
+ } else {
+ /* ignore 2#10XXXXXX */
+ continue;
+ }
+ if (numpoints) {
+ handle_potential_norm(&hp,savepoints,&numpoints,unipoint,&ret);
+ continue;
+ }
+ /* We are not building up any normalizations yet, look that we shouldn't start... */
+ if (is_candidate(unipoint)) {
+ handle_first_norm(savepoints,&numpoints,unipoint);
+ continue;
+ }
+ ret = CONS(hp,make_small(unipoint),ret);
+ hp += 2;
+ }
+ /* so, we'we looped to the beginning, do we have anything saved? */
+ if (numpoints) {
+ cleanup_norm(&hp,savepoints,numpoints,&ret);
+ }
+ if (hp_end != hp) {
+ HRelease(p,hp_end,hp);
+ }
+ return ret;
+}
+
/*
* The last step of characters_to_list, build a list from the buffer 'bytes' (created in the same way
* as for characters_to_utf8). All sizes are known in advance and most data will be held in a
@@ -1378,10 +1583,10 @@ static BIF_RETTYPE finalize_list_to_list(Process *p,
*/
free_restart(bytes);
- if (state == UTF8_INCOMPLETE) {
+ if (state == ERTS_UTF8_INCOMPLETE) {
hp = HAlloc(p,4);
ret = TUPLE3(hp,am_incomplete,converted,rest);
- } else if (state == UTF8_ERROR) {
+ } else if (state == ERTS_UTF8_ERROR) {
hp = HAlloc(p,4);
ret = TUPLE3(hp,am_error,converted,rest);
} else {
@@ -1408,7 +1613,7 @@ static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3)
/*
* Hooks into the process of decoding a binary depending on state.
- * If last_state is UTF8_ANALYZE_MORE, num_bytes_to_process
+ * If last_state is ERTS_UTF8_ANALYZE_MORE, num_bytes_to_process
* and num_resulting_chars will grow
* until we're done analyzing the binary. Then we'll eat
* the bytes to process, lowering num_bytes_to_process and num_resulting_chars,
@@ -1465,14 +1670,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
left = allowed_iterations(p);
- if (state == UTF8_ANALYZE_MORE) {
- state = analyze_utf8(bytes + num_bytes_to_process,
+ if (state == ERTS_UTF8_ANALYZE_MORE) {
+ state = erts_analyze_utf8(bytes + num_bytes_to_process,
size - num_bytes_to_process,
&endpos,&numchar,&left);
cost_to_proc(p,numchar);
num_resulting_chars += numchar;
num_bytes_to_process = endpos - bytes;
- if (state == UTF8_ANALYZE_MORE) {
+ if (state == ERTS_UTF8_ANALYZE_MORE) {
Eterm epos = erts_make_integer(num_bytes_to_process,p);
Eterm enumchar = erts_make_integer(num_resulting_chars,p);
erts_free_aligned_binary_bytes(temp_alloc);
@@ -1528,7 +1733,7 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
ErlSubBin *sb;
Eterm orig;
Uint offset;
- ASSERT(state != UTF8_OK);
+ ASSERT(state != ERTS_UTF8_OK);
hp = HAlloc(p, ERL_SUB_BIN_SIZE);
sb = (ErlSubBin *) hp;
ERTS_GET_REAL_BIN(orig_bin, orig, offset, bitoffs, bitsize);
@@ -1544,14 +1749,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
/* Done */
- if (state == UTF8_INCOMPLETE) {
+ if (state == ERTS_UTF8_INCOMPLETE) {
if (check_leftovers(bytes + num_bytes_to_process + num_processed_bytes,
b_sz) != 0) {
goto error_return;
}
hp = HAlloc(p,4);
ret = TUPLE3(hp,am_incomplete,converted,rest);
- } else if (state == UTF8_ERROR) {
+ } else if (state == ERTS_UTF8_ERROR) {
error_return:
hp = HAlloc(p,4);
ret = TUPLE3(hp,am_error,converted,rest);
@@ -1589,7 +1794,7 @@ static BIF_RETTYPE characters_to_list_trap_3(BIF_ALIST_3)
0U, /* nothing processed yet */
num_bytes_to_process,
num_resulting_chars,
- UTF8_ANALYZE_MORE, /* always this state here */
+ ERTS_UTF8_ANALYZE_MORE, /* always this state here */
NIL); /* Nothing built -> no tail yet */
}
@@ -1642,7 +1847,7 @@ static BIF_RETTYPE utf8_to_list(BIF_ALIST_1)
BIF_ERROR(BIF_P,BADARG);
}
return do_bif_utf8_to_list(BIF_P, BIF_ARG_1, 0U, 0U, 0U,
- UTF8_ANALYZE_MORE,NIL);
+ ERTS_UTF8_ANALYZE_MORE,NIL);
}
@@ -1728,8 +1933,8 @@ binary_to_atom(Process* p, Eterm bin, Eterm enc, int must_exist)
Uint n;
int reds_left = bin_size+1; /* Number of reductions left. */
- if (analyze_utf8(bytes, bin_size, &err_pos,
- &n, &reds_left) == UTF8_OK) {
+ if (erts_analyze_utf8(bytes, bin_size, &err_pos,
+ &n, &reds_left) == ERTS_UTF8_OK) {
/*
* Correct UTF-8 encoding, but too many characters to
* fit in an atom.
@@ -1813,3 +2018,616 @@ BIF_RETTYPE binary_to_existing_atom_2(BIF_ALIST_2)
{
return binary_to_atom(BIF_P, BIF_ARG_1, BIF_ARG_2, 1);
}
+
+/**********************************************************
+ * Simpler non-interruptable routines for UTF-8 and
+ * Windowish UTF-16 (restricted)
+ **********************************************************/
+/*
+ * This function is the heart of the Unicode support for
+ * open_port - spawn_executable. It converts both the name
+ * of the executable and the arguments according to the same rules
+ * as for filename conversion. That means as if your arguments are
+ * to be raw, you supply binaries, else unicode characters are allowed up to
+ * the encoding maximum (256 of the unicode max).
+ * Depending on the filename encoding standard, the vector is then
+ * converted to whatever is used, which might mean win_utf16 if on windows.
+ * Do not peek into the argument vector or filenam with ordinary
+ * string routines, that will certainly fail on some OS.
+ */
+
+char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int allow_empty)
+{
+ int encoding = erts_get_native_filename_encoding();
+ char* name_buf = NULL;
+
+ if (is_atom(name) || is_list(name) || (allow_empty && is_nil(name))) {
+ Sint need;
+ if ((need = erts_native_filename_need(name,encoding)) < 0) {
+ return NULL;
+ }
+ if (encoding == ERL_FILENAME_WIN_WCHAR) {
+ need += 2;
+ } else {
+ ++need;
+ }
+ name_buf = (char *) erts_alloc(alloc_type, need);
+ erts_native_filename_put(name,encoding,(byte *)name_buf);
+ name_buf[need-1] = 0;
+ if (encoding == ERL_FILENAME_WIN_WCHAR) {
+ name_buf[need-2] = 0;
+ }
+ } else if (is_binary(name)) {
+ byte *temp_alloc = NULL;
+ byte *bytes;
+ byte *err_pos;
+ Uint size,num_chars;
+
+ size = binary_size(name);
+ bytes = erts_get_aligned_binary_bytes(name, &temp_alloc);
+ if (encoding != ERL_FILENAME_WIN_WCHAR) {
+ /*Add 0 termination only*/
+ name_buf = (char *) erts_alloc(alloc_type, size+1);
+ memcpy(name_buf,bytes,size);
+ name_buf[size]=0;
+ } else if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK ||
+ erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) {
+ byte *p;
+ /* What to do now? Maybe latin1, so just take byte for byte instead */
+ name_buf = (char *) erts_alloc(alloc_type, (size+1)*2);
+ p = (byte *) name_buf;
+ while (size--) {
+ *p++ = *bytes++;
+ *p++ = 0;
+ }
+ *p++ = 0;
+ *p++ = 0;
+ } else { /* WIN_WCHAR and valid UTF8 */
+ name_buf = (char *) erts_alloc(alloc_type, (num_chars+1)*2);
+ erts_copy_utf8_to_utf16_little((byte *) name_buf, bytes, num_chars);
+ name_buf[num_chars*2] = 0;
+ name_buf[num_chars*2+1] = 0;
+ }
+ erts_free_aligned_binary_bytes(temp_alloc);
+ } else {
+ return NULL;
+ }
+ return name_buf;
+}
+
+
+Sint erts_native_filename_need(Eterm ioterm, int encoding)
+{
+ Eterm *objp;
+ Eterm obj;
+ DECLARE_ESTACK(stack);
+ Sint need = 0;
+
+ if (is_atom(ioterm)) {
+ Atom* ap;
+ int i;
+ ap = atom_tab(atom_val(ioterm));
+ switch (encoding) {
+ case ERL_FILENAME_LATIN1:
+ need = ap->len;
+ break;
+ case ERL_FILENAME_UTF8_MAC:
+ case ERL_FILENAME_UTF8:
+ for (i = 0; i < ap->len; i++) {
+ need += (ap->name[i] >= 0x80) ? 2 : 1;
+ }
+ break;
+ case ERL_FILENAME_WIN_WCHAR:
+ need = 2*(ap->len);
+ break;
+ default:
+ need = -1;
+ }
+ DESTROY_ESTACK(stack);
+ return need;
+ }
+
+ if (is_nil(ioterm)) {
+ DESTROY_ESTACK(stack);
+ return need;
+ }
+ if (!is_list(ioterm)) {
+ DESTROY_ESTACK(stack);
+ return (Sint) -1;
+ }
+ /* OK a list, needs to be processed in order, handling each flat list-level
+ as they occur, just like io_list_to_binary would */
+ ESTACK_PUSH(stack,ioterm);
+ while (!ESTACK_ISEMPTY(stack)) {
+ ioterm = ESTACK_POP(stack);
+ if (is_nil(ioterm)) {
+ /* ignore empty lists */
+ continue;
+ }
+ if(is_list(ioterm)) {
+L_Again: /* Restart with sublist, old listend was pushed on stack */
+ objp = list_val(ioterm);
+ obj = CAR(objp);
+ for(;;) { /* loop over one flat list of bytes and binaries
+ until sublist or list end is encountered */
+ if (is_small(obj)) { /* Always small */
+ for(;;) {
+ Uint x = unsigned_val(obj);
+ switch (encoding) {
+ case ERL_FILENAME_LATIN1:
+ if (x > 255) {
+ DESTROY_ESTACK(stack);
+ return ((Sint) -1);
+ }
+ need += 1;
+ break;
+ case ERL_FILENAME_UTF8_MAC:
+ case ERL_FILENAME_UTF8:
+ if (x < 0x80) {
+ need +=1;
+ } else if (x < 0x800) {
+ need += 2;
+ } else if (x < 0x10000) {
+ if ((x >= 0xD800 && x <= 0xDFFF) ||
+ (x == 0xFFFE) ||
+ (x == 0xFFFF)) { /* Invalid unicode range */
+ DESTROY_ESTACK(stack);
+ return ((Sint) -1);
+ }
+ need += 3;
+ } else if (x < 0x110000) {
+ need += 4;
+ } else {
+ DESTROY_ESTACK(stack);
+ return ((Sint) -1);
+ }
+ break;
+ case ERL_FILENAME_WIN_WCHAR:
+ if (x <= 0xffff) {
+ need += 2;
+ break;
+ } /* else fall throug to error */
+ default:
+ DESTROY_ESTACK(stack);
+ return ((Sint) -1);
+ }
+
+ /* everything else will give badarg later
+ in the process, so we dont check */
+ ioterm = CDR(objp);
+ if (!is_list(ioterm)) {
+ break;
+ }
+ objp = list_val(ioterm);
+ obj = CAR(objp);
+ if (!is_small(obj))
+ break;
+ }
+ } else if (is_nil(obj)) {
+ ioterm = CDR(objp);
+ if (!is_list(ioterm)) {
+ break;
+ }
+ objp = list_val(ioterm);
+ obj = CAR(objp);
+ } else if (is_list(obj)) {
+ /* push rest of list for later processing, start
+ again with sublist */
+ ESTACK_PUSH(stack,CDR(objp));
+ ioterm = obj;
+ goto L_Again;
+ } else {
+ DESTROY_ESTACK(stack);
+ return ((Sint) -1);
+ }
+ if (is_nil(ioterm) || !is_list(ioterm)) {
+ break;
+ }
+ } /* for(;;) */
+ } /* is_list(ioterm) */
+
+ if (!is_list(ioterm) && !is_nil(ioterm)) {
+ /* inproper list end */
+ DESTROY_ESTACK(stack);
+ return ((Sint) -1);
+ }
+ } /* while not estack empty */
+ DESTROY_ESTACK(stack);
+ return need;
+}
+
+void erts_native_filename_put(Eterm ioterm, int encoding, byte *p)
+{
+ Eterm *objp;
+ Eterm obj;
+ DECLARE_ESTACK(stack);
+
+ if (is_atom(ioterm)) {
+ Atom* ap;
+ int i;
+ ap = atom_tab(atom_val(ioterm));
+ switch (encoding) {
+ case ERL_FILENAME_LATIN1:
+ for (i = 0; i < ap->len; i++) {
+ *p++ = ap->name[i];
+ }
+ break;
+ case ERL_FILENAME_UTF8_MAC:
+ case ERL_FILENAME_UTF8:
+ for (i = 0; i < ap->len; i++) {
+ if(ap->name[i] < 0x80) {
+ *p++ = ap->name[i];
+ } else {
+ *p++ = (((ap->name[i]) >> 6) | ((byte) 0xC0));
+ *p++ = (((ap->name[i]) & 0x3F) | ((byte) 0x80));
+ }
+ }
+ break;
+ case ERL_FILENAME_WIN_WCHAR:
+ for (i = 0; i < ap->len; i++) {
+ /* Little endian */
+ *p++ = ap->name[i];
+ *p++ = 0;
+ }
+ break;
+ default:
+ ASSERT(0);
+ }
+ DESTROY_ESTACK(stack);
+ return;
+ }
+
+ if (is_nil(ioterm)) {
+ DESTROY_ESTACK(stack);
+ return;
+ }
+ ASSERT(is_list(ioterm));
+ /* OK a list, needs to be processed in order, handling each flat list-level
+ as they occur, just like io_list_to_binary would */
+ ESTACK_PUSH(stack,ioterm);
+ while (!ESTACK_ISEMPTY(stack)) {
+ ioterm = ESTACK_POP(stack);
+ if (is_nil(ioterm)) {
+ /* ignore empty lists */
+ continue;
+ }
+ if(is_list(ioterm)) {
+L_Again: /* Restart with sublist, old listend was pushed on stack */
+ objp = list_val(ioterm);
+ obj = CAR(objp);
+ for(;;) { /* loop over one flat list of bytes and binaries
+ until sublist or list end is encountered */
+ if (is_small(obj)) { /* Always small */
+ for(;;) {
+ Uint x = unsigned_val(obj);
+ switch (encoding) {
+ case ERL_FILENAME_LATIN1:
+ ASSERT( x < 256);
+ *p++ = (byte) x;
+ break;
+ case ERL_FILENAME_UTF8_MAC:
+ case ERL_FILENAME_UTF8:
+ if (x < 0x80) {
+ *p++ = (byte) x;
+ }
+ else if (x < 0x800) {
+ *p++ = (((byte) (x >> 6)) |
+ ((byte) 0xC0));
+ *p++ = (((byte) (x & 0x3F)) |
+ ((byte) 0x80));
+ } else if (x < 0x10000) {
+ ASSERT(!((x >= 0xD800 && x <= 0xDFFF) ||
+ (x == 0xFFFE) ||
+ (x == 0xFFFF)));
+ *p++ = (((byte) (x >> 12)) |
+ ((byte) 0xE0));
+ *p++ = ((((byte) (x >> 6)) & 0x3F) |
+ ((byte) 0x80));
+ *p++ = (((byte) (x & 0x3F)) |
+ ((byte) 0x80));
+ } else {
+ ASSERT(x < 0x110000);
+ *p++ = (((byte) (x >> 18)) |
+ ((byte) 0xF0));
+ *p++ = ((((byte) (x >> 12)) & 0x3F) |
+ ((byte) 0x80));
+ *p++ = ((((byte) (x >> 6)) & 0x3F) |
+ ((byte) 0x80));
+ *p++ = (((byte) (x & 0x3F)) |
+ ((byte) 0x80));
+ }
+ break;
+ case ERL_FILENAME_WIN_WCHAR:
+ ASSERT(x <= 0xFFFF);
+ *p++ = (byte) (x & 0xFFU);
+ *p++ = (byte) ((x >> 8) & 0xFFU);
+ break;
+ default:
+ ASSERT(0);
+ }
+
+ /* everything else will give badarg later
+ in the process, so we dont check */
+ ioterm = CDR(objp);
+ if (!is_list(ioterm)) {
+ break;
+ }
+ objp = list_val(ioterm);
+ obj = CAR(objp);
+ if (!is_small(obj))
+ break;
+ }
+ } else if (is_nil(obj)) {
+ ioterm = CDR(objp);
+ if (!is_list(ioterm)) {
+ break;
+ }
+ objp = list_val(ioterm);
+ obj = CAR(objp);
+ } else if (is_list(obj)) {
+ /* push rest of list for later processing, start
+ again with sublist */
+ ESTACK_PUSH(stack,CDR(objp));
+ ioterm = obj;
+ goto L_Again;
+ } else {
+ ASSERT(0);
+ }
+ if (is_nil(ioterm) || !is_list(ioterm)) {
+ break;
+ }
+ } /* for(;;) */
+ } /* is_list(ioterm) */
+
+ ASSERT(is_list(ioterm) || is_nil(ioterm));
+ } /* while not estack empty */
+ DESTROY_ESTACK(stack);
+ return;
+}
+void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars)
+{
+ Uint unipoint;
+
+ while (num_chars--) {
+ if (((*bytes) & ((byte) 0x80)) == 0) {
+ unipoint = (Uint) *bytes;
+ ++bytes;
+ } else if (((*bytes) & ((byte) 0xE0)) == 0xC0) {
+ unipoint =
+ (((Uint) ((*bytes) & ((byte) 0x1F))) << 6) |
+ ((Uint) (bytes[1] & ((byte) 0x3F)));
+ bytes += 2;
+ } else if (((*bytes) & ((byte) 0xF0)) == 0xE0) {
+ unipoint =
+ (((Uint) ((*bytes) & ((byte) 0xF))) << 12) |
+ (((Uint) (bytes[1] & ((byte) 0x3F))) << 6) |
+ ((Uint) (bytes[2] & ((byte) 0x3F)));
+ bytes +=3;
+ } else if (((*bytes) & ((byte) 0xF8)) == 0xF0) {
+ unipoint =
+ (((Uint) ((*bytes) & ((byte) 0x7))) << 18) |
+ (((Uint) (bytes[1] & ((byte) 0x3F))) << 12) |
+ (((Uint) (bytes[2] & ((byte) 0x3F))) << 6) |
+ ((Uint) (bytes[3] & ((byte) 0x3F)));
+ bytes += 4;
+ } else {
+ erl_exit(1,"Internal unicode error in prim_file:internal_name2native/1");
+ }
+ *target++ = (byte) (unipoint & 0xFF);
+ *target++ = (byte) ((unipoint >> 8) & 0xFF);
+ }
+}
+
+/*
+ * This internal bif converts a filename to whatever format is suitable for the file driver
+ * It also adds zero termination so that prim_file neednt bother with the character encoding
+ * of the file driver
+ */
+BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
+{
+ int encoding = erts_get_native_filename_encoding();
+ Sint need;
+ Eterm bin_term;
+ byte* bin_p;
+ /* Prim file explicitly does not allow atoms, although we could
+ very well cope with it. Instead of letting 'file' handle them,
+ it would probably be more efficient to handle them here. Subject to
+ change in R15. */
+ if (is_atom(BIF_ARG_1)) {
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ if (is_binary(BIF_ARG_1)) {
+ byte *temp_alloc = NULL;
+ byte *bytes;
+ byte *err_pos;
+ Uint size,num_chars;
+ /* Uninterpreted encoding except if windows widechar, in case we convert from
+ utf8 to win_wchar */
+ size = binary_size(BIF_ARG_1);
+ bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+ if (encoding != ERL_FILENAME_WIN_WCHAR) {
+ /*Add 0 termination only*/
+ bin_term = new_binary(BIF_P, NULL, size+1);
+ bin_p = binary_bytes(bin_term);
+ memcpy(bin_p,bytes,size);
+ bin_p[size]=0;
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_RET(bin_term);
+ }
+ /* In a wchar world, the emulator flags only affect how
+ binaries are interpreted when sent from the user. */
+ /* Determine real length and create a new binary */
+ if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK ||
+ erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) {
+ /* What to do now? Maybe latin1, so just take byte for byte instead */
+ bin_term = new_binary(BIF_P, 0, (size+1)*2);
+ bin_p = binary_bytes(bin_term);
+ while (size--) {
+ *bin_p++ = *bytes++;
+ *bin_p++ = 0;
+ }
+ *bin_p++ = 0;
+ *bin_p++ = 0;
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_RET(bin_term);
+ }
+ /* OK, UTF8 ok, number of characters is in num_chars */
+ bin_term = new_binary(BIF_P, 0, (num_chars+1)*2);
+ bin_p = binary_bytes(bin_term);
+ erts_copy_utf8_to_utf16_little(bin_p, bytes, num_chars);
+ /* zero termination */
+ bin_p[num_chars*2] = 0;
+ bin_p[num_chars*2+1] = 0;
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_RET(bin_term);
+ } /* binary */
+
+
+ if ((need = erts_native_filename_need(BIF_ARG_1,encoding)) < 0) {
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ if (encoding == ERL_FILENAME_WIN_WCHAR) {
+ need += 2;
+ } else {
+ ++need;
+ }
+
+ bin_term = new_binary(BIF_P, 0, need);
+ bin_p = binary_bytes(bin_term);
+ erts_native_filename_put(BIF_ARG_1,encoding,bin_p);
+ bin_p[need-1] = 0;
+ if (encoding == ERL_FILENAME_WIN_WCHAR) {
+ bin_p[need-2] = 0;
+ }
+ BIF_RET(bin_term);
+}
+
+BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1)
+{
+ Eterm real_bin;
+ Uint offset;
+ Uint size,num_chars;
+ Uint bitsize;
+ Uint bitoffs;
+ Eterm *hp;
+ byte *temp_alloc = NULL;
+ byte *bytes;
+ byte *err_pos;
+ Uint num_built; /* characters */
+ Uint num_eaten; /* bytes */
+ Eterm ret;
+ int mac = 0;
+
+ if (is_not_binary(BIF_ARG_1)) {
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ size = binary_size(BIF_ARG_1);
+ ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize);
+ if (bitsize != 0) {
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ if (size == 0) {
+ BIF_RET(NIL);
+ }
+ switch (erts_get_native_filename_encoding()) {
+ case ERL_FILENAME_LATIN1:
+ hp = HAlloc(BIF_P, 2 * size);
+ bytes = binary_bytes(real_bin)+offset;
+
+ BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs));
+ case ERL_FILENAME_UTF8_MAC:
+ mac = 1;
+ case ERL_FILENAME_UTF8:
+ bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+ if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) {
+ erts_free_aligned_binary_bytes(temp_alloc);
+ goto noconvert;
+ }
+ num_built = 0;
+ num_eaten = 0;
+ if (mac) {
+ ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size);
+ } else {
+ ret = do_utf8_to_list(BIF_P, num_chars, bytes, size, num_chars, &num_built, &num_eaten, NIL);
+ }
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_RET(ret);
+ case ERL_FILENAME_WIN_WCHAR:
+ bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+ if ((size % 2) != 0) { /* Panic fixup to avoid crashing the emulator */
+ size--;
+ hp = HAlloc(BIF_P, size+2);
+ ret = CONS(hp,make_small((Uint) bytes[size]),NIL);
+ hp += 2;
+ } else {
+ hp = HAlloc(BIF_P, size);
+ ret = NIL;
+ }
+ bytes += size-1;
+ while (size > 0) {
+ Uint x = ((Uint) *bytes--) << 8;
+ x |= ((Uint) *bytes--);
+ size -= 2;
+ ret = CONS(hp,make_small(x),ret);
+ hp += 2;
+ }
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_RET(ret);
+ default:
+ goto noconvert;
+ }
+ noconvert:
+ BIF_RET(BIF_ARG_1);
+}
+
+BIF_RETTYPE prim_file_internal_normalize_utf8_1(BIF_ALIST_1)
+{
+ Eterm real_bin;
+ Uint offset;
+ Uint size,num_chars;
+ Uint bitsize;
+ Uint bitoffs;
+ Eterm ret;
+ byte *temp_alloc = NULL;
+ byte *bytes;
+ byte *err_pos;
+
+ if (is_not_binary(BIF_ARG_1)) {
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ size = binary_size(BIF_ARG_1);
+ ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize);
+ if (bitsize != 0) {
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ if (size == 0) {
+ BIF_RET(NIL);
+ }
+ bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+ if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) {
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size);
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_RET(ret);
+}
+
+BIF_RETTYPE file_native_name_encoding_0(BIF_ALIST_0)
+{
+ switch (erts_get_native_filename_encoding()) {
+ case ERL_FILENAME_LATIN1:
+ BIF_RET(am_latin1);
+ case ERL_FILENAME_UTF8_MAC:
+ case ERL_FILENAME_UTF8:
+ BIF_RET(am_utf8);
+ case ERL_FILENAME_WIN_WCHAR:
+ if (erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) {
+ BIF_RET(am_latin1);
+ } else {
+ BIF_RET(am_utf8);
+ }
+ default:
+ BIF_RET(am_undefined);
+ }
+}
diff --git a/erts/emulator/beam/erl_unicode_normalize.h b/erts/emulator/beam/erl_unicode_normalize.h
new file mode 100644
index 0000000000..fb0a111ca2
--- /dev/null
+++ b/erts/emulator/beam/erl_unicode_normalize.h
@@ -0,0 +1,1687 @@
+/*
+* %CopyrightBegin%
+*
+* Copyright Ericsson AB 1999-2010. All Rights Reserved.
+*
+* The contents of this file are subject to the Erlang Public License,
+* Version 1.1, (the "License"); you may not use this file except in
+* compliance with the License. You should have received a copy of the
+* Erlang Public License along with this software. If not, it can be
+* retrieved online at http://www.erlang.org/.
+*
+* Software distributed under the License is distributed on an "AS IS"
+* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+* the License for the specific language governing rights and limitations
+* under the License.
+*
+* %CopyrightEnd%
+*/
+/*
+* This file is automatically generated by dec.erl, do not edit manually
+*/
+#define HASH_SIZE_FACTOR 2
+typedef struct _compose_entry {
+ Uint16 c;
+ Uint16 res;
+ Uint16 num_subs;
+ struct _compose_entry *subs;
+ int *hash;
+} CompEntry;
+
+static int compose_tab_size = 61;
+static int hash_compose_tab_0_15[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_0_15 */
+static CompEntry compose_tab_0_15[] = {
+{65, 7846, 0, NULL, NULL},
+{69, 7872, 0, NULL, NULL},
+{79, 7890, 0, NULL, NULL},
+{97, 7847, 0, NULL, NULL},
+{101, 7873, 0, NULL, NULL},
+{111, 7891, 0, NULL, NULL}
+}; /* compose_tab_0_15 */
+static int hash_compose_tab_0_16[8] =
+{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_0_16 */
+static CompEntry compose_tab_0_16[] = {
+{69, 7700, 0, NULL, NULL},
+{79, 7760, 0, NULL, NULL},
+{101, 7701, 0, NULL, NULL},
+{111, 7761, 0, NULL, NULL}
+}; /* compose_tab_0_16 */
+static int hash_compose_tab_0_17[4] =
+{-1,0,1,-1}; /* hash_compose_tab_0_17 */
+static CompEntry compose_tab_0_17[] = {
+{65, 7856, 0, NULL, NULL},
+{97, 7857, 0, NULL, NULL}
+}; /* compose_tab_0_17 */
+static int hash_compose_tab_0_18[8] =
+{-1,2,-1,-1,-1,0,1,3}; /* hash_compose_tab_0_18 */
+static CompEntry compose_tab_0_18[] = {
+{85, 475, 0, NULL, NULL},
+{117, 476, 0, NULL, NULL},
+{953, 8146, 0, NULL, NULL},
+{965, 8162, 0, NULL, NULL}
+}; /* compose_tab_0_18 */
+static int hash_compose_tab_0_19_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_19_0 */
+static CompEntry compose_tab_0_19_0[] = {
+{913, 8074, 0, NULL, NULL},
+{919, 8090, 0, NULL, NULL},
+{937, 8106, 0, NULL, NULL},
+{945, 8066, 0, NULL, NULL},
+{951, 8082, 0, NULL, NULL},
+{969, 8098, 0, NULL, NULL}
+}; /* compose_tab_0_19_0 */
+static int hash_compose_tab_0_19[28] =
+{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_0_19 */
+static CompEntry compose_tab_0_19[] = {
+{837, 0, 6, compose_tab_0_19_0, hash_compose_tab_0_19_0},
+{913, 7946, 0, NULL, NULL},
+{917, 7962, 0, NULL, NULL},
+{919, 7978, 0, NULL, NULL},
+{921, 7994, 0, NULL, NULL},
+{927, 8010, 0, NULL, NULL},
+{937, 8042, 0, NULL, NULL},
+{945, 7938, 0, NULL, NULL},
+{949, 7954, 0, NULL, NULL},
+{951, 7970, 0, NULL, NULL},
+{953, 7986, 0, NULL, NULL},
+{959, 8002, 0, NULL, NULL},
+{965, 8018, 0, NULL, NULL},
+{969, 8034, 0, NULL, NULL}
+}; /* compose_tab_0_19 */
+static int hash_compose_tab_0_20_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_20_0 */
+static CompEntry compose_tab_0_20_0[] = {
+{913, 8075, 0, NULL, NULL},
+{919, 8091, 0, NULL, NULL},
+{937, 8107, 0, NULL, NULL},
+{945, 8067, 0, NULL, NULL},
+{951, 8083, 0, NULL, NULL},
+{969, 8099, 0, NULL, NULL}
+}; /* compose_tab_0_20_0 */
+static int hash_compose_tab_0_20[30] =
+{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5,
+ 12}; /* hash_compose_tab_0_20 */
+static CompEntry compose_tab_0_20[] = {
+{837, 0, 6, compose_tab_0_20_0, hash_compose_tab_0_20_0},
+{913, 7947, 0, NULL, NULL},
+{917, 7963, 0, NULL, NULL},
+{919, 7979, 0, NULL, NULL},
+{921, 7995, 0, NULL, NULL},
+{927, 8011, 0, NULL, NULL},
+{933, 8027, 0, NULL, NULL},
+{937, 8043, 0, NULL, NULL},
+{945, 7939, 0, NULL, NULL},
+{949, 7955, 0, NULL, NULL},
+{951, 7971, 0, NULL, NULL},
+{953, 7987, 0, NULL, NULL},
+{959, 8003, 0, NULL, NULL},
+{965, 8019, 0, NULL, NULL},
+{969, 8035, 0, NULL, NULL}
+}; /* compose_tab_0_20 */
+static int hash_compose_tab_0_21[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_0_21 */
+static CompEntry compose_tab_0_21[] = {
+{79, 7900, 0, NULL, NULL},
+{85, 7914, 0, NULL, NULL},
+{111, 7901, 0, NULL, NULL},
+{117, 7915, 0, NULL, NULL}
+}; /* compose_tab_0_21 */
+static int hash_compose_tab_0_22[6] =
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_0_22 */
+static CompEntry compose_tab_0_22[] = {
+{945, 8114, 0, NULL, NULL},
+{951, 8130, 0, NULL, NULL},
+{969, 8178, 0, NULL, NULL}
+}; /* compose_tab_0_22 */
+static int hash_compose_tab_0[78] =
+{38,3,29,-1,-1,-1,-1,4,19,5,20,6,14,30,31,21,32,33,37,7,-1,-1,-1,8,34,-1,-1,9,
+ -1,35,-1,-1,-1,10,36,-1,-1,-1,-1,11,-1,12,-1,13,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,23,-1,22,-1,24,-1,25,-1,26,-1,0,-1,-1,15,1,16,27,17,2,18,28,-1,-1}; /* hash_compose_tab_0 */
+static CompEntry compose_tab_0[] = {
+{65, 192, 0, NULL, NULL},
+{69, 200, 0, NULL, NULL},
+{73, 204, 0, NULL, NULL},
+{79, 210, 0, NULL, NULL},
+{85, 217, 0, NULL, NULL},
+{87, 7808, 0, NULL, NULL},
+{89, 7922, 0, NULL, NULL},
+{97, 224, 0, NULL, NULL},
+{101, 232, 0, NULL, NULL},
+{105, 236, 0, NULL, NULL},
+{111, 242, 0, NULL, NULL},
+{117, 249, 0, NULL, NULL},
+{119, 7809, 0, NULL, NULL},
+{121, 7923, 0, NULL, NULL},
+{168, 8173, 0, NULL, NULL},
+{770, 0, 6, compose_tab_0_15, hash_compose_tab_0_15},
+{772, 0, 4, compose_tab_0_16, hash_compose_tab_0_16},
+{774, 0, 2, compose_tab_0_17, hash_compose_tab_0_17},
+{776, 0, 4, compose_tab_0_18, hash_compose_tab_0_18},
+{787, 0, 14, compose_tab_0_19, hash_compose_tab_0_19},
+{788, 0, 15, compose_tab_0_20, hash_compose_tab_0_20},
+{795, 0, 4, compose_tab_0_21, hash_compose_tab_0_21},
+{837, 0, 3, compose_tab_0_22, hash_compose_tab_0_22},
+{913, 8122, 0, NULL, NULL},
+{917, 8136, 0, NULL, NULL},
+{919, 8138, 0, NULL, NULL},
+{921, 8154, 0, NULL, NULL},
+{927, 8184, 0, NULL, NULL},
+{933, 8170, 0, NULL, NULL},
+{937, 8186, 0, NULL, NULL},
+{945, 8048, 0, NULL, NULL},
+{949, 8050, 0, NULL, NULL},
+{951, 8052, 0, NULL, NULL},
+{953, 8054, 0, NULL, NULL},
+{959, 8056, 0, NULL, NULL},
+{965, 8058, 0, NULL, NULL},
+{969, 8060, 0, NULL, NULL},
+{8127, 8141, 0, NULL, NULL},
+{8190, 8157, 0, NULL, NULL}
+}; /* compose_tab_0 */
+static int hash_compose_tab_1_39[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_1_39 */
+static CompEntry compose_tab_1_39[] = {
+{65, 7844, 0, NULL, NULL},
+{69, 7870, 0, NULL, NULL},
+{79, 7888, 0, NULL, NULL},
+{97, 7845, 0, NULL, NULL},
+{101, 7871, 0, NULL, NULL},
+{111, 7889, 0, NULL, NULL}
+}; /* compose_tab_1_39 */
+static int hash_compose_tab_1_40[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_40 */
+static CompEntry compose_tab_1_40[] = {
+{79, 7756, 0, NULL, NULL},
+{85, 7800, 0, NULL, NULL},
+{111, 7757, 0, NULL, NULL},
+{117, 7801, 0, NULL, NULL}
+}; /* compose_tab_1_40 */
+static int hash_compose_tab_1_41[8] =
+{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_1_41 */
+static CompEntry compose_tab_1_41[] = {
+{69, 7702, 0, NULL, NULL},
+{79, 7762, 0, NULL, NULL},
+{101, 7703, 0, NULL, NULL},
+{111, 7763, 0, NULL, NULL}
+}; /* compose_tab_1_41 */
+static int hash_compose_tab_1_42[4] =
+{-1,0,1,-1}; /* hash_compose_tab_1_42 */
+static CompEntry compose_tab_1_42[] = {
+{65, 7854, 0, NULL, NULL},
+{97, 7855, 0, NULL, NULL}
+}; /* compose_tab_1_42 */
+static int hash_compose_tab_1_43[12] =
+{-1,0,1,-1,-1,4,5,-1,-1,2,3,-1}; /* hash_compose_tab_1_43 */
+static CompEntry compose_tab_1_43[] = {
+{73, 7726, 0, NULL, NULL},
+{85, 471, 0, NULL, NULL},
+{105, 7727, 0, NULL, NULL},
+{117, 472, 0, NULL, NULL},
+{953, 8147, 0, NULL, NULL},
+{965, 8163, 0, NULL, NULL}
+}; /* compose_tab_1_43 */
+static int hash_compose_tab_1_44[4] =
+{-1,0,1,-1}; /* hash_compose_tab_1_44 */
+static CompEntry compose_tab_1_44[] = {
+{65, 506, 0, NULL, NULL},
+{97, 507, 0, NULL, NULL}
+}; /* compose_tab_1_44 */
+static int hash_compose_tab_1_45_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_45_0 */
+static CompEntry compose_tab_1_45_0[] = {
+{913, 8076, 0, NULL, NULL},
+{919, 8092, 0, NULL, NULL},
+{937, 8108, 0, NULL, NULL},
+{945, 8068, 0, NULL, NULL},
+{951, 8084, 0, NULL, NULL},
+{969, 8100, 0, NULL, NULL}
+}; /* compose_tab_1_45_0 */
+static int hash_compose_tab_1_45[28] =
+{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_1_45 */
+static CompEntry compose_tab_1_45[] = {
+{837, 0, 6, compose_tab_1_45_0, hash_compose_tab_1_45_0},
+{913, 7948, 0, NULL, NULL},
+{917, 7964, 0, NULL, NULL},
+{919, 7980, 0, NULL, NULL},
+{921, 7996, 0, NULL, NULL},
+{927, 8012, 0, NULL, NULL},
+{937, 8044, 0, NULL, NULL},
+{945, 7940, 0, NULL, NULL},
+{949, 7956, 0, NULL, NULL},
+{951, 7972, 0, NULL, NULL},
+{953, 7988, 0, NULL, NULL},
+{959, 8004, 0, NULL, NULL},
+{965, 8020, 0, NULL, NULL},
+{969, 8036, 0, NULL, NULL}
+}; /* compose_tab_1_45 */
+static int hash_compose_tab_1_46_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_46_0 */
+static CompEntry compose_tab_1_46_0[] = {
+{913, 8077, 0, NULL, NULL},
+{919, 8093, 0, NULL, NULL},
+{937, 8109, 0, NULL, NULL},
+{945, 8069, 0, NULL, NULL},
+{951, 8085, 0, NULL, NULL},
+{969, 8101, 0, NULL, NULL}
+}; /* compose_tab_1_46_0 */
+static int hash_compose_tab_1_46[30] =
+{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5,
+ 12}; /* hash_compose_tab_1_46 */
+static CompEntry compose_tab_1_46[] = {
+{837, 0, 6, compose_tab_1_46_0, hash_compose_tab_1_46_0},
+{913, 7949, 0, NULL, NULL},
+{917, 7965, 0, NULL, NULL},
+{919, 7981, 0, NULL, NULL},
+{921, 7997, 0, NULL, NULL},
+{927, 8013, 0, NULL, NULL},
+{933, 8029, 0, NULL, NULL},
+{937, 8045, 0, NULL, NULL},
+{945, 7941, 0, NULL, NULL},
+{949, 7957, 0, NULL, NULL},
+{951, 7973, 0, NULL, NULL},
+{953, 7989, 0, NULL, NULL},
+{959, 8005, 0, NULL, NULL},
+{965, 8021, 0, NULL, NULL},
+{969, 8037, 0, NULL, NULL}
+}; /* compose_tab_1_46 */
+static int hash_compose_tab_1_47[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_47 */
+static CompEntry compose_tab_1_47[] = {
+{79, 7898, 0, NULL, NULL},
+{85, 7912, 0, NULL, NULL},
+{111, 7899, 0, NULL, NULL},
+{117, 7913, 0, NULL, NULL}
+}; /* compose_tab_1_47 */
+static int hash_compose_tab_1_48[4] =
+{1,-1,-1,0}; /* hash_compose_tab_1_48 */
+static CompEntry compose_tab_1_48[] = {
+{67, 7688, 0, NULL, NULL},
+{99, 7689, 0, NULL, NULL}
+}; /* compose_tab_1_48 */
+static int hash_compose_tab_1_49[6] =
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_1_49 */
+static CompEntry compose_tab_1_49[] = {
+{945, 8116, 0, NULL, NULL},
+{951, 8132, 0, NULL, NULL},
+{959, 8180, 0, NULL, NULL}
+}; /* compose_tab_1_49 */
+static int hash_compose_tab_1[140] =
+{-1,-1,-1,-1,-1,-1,-1,68,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,34,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,35,-1,-1,-1,-1,64,-1,0,-1,1,-1,2,39,3,40,4,41,5,6,7,
+ 8,9,10,36,11,12,42,13,43,14,44,15,16,37,45,46,50,47,51,17,52,18,53,19,54,20,
+ 55,21,56,22,23,24,25,26,27,38,28,29,48,30,57,31,58,32,33,59,60,61,62,65,66,
+ 63,67,69,-1,-1,-1,-1,-1,49,-1,-1}; /* hash_compose_tab_1 */
+static CompEntry compose_tab_1[] = {
+{65, 193, 0, NULL, NULL},
+{67, 262, 0, NULL, NULL},
+{69, 201, 0, NULL, NULL},
+{71, 500, 0, NULL, NULL},
+{73, 205, 0, NULL, NULL},
+{75, 7728, 0, NULL, NULL},
+{76, 313, 0, NULL, NULL},
+{77, 7742, 0, NULL, NULL},
+{78, 323, 0, NULL, NULL},
+{79, 211, 0, NULL, NULL},
+{80, 7764, 0, NULL, NULL},
+{82, 340, 0, NULL, NULL},
+{83, 346, 0, NULL, NULL},
+{85, 218, 0, NULL, NULL},
+{87, 7810, 0, NULL, NULL},
+{89, 221, 0, NULL, NULL},
+{90, 377, 0, NULL, NULL},
+{97, 225, 0, NULL, NULL},
+{99, 263, 0, NULL, NULL},
+{101, 233, 0, NULL, NULL},
+{103, 501, 0, NULL, NULL},
+{105, 237, 0, NULL, NULL},
+{107, 7729, 0, NULL, NULL},
+{108, 314, 0, NULL, NULL},
+{109, 7743, 0, NULL, NULL},
+{110, 324, 0, NULL, NULL},
+{111, 243, 0, NULL, NULL},
+{112, 7765, 0, NULL, NULL},
+{114, 341, 0, NULL, NULL},
+{115, 347, 0, NULL, NULL},
+{117, 250, 0, NULL, NULL},
+{119, 7811, 0, NULL, NULL},
+{121, 253, 0, NULL, NULL},
+{122, 378, 0, NULL, NULL},
+{168, 8174, 0, NULL, NULL},
+{198, 508, 0, NULL, NULL},
+{216, 510, 0, NULL, NULL},
+{230, 509, 0, NULL, NULL},
+{248, 511, 0, NULL, NULL},
+{770, 0, 6, compose_tab_1_39, hash_compose_tab_1_39},
+{771, 0, 4, compose_tab_1_40, hash_compose_tab_1_40},
+{772, 0, 4, compose_tab_1_41, hash_compose_tab_1_41},
+{774, 0, 2, compose_tab_1_42, hash_compose_tab_1_42},
+{776, 0, 6, compose_tab_1_43, hash_compose_tab_1_43},
+{778, 0, 2, compose_tab_1_44, hash_compose_tab_1_44},
+{787, 0, 14, compose_tab_1_45, hash_compose_tab_1_45},
+{788, 0, 15, compose_tab_1_46, hash_compose_tab_1_46},
+{795, 0, 4, compose_tab_1_47, hash_compose_tab_1_47},
+{807, 0, 2, compose_tab_1_48, hash_compose_tab_1_48},
+{837, 0, 3, compose_tab_1_49, hash_compose_tab_1_49},
+{913, 8123, 0, NULL, NULL},
+{917, 8137, 0, NULL, NULL},
+{919, 8139, 0, NULL, NULL},
+{921, 8155, 0, NULL, NULL},
+{927, 8185, 0, NULL, NULL},
+{933, 8171, 0, NULL, NULL},
+{937, 8187, 0, NULL, NULL},
+{945, 8049, 0, NULL, NULL},
+{949, 8051, 0, NULL, NULL},
+{951, 8053, 0, NULL, NULL},
+{953, 8055, 0, NULL, NULL},
+{959, 8057, 0, NULL, NULL},
+{965, 8059, 0, NULL, NULL},
+{969, 8061, 0, NULL, NULL},
+{1043, 1027, 0, NULL, NULL},
+{1050, 1036, 0, NULL, NULL},
+{1075, 1107, 0, NULL, NULL},
+{1082, 1116, 0, NULL, NULL},
+{8127, 8142, 0, NULL, NULL},
+{8190, 8158, 0, NULL, NULL}
+}; /* compose_tab_1 */
+static int hash_compose_tab_2_26[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_2_26 */
+static CompEntry compose_tab_2_26[] = {
+{65, 7852, 0, NULL, NULL},
+{69, 7878, 0, NULL, NULL},
+{79, 7896, 0, NULL, NULL},
+{97, 7853, 0, NULL, NULL},
+{101, 7879, 0, NULL, NULL},
+{111, 7897, 0, NULL, NULL}
+}; /* compose_tab_2_26 */
+static int hash_compose_tab_2[54] =
+{-1,-1,-1,20,-1,-1,-1,21,-1,22,-1,0,23,1,24,2,25,3,4,5,6,-1,-1,-1,-1,7,-1,-1,
+ -1,8,-1,9,-1,10,-1,11,12,-1,-1,-1,-1,-1,-1,13,-1,14,-1,15,26,16,17,18,19,-1}; /* hash_compose_tab_2 */
+static CompEntry compose_tab_2[] = {
+{65, 194, 0, NULL, NULL},
+{67, 264, 0, NULL, NULL},
+{69, 202, 0, NULL, NULL},
+{71, 284, 0, NULL, NULL},
+{72, 292, 0, NULL, NULL},
+{73, 206, 0, NULL, NULL},
+{74, 308, 0, NULL, NULL},
+{79, 212, 0, NULL, NULL},
+{83, 348, 0, NULL, NULL},
+{85, 219, 0, NULL, NULL},
+{87, 372, 0, NULL, NULL},
+{89, 374, 0, NULL, NULL},
+{90, 7824, 0, NULL, NULL},
+{97, 226, 0, NULL, NULL},
+{99, 265, 0, NULL, NULL},
+{101, 234, 0, NULL, NULL},
+{103, 285, 0, NULL, NULL},
+{104, 293, 0, NULL, NULL},
+{105, 238, 0, NULL, NULL},
+{106, 309, 0, NULL, NULL},
+{111, 244, 0, NULL, NULL},
+{115, 349, 0, NULL, NULL},
+{117, 251, 0, NULL, NULL},
+{119, 373, 0, NULL, NULL},
+{121, 375, 0, NULL, NULL},
+{122, 7825, 0, NULL, NULL},
+{803, 0, 6, compose_tab_2_26, hash_compose_tab_2_26}
+}; /* compose_tab_2 */
+static int hash_compose_tab_3_16[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_3_16 */
+static CompEntry compose_tab_3_16[] = {
+{65, 7850, 0, NULL, NULL},
+{69, 7876, 0, NULL, NULL},
+{79, 7894, 0, NULL, NULL},
+{97, 7851, 0, NULL, NULL},
+{101, 7877, 0, NULL, NULL},
+{111, 7895, 0, NULL, NULL}
+}; /* compose_tab_3_16 */
+static int hash_compose_tab_3_17[4] =
+{-1,0,1,-1}; /* hash_compose_tab_3_17 */
+static CompEntry compose_tab_3_17[] = {
+{65, 7860, 0, NULL, NULL},
+{97, 7861, 0, NULL, NULL}
+}; /* compose_tab_3_17 */
+static int hash_compose_tab_3_18[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_3_18 */
+static CompEntry compose_tab_3_18[] = {
+{79, 7904, 0, NULL, NULL},
+{85, 7918, 0, NULL, NULL},
+{111, 7905, 0, NULL, NULL},
+{117, 7919, 0, NULL, NULL}
+}; /* compose_tab_3_18 */
+static int hash_compose_tab_3[38] =
+{-1,-1,3,4,13,14,-1,15,-1,5,6,16,-1,7,17,-1,-1,-1,-1,-1,-1,8,-1,-1,-1,9,-1,0,
+ -1,10,-1,1,-1,-1,11,2,12,18}; /* hash_compose_tab_3 */
+static CompEntry compose_tab_3[] = {
+{65, 195, 0, NULL, NULL},
+{69, 7868, 0, NULL, NULL},
+{73, 296, 0, NULL, NULL},
+{78, 209, 0, NULL, NULL},
+{79, 213, 0, NULL, NULL},
+{85, 360, 0, NULL, NULL},
+{86, 7804, 0, NULL, NULL},
+{89, 7928, 0, NULL, NULL},
+{97, 227, 0, NULL, NULL},
+{101, 7869, 0, NULL, NULL},
+{105, 297, 0, NULL, NULL},
+{110, 241, 0, NULL, NULL},
+{111, 245, 0, NULL, NULL},
+{117, 361, 0, NULL, NULL},
+{118, 7805, 0, NULL, NULL},
+{121, 7929, 0, NULL, NULL},
+{770, 0, 6, compose_tab_3_16, hash_compose_tab_3_16},
+{774, 0, 2, compose_tab_3_17, hash_compose_tab_3_17},
+{795, 0, 4, compose_tab_3_18, hash_compose_tab_3_18}
+}; /* compose_tab_3 */
+static int hash_compose_tab_4_14[4] =
+{-1,0,1,-1}; /* hash_compose_tab_4_14 */
+static CompEntry compose_tab_4_14[] = {
+{65, 480, 0, NULL, NULL},
+{97, 481, 0, NULL, NULL}
+}; /* compose_tab_4_14 */
+static int hash_compose_tab_4_15[8] =
+{-1,0,2,-1,-1,1,3,-1}; /* hash_compose_tab_4_15 */
+static CompEntry compose_tab_4_15[] = {
+{65, 478, 0, NULL, NULL},
+{85, 469, 0, NULL, NULL},
+{97, 479, 0, NULL, NULL},
+{117, 470, 0, NULL, NULL}
+}; /* compose_tab_4_15 */
+static int hash_compose_tab_4_16[8] =
+{-1,-1,1,3,0,2,-1,-1}; /* hash_compose_tab_4_16 */
+static CompEntry compose_tab_4_16[] = {
+{76, 7736, 0, NULL, NULL},
+{82, 7772, 0, NULL, NULL},
+{108, 7737, 0, NULL, NULL},
+{114, 7773, 0, NULL, NULL}
+}; /* compose_tab_4_16 */
+static int hash_compose_tab_4_17[4] =
+{1,-1,-1,0}; /* hash_compose_tab_4_17 */
+static CompEntry compose_tab_4_17[] = {
+{79, 492, 0, NULL, NULL},
+{111, 493, 0, NULL, NULL}
+}; /* compose_tab_4_17 */
+static int hash_compose_tab_4[56] =
+{-1,22,-1,-1,-1,11,13,-1,-1,0,-1,-1,-1,1,23,2,26,3,18,16,-1,-1,-1,4,17,19,-1,
+ 27,-1,5,12,-1,-1,-1,-1,-1,-1,20,-1,-1,24,6,-1,-1,-1,7,-1,8,14,9,15,21,25,-1,
+ -1,10}; /* hash_compose_tab_4 */
+static CompEntry compose_tab_4[] = {
+{65, 256, 0, NULL, NULL},
+{69, 274, 0, NULL, NULL},
+{71, 7712, 0, NULL, NULL},
+{73, 298, 0, NULL, NULL},
+{79, 332, 0, NULL, NULL},
+{85, 362, 0, NULL, NULL},
+{97, 257, 0, NULL, NULL},
+{101, 275, 0, NULL, NULL},
+{103, 7713, 0, NULL, NULL},
+{105, 299, 0, NULL, NULL},
+{111, 333, 0, NULL, NULL},
+{117, 363, 0, NULL, NULL},
+{198, 482, 0, NULL, NULL},
+{230, 483, 0, NULL, NULL},
+{775, 0, 2, compose_tab_4_14, hash_compose_tab_4_14},
+{776, 0, 4, compose_tab_4_15, hash_compose_tab_4_15},
+{803, 0, 4, compose_tab_4_16, hash_compose_tab_4_16},
+{808, 0, 2, compose_tab_4_17, hash_compose_tab_4_17},
+{913, 8121, 0, NULL, NULL},
+{921, 8153, 0, NULL, NULL},
+{933, 8169, 0, NULL, NULL},
+{945, 8113, 0, NULL, NULL},
+{953, 8145, 0, NULL, NULL},
+{965, 8161, 0, NULL, NULL},
+{1048, 1250, 0, NULL, NULL},
+{1059, 1262, 0, NULL, NULL},
+{1080, 1251, 0, NULL, NULL},
+{1091, 1263, 0, NULL, NULL}
+}; /* compose_tab_4 */
+static int hash_compose_tab_5_12[4] =
+{-1,0,1,-1}; /* hash_compose_tab_5_12 */
+static CompEntry compose_tab_5_12[] = {
+{65, 7862, 0, NULL, NULL},
+{97, 7863, 0, NULL, NULL}
+}; /* compose_tab_5_12 */
+static int hash_compose_tab_5_13[4] =
+{-1,0,1,-1}; /* hash_compose_tab_5_13 */
+static CompEntry compose_tab_5_13[] = {
+{69, 7708, 0, NULL, NULL},
+{101, 7709, 0, NULL, NULL}
+}; /* compose_tab_5_13 */
+static int hash_compose_tab_5[60] =
+{28,-1,-1,-1,-1,0,19,-1,-1,1,-1,2,29,3,14,-1,-1,-1,-1,4,20,15,-1,12,-1,5,21,
+ 13,22,23,-1,-1,-1,16,-1,-1,-1,6,-1,24,-1,7,-1,8,-1,9,17,-1,-1,-1,-1,10,25,18,
+ -1,-1,-1,11,26,27}; /* hash_compose_tab_5 */
+static CompEntry compose_tab_5[] = {
+{65, 258, 0, NULL, NULL},
+{69, 276, 0, NULL, NULL},
+{71, 286, 0, NULL, NULL},
+{73, 300, 0, NULL, NULL},
+{79, 334, 0, NULL, NULL},
+{85, 364, 0, NULL, NULL},
+{97, 259, 0, NULL, NULL},
+{101, 277, 0, NULL, NULL},
+{103, 287, 0, NULL, NULL},
+{105, 301, 0, NULL, NULL},
+{111, 335, 0, NULL, NULL},
+{117, 365, 0, NULL, NULL},
+{803, 0, 2, compose_tab_5_12, hash_compose_tab_5_12},
+{807, 0, 2, compose_tab_5_13, hash_compose_tab_5_13},
+{913, 8120, 0, NULL, NULL},
+{921, 8152, 0, NULL, NULL},
+{933, 8168, 0, NULL, NULL},
+{945, 8112, 0, NULL, NULL},
+{953, 8144, 0, NULL, NULL},
+{965, 8160, 0, NULL, NULL},
+{1040, 1232, 0, NULL, NULL},
+{1045, 1238, 0, NULL, NULL},
+{1046, 1217, 0, NULL, NULL},
+{1048, 1049, 0, NULL, NULL},
+{1059, 1038, 0, NULL, NULL},
+{1072, 1233, 0, NULL, NULL},
+{1077, 1239, 0, NULL, NULL},
+{1078, 1218, 0, NULL, NULL},
+{1080, 1081, 0, NULL, NULL},
+{1091, 1118, 0, NULL, NULL}
+}; /* compose_tab_5 */
+static int hash_compose_tab_6_36[4] =
+{1,-1,-1,0}; /* hash_compose_tab_6_36 */
+static CompEntry compose_tab_6_36[] = {
+{83, 7780, 0, NULL, NULL},
+{115, 7781, 0, NULL, NULL}
+}; /* compose_tab_6_36 */
+static int hash_compose_tab_6_38[4] =
+{1,-1,-1,0}; /* hash_compose_tab_6_38 */
+static CompEntry compose_tab_6_38[] = {
+{83, 7782, 0, NULL, NULL},
+{115, 7783, 0, NULL, NULL}
+}; /* compose_tab_6_38 */
+static int hash_compose_tab_6_39[4] =
+{1,-1,-1,0}; /* hash_compose_tab_6_39 */
+static CompEntry compose_tab_6_39[] = {
+{83, 7784, 0, NULL, NULL},
+{115, 7785, 0, NULL, NULL}
+}; /* compose_tab_6_39 */
+static int hash_compose_tab_6[80] =
+{10,-1,11,12,13,39,-1,14,15,16,17,-1,-1,-1,-1,-1,-1,-1,18,19,20,21,22,23,24,
+ -1,-1,-1,-1,25,26,-1,27,-1,28,29,30,-1,-1,31,32,33,34,-1,-1,-1,-1,-1,-1,36,
+ -1,-1,-1,-1,37,-1,-1,-1,-1,-1,38,-1,-1,35,-1,-1,0,1,2,3,4,5,6,7,-1,-1,-1,8,9,
+ -1}; /* hash_compose_tab_6 */
+static CompEntry compose_tab_6[] = {
+{66, 7682, 0, NULL, NULL},
+{67, 266, 0, NULL, NULL},
+{68, 7690, 0, NULL, NULL},
+{69, 278, 0, NULL, NULL},
+{70, 7710, 0, NULL, NULL},
+{71, 288, 0, NULL, NULL},
+{72, 7714, 0, NULL, NULL},
+{73, 304, 0, NULL, NULL},
+{77, 7744, 0, NULL, NULL},
+{78, 7748, 0, NULL, NULL},
+{80, 7766, 0, NULL, NULL},
+{82, 7768, 0, NULL, NULL},
+{83, 7776, 0, NULL, NULL},
+{84, 7786, 0, NULL, NULL},
+{87, 7814, 0, NULL, NULL},
+{88, 7818, 0, NULL, NULL},
+{89, 7822, 0, NULL, NULL},
+{90, 379, 0, NULL, NULL},
+{98, 7683, 0, NULL, NULL},
+{99, 267, 0, NULL, NULL},
+{100, 7691, 0, NULL, NULL},
+{101, 279, 0, NULL, NULL},
+{102, 7711, 0, NULL, NULL},
+{103, 289, 0, NULL, NULL},
+{104, 7715, 0, NULL, NULL},
+{109, 7745, 0, NULL, NULL},
+{110, 7749, 0, NULL, NULL},
+{112, 7767, 0, NULL, NULL},
+{114, 7769, 0, NULL, NULL},
+{115, 7777, 0, NULL, NULL},
+{116, 7787, 0, NULL, NULL},
+{119, 7815, 0, NULL, NULL},
+{120, 7819, 0, NULL, NULL},
+{121, 7823, 0, NULL, NULL},
+{122, 380, 0, NULL, NULL},
+{383, 7835, 0, NULL, NULL},
+{769, 0, 2, compose_tab_6_36, hash_compose_tab_6_36},
+{774, 784, 0, NULL, NULL},
+{780, 0, 2, compose_tab_6_38, hash_compose_tab_6_38},
+{803, 0, 2, compose_tab_6_39, hash_compose_tab_6_39}
+}; /* compose_tab_6 */
+static int hash_compose_tab_7_23[4] =
+{1,-1,-1,0}; /* hash_compose_tab_7_23 */
+static CompEntry compose_tab_7_23[] = {
+{79, 7758, 0, NULL, NULL},
+{111, 7759, 0, NULL, NULL}
+}; /* compose_tab_7_23 */
+static int hash_compose_tab_7_24[4] =
+{-1,0,1,-1}; /* hash_compose_tab_7_24 */
+static CompEntry compose_tab_7_24[] = {
+{85, 7802, 0, NULL, NULL},
+{117, 7803, 0, NULL, NULL}
+}; /* compose_tab_7_24 */
+static int hash_compose_tab_7[100] =
+{48,10,21,-1,11,12,-1,-1,-1,-1,49,13,-1,-1,-1,20,14,15,-1,16,17,18,25,-1,-1,
+ -1,-1,-1,-1,22,30,-1,-1,26,-1,-1,-1,-1,-1,-1,31,-1,-1,-1,-1,32,33,34,35,-1,
+ -1,-1,-1,27,36,-1,-1,-1,-1,37,-1,-1,-1,38,-1,0,28,39,-1,1,-1,23,2,3,24,40,-1,
+ 41,29,4,42,43,44,-1,-1,5,45,6,7,8,-1,46,-1,-1,-1,47,-1,9,-1,19}; /* hash_compose_tab_7 */
+static CompEntry compose_tab_7[] = {
+{65, 196, 0, NULL, NULL},
+{69, 203, 0, NULL, NULL},
+{72, 7718, 0, NULL, NULL},
+{73, 207, 0, NULL, NULL},
+{79, 214, 0, NULL, NULL},
+{85, 220, 0, NULL, NULL},
+{87, 7812, 0, NULL, NULL},
+{88, 7820, 0, NULL, NULL},
+{89, 376, 0, NULL, NULL},
+{97, 228, 0, NULL, NULL},
+{101, 235, 0, NULL, NULL},
+{104, 7719, 0, NULL, NULL},
+{105, 239, 0, NULL, NULL},
+{111, 246, 0, NULL, NULL},
+{116, 7831, 0, NULL, NULL},
+{117, 252, 0, NULL, NULL},
+{119, 7813, 0, NULL, NULL},
+{120, 7821, 0, NULL, NULL},
+{121, 255, 0, NULL, NULL},
+{399, 1242, 0, NULL, NULL},
+{415, 1258, 0, NULL, NULL},
+{601, 1243, 0, NULL, NULL},
+{629, 1259, 0, NULL, NULL},
+{771, 0, 2, compose_tab_7_23, hash_compose_tab_7_23},
+{772, 0, 2, compose_tab_7_24, hash_compose_tab_7_24},
+{921, 938, 0, NULL, NULL},
+{933, 939, 0, NULL, NULL},
+{953, 970, 0, NULL, NULL},
+{965, 971, 0, NULL, NULL},
+{978, 980, 0, NULL, NULL},
+{1030, 1031, 0, NULL, NULL},
+{1040, 1234, 0, NULL, NULL},
+{1045, 1025, 0, NULL, NULL},
+{1046, 1244, 0, NULL, NULL},
+{1047, 1246, 0, NULL, NULL},
+{1048, 1252, 0, NULL, NULL},
+{1054, 1254, 0, NULL, NULL},
+{1059, 1264, 0, NULL, NULL},
+{1063, 1268, 0, NULL, NULL},
+{1067, 1272, 0, NULL, NULL},
+{1072, 1235, 0, NULL, NULL},
+{1077, 1105, 0, NULL, NULL},
+{1078, 1245, 0, NULL, NULL},
+{1079, 1247, 0, NULL, NULL},
+{1080, 1253, 0, NULL, NULL},
+{1086, 1255, 0, NULL, NULL},
+{1091, 1265, 0, NULL, NULL},
+{1095, 1269, 0, NULL, NULL},
+{1099, 1273, 0, NULL, NULL},
+{1110, 1111, 0, NULL, NULL}
+}; /* compose_tab_7 */
+static int hash_compose_tab_8_12[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_8_12 */
+static CompEntry compose_tab_8_12[] = {
+{65, 7848, 0, NULL, NULL},
+{69, 7874, 0, NULL, NULL},
+{79, 7892, 0, NULL, NULL},
+{97, 7849, 0, NULL, NULL},
+{101, 7875, 0, NULL, NULL},
+{111, 7893, 0, NULL, NULL}
+}; /* compose_tab_8_12 */
+static int hash_compose_tab_8_13[4] =
+{-1,0,1,-1}; /* hash_compose_tab_8_13 */
+static CompEntry compose_tab_8_13[] = {
+{65, 7858, 0, NULL, NULL},
+{97, 7859, 0, NULL, NULL}
+}; /* compose_tab_8_13 */
+static int hash_compose_tab_8_14[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_8_14 */
+static CompEntry compose_tab_8_14[] = {
+{79, 7902, 0, NULL, NULL},
+{85, 7916, 0, NULL, NULL},
+{111, 7903, 0, NULL, NULL},
+{117, 7917, 0, NULL, NULL}
+}; /* compose_tab_8_14 */
+static int hash_compose_tab_8[30] =
+{-1,11,-1,-1,-1,0,-1,6,-1,1,-1,7,-1,2,-1,8,14,-1,-1,3,12,9,-1,-1,13,4,-1,10,
+ -1,5}; /* hash_compose_tab_8 */
+static CompEntry compose_tab_8[] = {
+{65, 7842, 0, NULL, NULL},
+{69, 7866, 0, NULL, NULL},
+{73, 7880, 0, NULL, NULL},
+{79, 7886, 0, NULL, NULL},
+{85, 7910, 0, NULL, NULL},
+{89, 7926, 0, NULL, NULL},
+{97, 7843, 0, NULL, NULL},
+{101, 7867, 0, NULL, NULL},
+{105, 7881, 0, NULL, NULL},
+{111, 7887, 0, NULL, NULL},
+{117, 7911, 0, NULL, NULL},
+{121, 7927, 0, NULL, NULL},
+{770, 0, 6, compose_tab_8_12, hash_compose_tab_8_12},
+{774, 0, 2, compose_tab_8_13, hash_compose_tab_8_13},
+{795, 0, 4, compose_tab_8_14, hash_compose_tab_8_14}
+}; /* compose_tab_8 */
+static int hash_compose_tab_9[12] =
+{-1,1,2,5,-1,0,-1,-1,-1,3,-1,4}; /* hash_compose_tab_9 */
+static CompEntry compose_tab_9[] = {
+{65, 197, 0, NULL, NULL},
+{85, 366, 0, NULL, NULL},
+{97, 229, 0, NULL, NULL},
+{117, 367, 0, NULL, NULL},
+{119, 7832, 0, NULL, NULL},
+{121, 7833, 0, NULL, NULL}
+}; /* compose_tab_9 */
+static int hash_compose_tab_10[12] =
+{-1,1,-1,2,4,-1,-1,0,-1,3,-1,5}; /* hash_compose_tab_10 */
+static CompEntry compose_tab_10[] = {
+{79, 336, 0, NULL, NULL},
+{85, 368, 0, NULL, NULL},
+{111, 337, 0, NULL, NULL},
+{117, 369, 0, NULL, NULL},
+{1059, 1266, 0, NULL, NULL},
+{1091, 1267, 0, NULL, NULL}
+}; /* compose_tab_10 */
+static int hash_compose_tab_11_33[4] =
+{-1,0,1,-1}; /* hash_compose_tab_11_33 */
+static CompEntry compose_tab_11_33[] = {
+{85, 473, 0, NULL, NULL},
+{117, 474, 0, NULL, NULL}
+}; /* compose_tab_11_33 */
+static int hash_compose_tab_11[68] =
+{2,3,-1,4,-1,5,-1,6,7,-1,8,9,-1,-1,10,11,12,13,-1,-1,-1,-1,14,-1,-1,-1,-1,-1,
+ 33,15,-1,16,17,18,31,19,-1,20,21,22,23,-1,24,25,-1,-1,26,27,28,29,32,-1,-1,
+ -1,30,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,-1,1}; /* hash_compose_tab_11 */
+static CompEntry compose_tab_11[] = {
+{65, 461, 0, NULL, NULL},
+{67, 268, 0, NULL, NULL},
+{68, 270, 0, NULL, NULL},
+{69, 282, 0, NULL, NULL},
+{71, 486, 0, NULL, NULL},
+{73, 463, 0, NULL, NULL},
+{75, 488, 0, NULL, NULL},
+{76, 317, 0, NULL, NULL},
+{78, 327, 0, NULL, NULL},
+{79, 465, 0, NULL, NULL},
+{82, 344, 0, NULL, NULL},
+{83, 352, 0, NULL, NULL},
+{84, 356, 0, NULL, NULL},
+{85, 467, 0, NULL, NULL},
+{90, 381, 0, NULL, NULL},
+{97, 462, 0, NULL, NULL},
+{99, 269, 0, NULL, NULL},
+{100, 271, 0, NULL, NULL},
+{101, 283, 0, NULL, NULL},
+{103, 487, 0, NULL, NULL},
+{105, 464, 0, NULL, NULL},
+{106, 496, 0, NULL, NULL},
+{107, 489, 0, NULL, NULL},
+{108, 318, 0, NULL, NULL},
+{110, 328, 0, NULL, NULL},
+{111, 466, 0, NULL, NULL},
+{114, 345, 0, NULL, NULL},
+{115, 353, 0, NULL, NULL},
+{116, 357, 0, NULL, NULL},
+{117, 468, 0, NULL, NULL},
+{122, 382, 0, NULL, NULL},
+{439, 494, 0, NULL, NULL},
+{658, 495, 0, NULL, NULL},
+{776, 0, 2, compose_tab_11_33, hash_compose_tab_11_33}
+}; /* compose_tab_11 */
+static int hash_compose_tab_12_1[4] =
+{-1,0,1,-1}; /* hash_compose_tab_12_1 */
+static CompEntry compose_tab_12_1[] = {
+{953, 912, 0, NULL, NULL},
+{965, 944, 0, NULL, NULL}
+}; /* compose_tab_12_1 */
+static int hash_compose_tab_12[34] =
+{11,4,12,5,-1,-1,-1,13,-1,6,-1,-1,-1,14,-1,7,-1,15,-1,8,-1,-1,-1,-1,-1,-1,16,
+ 9,1,2,-1,10,0,3}; /* hash_compose_tab_12 */
+static CompEntry compose_tab_12[] = {
+{168, 901, 0, NULL, NULL},
+{776, 0, 2, compose_tab_12_1, hash_compose_tab_12_1},
+{913, 902, 0, NULL, NULL},
+{917, 904, 0, NULL, NULL},
+{919, 905, 0, NULL, NULL},
+{921, 906, 0, NULL, NULL},
+{927, 908, 0, NULL, NULL},
+{933, 910, 0, NULL, NULL},
+{937, 911, 0, NULL, NULL},
+{945, 940, 0, NULL, NULL},
+{949, 941, 0, NULL, NULL},
+{951, 942, 0, NULL, NULL},
+{953, 943, 0, NULL, NULL},
+{959, 972, 0, NULL, NULL},
+{965, 973, 0, NULL, NULL},
+{969, 974, 0, NULL, NULL},
+{978, 979, 0, NULL, NULL}
+}; /* compose_tab_12 */
+static int hash_compose_tab_13[28] =
+{-1,5,10,-1,-1,11,-1,-1,-1,0,-1,-1,-1,1,6,-1,-1,2,7,-1,12,8,13,3,-1,-1,4,9}; /* hash_compose_tab_13 */
+static CompEntry compose_tab_13[] = {
+{65, 512, 0, NULL, NULL},
+{69, 516, 0, NULL, NULL},
+{73, 520, 0, NULL, NULL},
+{79, 524, 0, NULL, NULL},
+{82, 528, 0, NULL, NULL},
+{85, 532, 0, NULL, NULL},
+{97, 513, 0, NULL, NULL},
+{101, 517, 0, NULL, NULL},
+{105, 521, 0, NULL, NULL},
+{111, 525, 0, NULL, NULL},
+{114, 529, 0, NULL, NULL},
+{117, 533, 0, NULL, NULL},
+{1140, 1142, 0, NULL, NULL},
+{1141, 1143, 0, NULL, NULL}
+}; /* compose_tab_13 */
+static int hash_compose_tab_14[24] =
+{-1,2,6,-1,-1,7,-1,3,-1,8,4,-1,-1,5,-1,9,-1,0,10,-1,-1,1,11,-1}; /* hash_compose_tab_14 */
+static CompEntry compose_tab_14[] = {
+{65, 514, 0, NULL, NULL},
+{69, 518, 0, NULL, NULL},
+{73, 522, 0, NULL, NULL},
+{79, 526, 0, NULL, NULL},
+{82, 530, 0, NULL, NULL},
+{85, 534, 0, NULL, NULL},
+{97, 515, 0, NULL, NULL},
+{101, 519, 0, NULL, NULL},
+{105, 523, 0, NULL, NULL},
+{111, 527, 0, NULL, NULL},
+{114, 531, 0, NULL, NULL},
+{117, 535, 0, NULL, NULL}
+}; /* compose_tab_14 */
+static int hash_compose_tab_15_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_15_0 */
+static CompEntry compose_tab_15_0[] = {
+{913, 8072, 0, NULL, NULL},
+{919, 8088, 0, NULL, NULL},
+{937, 8104, 0, NULL, NULL},
+{945, 8064, 0, NULL, NULL},
+{951, 8080, 0, NULL, NULL},
+{969, 8096, 0, NULL, NULL}
+}; /* compose_tab_15_0 */
+static int hash_compose_tab_15[30] =
+{-1,12,-1,-1,-1,13,-1,6,-1,14,-1,-1,-1,1,-1,7,-1,2,-1,3,8,4,9,10,-1,-1,-1,0,5,
+ 11}; /* hash_compose_tab_15 */
+static CompEntry compose_tab_15[] = {
+{837, 0, 6, compose_tab_15_0, hash_compose_tab_15_0},
+{913, 7944, 0, NULL, NULL},
+{917, 7960, 0, NULL, NULL},
+{919, 7976, 0, NULL, NULL},
+{921, 7992, 0, NULL, NULL},
+{927, 8008, 0, NULL, NULL},
+{937, 8040, 0, NULL, NULL},
+{945, 7936, 0, NULL, NULL},
+{949, 7952, 0, NULL, NULL},
+{951, 7968, 0, NULL, NULL},
+{953, 7984, 0, NULL, NULL},
+{959, 8000, 0, NULL, NULL},
+{961, 8164, 0, NULL, NULL},
+{965, 8016, 0, NULL, NULL},
+{969, 8032, 0, NULL, NULL}
+}; /* compose_tab_15 */
+static int hash_compose_tab_16_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_16_0 */
+static CompEntry compose_tab_16_0[] = {
+{913, 8073, 0, NULL, NULL},
+{919, 8089, 0, NULL, NULL},
+{937, 8105, 0, NULL, NULL},
+{945, 8065, 0, NULL, NULL},
+{951, 8081, 0, NULL, NULL},
+{969, 8097, 0, NULL, NULL}
+}; /* compose_tab_16_0 */
+static int hash_compose_tab_16[34] =
+{11,3,12,4,-1,-1,-1,13,-1,5,14,6,-1,15,-1,7,-1,16,-1,8,-1,0,-1,-1,-1,-1,-1,9,
+ -1,1,-1,10,-1,2}; /* hash_compose_tab_16 */
+static CompEntry compose_tab_16[] = {
+{837, 0, 6, compose_tab_16_0, hash_compose_tab_16_0},
+{913, 7945, 0, NULL, NULL},
+{917, 7961, 0, NULL, NULL},
+{919, 7977, 0, NULL, NULL},
+{921, 7993, 0, NULL, NULL},
+{927, 8009, 0, NULL, NULL},
+{929, 8172, 0, NULL, NULL},
+{933, 8025, 0, NULL, NULL},
+{937, 8041, 0, NULL, NULL},
+{945, 7937, 0, NULL, NULL},
+{949, 7953, 0, NULL, NULL},
+{951, 7969, 0, NULL, NULL},
+{953, 7985, 0, NULL, NULL},
+{959, 8001, 0, NULL, NULL},
+{961, 8165, 0, NULL, NULL},
+{965, 8017, 0, NULL, NULL},
+{969, 8033, 0, NULL, NULL}
+}; /* compose_tab_16 */
+static int hash_compose_tab_17[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_17 */
+static CompEntry compose_tab_17[] = {
+{79, 416, 0, NULL, NULL},
+{85, 431, 0, NULL, NULL},
+{111, 417, 0, NULL, NULL},
+{117, 432, 0, NULL, NULL}
+}; /* compose_tab_17 */
+static int hash_compose_tab_18_38[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_18_38 */
+static CompEntry compose_tab_18_38[] = {
+{79, 7906, 0, NULL, NULL},
+{85, 7920, 0, NULL, NULL},
+{111, 7907, 0, NULL, NULL},
+{117, 7921, 0, NULL, NULL}
+}; /* compose_tab_18_38 */
+static int hash_compose_tab_18[78] =
+{9,10,-1,-1,11,12,13,14,15,16,-1,17,18,-1,-1,38,-1,-1,-1,19,20,-1,21,22,-1,-1,
+ 23,24,-1,25,26,27,28,29,-1,-1,30,31,32,33,34,35,-1,36,37,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,-1,2,3,-1,-1,4,5,-1,6,7,8}; /* hash_compose_tab_18 */
+static CompEntry compose_tab_18[] = {
+{65, 7840, 0, NULL, NULL},
+{66, 7684, 0, NULL, NULL},
+{68, 7692, 0, NULL, NULL},
+{69, 7864, 0, NULL, NULL},
+{72, 7716, 0, NULL, NULL},
+{73, 7882, 0, NULL, NULL},
+{75, 7730, 0, NULL, NULL},
+{76, 7734, 0, NULL, NULL},
+{77, 7746, 0, NULL, NULL},
+{78, 7750, 0, NULL, NULL},
+{79, 7884, 0, NULL, NULL},
+{82, 7770, 0, NULL, NULL},
+{83, 7778, 0, NULL, NULL},
+{84, 7788, 0, NULL, NULL},
+{85, 7908, 0, NULL, NULL},
+{86, 7806, 0, NULL, NULL},
+{87, 7816, 0, NULL, NULL},
+{89, 7924, 0, NULL, NULL},
+{90, 7826, 0, NULL, NULL},
+{97, 7841, 0, NULL, NULL},
+{98, 7685, 0, NULL, NULL},
+{100, 7693, 0, NULL, NULL},
+{101, 7865, 0, NULL, NULL},
+{104, 7717, 0, NULL, NULL},
+{105, 7883, 0, NULL, NULL},
+{107, 7731, 0, NULL, NULL},
+{108, 7735, 0, NULL, NULL},
+{109, 7747, 0, NULL, NULL},
+{110, 7751, 0, NULL, NULL},
+{111, 7885, 0, NULL, NULL},
+{114, 7771, 0, NULL, NULL},
+{115, 7779, 0, NULL, NULL},
+{116, 7789, 0, NULL, NULL},
+{117, 7909, 0, NULL, NULL},
+{118, 7807, 0, NULL, NULL},
+{119, 7817, 0, NULL, NULL},
+{121, 7925, 0, NULL, NULL},
+{122, 7827, 0, NULL, NULL},
+{795, 0, 4, compose_tab_18_38, hash_compose_tab_18_38}
+}; /* compose_tab_18 */
+static int hash_compose_tab_19[4] =
+{-1,0,1,-1}; /* hash_compose_tab_19 */
+static CompEntry compose_tab_19[] = {
+{85, 7794, 0, NULL, NULL},
+{117, 7795, 0, NULL, NULL}
+}; /* compose_tab_19 */
+static int hash_compose_tab_20[4] =
+{-1,0,1,-1}; /* hash_compose_tab_20 */
+static CompEntry compose_tab_20[] = {
+{65, 7680, 0, NULL, NULL},
+{97, 7681, 0, NULL, NULL}
+}; /* compose_tab_20 */
+static int hash_compose_tab_21[40] =
+{-1,-1,7,8,9,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,10,11,-1,-1,12,13,-1,
+ -1,0,1,14,15,2,3,16,17,4,5,18,6,19}; /* hash_compose_tab_21 */
+static CompEntry compose_tab_21[] = {
+{67, 199, 0, NULL, NULL},
+{68, 7696, 0, NULL, NULL},
+{71, 290, 0, NULL, NULL},
+{72, 7720, 0, NULL, NULL},
+{75, 310, 0, NULL, NULL},
+{76, 315, 0, NULL, NULL},
+{78, 325, 0, NULL, NULL},
+{82, 342, 0, NULL, NULL},
+{83, 350, 0, NULL, NULL},
+{84, 354, 0, NULL, NULL},
+{99, 231, 0, NULL, NULL},
+{100, 7697, 0, NULL, NULL},
+{103, 291, 0, NULL, NULL},
+{104, 7721, 0, NULL, NULL},
+{107, 311, 0, NULL, NULL},
+{108, 316, 0, NULL, NULL},
+{110, 326, 0, NULL, NULL},
+{114, 343, 0, NULL, NULL},
+{115, 351, 0, NULL, NULL},
+{116, 355, 0, NULL, NULL}
+}; /* compose_tab_21 */
+static int hash_compose_tab_22[20] =
+{-1,6,-1,-1,-1,0,4,7,-1,1,-1,8,-1,2,-1,-1,-1,5,9,3}; /* hash_compose_tab_22 */
+static CompEntry compose_tab_22[] = {
+{65, 260, 0, NULL, NULL},
+{69, 280, 0, NULL, NULL},
+{73, 302, 0, NULL, NULL},
+{79, 490, 0, NULL, NULL},
+{85, 370, 0, NULL, NULL},
+{97, 261, 0, NULL, NULL},
+{101, 281, 0, NULL, NULL},
+{105, 303, 0, NULL, NULL},
+{111, 491, 0, NULL, NULL},
+{117, 371, 0, NULL, NULL}
+}; /* compose_tab_22 */
+static int hash_compose_tab_23[24] =
+{-1,-1,-1,-1,2,6,3,7,-1,-1,-1,-1,4,5,8,9,-1,-1,-1,-1,0,1,10,11}; /* hash_compose_tab_23 */
+static CompEntry compose_tab_23[] = {
+{68, 7698, 0, NULL, NULL},
+{69, 7704, 0, NULL, NULL},
+{76, 7740, 0, NULL, NULL},
+{78, 7754, 0, NULL, NULL},
+{84, 7792, 0, NULL, NULL},
+{85, 7798, 0, NULL, NULL},
+{100, 7699, 0, NULL, NULL},
+{101, 7705, 0, NULL, NULL},
+{108, 7741, 0, NULL, NULL},
+{110, 7755, 0, NULL, NULL},
+{116, 7793, 0, NULL, NULL},
+{117, 7799, 0, NULL, NULL}
+}; /* compose_tab_23 */
+static int hash_compose_tab_24[4] =
+{0,1,-1,-1}; /* hash_compose_tab_24 */
+static CompEntry compose_tab_24[] = {
+{72, 7722, 0, NULL, NULL},
+{104, 7723, 0, NULL, NULL}
+}; /* compose_tab_24 */
+static int hash_compose_tab_25[12] =
+{-1,1,2,-1,-1,3,-1,-1,-1,0,4,5}; /* hash_compose_tab_25 */
+static CompEntry compose_tab_25[] = {
+{69, 7706, 0, NULL, NULL},
+{73, 7724, 0, NULL, NULL},
+{85, 7796, 0, NULL, NULL},
+{101, 7707, 0, NULL, NULL},
+{105, 7725, 0, NULL, NULL},
+{117, 7797, 0, NULL, NULL}
+}; /* compose_tab_25 */
+static int hash_compose_tab_26[34] =
+{1,-1,10,-1,-1,11,12,2,3,13,4,-1,14,-1,5,15,6,-1,-1,-1,16,-1,7,-1,-1,-1,-1,-1,
+ -1,-1,8,-1,0,9}; /* hash_compose_tab_26 */
+static CompEntry compose_tab_26[] = {
+{66, 7686, 0, NULL, NULL},
+{68, 7694, 0, NULL, NULL},
+{75, 7732, 0, NULL, NULL},
+{76, 7738, 0, NULL, NULL},
+{78, 7752, 0, NULL, NULL},
+{82, 7774, 0, NULL, NULL},
+{84, 7790, 0, NULL, NULL},
+{90, 7828, 0, NULL, NULL},
+{98, 7687, 0, NULL, NULL},
+{100, 7695, 0, NULL, NULL},
+{104, 7830, 0, NULL, NULL},
+{107, 7733, 0, NULL, NULL},
+{108, 7739, 0, NULL, NULL},
+{110, 7753, 0, NULL, NULL},
+{114, 7775, 0, NULL, NULL},
+{116, 7791, 0, NULL, NULL},
+{122, 7829, 0, NULL, NULL}
+}; /* compose_tab_26 */
+static int hash_compose_tab_27_1[4] =
+{-1,0,1,-1}; /* hash_compose_tab_27_1 */
+static CompEntry compose_tab_27_1[] = {
+{953, 8151, 0, NULL, NULL},
+{965, 8167, 0, NULL, NULL}
+}; /* compose_tab_27_1 */
+static int hash_compose_tab_27_2_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_2_0 */
+static CompEntry compose_tab_27_2_0[] = {
+{913, 8078, 0, NULL, NULL},
+{919, 8094, 0, NULL, NULL},
+{937, 8110, 0, NULL, NULL},
+{945, 8070, 0, NULL, NULL},
+{951, 8086, 0, NULL, NULL},
+{969, 8102, 0, NULL, NULL}
+}; /* compose_tab_27_2_0 */
+static int hash_compose_tab_27_2[20] =
+{-1,3,-1,-1,-1,5,8,-1,-1,9,-1,6,-1,1,7,-1,-1,0,4,2}; /* hash_compose_tab_27_2 */
+static CompEntry compose_tab_27_2[] = {
+{837, 0, 6, compose_tab_27_2_0, hash_compose_tab_27_2_0},
+{913, 7950, 0, NULL, NULL},
+{919, 7982, 0, NULL, NULL},
+{921, 7998, 0, NULL, NULL},
+{937, 8046, 0, NULL, NULL},
+{945, 7942, 0, NULL, NULL},
+{951, 7974, 0, NULL, NULL},
+{953, 7990, 0, NULL, NULL},
+{965, 8022, 0, NULL, NULL},
+{969, 8038, 0, NULL, NULL}
+}; /* compose_tab_27_2 */
+static int hash_compose_tab_27_3_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_3_0 */
+static CompEntry compose_tab_27_3_0[] = {
+{913, 8079, 0, NULL, NULL},
+{919, 8095, 0, NULL, NULL},
+{937, 8111, 0, NULL, NULL},
+{945, 8071, 0, NULL, NULL},
+{951, 8087, 0, NULL, NULL},
+{969, 8103, 0, NULL, NULL}
+}; /* compose_tab_27_3_0 */
+static int hash_compose_tab_27_3[22] =
+{-1,0,10,-1,-1,7,-1,8,-1,4,-1,1,-1,5,-1,-1,-1,2,-1,3,9,6}; /* hash_compose_tab_27_3 */
+static CompEntry compose_tab_27_3[] = {
+{837, 0, 6, compose_tab_27_3_0, hash_compose_tab_27_3_0},
+{913, 7951, 0, NULL, NULL},
+{919, 7983, 0, NULL, NULL},
+{921, 7999, 0, NULL, NULL},
+{933, 8031, 0, NULL, NULL},
+{937, 8047, 0, NULL, NULL},
+{945, 7943, 0, NULL, NULL},
+{951, 7975, 0, NULL, NULL},
+{953, 7991, 0, NULL, NULL},
+{965, 8023, 0, NULL, NULL},
+{969, 8039, 0, NULL, NULL}
+}; /* compose_tab_27_3 */
+static int hash_compose_tab_27_4[6] =
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_27_4 */
+static CompEntry compose_tab_27_4[] = {
+{945, 8119, 0, NULL, NULL},
+{951, 8135, 0, NULL, NULL},
+{969, 8183, 0, NULL, NULL}
+}; /* compose_tab_27_4 */
+static int hash_compose_tab_27[24] =
+{0,-1,-1,-1,-1,8,11,-1,1,5,9,-1,-1,-1,-1,6,10,7,-1,2,3,4,-1,-1}; /* hash_compose_tab_27 */
+static CompEntry compose_tab_27[] = {
+{168, 8129, 0, NULL, NULL},
+{776, 0, 2, compose_tab_27_1, hash_compose_tab_27_1},
+{787, 0, 10, compose_tab_27_2, hash_compose_tab_27_2},
+{788, 0, 11, compose_tab_27_3, hash_compose_tab_27_3},
+{837, 0, 3, compose_tab_27_4, hash_compose_tab_27_4},
+{945, 8118, 0, NULL, NULL},
+{951, 8134, 0, NULL, NULL},
+{953, 8150, 0, NULL, NULL},
+{965, 8166, 0, NULL, NULL},
+{969, 8182, 0, NULL, NULL},
+{8127, 8143, 0, NULL, NULL},
+{8190, 8159, 0, NULL, NULL}
+}; /* compose_tab_27 */
+static int hash_compose_tab_28[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_28 */
+static CompEntry compose_tab_28[] = {
+{913, 8124, 0, NULL, NULL},
+{919, 8140, 0, NULL, NULL},
+{937, 8188, 0, NULL, NULL},
+{945, 8115, 0, NULL, NULL},
+{951, 8131, 0, NULL, NULL},
+{969, 8179, 0, NULL, NULL}
+}; /* compose_tab_28 */
+static int hash_compose_tab_29[4] =
+{0,-1,1,-1}; /* hash_compose_tab_29 */
+static CompEntry compose_tab_29[] = {
+{1488, 64302, 0, NULL, NULL},
+{1522, 64287, 0, NULL, NULL}
+}; /* compose_tab_29 */
+static int hash_compose_tab_30[2] =
+{0,-1}; /* hash_compose_tab_30 */
+static CompEntry compose_tab_30[] = {
+{1488, 64303, 0, NULL, NULL}
+}; /* compose_tab_30 */
+static int hash_compose_tab_31[2] =
+{-1,0}; /* hash_compose_tab_31 */
+static CompEntry compose_tab_31[] = {
+{1493, 64331, 0, NULL, NULL}
+}; /* compose_tab_31 */
+static int hash_compose_tab_32[44] =
+{7,8,9,10,11,-1,12,-1,13,14,-1,15,16,-1,17,18,19,20,21,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,5,6,-1}; /* hash_compose_tab_32 */
+static CompEntry compose_tab_32[] = {
+{1488, 64304, 0, NULL, NULL},
+{1489, 64305, 0, NULL, NULL},
+{1490, 64306, 0, NULL, NULL},
+{1491, 64307, 0, NULL, NULL},
+{1492, 64308, 0, NULL, NULL},
+{1493, 64309, 0, NULL, NULL},
+{1494, 64310, 0, NULL, NULL},
+{1496, 64312, 0, NULL, NULL},
+{1497, 64313, 0, NULL, NULL},
+{1498, 64314, 0, NULL, NULL},
+{1499, 64315, 0, NULL, NULL},
+{1500, 64316, 0, NULL, NULL},
+{1502, 64318, 0, NULL, NULL},
+{1504, 64320, 0, NULL, NULL},
+{1505, 64321, 0, NULL, NULL},
+{1507, 64323, 0, NULL, NULL},
+{1508, 64324, 0, NULL, NULL},
+{1510, 64326, 0, NULL, NULL},
+{1511, 64327, 0, NULL, NULL},
+{1512, 64328, 0, NULL, NULL},
+{1513, 64329, 0, NULL, NULL},
+{1514, 64330, 0, NULL, NULL}
+}; /* compose_tab_32 */
+static int hash_compose_tab_33[6] =
+{-1,0,2,-1,-1,1}; /* hash_compose_tab_33 */
+static CompEntry compose_tab_33[] = {
+{1489, 64332, 0, NULL, NULL},
+{1499, 64333, 0, NULL, NULL},
+{1508, 64334, 0, NULL, NULL}
+}; /* compose_tab_33 */
+static int hash_compose_tab_34_0[2] =
+{-1,0}; /* hash_compose_tab_34_0 */
+static CompEntry compose_tab_34_0[] = {
+{1513, 64300, 0, NULL, NULL}
+}; /* compose_tab_34_0 */
+static int hash_compose_tab_34[4] =
+{0,1,-1,-1}; /* hash_compose_tab_34 */
+static CompEntry compose_tab_34[] = {
+{1468, 0, 1, compose_tab_34_0, hash_compose_tab_34_0},
+{1513, 64298, 0, NULL, NULL}
+}; /* compose_tab_34 */
+static int hash_compose_tab_35_0[2] =
+{-1,0}; /* hash_compose_tab_35_0 */
+static CompEntry compose_tab_35_0[] = {
+{1513, 64301, 0, NULL, NULL}
+}; /* compose_tab_35_0 */
+static int hash_compose_tab_35[4] =
+{0,1,-1,-1}; /* hash_compose_tab_35 */
+static CompEntry compose_tab_35[] = {
+{1468, 0, 1, compose_tab_35_0, hash_compose_tab_35_0},
+{1513, 64299, 0, NULL, NULL}
+}; /* compose_tab_35 */
+static int hash_compose_tab_36[22] =
+{3,10,-1,-1,-1,4,5,-1,-1,-1,-1,-1,6,-1,-1,0,1,2,7,8,9,-1}; /* hash_compose_tab_36 */
+static CompEntry compose_tab_36[] = {
+{2325, 2392, 0, NULL, NULL},
+{2326, 2393, 0, NULL, NULL},
+{2327, 2394, 0, NULL, NULL},
+{2332, 2395, 0, NULL, NULL},
+{2337, 2396, 0, NULL, NULL},
+{2338, 2397, 0, NULL, NULL},
+{2344, 2345, 0, NULL, NULL},
+{2347, 2398, 0, NULL, NULL},
+{2351, 2399, 0, NULL, NULL},
+{2352, 2353, 0, NULL, NULL},
+{2355, 2356, 0, NULL, NULL}
+}; /* compose_tab_36 */
+static int hash_compose_tab_37[8] =
+{-1,0,1,-1,2,-1,-1,3}; /* hash_compose_tab_37 */
+static CompEntry compose_tab_37[] = {
+{2465, 2524, 0, NULL, NULL},
+{2466, 2525, 0, NULL, NULL},
+{2476, 2480, 0, NULL, NULL},
+{2479, 2527, 0, NULL, NULL}
+}; /* compose_tab_37 */
+static int hash_compose_tab_38[2] =
+{-1,0}; /* hash_compose_tab_38 */
+static CompEntry compose_tab_38[] = {
+{2503, 2507, 0, NULL, NULL}
+}; /* compose_tab_38 */
+static int hash_compose_tab_39[2] =
+{-1,0}; /* hash_compose_tab_39 */
+static CompEntry compose_tab_39[] = {
+{2503, 2508, 0, NULL, NULL}
+}; /* compose_tab_39 */
+static int hash_compose_tab_40[10] =
+{-1,-1,0,1,3,4,-1,-1,2,-1}; /* hash_compose_tab_40 */
+static CompEntry compose_tab_40[] = {
+{2582, 2649, 0, NULL, NULL},
+{2583, 2650, 0, NULL, NULL},
+{2588, 2651, 0, NULL, NULL},
+{2593, 2652, 0, NULL, NULL},
+{2603, 2654, 0, NULL, NULL}
+}; /* compose_tab_40 */
+static int hash_compose_tab_41[6] =
+{1,2,-1,-1,-1,0}; /* hash_compose_tab_41 */
+static CompEntry compose_tab_41[] = {
+{2849, 2908, 0, NULL, NULL},
+{2850, 2909, 0, NULL, NULL},
+{2863, 2911, 0, NULL, NULL}
+}; /* compose_tab_41 */
+static int hash_compose_tab_42[2] =
+{-1,0}; /* hash_compose_tab_42 */
+static CompEntry compose_tab_42[] = {
+{2887, 2891, 0, NULL, NULL}
+}; /* compose_tab_42 */
+static int hash_compose_tab_43[2] =
+{-1,0}; /* hash_compose_tab_43 */
+static CompEntry compose_tab_43[] = {
+{2887, 2888, 0, NULL, NULL}
+}; /* compose_tab_43 */
+static int hash_compose_tab_44[2] =
+{-1,0}; /* hash_compose_tab_44 */
+static CompEntry compose_tab_44[] = {
+{2887, 2892, 0, NULL, NULL}
+}; /* compose_tab_44 */
+static int hash_compose_tab_45[4] =
+{-1,-1,0,1}; /* hash_compose_tab_45 */
+static CompEntry compose_tab_45[] = {
+{3014, 3018, 0, NULL, NULL},
+{3015, 3019, 0, NULL, NULL}
+}; /* compose_tab_45 */
+static int hash_compose_tab_46[4] =
+{-1,-1,0,1}; /* hash_compose_tab_46 */
+static CompEntry compose_tab_46[] = {
+{2962, 2964, 0, NULL, NULL},
+{3014, 3020, 0, NULL, NULL}
+}; /* compose_tab_46 */
+static int hash_compose_tab_47[2] =
+{0,-1}; /* hash_compose_tab_47 */
+static CompEntry compose_tab_47[] = {
+{3142, 3144, 0, NULL, NULL}
+}; /* compose_tab_47 */
+static int hash_compose_tab_48[2] =
+{0,-1}; /* hash_compose_tab_48 */
+static CompEntry compose_tab_48[] = {
+{3270, 3274, 0, NULL, NULL}
+}; /* compose_tab_48 */
+static int hash_compose_tab_49_1[2] =
+{0,-1}; /* hash_compose_tab_49_1 */
+static CompEntry compose_tab_49_1[] = {
+{3270, 3275, 0, NULL, NULL}
+}; /* compose_tab_49_1 */
+static int hash_compose_tab_49[6] =
+{2,-1,1,-1,-1,0}; /* hash_compose_tab_49 */
+static CompEntry compose_tab_49[] = {
+{3263, 3264, 0, NULL, NULL},
+{3266, 0, 1, compose_tab_49_1, hash_compose_tab_49_1},
+{3270, 3271, 0, NULL, NULL}
+}; /* compose_tab_49 */
+static int hash_compose_tab_50[2] =
+{0,-1}; /* hash_compose_tab_50 */
+static CompEntry compose_tab_50[] = {
+{3270, 3272, 0, NULL, NULL}
+}; /* compose_tab_50 */
+static int hash_compose_tab_51[4] =
+{-1,-1,0,1}; /* hash_compose_tab_51 */
+static CompEntry compose_tab_51[] = {
+{3398, 3402, 0, NULL, NULL},
+{3399, 3403, 0, NULL, NULL}
+}; /* compose_tab_51 */
+static int hash_compose_tab_52[2] =
+{0,-1}; /* hash_compose_tab_52 */
+static CompEntry compose_tab_52[] = {
+{3398, 3404, 0, NULL, NULL}
+}; /* compose_tab_52 */
+static int hash_compose_tab_53[2] =
+{-1,0}; /* hash_compose_tab_53 */
+static CompEntry compose_tab_53[] = {
+{3661, 3635, 0, NULL, NULL}
+}; /* compose_tab_53 */
+static int hash_compose_tab_54[2] =
+{-1,0}; /* hash_compose_tab_54 */
+static CompEntry compose_tab_54[] = {
+{3789, 3763, 0, NULL, NULL}
+}; /* compose_tab_54 */
+static int hash_compose_tab_55_2[4] =
+{-1,-1,0,1}; /* hash_compose_tab_55_2 */
+static CompEntry compose_tab_55_2[] = {
+{4018, 3959, 0, NULL, NULL},
+{4019, 3961, 0, NULL, NULL}
+}; /* compose_tab_55_2 */
+static int hash_compose_tab_55[6] =
+{0,-1,1,2,-1,-1}; /* hash_compose_tab_55 */
+static CompEntry compose_tab_55[] = {
+{3954, 3955, 0, NULL, NULL},
+{3956, 3957, 0, NULL, NULL},
+{3968, 0, 2, compose_tab_55_2, hash_compose_tab_55_2}
+}; /* compose_tab_55 */
+static int hash_compose_tab_56[4] =
+{-1,-1,0,1}; /* hash_compose_tab_56 */
+static CompEntry compose_tab_56[] = {
+{4018, 3958, 0, NULL, NULL},
+{4019, 3960, 0, NULL, NULL}
+}; /* compose_tab_56 */
+static int hash_compose_tab_57[4] =
+{0,1,-1,-1}; /* hash_compose_tab_57 */
+static CompEntry compose_tab_57[] = {
+{3904, 3945, 0, NULL, NULL},
+{3984, 4025, 0, NULL, NULL}
+}; /* compose_tab_57 */
+static int hash_compose_tab_58[20] =
+{-1,2,7,-1,-1,-1,0,3,5,8,-1,4,9,-1,-1,-1,1,6,-1,-1}; /* hash_compose_tab_58 */
+static CompEntry compose_tab_58[] = {
+{3906, 3907, 0, NULL, NULL},
+{3916, 3917, 0, NULL, NULL},
+{3921, 3922, 0, NULL, NULL},
+{3926, 3927, 0, NULL, NULL},
+{3931, 3932, 0, NULL, NULL},
+{3986, 3987, 0, NULL, NULL},
+{3996, 3997, 0, NULL, NULL},
+{4001, 4002, 0, NULL, NULL},
+{4006, 4007, 0, NULL, NULL},
+{4011, 4012, 0, NULL, NULL}
+}; /* compose_tab_58 */
+static int hash_compose_tab_59[96] =
+{33,12,34,-1,13,35,14,36,15,37,-1,-1,-1,-1,-1,16,38,-1,17,39,-1,18,40,-1,19,
+ 41,-1,20,42,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,43,44,45,
+ 46,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,21,47,-1,-1,-1,-1,-1,-1,-1,0,22,-1,-1,-1,1,
+ 23,2,24,3,25,4,26,5,27,6,28,7,29,8,30,9,31,10,32,11}; /* hash_compose_tab_59 */
+static CompEntry compose_tab_59[] = {
+{12358, 12436, 0, NULL, NULL},
+{12363, 12364, 0, NULL, NULL},
+{12365, 12366, 0, NULL, NULL},
+{12367, 12368, 0, NULL, NULL},
+{12369, 12370, 0, NULL, NULL},
+{12371, 12372, 0, NULL, NULL},
+{12373, 12374, 0, NULL, NULL},
+{12375, 12376, 0, NULL, NULL},
+{12377, 12378, 0, NULL, NULL},
+{12379, 12380, 0, NULL, NULL},
+{12381, 12382, 0, NULL, NULL},
+{12383, 12384, 0, NULL, NULL},
+{12385, 12386, 0, NULL, NULL},
+{12388, 12389, 0, NULL, NULL},
+{12390, 12391, 0, NULL, NULL},
+{12392, 12393, 0, NULL, NULL},
+{12399, 12400, 0, NULL, NULL},
+{12402, 12403, 0, NULL, NULL},
+{12405, 12406, 0, NULL, NULL},
+{12408, 12409, 0, NULL, NULL},
+{12411, 12412, 0, NULL, NULL},
+{12445, 12446, 0, NULL, NULL},
+{12454, 12532, 0, NULL, NULL},
+{12459, 12460, 0, NULL, NULL},
+{12461, 12462, 0, NULL, NULL},
+{12463, 12464, 0, NULL, NULL},
+{12465, 12466, 0, NULL, NULL},
+{12467, 12468, 0, NULL, NULL},
+{12469, 12470, 0, NULL, NULL},
+{12471, 12472, 0, NULL, NULL},
+{12473, 12474, 0, NULL, NULL},
+{12475, 12476, 0, NULL, NULL},
+{12477, 12478, 0, NULL, NULL},
+{12479, 12480, 0, NULL, NULL},
+{12481, 12482, 0, NULL, NULL},
+{12484, 12485, 0, NULL, NULL},
+{12486, 12487, 0, NULL, NULL},
+{12488, 12489, 0, NULL, NULL},
+{12495, 12496, 0, NULL, NULL},
+{12498, 12499, 0, NULL, NULL},
+{12501, 12502, 0, NULL, NULL},
+{12504, 12505, 0, NULL, NULL},
+{12507, 12508, 0, NULL, NULL},
+{12527, 12535, 0, NULL, NULL},
+{12528, 12536, 0, NULL, NULL},
+{12529, 12537, 0, NULL, NULL},
+{12530, 12538, 0, NULL, NULL},
+{12541, 12542, 0, NULL, NULL}
+}; /* compose_tab_59 */
+static int hash_compose_tab_60[20] =
+{-1,7,1,-1,8,2,-1,9,3,-1,-1,4,-1,-1,-1,5,-1,-1,6,0}; /* hash_compose_tab_60 */
+static CompEntry compose_tab_60[] = {
+{12399, 12401, 0, NULL, NULL},
+{12402, 12404, 0, NULL, NULL},
+{12405, 12407, 0, NULL, NULL},
+{12408, 12410, 0, NULL, NULL},
+{12411, 12413, 0, NULL, NULL},
+{12495, 12497, 0, NULL, NULL},
+{12498, 12500, 0, NULL, NULL},
+{12501, 12503, 0, NULL, NULL},
+{12504, 12506, 0, NULL, NULL},
+{12507, 12509, 0, NULL, NULL}
+}; /* compose_tab_60 */
+static int hash_compose_tab[122] =
+{30,31,52,60,32,-1,-1,33,-1,34,35,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,-1,5,6,7,8,9,10,11,12,36,13,37,14,
+ 38,15,16,55,40,-1,-1,-1,-1,17,56,-1,-1,-1,-1,-1,41,18,19,20,42,21,22,-1,45,
+ 39,-1,23,24,-1,25,26,-1,-1,-1,-1,-1,-1,-1,-1,48,-1,43,44,51,53,-1,-1,27,46,
+ 54,28,-1,-1,47,-1,-1,-1,-1,49,50,-1,-1,57,-1,58,59,29}; /* hash_compose_tab */
+static CompEntry compose_tab[] = {
+{768, 0, 39, compose_tab_0, hash_compose_tab_0},
+{769, 0, 70, compose_tab_1, hash_compose_tab_1},
+{770, 0, 27, compose_tab_2, hash_compose_tab_2},
+{771, 0, 19, compose_tab_3, hash_compose_tab_3},
+{772, 0, 28, compose_tab_4, hash_compose_tab_4},
+{774, 0, 30, compose_tab_5, hash_compose_tab_5},
+{775, 0, 40, compose_tab_6, hash_compose_tab_6},
+{776, 0, 50, compose_tab_7, hash_compose_tab_7},
+{777, 0, 15, compose_tab_8, hash_compose_tab_8},
+{778, 0, 6, compose_tab_9, hash_compose_tab_9},
+{779, 0, 6, compose_tab_10, hash_compose_tab_10},
+{780, 0, 34, compose_tab_11, hash_compose_tab_11},
+{781, 0, 17, compose_tab_12, hash_compose_tab_12},
+{783, 0, 14, compose_tab_13, hash_compose_tab_13},
+{785, 0, 12, compose_tab_14, hash_compose_tab_14},
+{787, 0, 15, compose_tab_15, hash_compose_tab_15},
+{788, 0, 17, compose_tab_16, hash_compose_tab_16},
+{795, 0, 4, compose_tab_17, hash_compose_tab_17},
+{803, 0, 39, compose_tab_18, hash_compose_tab_18},
+{804, 0, 2, compose_tab_19, hash_compose_tab_19},
+{805, 0, 2, compose_tab_20, hash_compose_tab_20},
+{807, 0, 20, compose_tab_21, hash_compose_tab_21},
+{808, 0, 10, compose_tab_22, hash_compose_tab_22},
+{813, 0, 12, compose_tab_23, hash_compose_tab_23},
+{814, 0, 2, compose_tab_24, hash_compose_tab_24},
+{816, 0, 6, compose_tab_25, hash_compose_tab_25},
+{817, 0, 17, compose_tab_26, hash_compose_tab_26},
+{834, 0, 12, compose_tab_27, hash_compose_tab_27},
+{837, 0, 6, compose_tab_28, hash_compose_tab_28},
+{1463, 0, 2, compose_tab_29, hash_compose_tab_29},
+{1464, 0, 1, compose_tab_30, hash_compose_tab_30},
+{1465, 0, 1, compose_tab_31, hash_compose_tab_31},
+{1468, 0, 22, compose_tab_32, hash_compose_tab_32},
+{1471, 0, 3, compose_tab_33, hash_compose_tab_33},
+{1473, 0, 2, compose_tab_34, hash_compose_tab_34},
+{1474, 0, 2, compose_tab_35, hash_compose_tab_35},
+{2364, 0, 11, compose_tab_36, hash_compose_tab_36},
+{2492, 0, 4, compose_tab_37, hash_compose_tab_37},
+{2494, 0, 1, compose_tab_38, hash_compose_tab_38},
+{2519, 0, 1, compose_tab_39, hash_compose_tab_39},
+{2620, 0, 5, compose_tab_40, hash_compose_tab_40},
+{2876, 0, 3, compose_tab_41, hash_compose_tab_41},
+{2878, 0, 1, compose_tab_42, hash_compose_tab_42},
+{2902, 0, 1, compose_tab_43, hash_compose_tab_43},
+{2903, 0, 1, compose_tab_44, hash_compose_tab_44},
+{3006, 0, 2, compose_tab_45, hash_compose_tab_45},
+{3031, 0, 2, compose_tab_46, hash_compose_tab_46},
+{3158, 0, 1, compose_tab_47, hash_compose_tab_47},
+{3266, 0, 1, compose_tab_48, hash_compose_tab_48},
+{3285, 0, 3, compose_tab_49, hash_compose_tab_49},
+{3286, 0, 1, compose_tab_50, hash_compose_tab_50},
+{3390, 0, 2, compose_tab_51, hash_compose_tab_51},
+{3415, 0, 1, compose_tab_52, hash_compose_tab_52},
+{3634, 0, 1, compose_tab_53, hash_compose_tab_53},
+{3762, 0, 1, compose_tab_54, hash_compose_tab_54},
+{3953, 0, 3, compose_tab_55, hash_compose_tab_55},
+{3968, 0, 2, compose_tab_56, hash_compose_tab_56},
+{4021, 0, 2, compose_tab_57, hash_compose_tab_57},
+{4023, 0, 10, compose_tab_58, hash_compose_tab_58},
+{12441, 0, 48, compose_tab_59, hash_compose_tab_59},
+{12442, 0, 10, compose_tab_60, hash_compose_tab_60}
+}; /* compose_tab */
+#define COMP_CANDIDATE_MAP_OFFSET 24
+static Uint32 comp_candidate_map[] = {
+ 0x081ABFDFU,
+ 0x000361B8U,
+ 0x00000024U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x93800000U,
+ 0x00000006U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x10000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x50000000U,
+ 0x00800000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x10000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x50000000U,
+ 0x00C00000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x40000000U,
+ 0x00800000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00400000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00600004U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x40000000U,
+ 0x00800000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00040000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00040000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00020000U,
+ 0x00000001U,
+ 0x00A00000U
+};
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 89c6625550..524db2a2eb 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1596,6 +1596,19 @@ Sint erts_binary_set_loop_limit(Sint limit);
/* erl_unicode.c */
void erts_init_unicode(void);
Sint erts_unicode_set_loop_limit(Sint limit);
+
+void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) ;
+Sint erts_native_filename_need(Eterm ioterm, int encoding);
+void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars);
+int erts_analyze_utf8(byte *source, Uint size,
+ byte **err_pos, Uint *num_chars, int *left);
+char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int allow_empty);
+
+#define ERTS_UTF8_OK 0
+#define ERTS_UTF8_INCOMPLETE 1
+#define ERTS_UTF8_ERROR 2
+#define ERTS_UTF8_ANALYZE_MORE 3
+
/* erl_trace.c */
void erts_init_trace(void);
void erts_trace_check_exiting(Eterm exiting);
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 0d15272aa8..27c5f99320 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1253,6 +1253,22 @@ char* win32_errorstr(int);
#endif
+/************************************************************************
+ * Find out the native filename encoding of the process (look at locale of
+ * Unix processes and just do UTF16 on windows
+ ************************************************************************/
+#define ERL_FILENAME_UNKNOWN 0
+#define ERL_FILENAME_LATIN1 1
+#define ERL_FILENAME_UTF8 2
+#define ERL_FILENAME_UTF8_MAC 3
+#define ERL_FILENAME_WIN_WCHAR 4
+
+int erts_get_native_filename_encoding(void);
+/* The set function is only to be used by erl_init! */
+void erts_set_user_requested_filename_encoding(int encoding);
+int erts_get_user_requested_filename_encoding(void);
+
+void erts_init_sys_common_misc(void);
#endif