aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam')
-rw-r--r--erts/emulator/beam/bif.tab7
-rw-r--r--erts/emulator/beam/binary.c10
-rw-r--r--erts/emulator/beam/erl_bif_binary.c2
-rw-r--r--erts/emulator/beam/erl_bif_port.c91
-rw-r--r--erts/emulator/beam/erl_binary.h2
-rw-r--r--erts/emulator/beam/erl_init.c23
-rw-r--r--erts/emulator/beam/erl_unicode.c902
-rw-r--r--erts/emulator/beam/erl_unicode_normalize.h1687
-rw-r--r--erts/emulator/beam/global.h13
-rw-r--r--erts/emulator/beam/sys.h16
10 files changed, 2652 insertions, 101 deletions
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 0674aae77f..60b4b1946b 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -795,6 +795,13 @@ bif erlang:nif_error/1
bif erlang:nif_error/2
#
+# Helpers for unicode filenames
+#
+bif prim_file:internal_name2native/1
+bif prim_file:internal_native2name/1
+bif prim_file:internal_normalize_utf8/1
+bif file:native_name_encoding/0
+#
# Obsolete
#
diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c
index 8ee8fbcb29..4be869f269 100644
--- a/erts/emulator/beam/binary.c
+++ b/erts/emulator/beam/binary.c
@@ -217,8 +217,8 @@ erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, ErtsAlcType_t al
return bytes;
}
-static Eterm
-bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs)
+Eterm
+erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs)
{
if (bitoffs == 0) {
while (size) {
@@ -263,7 +263,7 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1)
Eterm* hp = HAlloc(BIF_P, 2 * size);
byte* bytes = binary_bytes(real_bin)+offset;
- BIF_RET(bin_bytes_to_list(NIL, hp, bytes, size, bitoffs));
+ BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs));
}
error:
@@ -295,7 +295,7 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3)
}
i = stop-start+1;
hp = HAlloc(BIF_P, 2*i);
- BIF_RET(bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs));
+ BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs));
error:
BIF_ERROR(BIF_P, BADARG);
@@ -339,7 +339,7 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1)
previous = CONS(hp, make_binary(last), previous);
hp += 2;
}
- BIF_RET(bin_bytes_to_list(previous, hp, bytes, size, bitoffs));
+ BIF_RET(erts_bin_bytes_to_list(previous, hp, bytes, size, bitoffs));
}
diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c
index b6a445c55c..684fa5d12f 100644
--- a/erts/emulator/beam/erl_bif_binary.c
+++ b/erts/emulator/beam/erl_bif_binary.c
@@ -1477,7 +1477,7 @@ BIF_RETTYPE binary_matches_3(BIF_ALIST_3)
goto badarg;
}
if (hsend == 0) {
- BIF_RET(am_nomatch);
+ BIF_RET(NIL);
}
if (is_tuple(BIF_ARG_2)) {
tp = tuple_val(BIF_ARG_2);
diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c
index 378c5e73fd..fbc92b9730 100644
--- a/erts/emulator/beam/erl_bif_port.c
+++ b/erts/emulator/beam/erl_bif_port.c
@@ -610,6 +610,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
int binary_io;
int soft_eof;
Sint linebuf;
+ Eterm edir = NIL;
byte dir[MAXPATHLEN];
/* These are the defaults */
@@ -686,19 +687,10 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
} else if (option == am_arg0) {
char *a0;
- int n;
- if (is_nil(*tp)) {
- n = 0;
- } else if( (n = is_string(*tp)) == 0) {
+
+ if ((a0 = erts_convert_filename_to_native(*tp, ERTS_ALC_T_TMP, 1)) == NULL) {
goto badarg;
}
- a0 = (char *) erts_alloc(ERTS_ALC_T_TMP,
- (n + 1) * sizeof(byte));
- if (intlist_to_buf(*tp, a0, n) != n) {
- erl_exit(1, "%s:%d: Internal error\n",
- __FILE__, __LINE__);
- }
- a0[n] = '\0';
if (opts.argv == NULL) {
opts.argv = erts_alloc(ERTS_ALC_T_TMP,
2 * sizeof(char **));
@@ -711,22 +703,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
opts.argv[0] = a0;
}
} else if (option == am_cd) {
- Eterm iolist;
- DeclareTmpHeap(heap,4,p);
- int r;
-
- UseTmpHeap(4,p);
- heap[0] = *tp;
- heap[1] = make_list(heap+2);
- heap[2] = make_small(0);
- heap[3] = NIL;
- iolist = make_list(heap);
- r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN);
- UnUseTmpHeap(4,p);
- if (r < 0) {
- goto badarg;
- }
- opts.wd = (char *) dir;
+ edir = *tp;
} else {
goto badarg;
}
@@ -838,19 +815,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
goto badarg;
}
name = tp[1];
- if (is_atom(name)) {
- name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP,
- atom_tab(atom_val(name))->len+1);
- sys_memcpy((void *) name_buf,
- (void *) atom_tab(atom_val(name))->name,
- atom_tab(atom_val(name))->len);
- name_buf[atom_tab(atom_val(name))->len] = '\0';
- } else if ((i = is_string(name))) {
- name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1);
- if (intlist_to_buf(name, name_buf, i) != i)
- erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
- name_buf[i] = '\0';
- } else {
+ if ((name_buf = erts_convert_filename_to_native(name,ERTS_ALC_T_TMP,0)) == NULL) {
goto badarg;
}
opts.spawn_type = ERTS_SPAWN_EXECUTABLE;
@@ -892,7 +857,33 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
/* Argument vector only if explicit spawn_executable */
goto badarg;
}
-
+
+ if (edir != NIL) {
+ /* A working directory is expressed differently if spawn_executable, i.e. Unicode is handles
+ for spawn_executable... */
+ if (opts.spawn_type != ERTS_SPAWN_EXECUTABLE) {
+ Eterm iolist;
+ DeclareTmpHeap(heap,4,p);
+ int r;
+
+ UseTmpHeap(4,p);
+ heap[0] = edir;
+ heap[1] = make_list(heap+2);
+ heap[2] = make_small(0);
+ heap[3] = NIL;
+ iolist = make_list(heap);
+ r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN);
+ UnUseTmpHeap(4,p);
+ if (r < 0) {
+ goto badarg;
+ }
+ opts.wd = (char *) dir;
+ } else {
+ if ((opts.wd = erts_convert_filename_to_native(edir,ERTS_ALC_T_TMP,0)) == NULL) {
+ goto badarg;
+ }
+ }
+ }
if (driver != &spawn_driver && opts.exit_status) {
goto badarg;
@@ -941,6 +932,9 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
if (opts.argv) {
free_args(opts.argv);
}
+ if (opts.wd && opts.wd != ((char *)dir)) {
+ erts_free(ERTS_ALC_T_TMP, (void *) opts.wd);
+ }
return port_num;
badarg:
@@ -950,6 +944,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump)
#undef OPEN_PORT_ERROR
}
+/* Arguments can be given i unicode and as raw binaries, convert filename is used to convert */
static char **convert_args(Eterm l)
{
char **pp;
@@ -966,22 +961,14 @@ static char **convert_args(Eterm l)
pp[i++] = erts_default_arg0;
while (is_list(l)) {
str = CAR(list_val(l));
-
- if (is_nil(str)) {
- n = 0;
- } else if( (n = is_string(str)) == 0) {
- /* Not a string... */
+ if ((b = erts_convert_filename_to_native(str,ERTS_ALC_T_TMP,1)) == NULL) {
int j;
for (j = 1; j < i; ++j)
erts_free(ERTS_ALC_T_TMP, pp[j]);
erts_free(ERTS_ALC_T_TMP, pp);
return NULL;
- }
- b = (char *) erts_alloc(ERTS_ALC_T_TMP, (n + 1) * sizeof(byte));
- pp[i++] = (char *) b;
- if (intlist_to_buf(str, b, n) != n)
- erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__);
- b[n] = '\0';
+ }
+ pp[i++] = b;
l = CDR(list_val(l));
}
pp[i] = NULL;
diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h
index a569fe2e85..bdf0fe23fc 100644
--- a/erts/emulator/beam/erl_binary.h
+++ b/erts/emulator/beam/erl_binary.h
@@ -152,6 +152,8 @@ do { \
void erts_init_binary(void);
byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, ErtsAlcType_t, unsigned extra);
+/* Used by unicode module */
+Eterm erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs);
/*
* Common implementation for erlang:list_to_binary/1 and binary:list_to_bin/1
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index a9f4f041ac..464ee750f7 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -251,6 +251,7 @@ erl_init(int ncpu)
erts_init_monitors();
erts_init_gc();
init_time();
+ erts_init_sys_common_misc();
erts_init_process(ncpu);
erts_init_scheduling(use_multi_run_queue,
no_schedulers,
@@ -907,7 +908,27 @@ erl_start(int argc, char **argv)
VERBOSE(DEBUG_SYSTEM,
("using display items %d\n",display_items));
break;
-
+ case 'f':
+ if (!strncmp(argv[i],"-fn",3)) {
+ arg = get_arg(argv[i]+3, argv[i+1], &i);
+ switch (*arg) {
+ case 'u':
+ erts_set_user_requested_filename_encoding(ERL_FILENAME_UTF8);
+ break;
+ case 'l':
+ erts_set_user_requested_filename_encoding(ERL_FILENAME_LATIN1);
+ break;
+ case 'a':
+ erts_set_user_requested_filename_encoding(ERL_FILENAME_UNKNOWN);
+ default:
+ erts_fprintf(stderr, "bad filename encoding %s, can be (l,u or a)\n", arg);
+ erts_usage();
+ }
+ break;
+ } else {
+ erts_fprintf(stderr, "%s unknown flag %s\n", argv[0], argv[i]);
+ erts_usage();
+ }
case 'l':
display_loads++;
break;
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index d01a3661f9..72207df621 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -30,6 +30,8 @@
#include "big.h"
#include "erl_unicode.h"
+#include "erl_unicode_normalize.h"
+
typedef struct _restart_context {
byte *bytes;
@@ -54,13 +56,6 @@ static BIF_RETTYPE finalize_list_to_list(Process *p,
Uint num_resulting_chars,
int state, int left,
Eterm tail);
-static int analyze_utf8(byte *source, Uint size,
- byte **err_pos, Uint *num_chars, int *left);
-#define UTF8_OK 0
-#define UTF8_INCOMPLETE 1
-#define UTF8_ERROR 2
-#define UTF8_ANALYZE_MORE 3
-
static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3);
static BIF_RETTYPE characters_to_list_trap_1(BIF_ALIST_3);
static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3);
@@ -463,7 +458,7 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */
}
objp = list_val(ioterm);
obj = CAR(objp);
- if (!is_byte(obj))
+ if (!is_small(obj))
break;
}
} else if (is_nil(obj)) {
@@ -970,11 +965,11 @@ static int is_valid_utf8(Eterm orig_bin)
bytes = erts_get_aligned_binary_bytes(orig_bin, &temp_alloc);
}
size = binary_size(orig_bin);
- ret = analyze_utf8(bytes,
+ ret = erts_analyze_utf8(bytes,
size,
&endpos,&numchar,NULL);
erts_free_aligned_binary_bytes(temp_alloc);
- return (ret == UTF8_OK);
+ return (ret == ERTS_UTF8_OK);
}
BIF_RETTYPE unicode_characters_to_binary_2(BIF_ALIST_2)
@@ -1084,14 +1079,14 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char
hp += 2;
rest_term = CONS(hp,leftover_bin,rest_term);
}
- BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, UTF8_ERROR, left, NIL));
+ BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, ERTS_UTF8_ERROR, left, NIL));
} else if (rest_term == NIL && num_leftovers != 0) {
Eterm leftover_bin = new_binary(p, leftover, num_leftovers);
if (check_leftovers(leftover,num_leftovers) != 0) {
- BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_ERROR,
+ BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_ERROR,
left, NIL));
} else {
- BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_INCOMPLETE,
+ BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_INCOMPLETE,
left, NIL));
}
} else { /* All OK */
@@ -1107,11 +1102,11 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char
rc.num_processed_bytes = 0; /* not used */
rc.num_bytes_to_process = pos;
rc.num_resulting_chars = characters;
- rc.state = UTF8_OK; /* not used */
+ rc.state = ERTS_UTF8_OK; /* not used */
BIF_TRAP3(&characters_to_list_trap_1_exp, p, make_magic_bin_for_restart(p,&rc),
rest_term, latin1);
} else { /* Success */
- BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, UTF8_OK, left, NIL));
+ BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, ERTS_UTF8_OK, left, NIL));
}
}
}
@@ -1205,7 +1200,7 @@ BIF_RETTYPE unicode_characters_to_list_2(BIF_ALIST_2)
* When input to characters_to_list is a plain binary and the format is 'unicode', we do
* a faster analyze and size count with this function.
*/
-static int analyze_utf8(byte *source, Uint size,
+int erts_analyze_utf8(byte *source, Uint size,
byte **err_pos, Uint *num_chars, int *left)
{
*err_pos = source;
@@ -1216,60 +1211,60 @@ static int analyze_utf8(byte *source, Uint size,
--size;
} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
if (size < 2) {
- return UTF8_INCOMPLETE;
+ return ERTS_UTF8_INCOMPLETE;
}
if (((source[1] & ((byte) 0xC0)) != 0x80) ||
((*source) < 0xC2) /* overlong */) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
source += 2;
size -= 2;
} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
if (size < 3) {
- return UTF8_INCOMPLETE;
+ return ERTS_UTF8_INCOMPLETE;
}
if (((source[1] & ((byte) 0xC0)) != 0x80) ||
((source[2] & ((byte) 0xC0)) != 0x80) ||
(((*source) == 0xE0) && (source[1] < 0xA0)) /* overlong */ ) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
if ((((*source) & ((byte) 0xF)) == 0xD) &&
((source[1] & 0x20) != 0)) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
if (((*source) == 0xEF) && (source[1] == 0xBF) &&
((source[2] == 0xBE) || (source[2] == 0xBF))) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
source += 3;
size -= 3;
} else if (((*source) & ((byte) 0xF8)) == 0xF0) {
if (size < 4) {
- return UTF8_INCOMPLETE;
+ return ERTS_UTF8_INCOMPLETE;
}
if (((source[1] & ((byte) 0xC0)) != 0x80) ||
((source[2] & ((byte) 0xC0)) != 0x80) ||
((source[3] & ((byte) 0xC0)) != 0x80) ||
(((*source) == 0xF0) && (source[1] < 0x90)) /* overlong */) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
if ((((*source) & ((byte)0x7)) > 0x4U) ||
((((*source) & ((byte)0x7)) == 0x4U) &&
((source[1] & ((byte)0x3F)) > 0xFU))) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
source += 4;
size -= 4;
} else {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
++(*num_chars);
*err_pos = source;
if (left && --(*left) <= 0) {
- return UTF8_ANALYZE_MORE;
+ return ERTS_UTF8_ANALYZE_MORE;
}
}
- return UTF8_OK;
+ return ERTS_UTF8_OK;
}
/*
@@ -1304,7 +1299,7 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz,
} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
unipoint =
(((Uint) ((*source) & ((byte) 0x1F))) << 6) |
- ((Uint) (source[1] & ((byte) 0x3F)));
+ ((Uint) (source[1] & ((byte) 0x3F)));
} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
unipoint =
(((Uint) ((*source) & ((byte) 0xF))) << 12) |
@@ -1330,6 +1325,216 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz,
return ret;
}
+static int is_candidate(Uint cp)
+{
+ int index,pos;
+ if (cp < 768) return 0;
+ if (cp > 4023) {
+ if (cp == 12441 || cp == 12442) return 1;
+ return 0;
+ }
+ index = cp / 32 - COMP_CANDIDATE_MAP_OFFSET;
+ pos = cp % 32;
+ return !!(comp_candidate_map[index] & (1UL << pos));
+}
+
+static int hashsearch(int *htab, int htab_size, CompEntry *cv, Uint16 c)
+{
+ int bucket = c % htab_size;
+ while (htab[bucket] != -1 && cv[htab[bucket]].c != c)
+ bucket = (bucket + 1) % htab_size;
+ return htab[bucket];
+}
+
+#define TRANSLATE_NO 0
+#define TRANSLATE_MAYBE -1
+
+/* The s array is reversed */
+static int translate(Uint16 *s, int slen, Uint16 *res)
+{
+ /* Go backwards through buffer and match against tree */
+ int pos = 0;
+ CompEntry *cv = compose_tab;
+ int *hc = hash_compose_tab;
+ int cvs = compose_tab_size;
+ int x;
+ while (pos < slen) {
+ x = hashsearch(hc,cvs*HASH_SIZE_FACTOR,cv,s[pos]);
+ if (x < 0) {
+ return TRANSLATE_NO;
+ }
+ if (cv[x].res) {
+ *res = cv[x].res;
+ return pos;
+ }
+ cvs = cv[x].num_subs;
+ hc = cv[x].hash;
+ cv = cv[x].subs;
+ ++pos;
+ }
+ return TRANSLATE_MAYBE;
+}
+
+static void handle_first_norm(Uint16 *savepoints, int *numpointsp, Uint unipoint)
+{
+ /*erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) *numpointsp);*/
+ *numpointsp = 1;
+ savepoints[0] = (Uint16) unipoint;
+}
+
+static void cleanup_norm(Eterm **hpp, Uint16 *savepoints, int numpoints, Eterm *retp)
+{
+ Eterm *hp = *hpp;
+ int res,i;
+ Uint16 newpoint;
+ Eterm ret = *retp;
+
+ ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+ hp += 2;
+
+ for (i = 1;i < numpoints;) {
+ if(!is_candidate(savepoints[i]) ||
+ ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) {
+ ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+ hp += 2;
+ ++i;
+ } else {
+ ret = CONS(hp,make_small((Uint) newpoint),ret);
+ hp += 2;
+ i += res;
+ }
+ }
+ *retp = ret;
+}
+
+static void handle_potential_norm(Eterm **hpp, Uint16 *savepoints, int *numpointsp, Uint unipoint, Eterm *retp)
+{
+ Eterm *hp = *hpp;
+ int numpoints = *numpointsp;
+ int res,i;
+ Uint16 newpoint;
+ Eterm ret = *retp;
+
+ /* erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) numpoints);*/
+ if ((unipoint >> 16) == 0) { /* otherwise we're done here */
+ savepoints[numpoints++] = (Uint16) unipoint;
+ res = translate(savepoints,numpoints,&newpoint);
+ if (res == TRANSLATE_NO) {
+ ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+ hp += 2;
+ for (i = 1;i < numpoints;) {
+ if(!is_candidate(savepoints[i]) ||
+ ((res = translate(savepoints+i,numpoints - i, &newpoint)) == 0)) {
+ ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+ hp += 2;
+ ++i;
+ } else if (res > 0) {
+ ret = CONS(hp,make_small((Uint) newpoint),ret);
+ hp += 2;
+ i += res;
+ } else { /* res < 0 */
+ /* A "maybe", means we are not done yet */
+ int j = 0;
+ while (i < numpoints) {
+ savepoints[j++] = savepoints[i++];
+ }
+ numpoints = j;
+ goto breakaway;
+ }
+ }
+ numpoints = 0;
+ breakaway:
+ ;
+ } else if (res > 0) {
+ numpoints = 0;
+ ret = CONS(hp,make_small((Uint) newpoint),ret);
+ hp += 2;
+ } /* < 0 means go on */
+ } else {
+ /* Unconditional rollup, this character is larger than 16 bit */
+ ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+ hp += 2;
+
+ for (i = 1;i < numpoints;) {
+ if(!is_candidate(savepoints[i]) ||
+ ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) {
+ ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+ hp += 2;
+ ++i;
+ } else {
+ ret = CONS(hp,make_small((Uint) newpoint),ret);
+ hp += 2;
+ i += res;
+ }
+ }
+ ret = CONS(hp,make_small(unipoint),ret);
+ hp += 2;
+ numpoints = 0;
+ }
+ *hpp = hp;
+ *numpointsp = numpoints;
+ *retp = ret;
+}
+
+static Eterm do_utf8_to_list_normalize(Process *p, Uint num, byte *bytes, Uint sz)
+{
+ Eterm *hp,*hp_end;
+ Eterm ret;
+ byte *source;
+ Uint unipoint;
+ Uint16 savepoints[4];
+ int numpoints = 0;
+
+ ASSERT(num > 0);
+
+ hp = HAlloc(p,num * 2); /* May be to much */
+ hp_end = hp + num * 2;
+ ret = NIL;
+ source = bytes + sz;
+ while(--source >= bytes) {
+ if (((*source) & ((byte) 0x80)) == 0) {
+ unipoint = (Uint) *source;
+ } else if (((*source) & ((byte) 0xE0)) == 0xC0) {
+ unipoint =
+ (((Uint) ((*source) & ((byte) 0x1F))) << 6) |
+ ((Uint) (source[1] & ((byte) 0x3F)));
+ } else if (((*source) & ((byte) 0xF0)) == 0xE0) {
+ unipoint =
+ (((Uint) ((*source) & ((byte) 0xF))) << 12) |
+ (((Uint) (source[1] & ((byte) 0x3F))) << 6) |
+ ((Uint) (source[2] & ((byte) 0x3F)));
+ } else if (((*source) & ((byte) 0xF8)) == 0xF0) {
+ unipoint =
+ (((Uint) ((*source) & ((byte) 0x7))) << 18) |
+ (((Uint) (source[1] & ((byte) 0x3F))) << 12) |
+ (((Uint) (source[2] & ((byte) 0x3F))) << 6) |
+ ((Uint) (source[3] & ((byte) 0x3F)));
+ } else {
+ /* ignore 2#10XXXXXX */
+ continue;
+ }
+ if (numpoints) {
+ handle_potential_norm(&hp,savepoints,&numpoints,unipoint,&ret);
+ continue;
+ }
+ /* We are not building up any normalizations yet, look that we shouldn't start... */
+ if (is_candidate(unipoint)) {
+ handle_first_norm(savepoints,&numpoints,unipoint);
+ continue;
+ }
+ ret = CONS(hp,make_small(unipoint),ret);
+ hp += 2;
+ }
+ /* so, we'we looped to the beginning, do we have anything saved? */
+ if (numpoints) {
+ cleanup_norm(&hp,savepoints,numpoints,&ret);
+ }
+ if (hp_end != hp) {
+ HRelease(p,hp_end,hp);
+ }
+ return ret;
+}
+
/*
* The last step of characters_to_list, build a list from the buffer 'bytes' (created in the same way
* as for characters_to_utf8). All sizes are known in advance and most data will be held in a
@@ -1378,10 +1583,10 @@ static BIF_RETTYPE finalize_list_to_list(Process *p,
*/
free_restart(bytes);
- if (state == UTF8_INCOMPLETE) {
+ if (state == ERTS_UTF8_INCOMPLETE) {
hp = HAlloc(p,4);
ret = TUPLE3(hp,am_incomplete,converted,rest);
- } else if (state == UTF8_ERROR) {
+ } else if (state == ERTS_UTF8_ERROR) {
hp = HAlloc(p,4);
ret = TUPLE3(hp,am_error,converted,rest);
} else {
@@ -1408,7 +1613,7 @@ static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3)
/*
* Hooks into the process of decoding a binary depending on state.
- * If last_state is UTF8_ANALYZE_MORE, num_bytes_to_process
+ * If last_state is ERTS_UTF8_ANALYZE_MORE, num_bytes_to_process
* and num_resulting_chars will grow
* until we're done analyzing the binary. Then we'll eat
* the bytes to process, lowering num_bytes_to_process and num_resulting_chars,
@@ -1465,14 +1670,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
left = allowed_iterations(p);
- if (state == UTF8_ANALYZE_MORE) {
- state = analyze_utf8(bytes + num_bytes_to_process,
+ if (state == ERTS_UTF8_ANALYZE_MORE) {
+ state = erts_analyze_utf8(bytes + num_bytes_to_process,
size - num_bytes_to_process,
&endpos,&numchar,&left);
cost_to_proc(p,numchar);
num_resulting_chars += numchar;
num_bytes_to_process = endpos - bytes;
- if (state == UTF8_ANALYZE_MORE) {
+ if (state == ERTS_UTF8_ANALYZE_MORE) {
Eterm epos = erts_make_integer(num_bytes_to_process,p);
Eterm enumchar = erts_make_integer(num_resulting_chars,p);
erts_free_aligned_binary_bytes(temp_alloc);
@@ -1528,7 +1733,7 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
ErlSubBin *sb;
Eterm orig;
Uint offset;
- ASSERT(state != UTF8_OK);
+ ASSERT(state != ERTS_UTF8_OK);
hp = HAlloc(p, ERL_SUB_BIN_SIZE);
sb = (ErlSubBin *) hp;
ERTS_GET_REAL_BIN(orig_bin, orig, offset, bitoffs, bitsize);
@@ -1544,14 +1749,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
/* Done */
- if (state == UTF8_INCOMPLETE) {
+ if (state == ERTS_UTF8_INCOMPLETE) {
if (check_leftovers(bytes + num_bytes_to_process + num_processed_bytes,
b_sz) != 0) {
goto error_return;
}
hp = HAlloc(p,4);
ret = TUPLE3(hp,am_incomplete,converted,rest);
- } else if (state == UTF8_ERROR) {
+ } else if (state == ERTS_UTF8_ERROR) {
error_return:
hp = HAlloc(p,4);
ret = TUPLE3(hp,am_error,converted,rest);
@@ -1589,7 +1794,7 @@ static BIF_RETTYPE characters_to_list_trap_3(BIF_ALIST_3)
0U, /* nothing processed yet */
num_bytes_to_process,
num_resulting_chars,
- UTF8_ANALYZE_MORE, /* always this state here */
+ ERTS_UTF8_ANALYZE_MORE, /* always this state here */
NIL); /* Nothing built -> no tail yet */
}
@@ -1642,7 +1847,7 @@ static BIF_RETTYPE utf8_to_list(BIF_ALIST_1)
BIF_ERROR(BIF_P,BADARG);
}
return do_bif_utf8_to_list(BIF_P, BIF_ARG_1, 0U, 0U, 0U,
- UTF8_ANALYZE_MORE,NIL);
+ ERTS_UTF8_ANALYZE_MORE,NIL);
}
@@ -1728,8 +1933,8 @@ binary_to_atom(Process* p, Eterm bin, Eterm enc, int must_exist)
Uint n;
int reds_left = bin_size+1; /* Number of reductions left. */
- if (analyze_utf8(bytes, bin_size, &err_pos,
- &n, &reds_left) == UTF8_OK) {
+ if (erts_analyze_utf8(bytes, bin_size, &err_pos,
+ &n, &reds_left) == ERTS_UTF8_OK) {
/*
* Correct UTF-8 encoding, but too many characters to
* fit in an atom.
@@ -1813,3 +2018,616 @@ BIF_RETTYPE binary_to_existing_atom_2(BIF_ALIST_2)
{
return binary_to_atom(BIF_P, BIF_ARG_1, BIF_ARG_2, 1);
}
+
+/**********************************************************
+ * Simpler non-interruptable routines for UTF-8 and
+ * Windowish UTF-16 (restricted)
+ **********************************************************/
+/*
+ * This function is the heart of the Unicode support for
+ * open_port - spawn_executable. It converts both the name
+ * of the executable and the arguments according to the same rules
+ * as for filename conversion. That means as if your arguments are
+ * to be raw, you supply binaries, else unicode characters are allowed up to
+ * the encoding maximum (256 of the unicode max).
+ * Depending on the filename encoding standard, the vector is then
+ * converted to whatever is used, which might mean win_utf16 if on windows.
+ * Do not peek into the argument vector or filenam with ordinary
+ * string routines, that will certainly fail on some OS.
+ */
+
+char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int allow_empty)
+{
+ int encoding = erts_get_native_filename_encoding();
+ char* name_buf = NULL;
+
+ if (is_atom(name) || is_list(name) || (allow_empty && is_nil(name))) {
+ Sint need;
+ if ((need = erts_native_filename_need(name,encoding)) < 0) {
+ return NULL;
+ }
+ if (encoding == ERL_FILENAME_WIN_WCHAR) {
+ need += 2;
+ } else {
+ ++need;
+ }
+ name_buf = (char *) erts_alloc(alloc_type, need);
+ erts_native_filename_put(name,encoding,(byte *)name_buf);
+ name_buf[need-1] = 0;
+ if (encoding == ERL_FILENAME_WIN_WCHAR) {
+ name_buf[need-2] = 0;
+ }
+ } else if (is_binary(name)) {
+ byte *temp_alloc = NULL;
+ byte *bytes;
+ byte *err_pos;
+ Uint size,num_chars;
+
+ size = binary_size(name);
+ bytes = erts_get_aligned_binary_bytes(name, &temp_alloc);
+ if (encoding != ERL_FILENAME_WIN_WCHAR) {
+ /*Add 0 termination only*/
+ name_buf = (char *) erts_alloc(alloc_type, size+1);
+ memcpy(name_buf,bytes,size);
+ name_buf[size]=0;
+ } else if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK ||
+ erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) {
+ byte *p;
+ /* What to do now? Maybe latin1, so just take byte for byte instead */
+ name_buf = (char *) erts_alloc(alloc_type, (size+1)*2);
+ p = (byte *) name_buf;
+ while (size--) {
+ *p++ = *bytes++;
+ *p++ = 0;
+ }
+ *p++ = 0;
+ *p++ = 0;
+ } else { /* WIN_WCHAR and valid UTF8 */
+ name_buf = (char *) erts_alloc(alloc_type, (num_chars+1)*2);
+ erts_copy_utf8_to_utf16_little((byte *) name_buf, bytes, num_chars);
+ name_buf[num_chars*2] = 0;
+ name_buf[num_chars*2+1] = 0;
+ }
+ erts_free_aligned_binary_bytes(temp_alloc);
+ } else {
+ return NULL;
+ }
+ return name_buf;
+}
+
+
+Sint erts_native_filename_need(Eterm ioterm, int encoding)
+{
+ Eterm *objp;
+ Eterm obj;
+ DECLARE_ESTACK(stack);
+ Sint need = 0;
+
+ if (is_atom(ioterm)) {
+ Atom* ap;
+ int i;
+ ap = atom_tab(atom_val(ioterm));
+ switch (encoding) {
+ case ERL_FILENAME_LATIN1:
+ need = ap->len;
+ break;
+ case ERL_FILENAME_UTF8_MAC:
+ case ERL_FILENAME_UTF8:
+ for (i = 0; i < ap->len; i++) {
+ need += (ap->name[i] >= 0x80) ? 2 : 1;
+ }
+ break;
+ case ERL_FILENAME_WIN_WCHAR:
+ need = 2*(ap->len);
+ break;
+ default:
+ need = -1;
+ }
+ DESTROY_ESTACK(stack);
+ return need;
+ }
+
+ if (is_nil(ioterm)) {
+ DESTROY_ESTACK(stack);
+ return need;
+ }
+ if (!is_list(ioterm)) {
+ DESTROY_ESTACK(stack);
+ return (Sint) -1;
+ }
+ /* OK a list, needs to be processed in order, handling each flat list-level
+ as they occur, just like io_list_to_binary would */
+ ESTACK_PUSH(stack,ioterm);
+ while (!ESTACK_ISEMPTY(stack)) {
+ ioterm = ESTACK_POP(stack);
+ if (is_nil(ioterm)) {
+ /* ignore empty lists */
+ continue;
+ }
+ if(is_list(ioterm)) {
+L_Again: /* Restart with sublist, old listend was pushed on stack */
+ objp = list_val(ioterm);
+ obj = CAR(objp);
+ for(;;) { /* loop over one flat list of bytes and binaries
+ until sublist or list end is encountered */
+ if (is_small(obj)) { /* Always small */
+ for(;;) {
+ Uint x = unsigned_val(obj);
+ switch (encoding) {
+ case ERL_FILENAME_LATIN1:
+ if (x > 255) {
+ DESTROY_ESTACK(stack);
+ return ((Sint) -1);
+ }
+ need += 1;
+ break;
+ case ERL_FILENAME_UTF8_MAC:
+ case ERL_FILENAME_UTF8:
+ if (x < 0x80) {
+ need +=1;
+ } else if (x < 0x800) {
+ need += 2;
+ } else if (x < 0x10000) {
+ if ((x >= 0xD800 && x <= 0xDFFF) ||
+ (x == 0xFFFE) ||
+ (x == 0xFFFF)) { /* Invalid unicode range */
+ DESTROY_ESTACK(stack);
+ return ((Sint) -1);
+ }
+ need += 3;
+ } else if (x < 0x110000) {
+ need += 4;
+ } else {
+ DESTROY_ESTACK(stack);
+ return ((Sint) -1);
+ }
+ break;
+ case ERL_FILENAME_WIN_WCHAR:
+ if (x <= 0xffff) {
+ need += 2;
+ break;
+ } /* else fall throug to error */
+ default:
+ DESTROY_ESTACK(stack);
+ return ((Sint) -1);
+ }
+
+ /* everything else will give badarg later
+ in the process, so we dont check */
+ ioterm = CDR(objp);
+ if (!is_list(ioterm)) {
+ break;
+ }
+ objp = list_val(ioterm);
+ obj = CAR(objp);
+ if (!is_small(obj))
+ break;
+ }
+ } else if (is_nil(obj)) {
+ ioterm = CDR(objp);
+ if (!is_list(ioterm)) {
+ break;
+ }
+ objp = list_val(ioterm);
+ obj = CAR(objp);
+ } else if (is_list(obj)) {
+ /* push rest of list for later processing, start
+ again with sublist */
+ ESTACK_PUSH(stack,CDR(objp));
+ ioterm = obj;
+ goto L_Again;
+ } else {
+ DESTROY_ESTACK(stack);
+ return ((Sint) -1);
+ }
+ if (is_nil(ioterm) || !is_list(ioterm)) {
+ break;
+ }
+ } /* for(;;) */
+ } /* is_list(ioterm) */
+
+ if (!is_list(ioterm) && !is_nil(ioterm)) {
+ /* inproper list end */
+ DESTROY_ESTACK(stack);
+ return ((Sint) -1);
+ }
+ } /* while not estack empty */
+ DESTROY_ESTACK(stack);
+ return need;
+}
+
+void erts_native_filename_put(Eterm ioterm, int encoding, byte *p)
+{
+ Eterm *objp;
+ Eterm obj;
+ DECLARE_ESTACK(stack);
+
+ if (is_atom(ioterm)) {
+ Atom* ap;
+ int i;
+ ap = atom_tab(atom_val(ioterm));
+ switch (encoding) {
+ case ERL_FILENAME_LATIN1:
+ for (i = 0; i < ap->len; i++) {
+ *p++ = ap->name[i];
+ }
+ break;
+ case ERL_FILENAME_UTF8_MAC:
+ case ERL_FILENAME_UTF8:
+ for (i = 0; i < ap->len; i++) {
+ if(ap->name[i] < 0x80) {
+ *p++ = ap->name[i];
+ } else {
+ *p++ = (((ap->name[i]) >> 6) | ((byte) 0xC0));
+ *p++ = (((ap->name[i]) & 0x3F) | ((byte) 0x80));
+ }
+ }
+ break;
+ case ERL_FILENAME_WIN_WCHAR:
+ for (i = 0; i < ap->len; i++) {
+ /* Little endian */
+ *p++ = ap->name[i];
+ *p++ = 0;
+ }
+ break;
+ default:
+ ASSERT(0);
+ }
+ DESTROY_ESTACK(stack);
+ return;
+ }
+
+ if (is_nil(ioterm)) {
+ DESTROY_ESTACK(stack);
+ return;
+ }
+ ASSERT(is_list(ioterm));
+ /* OK a list, needs to be processed in order, handling each flat list-level
+ as they occur, just like io_list_to_binary would */
+ ESTACK_PUSH(stack,ioterm);
+ while (!ESTACK_ISEMPTY(stack)) {
+ ioterm = ESTACK_POP(stack);
+ if (is_nil(ioterm)) {
+ /* ignore empty lists */
+ continue;
+ }
+ if(is_list(ioterm)) {
+L_Again: /* Restart with sublist, old listend was pushed on stack */
+ objp = list_val(ioterm);
+ obj = CAR(objp);
+ for(;;) { /* loop over one flat list of bytes and binaries
+ until sublist or list end is encountered */
+ if (is_small(obj)) { /* Always small */
+ for(;;) {
+ Uint x = unsigned_val(obj);
+ switch (encoding) {
+ case ERL_FILENAME_LATIN1:
+ ASSERT( x < 256);
+ *p++ = (byte) x;
+ break;
+ case ERL_FILENAME_UTF8_MAC:
+ case ERL_FILENAME_UTF8:
+ if (x < 0x80) {
+ *p++ = (byte) x;
+ }
+ else if (x < 0x800) {
+ *p++ = (((byte) (x >> 6)) |
+ ((byte) 0xC0));
+ *p++ = (((byte) (x & 0x3F)) |
+ ((byte) 0x80));
+ } else if (x < 0x10000) {
+ ASSERT(!((x >= 0xD800 && x <= 0xDFFF) ||
+ (x == 0xFFFE) ||
+ (x == 0xFFFF)));
+ *p++ = (((byte) (x >> 12)) |
+ ((byte) 0xE0));
+ *p++ = ((((byte) (x >> 6)) & 0x3F) |
+ ((byte) 0x80));
+ *p++ = (((byte) (x & 0x3F)) |
+ ((byte) 0x80));
+ } else {
+ ASSERT(x < 0x110000);
+ *p++ = (((byte) (x >> 18)) |
+ ((byte) 0xF0));
+ *p++ = ((((byte) (x >> 12)) & 0x3F) |
+ ((byte) 0x80));
+ *p++ = ((((byte) (x >> 6)) & 0x3F) |
+ ((byte) 0x80));
+ *p++ = (((byte) (x & 0x3F)) |
+ ((byte) 0x80));
+ }
+ break;
+ case ERL_FILENAME_WIN_WCHAR:
+ ASSERT(x <= 0xFFFF);
+ *p++ = (byte) (x & 0xFFU);
+ *p++ = (byte) ((x >> 8) & 0xFFU);
+ break;
+ default:
+ ASSERT(0);
+ }
+
+ /* everything else will give badarg later
+ in the process, so we dont check */
+ ioterm = CDR(objp);
+ if (!is_list(ioterm)) {
+ break;
+ }
+ objp = list_val(ioterm);
+ obj = CAR(objp);
+ if (!is_small(obj))
+ break;
+ }
+ } else if (is_nil(obj)) {
+ ioterm = CDR(objp);
+ if (!is_list(ioterm)) {
+ break;
+ }
+ objp = list_val(ioterm);
+ obj = CAR(objp);
+ } else if (is_list(obj)) {
+ /* push rest of list for later processing, start
+ again with sublist */
+ ESTACK_PUSH(stack,CDR(objp));
+ ioterm = obj;
+ goto L_Again;
+ } else {
+ ASSERT(0);
+ }
+ if (is_nil(ioterm) || !is_list(ioterm)) {
+ break;
+ }
+ } /* for(;;) */
+ } /* is_list(ioterm) */
+
+ ASSERT(is_list(ioterm) || is_nil(ioterm));
+ } /* while not estack empty */
+ DESTROY_ESTACK(stack);
+ return;
+}
+void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars)
+{
+ Uint unipoint;
+
+ while (num_chars--) {
+ if (((*bytes) & ((byte) 0x80)) == 0) {
+ unipoint = (Uint) *bytes;
+ ++bytes;
+ } else if (((*bytes) & ((byte) 0xE0)) == 0xC0) {
+ unipoint =
+ (((Uint) ((*bytes) & ((byte) 0x1F))) << 6) |
+ ((Uint) (bytes[1] & ((byte) 0x3F)));
+ bytes += 2;
+ } else if (((*bytes) & ((byte) 0xF0)) == 0xE0) {
+ unipoint =
+ (((Uint) ((*bytes) & ((byte) 0xF))) << 12) |
+ (((Uint) (bytes[1] & ((byte) 0x3F))) << 6) |
+ ((Uint) (bytes[2] & ((byte) 0x3F)));
+ bytes +=3;
+ } else if (((*bytes) & ((byte) 0xF8)) == 0xF0) {
+ unipoint =
+ (((Uint) ((*bytes) & ((byte) 0x7))) << 18) |
+ (((Uint) (bytes[1] & ((byte) 0x3F))) << 12) |
+ (((Uint) (bytes[2] & ((byte) 0x3F))) << 6) |
+ ((Uint) (bytes[3] & ((byte) 0x3F)));
+ bytes += 4;
+ } else {
+ erl_exit(1,"Internal unicode error in prim_file:internal_name2native/1");
+ }
+ *target++ = (byte) (unipoint & 0xFF);
+ *target++ = (byte) ((unipoint >> 8) & 0xFF);
+ }
+}
+
+/*
+ * This internal bif converts a filename to whatever format is suitable for the file driver
+ * It also adds zero termination so that prim_file neednt bother with the character encoding
+ * of the file driver
+ */
+BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
+{
+ int encoding = erts_get_native_filename_encoding();
+ Sint need;
+ Eterm bin_term;
+ byte* bin_p;
+ /* Prim file explicitly does not allow atoms, although we could
+ very well cope with it. Instead of letting 'file' handle them,
+ it would probably be more efficient to handle them here. Subject to
+ change in R15. */
+ if (is_atom(BIF_ARG_1)) {
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ if (is_binary(BIF_ARG_1)) {
+ byte *temp_alloc = NULL;
+ byte *bytes;
+ byte *err_pos;
+ Uint size,num_chars;
+ /* Uninterpreted encoding except if windows widechar, in case we convert from
+ utf8 to win_wchar */
+ size = binary_size(BIF_ARG_1);
+ bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+ if (encoding != ERL_FILENAME_WIN_WCHAR) {
+ /*Add 0 termination only*/
+ bin_term = new_binary(BIF_P, NULL, size+1);
+ bin_p = binary_bytes(bin_term);
+ memcpy(bin_p,bytes,size);
+ bin_p[size]=0;
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_RET(bin_term);
+ }
+ /* In a wchar world, the emulator flags only affect how
+ binaries are interpreted when sent from the user. */
+ /* Determine real length and create a new binary */
+ if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK ||
+ erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) {
+ /* What to do now? Maybe latin1, so just take byte for byte instead */
+ bin_term = new_binary(BIF_P, 0, (size+1)*2);
+ bin_p = binary_bytes(bin_term);
+ while (size--) {
+ *bin_p++ = *bytes++;
+ *bin_p++ = 0;
+ }
+ *bin_p++ = 0;
+ *bin_p++ = 0;
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_RET(bin_term);
+ }
+ /* OK, UTF8 ok, number of characters is in num_chars */
+ bin_term = new_binary(BIF_P, 0, (num_chars+1)*2);
+ bin_p = binary_bytes(bin_term);
+ erts_copy_utf8_to_utf16_little(bin_p, bytes, num_chars);
+ /* zero termination */
+ bin_p[num_chars*2] = 0;
+ bin_p[num_chars*2+1] = 0;
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_RET(bin_term);
+ } /* binary */
+
+
+ if ((need = erts_native_filename_need(BIF_ARG_1,encoding)) < 0) {
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ if (encoding == ERL_FILENAME_WIN_WCHAR) {
+ need += 2;
+ } else {
+ ++need;
+ }
+
+ bin_term = new_binary(BIF_P, 0, need);
+ bin_p = binary_bytes(bin_term);
+ erts_native_filename_put(BIF_ARG_1,encoding,bin_p);
+ bin_p[need-1] = 0;
+ if (encoding == ERL_FILENAME_WIN_WCHAR) {
+ bin_p[need-2] = 0;
+ }
+ BIF_RET(bin_term);
+}
+
+BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1)
+{
+ Eterm real_bin;
+ Uint offset;
+ Uint size,num_chars;
+ Uint bitsize;
+ Uint bitoffs;
+ Eterm *hp;
+ byte *temp_alloc = NULL;
+ byte *bytes;
+ byte *err_pos;
+ Uint num_built; /* characters */
+ Uint num_eaten; /* bytes */
+ Eterm ret;
+ int mac = 0;
+
+ if (is_not_binary(BIF_ARG_1)) {
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ size = binary_size(BIF_ARG_1);
+ ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize);
+ if (bitsize != 0) {
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ if (size == 0) {
+ BIF_RET(NIL);
+ }
+ switch (erts_get_native_filename_encoding()) {
+ case ERL_FILENAME_LATIN1:
+ hp = HAlloc(BIF_P, 2 * size);
+ bytes = binary_bytes(real_bin)+offset;
+
+ BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs));
+ case ERL_FILENAME_UTF8_MAC:
+ mac = 1;
+ case ERL_FILENAME_UTF8:
+ bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+ if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) {
+ erts_free_aligned_binary_bytes(temp_alloc);
+ goto noconvert;
+ }
+ num_built = 0;
+ num_eaten = 0;
+ if (mac) {
+ ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size);
+ } else {
+ ret = do_utf8_to_list(BIF_P, num_chars, bytes, size, num_chars, &num_built, &num_eaten, NIL);
+ }
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_RET(ret);
+ case ERL_FILENAME_WIN_WCHAR:
+ bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+ if ((size % 2) != 0) { /* Panic fixup to avoid crashing the emulator */
+ size--;
+ hp = HAlloc(BIF_P, size+2);
+ ret = CONS(hp,make_small((Uint) bytes[size]),NIL);
+ hp += 2;
+ } else {
+ hp = HAlloc(BIF_P, size);
+ ret = NIL;
+ }
+ bytes += size-1;
+ while (size > 0) {
+ Uint x = ((Uint) *bytes--) << 8;
+ x |= ((Uint) *bytes--);
+ size -= 2;
+ ret = CONS(hp,make_small(x),ret);
+ hp += 2;
+ }
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_RET(ret);
+ default:
+ goto noconvert;
+ }
+ noconvert:
+ BIF_RET(BIF_ARG_1);
+}
+
+BIF_RETTYPE prim_file_internal_normalize_utf8_1(BIF_ALIST_1)
+{
+ Eterm real_bin;
+ Uint offset;
+ Uint size,num_chars;
+ Uint bitsize;
+ Uint bitoffs;
+ Eterm ret;
+ byte *temp_alloc = NULL;
+ byte *bytes;
+ byte *err_pos;
+
+ if (is_not_binary(BIF_ARG_1)) {
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ size = binary_size(BIF_ARG_1);
+ ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize);
+ if (bitsize != 0) {
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ if (size == 0) {
+ BIF_RET(NIL);
+ }
+ bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+ if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) {
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size);
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_RET(ret);
+}
+
+BIF_RETTYPE file_native_name_encoding_0(BIF_ALIST_0)
+{
+ switch (erts_get_native_filename_encoding()) {
+ case ERL_FILENAME_LATIN1:
+ BIF_RET(am_latin1);
+ case ERL_FILENAME_UTF8_MAC:
+ case ERL_FILENAME_UTF8:
+ BIF_RET(am_utf8);
+ case ERL_FILENAME_WIN_WCHAR:
+ if (erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) {
+ BIF_RET(am_latin1);
+ } else {
+ BIF_RET(am_utf8);
+ }
+ default:
+ BIF_RET(am_undefined);
+ }
+}
diff --git a/erts/emulator/beam/erl_unicode_normalize.h b/erts/emulator/beam/erl_unicode_normalize.h
new file mode 100644
index 0000000000..fb0a111ca2
--- /dev/null
+++ b/erts/emulator/beam/erl_unicode_normalize.h
@@ -0,0 +1,1687 @@
+/*
+* %CopyrightBegin%
+*
+* Copyright Ericsson AB 1999-2010. All Rights Reserved.
+*
+* The contents of this file are subject to the Erlang Public License,
+* Version 1.1, (the "License"); you may not use this file except in
+* compliance with the License. You should have received a copy of the
+* Erlang Public License along with this software. If not, it can be
+* retrieved online at http://www.erlang.org/.
+*
+* Software distributed under the License is distributed on an "AS IS"
+* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+* the License for the specific language governing rights and limitations
+* under the License.
+*
+* %CopyrightEnd%
+*/
+/*
+* This file is automatically generated by dec.erl, do not edit manually
+*/
+#define HASH_SIZE_FACTOR 2
+typedef struct _compose_entry {
+ Uint16 c;
+ Uint16 res;
+ Uint16 num_subs;
+ struct _compose_entry *subs;
+ int *hash;
+} CompEntry;
+
+static int compose_tab_size = 61;
+static int hash_compose_tab_0_15[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_0_15 */
+static CompEntry compose_tab_0_15[] = {
+{65, 7846, 0, NULL, NULL},
+{69, 7872, 0, NULL, NULL},
+{79, 7890, 0, NULL, NULL},
+{97, 7847, 0, NULL, NULL},
+{101, 7873, 0, NULL, NULL},
+{111, 7891, 0, NULL, NULL}
+}; /* compose_tab_0_15 */
+static int hash_compose_tab_0_16[8] =
+{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_0_16 */
+static CompEntry compose_tab_0_16[] = {
+{69, 7700, 0, NULL, NULL},
+{79, 7760, 0, NULL, NULL},
+{101, 7701, 0, NULL, NULL},
+{111, 7761, 0, NULL, NULL}
+}; /* compose_tab_0_16 */
+static int hash_compose_tab_0_17[4] =
+{-1,0,1,-1}; /* hash_compose_tab_0_17 */
+static CompEntry compose_tab_0_17[] = {
+{65, 7856, 0, NULL, NULL},
+{97, 7857, 0, NULL, NULL}
+}; /* compose_tab_0_17 */
+static int hash_compose_tab_0_18[8] =
+{-1,2,-1,-1,-1,0,1,3}; /* hash_compose_tab_0_18 */
+static CompEntry compose_tab_0_18[] = {
+{85, 475, 0, NULL, NULL},
+{117, 476, 0, NULL, NULL},
+{953, 8146, 0, NULL, NULL},
+{965, 8162, 0, NULL, NULL}
+}; /* compose_tab_0_18 */
+static int hash_compose_tab_0_19_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_19_0 */
+static CompEntry compose_tab_0_19_0[] = {
+{913, 8074, 0, NULL, NULL},
+{919, 8090, 0, NULL, NULL},
+{937, 8106, 0, NULL, NULL},
+{945, 8066, 0, NULL, NULL},
+{951, 8082, 0, NULL, NULL},
+{969, 8098, 0, NULL, NULL}
+}; /* compose_tab_0_19_0 */
+static int hash_compose_tab_0_19[28] =
+{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_0_19 */
+static CompEntry compose_tab_0_19[] = {
+{837, 0, 6, compose_tab_0_19_0, hash_compose_tab_0_19_0},
+{913, 7946, 0, NULL, NULL},
+{917, 7962, 0, NULL, NULL},
+{919, 7978, 0, NULL, NULL},
+{921, 7994, 0, NULL, NULL},
+{927, 8010, 0, NULL, NULL},
+{937, 8042, 0, NULL, NULL},
+{945, 7938, 0, NULL, NULL},
+{949, 7954, 0, NULL, NULL},
+{951, 7970, 0, NULL, NULL},
+{953, 7986, 0, NULL, NULL},
+{959, 8002, 0, NULL, NULL},
+{965, 8018, 0, NULL, NULL},
+{969, 8034, 0, NULL, NULL}
+}; /* compose_tab_0_19 */
+static int hash_compose_tab_0_20_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_20_0 */
+static CompEntry compose_tab_0_20_0[] = {
+{913, 8075, 0, NULL, NULL},
+{919, 8091, 0, NULL, NULL},
+{937, 8107, 0, NULL, NULL},
+{945, 8067, 0, NULL, NULL},
+{951, 8083, 0, NULL, NULL},
+{969, 8099, 0, NULL, NULL}
+}; /* compose_tab_0_20_0 */
+static int hash_compose_tab_0_20[30] =
+{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5,
+ 12}; /* hash_compose_tab_0_20 */
+static CompEntry compose_tab_0_20[] = {
+{837, 0, 6, compose_tab_0_20_0, hash_compose_tab_0_20_0},
+{913, 7947, 0, NULL, NULL},
+{917, 7963, 0, NULL, NULL},
+{919, 7979, 0, NULL, NULL},
+{921, 7995, 0, NULL, NULL},
+{927, 8011, 0, NULL, NULL},
+{933, 8027, 0, NULL, NULL},
+{937, 8043, 0, NULL, NULL},
+{945, 7939, 0, NULL, NULL},
+{949, 7955, 0, NULL, NULL},
+{951, 7971, 0, NULL, NULL},
+{953, 7987, 0, NULL, NULL},
+{959, 8003, 0, NULL, NULL},
+{965, 8019, 0, NULL, NULL},
+{969, 8035, 0, NULL, NULL}
+}; /* compose_tab_0_20 */
+static int hash_compose_tab_0_21[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_0_21 */
+static CompEntry compose_tab_0_21[] = {
+{79, 7900, 0, NULL, NULL},
+{85, 7914, 0, NULL, NULL},
+{111, 7901, 0, NULL, NULL},
+{117, 7915, 0, NULL, NULL}
+}; /* compose_tab_0_21 */
+static int hash_compose_tab_0_22[6] =
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_0_22 */
+static CompEntry compose_tab_0_22[] = {
+{945, 8114, 0, NULL, NULL},
+{951, 8130, 0, NULL, NULL},
+{969, 8178, 0, NULL, NULL}
+}; /* compose_tab_0_22 */
+static int hash_compose_tab_0[78] =
+{38,3,29,-1,-1,-1,-1,4,19,5,20,6,14,30,31,21,32,33,37,7,-1,-1,-1,8,34,-1,-1,9,
+ -1,35,-1,-1,-1,10,36,-1,-1,-1,-1,11,-1,12,-1,13,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,23,-1,22,-1,24,-1,25,-1,26,-1,0,-1,-1,15,1,16,27,17,2,18,28,-1,-1}; /* hash_compose_tab_0 */
+static CompEntry compose_tab_0[] = {
+{65, 192, 0, NULL, NULL},
+{69, 200, 0, NULL, NULL},
+{73, 204, 0, NULL, NULL},
+{79, 210, 0, NULL, NULL},
+{85, 217, 0, NULL, NULL},
+{87, 7808, 0, NULL, NULL},
+{89, 7922, 0, NULL, NULL},
+{97, 224, 0, NULL, NULL},
+{101, 232, 0, NULL, NULL},
+{105, 236, 0, NULL, NULL},
+{111, 242, 0, NULL, NULL},
+{117, 249, 0, NULL, NULL},
+{119, 7809, 0, NULL, NULL},
+{121, 7923, 0, NULL, NULL},
+{168, 8173, 0, NULL, NULL},
+{770, 0, 6, compose_tab_0_15, hash_compose_tab_0_15},
+{772, 0, 4, compose_tab_0_16, hash_compose_tab_0_16},
+{774, 0, 2, compose_tab_0_17, hash_compose_tab_0_17},
+{776, 0, 4, compose_tab_0_18, hash_compose_tab_0_18},
+{787, 0, 14, compose_tab_0_19, hash_compose_tab_0_19},
+{788, 0, 15, compose_tab_0_20, hash_compose_tab_0_20},
+{795, 0, 4, compose_tab_0_21, hash_compose_tab_0_21},
+{837, 0, 3, compose_tab_0_22, hash_compose_tab_0_22},
+{913, 8122, 0, NULL, NULL},
+{917, 8136, 0, NULL, NULL},
+{919, 8138, 0, NULL, NULL},
+{921, 8154, 0, NULL, NULL},
+{927, 8184, 0, NULL, NULL},
+{933, 8170, 0, NULL, NULL},
+{937, 8186, 0, NULL, NULL},
+{945, 8048, 0, NULL, NULL},
+{949, 8050, 0, NULL, NULL},
+{951, 8052, 0, NULL, NULL},
+{953, 8054, 0, NULL, NULL},
+{959, 8056, 0, NULL, NULL},
+{965, 8058, 0, NULL, NULL},
+{969, 8060, 0, NULL, NULL},
+{8127, 8141, 0, NULL, NULL},
+{8190, 8157, 0, NULL, NULL}
+}; /* compose_tab_0 */
+static int hash_compose_tab_1_39[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_1_39 */
+static CompEntry compose_tab_1_39[] = {
+{65, 7844, 0, NULL, NULL},
+{69, 7870, 0, NULL, NULL},
+{79, 7888, 0, NULL, NULL},
+{97, 7845, 0, NULL, NULL},
+{101, 7871, 0, NULL, NULL},
+{111, 7889, 0, NULL, NULL}
+}; /* compose_tab_1_39 */
+static int hash_compose_tab_1_40[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_40 */
+static CompEntry compose_tab_1_40[] = {
+{79, 7756, 0, NULL, NULL},
+{85, 7800, 0, NULL, NULL},
+{111, 7757, 0, NULL, NULL},
+{117, 7801, 0, NULL, NULL}
+}; /* compose_tab_1_40 */
+static int hash_compose_tab_1_41[8] =
+{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_1_41 */
+static CompEntry compose_tab_1_41[] = {
+{69, 7702, 0, NULL, NULL},
+{79, 7762, 0, NULL, NULL},
+{101, 7703, 0, NULL, NULL},
+{111, 7763, 0, NULL, NULL}
+}; /* compose_tab_1_41 */
+static int hash_compose_tab_1_42[4] =
+{-1,0,1,-1}; /* hash_compose_tab_1_42 */
+static CompEntry compose_tab_1_42[] = {
+{65, 7854, 0, NULL, NULL},
+{97, 7855, 0, NULL, NULL}
+}; /* compose_tab_1_42 */
+static int hash_compose_tab_1_43[12] =
+{-1,0,1,-1,-1,4,5,-1,-1,2,3,-1}; /* hash_compose_tab_1_43 */
+static CompEntry compose_tab_1_43[] = {
+{73, 7726, 0, NULL, NULL},
+{85, 471, 0, NULL, NULL},
+{105, 7727, 0, NULL, NULL},
+{117, 472, 0, NULL, NULL},
+{953, 8147, 0, NULL, NULL},
+{965, 8163, 0, NULL, NULL}
+}; /* compose_tab_1_43 */
+static int hash_compose_tab_1_44[4] =
+{-1,0,1,-1}; /* hash_compose_tab_1_44 */
+static CompEntry compose_tab_1_44[] = {
+{65, 506, 0, NULL, NULL},
+{97, 507, 0, NULL, NULL}
+}; /* compose_tab_1_44 */
+static int hash_compose_tab_1_45_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_45_0 */
+static CompEntry compose_tab_1_45_0[] = {
+{913, 8076, 0, NULL, NULL},
+{919, 8092, 0, NULL, NULL},
+{937, 8108, 0, NULL, NULL},
+{945, 8068, 0, NULL, NULL},
+{951, 8084, 0, NULL, NULL},
+{969, 8100, 0, NULL, NULL}
+}; /* compose_tab_1_45_0 */
+static int hash_compose_tab_1_45[28] =
+{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_1_45 */
+static CompEntry compose_tab_1_45[] = {
+{837, 0, 6, compose_tab_1_45_0, hash_compose_tab_1_45_0},
+{913, 7948, 0, NULL, NULL},
+{917, 7964, 0, NULL, NULL},
+{919, 7980, 0, NULL, NULL},
+{921, 7996, 0, NULL, NULL},
+{927, 8012, 0, NULL, NULL},
+{937, 8044, 0, NULL, NULL},
+{945, 7940, 0, NULL, NULL},
+{949, 7956, 0, NULL, NULL},
+{951, 7972, 0, NULL, NULL},
+{953, 7988, 0, NULL, NULL},
+{959, 8004, 0, NULL, NULL},
+{965, 8020, 0, NULL, NULL},
+{969, 8036, 0, NULL, NULL}
+}; /* compose_tab_1_45 */
+static int hash_compose_tab_1_46_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_46_0 */
+static CompEntry compose_tab_1_46_0[] = {
+{913, 8077, 0, NULL, NULL},
+{919, 8093, 0, NULL, NULL},
+{937, 8109, 0, NULL, NULL},
+{945, 8069, 0, NULL, NULL},
+{951, 8085, 0, NULL, NULL},
+{969, 8101, 0, NULL, NULL}
+}; /* compose_tab_1_46_0 */
+static int hash_compose_tab_1_46[30] =
+{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5,
+ 12}; /* hash_compose_tab_1_46 */
+static CompEntry compose_tab_1_46[] = {
+{837, 0, 6, compose_tab_1_46_0, hash_compose_tab_1_46_0},
+{913, 7949, 0, NULL, NULL},
+{917, 7965, 0, NULL, NULL},
+{919, 7981, 0, NULL, NULL},
+{921, 7997, 0, NULL, NULL},
+{927, 8013, 0, NULL, NULL},
+{933, 8029, 0, NULL, NULL},
+{937, 8045, 0, NULL, NULL},
+{945, 7941, 0, NULL, NULL},
+{949, 7957, 0, NULL, NULL},
+{951, 7973, 0, NULL, NULL},
+{953, 7989, 0, NULL, NULL},
+{959, 8005, 0, NULL, NULL},
+{965, 8021, 0, NULL, NULL},
+{969, 8037, 0, NULL, NULL}
+}; /* compose_tab_1_46 */
+static int hash_compose_tab_1_47[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_47 */
+static CompEntry compose_tab_1_47[] = {
+{79, 7898, 0, NULL, NULL},
+{85, 7912, 0, NULL, NULL},
+{111, 7899, 0, NULL, NULL},
+{117, 7913, 0, NULL, NULL}
+}; /* compose_tab_1_47 */
+static int hash_compose_tab_1_48[4] =
+{1,-1,-1,0}; /* hash_compose_tab_1_48 */
+static CompEntry compose_tab_1_48[] = {
+{67, 7688, 0, NULL, NULL},
+{99, 7689, 0, NULL, NULL}
+}; /* compose_tab_1_48 */
+static int hash_compose_tab_1_49[6] =
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_1_49 */
+static CompEntry compose_tab_1_49[] = {
+{945, 8116, 0, NULL, NULL},
+{951, 8132, 0, NULL, NULL},
+{959, 8180, 0, NULL, NULL}
+}; /* compose_tab_1_49 */
+static int hash_compose_tab_1[140] =
+{-1,-1,-1,-1,-1,-1,-1,68,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,34,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,35,-1,-1,-1,-1,64,-1,0,-1,1,-1,2,39,3,40,4,41,5,6,7,
+ 8,9,10,36,11,12,42,13,43,14,44,15,16,37,45,46,50,47,51,17,52,18,53,19,54,20,
+ 55,21,56,22,23,24,25,26,27,38,28,29,48,30,57,31,58,32,33,59,60,61,62,65,66,
+ 63,67,69,-1,-1,-1,-1,-1,49,-1,-1}; /* hash_compose_tab_1 */
+static CompEntry compose_tab_1[] = {
+{65, 193, 0, NULL, NULL},
+{67, 262, 0, NULL, NULL},
+{69, 201, 0, NULL, NULL},
+{71, 500, 0, NULL, NULL},
+{73, 205, 0, NULL, NULL},
+{75, 7728, 0, NULL, NULL},
+{76, 313, 0, NULL, NULL},
+{77, 7742, 0, NULL, NULL},
+{78, 323, 0, NULL, NULL},
+{79, 211, 0, NULL, NULL},
+{80, 7764, 0, NULL, NULL},
+{82, 340, 0, NULL, NULL},
+{83, 346, 0, NULL, NULL},
+{85, 218, 0, NULL, NULL},
+{87, 7810, 0, NULL, NULL},
+{89, 221, 0, NULL, NULL},
+{90, 377, 0, NULL, NULL},
+{97, 225, 0, NULL, NULL},
+{99, 263, 0, NULL, NULL},
+{101, 233, 0, NULL, NULL},
+{103, 501, 0, NULL, NULL},
+{105, 237, 0, NULL, NULL},
+{107, 7729, 0, NULL, NULL},
+{108, 314, 0, NULL, NULL},
+{109, 7743, 0, NULL, NULL},
+{110, 324, 0, NULL, NULL},
+{111, 243, 0, NULL, NULL},
+{112, 7765, 0, NULL, NULL},
+{114, 341, 0, NULL, NULL},
+{115, 347, 0, NULL, NULL},
+{117, 250, 0, NULL, NULL},
+{119, 7811, 0, NULL, NULL},
+{121, 253, 0, NULL, NULL},
+{122, 378, 0, NULL, NULL},
+{168, 8174, 0, NULL, NULL},
+{198, 508, 0, NULL, NULL},
+{216, 510, 0, NULL, NULL},
+{230, 509, 0, NULL, NULL},
+{248, 511, 0, NULL, NULL},
+{770, 0, 6, compose_tab_1_39, hash_compose_tab_1_39},
+{771, 0, 4, compose_tab_1_40, hash_compose_tab_1_40},
+{772, 0, 4, compose_tab_1_41, hash_compose_tab_1_41},
+{774, 0, 2, compose_tab_1_42, hash_compose_tab_1_42},
+{776, 0, 6, compose_tab_1_43, hash_compose_tab_1_43},
+{778, 0, 2, compose_tab_1_44, hash_compose_tab_1_44},
+{787, 0, 14, compose_tab_1_45, hash_compose_tab_1_45},
+{788, 0, 15, compose_tab_1_46, hash_compose_tab_1_46},
+{795, 0, 4, compose_tab_1_47, hash_compose_tab_1_47},
+{807, 0, 2, compose_tab_1_48, hash_compose_tab_1_48},
+{837, 0, 3, compose_tab_1_49, hash_compose_tab_1_49},
+{913, 8123, 0, NULL, NULL},
+{917, 8137, 0, NULL, NULL},
+{919, 8139, 0, NULL, NULL},
+{921, 8155, 0, NULL, NULL},
+{927, 8185, 0, NULL, NULL},
+{933, 8171, 0, NULL, NULL},
+{937, 8187, 0, NULL, NULL},
+{945, 8049, 0, NULL, NULL},
+{949, 8051, 0, NULL, NULL},
+{951, 8053, 0, NULL, NULL},
+{953, 8055, 0, NULL, NULL},
+{959, 8057, 0, NULL, NULL},
+{965, 8059, 0, NULL, NULL},
+{969, 8061, 0, NULL, NULL},
+{1043, 1027, 0, NULL, NULL},
+{1050, 1036, 0, NULL, NULL},
+{1075, 1107, 0, NULL, NULL},
+{1082, 1116, 0, NULL, NULL},
+{8127, 8142, 0, NULL, NULL},
+{8190, 8158, 0, NULL, NULL}
+}; /* compose_tab_1 */
+static int hash_compose_tab_2_26[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_2_26 */
+static CompEntry compose_tab_2_26[] = {
+{65, 7852, 0, NULL, NULL},
+{69, 7878, 0, NULL, NULL},
+{79, 7896, 0, NULL, NULL},
+{97, 7853, 0, NULL, NULL},
+{101, 7879, 0, NULL, NULL},
+{111, 7897, 0, NULL, NULL}
+}; /* compose_tab_2_26 */
+static int hash_compose_tab_2[54] =
+{-1,-1,-1,20,-1,-1,-1,21,-1,22,-1,0,23,1,24,2,25,3,4,5,6,-1,-1,-1,-1,7,-1,-1,
+ -1,8,-1,9,-1,10,-1,11,12,-1,-1,-1,-1,-1,-1,13,-1,14,-1,15,26,16,17,18,19,-1}; /* hash_compose_tab_2 */
+static CompEntry compose_tab_2[] = {
+{65, 194, 0, NULL, NULL},
+{67, 264, 0, NULL, NULL},
+{69, 202, 0, NULL, NULL},
+{71, 284, 0, NULL, NULL},
+{72, 292, 0, NULL, NULL},
+{73, 206, 0, NULL, NULL},
+{74, 308, 0, NULL, NULL},
+{79, 212, 0, NULL, NULL},
+{83, 348, 0, NULL, NULL},
+{85, 219, 0, NULL, NULL},
+{87, 372, 0, NULL, NULL},
+{89, 374, 0, NULL, NULL},
+{90, 7824, 0, NULL, NULL},
+{97, 226, 0, NULL, NULL},
+{99, 265, 0, NULL, NULL},
+{101, 234, 0, NULL, NULL},
+{103, 285, 0, NULL, NULL},
+{104, 293, 0, NULL, NULL},
+{105, 238, 0, NULL, NULL},
+{106, 309, 0, NULL, NULL},
+{111, 244, 0, NULL, NULL},
+{115, 349, 0, NULL, NULL},
+{117, 251, 0, NULL, NULL},
+{119, 373, 0, NULL, NULL},
+{121, 375, 0, NULL, NULL},
+{122, 7825, 0, NULL, NULL},
+{803, 0, 6, compose_tab_2_26, hash_compose_tab_2_26}
+}; /* compose_tab_2 */
+static int hash_compose_tab_3_16[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_3_16 */
+static CompEntry compose_tab_3_16[] = {
+{65, 7850, 0, NULL, NULL},
+{69, 7876, 0, NULL, NULL},
+{79, 7894, 0, NULL, NULL},
+{97, 7851, 0, NULL, NULL},
+{101, 7877, 0, NULL, NULL},
+{111, 7895, 0, NULL, NULL}
+}; /* compose_tab_3_16 */
+static int hash_compose_tab_3_17[4] =
+{-1,0,1,-1}; /* hash_compose_tab_3_17 */
+static CompEntry compose_tab_3_17[] = {
+{65, 7860, 0, NULL, NULL},
+{97, 7861, 0, NULL, NULL}
+}; /* compose_tab_3_17 */
+static int hash_compose_tab_3_18[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_3_18 */
+static CompEntry compose_tab_3_18[] = {
+{79, 7904, 0, NULL, NULL},
+{85, 7918, 0, NULL, NULL},
+{111, 7905, 0, NULL, NULL},
+{117, 7919, 0, NULL, NULL}
+}; /* compose_tab_3_18 */
+static int hash_compose_tab_3[38] =
+{-1,-1,3,4,13,14,-1,15,-1,5,6,16,-1,7,17,-1,-1,-1,-1,-1,-1,8,-1,-1,-1,9,-1,0,
+ -1,10,-1,1,-1,-1,11,2,12,18}; /* hash_compose_tab_3 */
+static CompEntry compose_tab_3[] = {
+{65, 195, 0, NULL, NULL},
+{69, 7868, 0, NULL, NULL},
+{73, 296, 0, NULL, NULL},
+{78, 209, 0, NULL, NULL},
+{79, 213, 0, NULL, NULL},
+{85, 360, 0, NULL, NULL},
+{86, 7804, 0, NULL, NULL},
+{89, 7928, 0, NULL, NULL},
+{97, 227, 0, NULL, NULL},
+{101, 7869, 0, NULL, NULL},
+{105, 297, 0, NULL, NULL},
+{110, 241, 0, NULL, NULL},
+{111, 245, 0, NULL, NULL},
+{117, 361, 0, NULL, NULL},
+{118, 7805, 0, NULL, NULL},
+{121, 7929, 0, NULL, NULL},
+{770, 0, 6, compose_tab_3_16, hash_compose_tab_3_16},
+{774, 0, 2, compose_tab_3_17, hash_compose_tab_3_17},
+{795, 0, 4, compose_tab_3_18, hash_compose_tab_3_18}
+}; /* compose_tab_3 */
+static int hash_compose_tab_4_14[4] =
+{-1,0,1,-1}; /* hash_compose_tab_4_14 */
+static CompEntry compose_tab_4_14[] = {
+{65, 480, 0, NULL, NULL},
+{97, 481, 0, NULL, NULL}
+}; /* compose_tab_4_14 */
+static int hash_compose_tab_4_15[8] =
+{-1,0,2,-1,-1,1,3,-1}; /* hash_compose_tab_4_15 */
+static CompEntry compose_tab_4_15[] = {
+{65, 478, 0, NULL, NULL},
+{85, 469, 0, NULL, NULL},
+{97, 479, 0, NULL, NULL},
+{117, 470, 0, NULL, NULL}
+}; /* compose_tab_4_15 */
+static int hash_compose_tab_4_16[8] =
+{-1,-1,1,3,0,2,-1,-1}; /* hash_compose_tab_4_16 */
+static CompEntry compose_tab_4_16[] = {
+{76, 7736, 0, NULL, NULL},
+{82, 7772, 0, NULL, NULL},
+{108, 7737, 0, NULL, NULL},
+{114, 7773, 0, NULL, NULL}
+}; /* compose_tab_4_16 */
+static int hash_compose_tab_4_17[4] =
+{1,-1,-1,0}; /* hash_compose_tab_4_17 */
+static CompEntry compose_tab_4_17[] = {
+{79, 492, 0, NULL, NULL},
+{111, 493, 0, NULL, NULL}
+}; /* compose_tab_4_17 */
+static int hash_compose_tab_4[56] =
+{-1,22,-1,-1,-1,11,13,-1,-1,0,-1,-1,-1,1,23,2,26,3,18,16,-1,-1,-1,4,17,19,-1,
+ 27,-1,5,12,-1,-1,-1,-1,-1,-1,20,-1,-1,24,6,-1,-1,-1,7,-1,8,14,9,15,21,25,-1,
+ -1,10}; /* hash_compose_tab_4 */
+static CompEntry compose_tab_4[] = {
+{65, 256, 0, NULL, NULL},
+{69, 274, 0, NULL, NULL},
+{71, 7712, 0, NULL, NULL},
+{73, 298, 0, NULL, NULL},
+{79, 332, 0, NULL, NULL},
+{85, 362, 0, NULL, NULL},
+{97, 257, 0, NULL, NULL},
+{101, 275, 0, NULL, NULL},
+{103, 7713, 0, NULL, NULL},
+{105, 299, 0, NULL, NULL},
+{111, 333, 0, NULL, NULL},
+{117, 363, 0, NULL, NULL},
+{198, 482, 0, NULL, NULL},
+{230, 483, 0, NULL, NULL},
+{775, 0, 2, compose_tab_4_14, hash_compose_tab_4_14},
+{776, 0, 4, compose_tab_4_15, hash_compose_tab_4_15},
+{803, 0, 4, compose_tab_4_16, hash_compose_tab_4_16},
+{808, 0, 2, compose_tab_4_17, hash_compose_tab_4_17},
+{913, 8121, 0, NULL, NULL},
+{921, 8153, 0, NULL, NULL},
+{933, 8169, 0, NULL, NULL},
+{945, 8113, 0, NULL, NULL},
+{953, 8145, 0, NULL, NULL},
+{965, 8161, 0, NULL, NULL},
+{1048, 1250, 0, NULL, NULL},
+{1059, 1262, 0, NULL, NULL},
+{1080, 1251, 0, NULL, NULL},
+{1091, 1263, 0, NULL, NULL}
+}; /* compose_tab_4 */
+static int hash_compose_tab_5_12[4] =
+{-1,0,1,-1}; /* hash_compose_tab_5_12 */
+static CompEntry compose_tab_5_12[] = {
+{65, 7862, 0, NULL, NULL},
+{97, 7863, 0, NULL, NULL}
+}; /* compose_tab_5_12 */
+static int hash_compose_tab_5_13[4] =
+{-1,0,1,-1}; /* hash_compose_tab_5_13 */
+static CompEntry compose_tab_5_13[] = {
+{69, 7708, 0, NULL, NULL},
+{101, 7709, 0, NULL, NULL}
+}; /* compose_tab_5_13 */
+static int hash_compose_tab_5[60] =
+{28,-1,-1,-1,-1,0,19,-1,-1,1,-1,2,29,3,14,-1,-1,-1,-1,4,20,15,-1,12,-1,5,21,
+ 13,22,23,-1,-1,-1,16,-1,-1,-1,6,-1,24,-1,7,-1,8,-1,9,17,-1,-1,-1,-1,10,25,18,
+ -1,-1,-1,11,26,27}; /* hash_compose_tab_5 */
+static CompEntry compose_tab_5[] = {
+{65, 258, 0, NULL, NULL},
+{69, 276, 0, NULL, NULL},
+{71, 286, 0, NULL, NULL},
+{73, 300, 0, NULL, NULL},
+{79, 334, 0, NULL, NULL},
+{85, 364, 0, NULL, NULL},
+{97, 259, 0, NULL, NULL},
+{101, 277, 0, NULL, NULL},
+{103, 287, 0, NULL, NULL},
+{105, 301, 0, NULL, NULL},
+{111, 335, 0, NULL, NULL},
+{117, 365, 0, NULL, NULL},
+{803, 0, 2, compose_tab_5_12, hash_compose_tab_5_12},
+{807, 0, 2, compose_tab_5_13, hash_compose_tab_5_13},
+{913, 8120, 0, NULL, NULL},
+{921, 8152, 0, NULL, NULL},
+{933, 8168, 0, NULL, NULL},
+{945, 8112, 0, NULL, NULL},
+{953, 8144, 0, NULL, NULL},
+{965, 8160, 0, NULL, NULL},
+{1040, 1232, 0, NULL, NULL},
+{1045, 1238, 0, NULL, NULL},
+{1046, 1217, 0, NULL, NULL},
+{1048, 1049, 0, NULL, NULL},
+{1059, 1038, 0, NULL, NULL},
+{1072, 1233, 0, NULL, NULL},
+{1077, 1239, 0, NULL, NULL},
+{1078, 1218, 0, NULL, NULL},
+{1080, 1081, 0, NULL, NULL},
+{1091, 1118, 0, NULL, NULL}
+}; /* compose_tab_5 */
+static int hash_compose_tab_6_36[4] =
+{1,-1,-1,0}; /* hash_compose_tab_6_36 */
+static CompEntry compose_tab_6_36[] = {
+{83, 7780, 0, NULL, NULL},
+{115, 7781, 0, NULL, NULL}
+}; /* compose_tab_6_36 */
+static int hash_compose_tab_6_38[4] =
+{1,-1,-1,0}; /* hash_compose_tab_6_38 */
+static CompEntry compose_tab_6_38[] = {
+{83, 7782, 0, NULL, NULL},
+{115, 7783, 0, NULL, NULL}
+}; /* compose_tab_6_38 */
+static int hash_compose_tab_6_39[4] =
+{1,-1,-1,0}; /* hash_compose_tab_6_39 */
+static CompEntry compose_tab_6_39[] = {
+{83, 7784, 0, NULL, NULL},
+{115, 7785, 0, NULL, NULL}
+}; /* compose_tab_6_39 */
+static int hash_compose_tab_6[80] =
+{10,-1,11,12,13,39,-1,14,15,16,17,-1,-1,-1,-1,-1,-1,-1,18,19,20,21,22,23,24,
+ -1,-1,-1,-1,25,26,-1,27,-1,28,29,30,-1,-1,31,32,33,34,-1,-1,-1,-1,-1,-1,36,
+ -1,-1,-1,-1,37,-1,-1,-1,-1,-1,38,-1,-1,35,-1,-1,0,1,2,3,4,5,6,7,-1,-1,-1,8,9,
+ -1}; /* hash_compose_tab_6 */
+static CompEntry compose_tab_6[] = {
+{66, 7682, 0, NULL, NULL},
+{67, 266, 0, NULL, NULL},
+{68, 7690, 0, NULL, NULL},
+{69, 278, 0, NULL, NULL},
+{70, 7710, 0, NULL, NULL},
+{71, 288, 0, NULL, NULL},
+{72, 7714, 0, NULL, NULL},
+{73, 304, 0, NULL, NULL},
+{77, 7744, 0, NULL, NULL},
+{78, 7748, 0, NULL, NULL},
+{80, 7766, 0, NULL, NULL},
+{82, 7768, 0, NULL, NULL},
+{83, 7776, 0, NULL, NULL},
+{84, 7786, 0, NULL, NULL},
+{87, 7814, 0, NULL, NULL},
+{88, 7818, 0, NULL, NULL},
+{89, 7822, 0, NULL, NULL},
+{90, 379, 0, NULL, NULL},
+{98, 7683, 0, NULL, NULL},
+{99, 267, 0, NULL, NULL},
+{100, 7691, 0, NULL, NULL},
+{101, 279, 0, NULL, NULL},
+{102, 7711, 0, NULL, NULL},
+{103, 289, 0, NULL, NULL},
+{104, 7715, 0, NULL, NULL},
+{109, 7745, 0, NULL, NULL},
+{110, 7749, 0, NULL, NULL},
+{112, 7767, 0, NULL, NULL},
+{114, 7769, 0, NULL, NULL},
+{115, 7777, 0, NULL, NULL},
+{116, 7787, 0, NULL, NULL},
+{119, 7815, 0, NULL, NULL},
+{120, 7819, 0, NULL, NULL},
+{121, 7823, 0, NULL, NULL},
+{122, 380, 0, NULL, NULL},
+{383, 7835, 0, NULL, NULL},
+{769, 0, 2, compose_tab_6_36, hash_compose_tab_6_36},
+{774, 784, 0, NULL, NULL},
+{780, 0, 2, compose_tab_6_38, hash_compose_tab_6_38},
+{803, 0, 2, compose_tab_6_39, hash_compose_tab_6_39}
+}; /* compose_tab_6 */
+static int hash_compose_tab_7_23[4] =
+{1,-1,-1,0}; /* hash_compose_tab_7_23 */
+static CompEntry compose_tab_7_23[] = {
+{79, 7758, 0, NULL, NULL},
+{111, 7759, 0, NULL, NULL}
+}; /* compose_tab_7_23 */
+static int hash_compose_tab_7_24[4] =
+{-1,0,1,-1}; /* hash_compose_tab_7_24 */
+static CompEntry compose_tab_7_24[] = {
+{85, 7802, 0, NULL, NULL},
+{117, 7803, 0, NULL, NULL}
+}; /* compose_tab_7_24 */
+static int hash_compose_tab_7[100] =
+{48,10,21,-1,11,12,-1,-1,-1,-1,49,13,-1,-1,-1,20,14,15,-1,16,17,18,25,-1,-1,
+ -1,-1,-1,-1,22,30,-1,-1,26,-1,-1,-1,-1,-1,-1,31,-1,-1,-1,-1,32,33,34,35,-1,
+ -1,-1,-1,27,36,-1,-1,-1,-1,37,-1,-1,-1,38,-1,0,28,39,-1,1,-1,23,2,3,24,40,-1,
+ 41,29,4,42,43,44,-1,-1,5,45,6,7,8,-1,46,-1,-1,-1,47,-1,9,-1,19}; /* hash_compose_tab_7 */
+static CompEntry compose_tab_7[] = {
+{65, 196, 0, NULL, NULL},
+{69, 203, 0, NULL, NULL},
+{72, 7718, 0, NULL, NULL},
+{73, 207, 0, NULL, NULL},
+{79, 214, 0, NULL, NULL},
+{85, 220, 0, NULL, NULL},
+{87, 7812, 0, NULL, NULL},
+{88, 7820, 0, NULL, NULL},
+{89, 376, 0, NULL, NULL},
+{97, 228, 0, NULL, NULL},
+{101, 235, 0, NULL, NULL},
+{104, 7719, 0, NULL, NULL},
+{105, 239, 0, NULL, NULL},
+{111, 246, 0, NULL, NULL},
+{116, 7831, 0, NULL, NULL},
+{117, 252, 0, NULL, NULL},
+{119, 7813, 0, NULL, NULL},
+{120, 7821, 0, NULL, NULL},
+{121, 255, 0, NULL, NULL},
+{399, 1242, 0, NULL, NULL},
+{415, 1258, 0, NULL, NULL},
+{601, 1243, 0, NULL, NULL},
+{629, 1259, 0, NULL, NULL},
+{771, 0, 2, compose_tab_7_23, hash_compose_tab_7_23},
+{772, 0, 2, compose_tab_7_24, hash_compose_tab_7_24},
+{921, 938, 0, NULL, NULL},
+{933, 939, 0, NULL, NULL},
+{953, 970, 0, NULL, NULL},
+{965, 971, 0, NULL, NULL},
+{978, 980, 0, NULL, NULL},
+{1030, 1031, 0, NULL, NULL},
+{1040, 1234, 0, NULL, NULL},
+{1045, 1025, 0, NULL, NULL},
+{1046, 1244, 0, NULL, NULL},
+{1047, 1246, 0, NULL, NULL},
+{1048, 1252, 0, NULL, NULL},
+{1054, 1254, 0, NULL, NULL},
+{1059, 1264, 0, NULL, NULL},
+{1063, 1268, 0, NULL, NULL},
+{1067, 1272, 0, NULL, NULL},
+{1072, 1235, 0, NULL, NULL},
+{1077, 1105, 0, NULL, NULL},
+{1078, 1245, 0, NULL, NULL},
+{1079, 1247, 0, NULL, NULL},
+{1080, 1253, 0, NULL, NULL},
+{1086, 1255, 0, NULL, NULL},
+{1091, 1265, 0, NULL, NULL},
+{1095, 1269, 0, NULL, NULL},
+{1099, 1273, 0, NULL, NULL},
+{1110, 1111, 0, NULL, NULL}
+}; /* compose_tab_7 */
+static int hash_compose_tab_8_12[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_8_12 */
+static CompEntry compose_tab_8_12[] = {
+{65, 7848, 0, NULL, NULL},
+{69, 7874, 0, NULL, NULL},
+{79, 7892, 0, NULL, NULL},
+{97, 7849, 0, NULL, NULL},
+{101, 7875, 0, NULL, NULL},
+{111, 7893, 0, NULL, NULL}
+}; /* compose_tab_8_12 */
+static int hash_compose_tab_8_13[4] =
+{-1,0,1,-1}; /* hash_compose_tab_8_13 */
+static CompEntry compose_tab_8_13[] = {
+{65, 7858, 0, NULL, NULL},
+{97, 7859, 0, NULL, NULL}
+}; /* compose_tab_8_13 */
+static int hash_compose_tab_8_14[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_8_14 */
+static CompEntry compose_tab_8_14[] = {
+{79, 7902, 0, NULL, NULL},
+{85, 7916, 0, NULL, NULL},
+{111, 7903, 0, NULL, NULL},
+{117, 7917, 0, NULL, NULL}
+}; /* compose_tab_8_14 */
+static int hash_compose_tab_8[30] =
+{-1,11,-1,-1,-1,0,-1,6,-1,1,-1,7,-1,2,-1,8,14,-1,-1,3,12,9,-1,-1,13,4,-1,10,
+ -1,5}; /* hash_compose_tab_8 */
+static CompEntry compose_tab_8[] = {
+{65, 7842, 0, NULL, NULL},
+{69, 7866, 0, NULL, NULL},
+{73, 7880, 0, NULL, NULL},
+{79, 7886, 0, NULL, NULL},
+{85, 7910, 0, NULL, NULL},
+{89, 7926, 0, NULL, NULL},
+{97, 7843, 0, NULL, NULL},
+{101, 7867, 0, NULL, NULL},
+{105, 7881, 0, NULL, NULL},
+{111, 7887, 0, NULL, NULL},
+{117, 7911, 0, NULL, NULL},
+{121, 7927, 0, NULL, NULL},
+{770, 0, 6, compose_tab_8_12, hash_compose_tab_8_12},
+{774, 0, 2, compose_tab_8_13, hash_compose_tab_8_13},
+{795, 0, 4, compose_tab_8_14, hash_compose_tab_8_14}
+}; /* compose_tab_8 */
+static int hash_compose_tab_9[12] =
+{-1,1,2,5,-1,0,-1,-1,-1,3,-1,4}; /* hash_compose_tab_9 */
+static CompEntry compose_tab_9[] = {
+{65, 197, 0, NULL, NULL},
+{85, 366, 0, NULL, NULL},
+{97, 229, 0, NULL, NULL},
+{117, 367, 0, NULL, NULL},
+{119, 7832, 0, NULL, NULL},
+{121, 7833, 0, NULL, NULL}
+}; /* compose_tab_9 */
+static int hash_compose_tab_10[12] =
+{-1,1,-1,2,4,-1,-1,0,-1,3,-1,5}; /* hash_compose_tab_10 */
+static CompEntry compose_tab_10[] = {
+{79, 336, 0, NULL, NULL},
+{85, 368, 0, NULL, NULL},
+{111, 337, 0, NULL, NULL},
+{117, 369, 0, NULL, NULL},
+{1059, 1266, 0, NULL, NULL},
+{1091, 1267, 0, NULL, NULL}
+}; /* compose_tab_10 */
+static int hash_compose_tab_11_33[4] =
+{-1,0,1,-1}; /* hash_compose_tab_11_33 */
+static CompEntry compose_tab_11_33[] = {
+{85, 473, 0, NULL, NULL},
+{117, 474, 0, NULL, NULL}
+}; /* compose_tab_11_33 */
+static int hash_compose_tab_11[68] =
+{2,3,-1,4,-1,5,-1,6,7,-1,8,9,-1,-1,10,11,12,13,-1,-1,-1,-1,14,-1,-1,-1,-1,-1,
+ 33,15,-1,16,17,18,31,19,-1,20,21,22,23,-1,24,25,-1,-1,26,27,28,29,32,-1,-1,
+ -1,30,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,-1,1}; /* hash_compose_tab_11 */
+static CompEntry compose_tab_11[] = {
+{65, 461, 0, NULL, NULL},
+{67, 268, 0, NULL, NULL},
+{68, 270, 0, NULL, NULL},
+{69, 282, 0, NULL, NULL},
+{71, 486, 0, NULL, NULL},
+{73, 463, 0, NULL, NULL},
+{75, 488, 0, NULL, NULL},
+{76, 317, 0, NULL, NULL},
+{78, 327, 0, NULL, NULL},
+{79, 465, 0, NULL, NULL},
+{82, 344, 0, NULL, NULL},
+{83, 352, 0, NULL, NULL},
+{84, 356, 0, NULL, NULL},
+{85, 467, 0, NULL, NULL},
+{90, 381, 0, NULL, NULL},
+{97, 462, 0, NULL, NULL},
+{99, 269, 0, NULL, NULL},
+{100, 271, 0, NULL, NULL},
+{101, 283, 0, NULL, NULL},
+{103, 487, 0, NULL, NULL},
+{105, 464, 0, NULL, NULL},
+{106, 496, 0, NULL, NULL},
+{107, 489, 0, NULL, NULL},
+{108, 318, 0, NULL, NULL},
+{110, 328, 0, NULL, NULL},
+{111, 466, 0, NULL, NULL},
+{114, 345, 0, NULL, NULL},
+{115, 353, 0, NULL, NULL},
+{116, 357, 0, NULL, NULL},
+{117, 468, 0, NULL, NULL},
+{122, 382, 0, NULL, NULL},
+{439, 494, 0, NULL, NULL},
+{658, 495, 0, NULL, NULL},
+{776, 0, 2, compose_tab_11_33, hash_compose_tab_11_33}
+}; /* compose_tab_11 */
+static int hash_compose_tab_12_1[4] =
+{-1,0,1,-1}; /* hash_compose_tab_12_1 */
+static CompEntry compose_tab_12_1[] = {
+{953, 912, 0, NULL, NULL},
+{965, 944, 0, NULL, NULL}
+}; /* compose_tab_12_1 */
+static int hash_compose_tab_12[34] =
+{11,4,12,5,-1,-1,-1,13,-1,6,-1,-1,-1,14,-1,7,-1,15,-1,8,-1,-1,-1,-1,-1,-1,16,
+ 9,1,2,-1,10,0,3}; /* hash_compose_tab_12 */
+static CompEntry compose_tab_12[] = {
+{168, 901, 0, NULL, NULL},
+{776, 0, 2, compose_tab_12_1, hash_compose_tab_12_1},
+{913, 902, 0, NULL, NULL},
+{917, 904, 0, NULL, NULL},
+{919, 905, 0, NULL, NULL},
+{921, 906, 0, NULL, NULL},
+{927, 908, 0, NULL, NULL},
+{933, 910, 0, NULL, NULL},
+{937, 911, 0, NULL, NULL},
+{945, 940, 0, NULL, NULL},
+{949, 941, 0, NULL, NULL},
+{951, 942, 0, NULL, NULL},
+{953, 943, 0, NULL, NULL},
+{959, 972, 0, NULL, NULL},
+{965, 973, 0, NULL, NULL},
+{969, 974, 0, NULL, NULL},
+{978, 979, 0, NULL, NULL}
+}; /* compose_tab_12 */
+static int hash_compose_tab_13[28] =
+{-1,5,10,-1,-1,11,-1,-1,-1,0,-1,-1,-1,1,6,-1,-1,2,7,-1,12,8,13,3,-1,-1,4,9}; /* hash_compose_tab_13 */
+static CompEntry compose_tab_13[] = {
+{65, 512, 0, NULL, NULL},
+{69, 516, 0, NULL, NULL},
+{73, 520, 0, NULL, NULL},
+{79, 524, 0, NULL, NULL},
+{82, 528, 0, NULL, NULL},
+{85, 532, 0, NULL, NULL},
+{97, 513, 0, NULL, NULL},
+{101, 517, 0, NULL, NULL},
+{105, 521, 0, NULL, NULL},
+{111, 525, 0, NULL, NULL},
+{114, 529, 0, NULL, NULL},
+{117, 533, 0, NULL, NULL},
+{1140, 1142, 0, NULL, NULL},
+{1141, 1143, 0, NULL, NULL}
+}; /* compose_tab_13 */
+static int hash_compose_tab_14[24] =
+{-1,2,6,-1,-1,7,-1,3,-1,8,4,-1,-1,5,-1,9,-1,0,10,-1,-1,1,11,-1}; /* hash_compose_tab_14 */
+static CompEntry compose_tab_14[] = {
+{65, 514, 0, NULL, NULL},
+{69, 518, 0, NULL, NULL},
+{73, 522, 0, NULL, NULL},
+{79, 526, 0, NULL, NULL},
+{82, 530, 0, NULL, NULL},
+{85, 534, 0, NULL, NULL},
+{97, 515, 0, NULL, NULL},
+{101, 519, 0, NULL, NULL},
+{105, 523, 0, NULL, NULL},
+{111, 527, 0, NULL, NULL},
+{114, 531, 0, NULL, NULL},
+{117, 535, 0, NULL, NULL}
+}; /* compose_tab_14 */
+static int hash_compose_tab_15_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_15_0 */
+static CompEntry compose_tab_15_0[] = {
+{913, 8072, 0, NULL, NULL},
+{919, 8088, 0, NULL, NULL},
+{937, 8104, 0, NULL, NULL},
+{945, 8064, 0, NULL, NULL},
+{951, 8080, 0, NULL, NULL},
+{969, 8096, 0, NULL, NULL}
+}; /* compose_tab_15_0 */
+static int hash_compose_tab_15[30] =
+{-1,12,-1,-1,-1,13,-1,6,-1,14,-1,-1,-1,1,-1,7,-1,2,-1,3,8,4,9,10,-1,-1,-1,0,5,
+ 11}; /* hash_compose_tab_15 */
+static CompEntry compose_tab_15[] = {
+{837, 0, 6, compose_tab_15_0, hash_compose_tab_15_0},
+{913, 7944, 0, NULL, NULL},
+{917, 7960, 0, NULL, NULL},
+{919, 7976, 0, NULL, NULL},
+{921, 7992, 0, NULL, NULL},
+{927, 8008, 0, NULL, NULL},
+{937, 8040, 0, NULL, NULL},
+{945, 7936, 0, NULL, NULL},
+{949, 7952, 0, NULL, NULL},
+{951, 7968, 0, NULL, NULL},
+{953, 7984, 0, NULL, NULL},
+{959, 8000, 0, NULL, NULL},
+{961, 8164, 0, NULL, NULL},
+{965, 8016, 0, NULL, NULL},
+{969, 8032, 0, NULL, NULL}
+}; /* compose_tab_15 */
+static int hash_compose_tab_16_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_16_0 */
+static CompEntry compose_tab_16_0[] = {
+{913, 8073, 0, NULL, NULL},
+{919, 8089, 0, NULL, NULL},
+{937, 8105, 0, NULL, NULL},
+{945, 8065, 0, NULL, NULL},
+{951, 8081, 0, NULL, NULL},
+{969, 8097, 0, NULL, NULL}
+}; /* compose_tab_16_0 */
+static int hash_compose_tab_16[34] =
+{11,3,12,4,-1,-1,-1,13,-1,5,14,6,-1,15,-1,7,-1,16,-1,8,-1,0,-1,-1,-1,-1,-1,9,
+ -1,1,-1,10,-1,2}; /* hash_compose_tab_16 */
+static CompEntry compose_tab_16[] = {
+{837, 0, 6, compose_tab_16_0, hash_compose_tab_16_0},
+{913, 7945, 0, NULL, NULL},
+{917, 7961, 0, NULL, NULL},
+{919, 7977, 0, NULL, NULL},
+{921, 7993, 0, NULL, NULL},
+{927, 8009, 0, NULL, NULL},
+{929, 8172, 0, NULL, NULL},
+{933, 8025, 0, NULL, NULL},
+{937, 8041, 0, NULL, NULL},
+{945, 7937, 0, NULL, NULL},
+{949, 7953, 0, NULL, NULL},
+{951, 7969, 0, NULL, NULL},
+{953, 7985, 0, NULL, NULL},
+{959, 8001, 0, NULL, NULL},
+{961, 8165, 0, NULL, NULL},
+{965, 8017, 0, NULL, NULL},
+{969, 8033, 0, NULL, NULL}
+}; /* compose_tab_16 */
+static int hash_compose_tab_17[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_17 */
+static CompEntry compose_tab_17[] = {
+{79, 416, 0, NULL, NULL},
+{85, 431, 0, NULL, NULL},
+{111, 417, 0, NULL, NULL},
+{117, 432, 0, NULL, NULL}
+}; /* compose_tab_17 */
+static int hash_compose_tab_18_38[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_18_38 */
+static CompEntry compose_tab_18_38[] = {
+{79, 7906, 0, NULL, NULL},
+{85, 7920, 0, NULL, NULL},
+{111, 7907, 0, NULL, NULL},
+{117, 7921, 0, NULL, NULL}
+}; /* compose_tab_18_38 */
+static int hash_compose_tab_18[78] =
+{9,10,-1,-1,11,12,13,14,15,16,-1,17,18,-1,-1,38,-1,-1,-1,19,20,-1,21,22,-1,-1,
+ 23,24,-1,25,26,27,28,29,-1,-1,30,31,32,33,34,35,-1,36,37,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,-1,2,3,-1,-1,4,5,-1,6,7,8}; /* hash_compose_tab_18 */
+static CompEntry compose_tab_18[] = {
+{65, 7840, 0, NULL, NULL},
+{66, 7684, 0, NULL, NULL},
+{68, 7692, 0, NULL, NULL},
+{69, 7864, 0, NULL, NULL},
+{72, 7716, 0, NULL, NULL},
+{73, 7882, 0, NULL, NULL},
+{75, 7730, 0, NULL, NULL},
+{76, 7734, 0, NULL, NULL},
+{77, 7746, 0, NULL, NULL},
+{78, 7750, 0, NULL, NULL},
+{79, 7884, 0, NULL, NULL},
+{82, 7770, 0, NULL, NULL},
+{83, 7778, 0, NULL, NULL},
+{84, 7788, 0, NULL, NULL},
+{85, 7908, 0, NULL, NULL},
+{86, 7806, 0, NULL, NULL},
+{87, 7816, 0, NULL, NULL},
+{89, 7924, 0, NULL, NULL},
+{90, 7826, 0, NULL, NULL},
+{97, 7841, 0, NULL, NULL},
+{98, 7685, 0, NULL, NULL},
+{100, 7693, 0, NULL, NULL},
+{101, 7865, 0, NULL, NULL},
+{104, 7717, 0, NULL, NULL},
+{105, 7883, 0, NULL, NULL},
+{107, 7731, 0, NULL, NULL},
+{108, 7735, 0, NULL, NULL},
+{109, 7747, 0, NULL, NULL},
+{110, 7751, 0, NULL, NULL},
+{111, 7885, 0, NULL, NULL},
+{114, 7771, 0, NULL, NULL},
+{115, 7779, 0, NULL, NULL},
+{116, 7789, 0, NULL, NULL},
+{117, 7909, 0, NULL, NULL},
+{118, 7807, 0, NULL, NULL},
+{119, 7817, 0, NULL, NULL},
+{121, 7925, 0, NULL, NULL},
+{122, 7827, 0, NULL, NULL},
+{795, 0, 4, compose_tab_18_38, hash_compose_tab_18_38}
+}; /* compose_tab_18 */
+static int hash_compose_tab_19[4] =
+{-1,0,1,-1}; /* hash_compose_tab_19 */
+static CompEntry compose_tab_19[] = {
+{85, 7794, 0, NULL, NULL},
+{117, 7795, 0, NULL, NULL}
+}; /* compose_tab_19 */
+static int hash_compose_tab_20[4] =
+{-1,0,1,-1}; /* hash_compose_tab_20 */
+static CompEntry compose_tab_20[] = {
+{65, 7680, 0, NULL, NULL},
+{97, 7681, 0, NULL, NULL}
+}; /* compose_tab_20 */
+static int hash_compose_tab_21[40] =
+{-1,-1,7,8,9,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,10,11,-1,-1,12,13,-1,
+ -1,0,1,14,15,2,3,16,17,4,5,18,6,19}; /* hash_compose_tab_21 */
+static CompEntry compose_tab_21[] = {
+{67, 199, 0, NULL, NULL},
+{68, 7696, 0, NULL, NULL},
+{71, 290, 0, NULL, NULL},
+{72, 7720, 0, NULL, NULL},
+{75, 310, 0, NULL, NULL},
+{76, 315, 0, NULL, NULL},
+{78, 325, 0, NULL, NULL},
+{82, 342, 0, NULL, NULL},
+{83, 350, 0, NULL, NULL},
+{84, 354, 0, NULL, NULL},
+{99, 231, 0, NULL, NULL},
+{100, 7697, 0, NULL, NULL},
+{103, 291, 0, NULL, NULL},
+{104, 7721, 0, NULL, NULL},
+{107, 311, 0, NULL, NULL},
+{108, 316, 0, NULL, NULL},
+{110, 326, 0, NULL, NULL},
+{114, 343, 0, NULL, NULL},
+{115, 351, 0, NULL, NULL},
+{116, 355, 0, NULL, NULL}
+}; /* compose_tab_21 */
+static int hash_compose_tab_22[20] =
+{-1,6,-1,-1,-1,0,4,7,-1,1,-1,8,-1,2,-1,-1,-1,5,9,3}; /* hash_compose_tab_22 */
+static CompEntry compose_tab_22[] = {
+{65, 260, 0, NULL, NULL},
+{69, 280, 0, NULL, NULL},
+{73, 302, 0, NULL, NULL},
+{79, 490, 0, NULL, NULL},
+{85, 370, 0, NULL, NULL},
+{97, 261, 0, NULL, NULL},
+{101, 281, 0, NULL, NULL},
+{105, 303, 0, NULL, NULL},
+{111, 491, 0, NULL, NULL},
+{117, 371, 0, NULL, NULL}
+}; /* compose_tab_22 */
+static int hash_compose_tab_23[24] =
+{-1,-1,-1,-1,2,6,3,7,-1,-1,-1,-1,4,5,8,9,-1,-1,-1,-1,0,1,10,11}; /* hash_compose_tab_23 */
+static CompEntry compose_tab_23[] = {
+{68, 7698, 0, NULL, NULL},
+{69, 7704, 0, NULL, NULL},
+{76, 7740, 0, NULL, NULL},
+{78, 7754, 0, NULL, NULL},
+{84, 7792, 0, NULL, NULL},
+{85, 7798, 0, NULL, NULL},
+{100, 7699, 0, NULL, NULL},
+{101, 7705, 0, NULL, NULL},
+{108, 7741, 0, NULL, NULL},
+{110, 7755, 0, NULL, NULL},
+{116, 7793, 0, NULL, NULL},
+{117, 7799, 0, NULL, NULL}
+}; /* compose_tab_23 */
+static int hash_compose_tab_24[4] =
+{0,1,-1,-1}; /* hash_compose_tab_24 */
+static CompEntry compose_tab_24[] = {
+{72, 7722, 0, NULL, NULL},
+{104, 7723, 0, NULL, NULL}
+}; /* compose_tab_24 */
+static int hash_compose_tab_25[12] =
+{-1,1,2,-1,-1,3,-1,-1,-1,0,4,5}; /* hash_compose_tab_25 */
+static CompEntry compose_tab_25[] = {
+{69, 7706, 0, NULL, NULL},
+{73, 7724, 0, NULL, NULL},
+{85, 7796, 0, NULL, NULL},
+{101, 7707, 0, NULL, NULL},
+{105, 7725, 0, NULL, NULL},
+{117, 7797, 0, NULL, NULL}
+}; /* compose_tab_25 */
+static int hash_compose_tab_26[34] =
+{1,-1,10,-1,-1,11,12,2,3,13,4,-1,14,-1,5,15,6,-1,-1,-1,16,-1,7,-1,-1,-1,-1,-1,
+ -1,-1,8,-1,0,9}; /* hash_compose_tab_26 */
+static CompEntry compose_tab_26[] = {
+{66, 7686, 0, NULL, NULL},
+{68, 7694, 0, NULL, NULL},
+{75, 7732, 0, NULL, NULL},
+{76, 7738, 0, NULL, NULL},
+{78, 7752, 0, NULL, NULL},
+{82, 7774, 0, NULL, NULL},
+{84, 7790, 0, NULL, NULL},
+{90, 7828, 0, NULL, NULL},
+{98, 7687, 0, NULL, NULL},
+{100, 7695, 0, NULL, NULL},
+{104, 7830, 0, NULL, NULL},
+{107, 7733, 0, NULL, NULL},
+{108, 7739, 0, NULL, NULL},
+{110, 7753, 0, NULL, NULL},
+{114, 7775, 0, NULL, NULL},
+{116, 7791, 0, NULL, NULL},
+{122, 7829, 0, NULL, NULL}
+}; /* compose_tab_26 */
+static int hash_compose_tab_27_1[4] =
+{-1,0,1,-1}; /* hash_compose_tab_27_1 */
+static CompEntry compose_tab_27_1[] = {
+{953, 8151, 0, NULL, NULL},
+{965, 8167, 0, NULL, NULL}
+}; /* compose_tab_27_1 */
+static int hash_compose_tab_27_2_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_2_0 */
+static CompEntry compose_tab_27_2_0[] = {
+{913, 8078, 0, NULL, NULL},
+{919, 8094, 0, NULL, NULL},
+{937, 8110, 0, NULL, NULL},
+{945, 8070, 0, NULL, NULL},
+{951, 8086, 0, NULL, NULL},
+{969, 8102, 0, NULL, NULL}
+}; /* compose_tab_27_2_0 */
+static int hash_compose_tab_27_2[20] =
+{-1,3,-1,-1,-1,5,8,-1,-1,9,-1,6,-1,1,7,-1,-1,0,4,2}; /* hash_compose_tab_27_2 */
+static CompEntry compose_tab_27_2[] = {
+{837, 0, 6, compose_tab_27_2_0, hash_compose_tab_27_2_0},
+{913, 7950, 0, NULL, NULL},
+{919, 7982, 0, NULL, NULL},
+{921, 7998, 0, NULL, NULL},
+{937, 8046, 0, NULL, NULL},
+{945, 7942, 0, NULL, NULL},
+{951, 7974, 0, NULL, NULL},
+{953, 7990, 0, NULL, NULL},
+{965, 8022, 0, NULL, NULL},
+{969, 8038, 0, NULL, NULL}
+}; /* compose_tab_27_2 */
+static int hash_compose_tab_27_3_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_3_0 */
+static CompEntry compose_tab_27_3_0[] = {
+{913, 8079, 0, NULL, NULL},
+{919, 8095, 0, NULL, NULL},
+{937, 8111, 0, NULL, NULL},
+{945, 8071, 0, NULL, NULL},
+{951, 8087, 0, NULL, NULL},
+{969, 8103, 0, NULL, NULL}
+}; /* compose_tab_27_3_0 */
+static int hash_compose_tab_27_3[22] =
+{-1,0,10,-1,-1,7,-1,8,-1,4,-1,1,-1,5,-1,-1,-1,2,-1,3,9,6}; /* hash_compose_tab_27_3 */
+static CompEntry compose_tab_27_3[] = {
+{837, 0, 6, compose_tab_27_3_0, hash_compose_tab_27_3_0},
+{913, 7951, 0, NULL, NULL},
+{919, 7983, 0, NULL, NULL},
+{921, 7999, 0, NULL, NULL},
+{933, 8031, 0, NULL, NULL},
+{937, 8047, 0, NULL, NULL},
+{945, 7943, 0, NULL, NULL},
+{951, 7975, 0, NULL, NULL},
+{953, 7991, 0, NULL, NULL},
+{965, 8023, 0, NULL, NULL},
+{969, 8039, 0, NULL, NULL}
+}; /* compose_tab_27_3 */
+static int hash_compose_tab_27_4[6] =
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_27_4 */
+static CompEntry compose_tab_27_4[] = {
+{945, 8119, 0, NULL, NULL},
+{951, 8135, 0, NULL, NULL},
+{969, 8183, 0, NULL, NULL}
+}; /* compose_tab_27_4 */
+static int hash_compose_tab_27[24] =
+{0,-1,-1,-1,-1,8,11,-1,1,5,9,-1,-1,-1,-1,6,10,7,-1,2,3,4,-1,-1}; /* hash_compose_tab_27 */
+static CompEntry compose_tab_27[] = {
+{168, 8129, 0, NULL, NULL},
+{776, 0, 2, compose_tab_27_1, hash_compose_tab_27_1},
+{787, 0, 10, compose_tab_27_2, hash_compose_tab_27_2},
+{788, 0, 11, compose_tab_27_3, hash_compose_tab_27_3},
+{837, 0, 3, compose_tab_27_4, hash_compose_tab_27_4},
+{945, 8118, 0, NULL, NULL},
+{951, 8134, 0, NULL, NULL},
+{953, 8150, 0, NULL, NULL},
+{965, 8166, 0, NULL, NULL},
+{969, 8182, 0, NULL, NULL},
+{8127, 8143, 0, NULL, NULL},
+{8190, 8159, 0, NULL, NULL}
+}; /* compose_tab_27 */
+static int hash_compose_tab_28[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_28 */
+static CompEntry compose_tab_28[] = {
+{913, 8124, 0, NULL, NULL},
+{919, 8140, 0, NULL, NULL},
+{937, 8188, 0, NULL, NULL},
+{945, 8115, 0, NULL, NULL},
+{951, 8131, 0, NULL, NULL},
+{969, 8179, 0, NULL, NULL}
+}; /* compose_tab_28 */
+static int hash_compose_tab_29[4] =
+{0,-1,1,-1}; /* hash_compose_tab_29 */
+static CompEntry compose_tab_29[] = {
+{1488, 64302, 0, NULL, NULL},
+{1522, 64287, 0, NULL, NULL}
+}; /* compose_tab_29 */
+static int hash_compose_tab_30[2] =
+{0,-1}; /* hash_compose_tab_30 */
+static CompEntry compose_tab_30[] = {
+{1488, 64303, 0, NULL, NULL}
+}; /* compose_tab_30 */
+static int hash_compose_tab_31[2] =
+{-1,0}; /* hash_compose_tab_31 */
+static CompEntry compose_tab_31[] = {
+{1493, 64331, 0, NULL, NULL}
+}; /* compose_tab_31 */
+static int hash_compose_tab_32[44] =
+{7,8,9,10,11,-1,12,-1,13,14,-1,15,16,-1,17,18,19,20,21,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,5,6,-1}; /* hash_compose_tab_32 */
+static CompEntry compose_tab_32[] = {
+{1488, 64304, 0, NULL, NULL},
+{1489, 64305, 0, NULL, NULL},
+{1490, 64306, 0, NULL, NULL},
+{1491, 64307, 0, NULL, NULL},
+{1492, 64308, 0, NULL, NULL},
+{1493, 64309, 0, NULL, NULL},
+{1494, 64310, 0, NULL, NULL},
+{1496, 64312, 0, NULL, NULL},
+{1497, 64313, 0, NULL, NULL},
+{1498, 64314, 0, NULL, NULL},
+{1499, 64315, 0, NULL, NULL},
+{1500, 64316, 0, NULL, NULL},
+{1502, 64318, 0, NULL, NULL},
+{1504, 64320, 0, NULL, NULL},
+{1505, 64321, 0, NULL, NULL},
+{1507, 64323, 0, NULL, NULL},
+{1508, 64324, 0, NULL, NULL},
+{1510, 64326, 0, NULL, NULL},
+{1511, 64327, 0, NULL, NULL},
+{1512, 64328, 0, NULL, NULL},
+{1513, 64329, 0, NULL, NULL},
+{1514, 64330, 0, NULL, NULL}
+}; /* compose_tab_32 */
+static int hash_compose_tab_33[6] =
+{-1,0,2,-1,-1,1}; /* hash_compose_tab_33 */
+static CompEntry compose_tab_33[] = {
+{1489, 64332, 0, NULL, NULL},
+{1499, 64333, 0, NULL, NULL},
+{1508, 64334, 0, NULL, NULL}
+}; /* compose_tab_33 */
+static int hash_compose_tab_34_0[2] =
+{-1,0}; /* hash_compose_tab_34_0 */
+static CompEntry compose_tab_34_0[] = {
+{1513, 64300, 0, NULL, NULL}
+}; /* compose_tab_34_0 */
+static int hash_compose_tab_34[4] =
+{0,1,-1,-1}; /* hash_compose_tab_34 */
+static CompEntry compose_tab_34[] = {
+{1468, 0, 1, compose_tab_34_0, hash_compose_tab_34_0},
+{1513, 64298, 0, NULL, NULL}
+}; /* compose_tab_34 */
+static int hash_compose_tab_35_0[2] =
+{-1,0}; /* hash_compose_tab_35_0 */
+static CompEntry compose_tab_35_0[] = {
+{1513, 64301, 0, NULL, NULL}
+}; /* compose_tab_35_0 */
+static int hash_compose_tab_35[4] =
+{0,1,-1,-1}; /* hash_compose_tab_35 */
+static CompEntry compose_tab_35[] = {
+{1468, 0, 1, compose_tab_35_0, hash_compose_tab_35_0},
+{1513, 64299, 0, NULL, NULL}
+}; /* compose_tab_35 */
+static int hash_compose_tab_36[22] =
+{3,10,-1,-1,-1,4,5,-1,-1,-1,-1,-1,6,-1,-1,0,1,2,7,8,9,-1}; /* hash_compose_tab_36 */
+static CompEntry compose_tab_36[] = {
+{2325, 2392, 0, NULL, NULL},
+{2326, 2393, 0, NULL, NULL},
+{2327, 2394, 0, NULL, NULL},
+{2332, 2395, 0, NULL, NULL},
+{2337, 2396, 0, NULL, NULL},
+{2338, 2397, 0, NULL, NULL},
+{2344, 2345, 0, NULL, NULL},
+{2347, 2398, 0, NULL, NULL},
+{2351, 2399, 0, NULL, NULL},
+{2352, 2353, 0, NULL, NULL},
+{2355, 2356, 0, NULL, NULL}
+}; /* compose_tab_36 */
+static int hash_compose_tab_37[8] =
+{-1,0,1,-1,2,-1,-1,3}; /* hash_compose_tab_37 */
+static CompEntry compose_tab_37[] = {
+{2465, 2524, 0, NULL, NULL},
+{2466, 2525, 0, NULL, NULL},
+{2476, 2480, 0, NULL, NULL},
+{2479, 2527, 0, NULL, NULL}
+}; /* compose_tab_37 */
+static int hash_compose_tab_38[2] =
+{-1,0}; /* hash_compose_tab_38 */
+static CompEntry compose_tab_38[] = {
+{2503, 2507, 0, NULL, NULL}
+}; /* compose_tab_38 */
+static int hash_compose_tab_39[2] =
+{-1,0}; /* hash_compose_tab_39 */
+static CompEntry compose_tab_39[] = {
+{2503, 2508, 0, NULL, NULL}
+}; /* compose_tab_39 */
+static int hash_compose_tab_40[10] =
+{-1,-1,0,1,3,4,-1,-1,2,-1}; /* hash_compose_tab_40 */
+static CompEntry compose_tab_40[] = {
+{2582, 2649, 0, NULL, NULL},
+{2583, 2650, 0, NULL, NULL},
+{2588, 2651, 0, NULL, NULL},
+{2593, 2652, 0, NULL, NULL},
+{2603, 2654, 0, NULL, NULL}
+}; /* compose_tab_40 */
+static int hash_compose_tab_41[6] =
+{1,2,-1,-1,-1,0}; /* hash_compose_tab_41 */
+static CompEntry compose_tab_41[] = {
+{2849, 2908, 0, NULL, NULL},
+{2850, 2909, 0, NULL, NULL},
+{2863, 2911, 0, NULL, NULL}
+}; /* compose_tab_41 */
+static int hash_compose_tab_42[2] =
+{-1,0}; /* hash_compose_tab_42 */
+static CompEntry compose_tab_42[] = {
+{2887, 2891, 0, NULL, NULL}
+}; /* compose_tab_42 */
+static int hash_compose_tab_43[2] =
+{-1,0}; /* hash_compose_tab_43 */
+static CompEntry compose_tab_43[] = {
+{2887, 2888, 0, NULL, NULL}
+}; /* compose_tab_43 */
+static int hash_compose_tab_44[2] =
+{-1,0}; /* hash_compose_tab_44 */
+static CompEntry compose_tab_44[] = {
+{2887, 2892, 0, NULL, NULL}
+}; /* compose_tab_44 */
+static int hash_compose_tab_45[4] =
+{-1,-1,0,1}; /* hash_compose_tab_45 */
+static CompEntry compose_tab_45[] = {
+{3014, 3018, 0, NULL, NULL},
+{3015, 3019, 0, NULL, NULL}
+}; /* compose_tab_45 */
+static int hash_compose_tab_46[4] =
+{-1,-1,0,1}; /* hash_compose_tab_46 */
+static CompEntry compose_tab_46[] = {
+{2962, 2964, 0, NULL, NULL},
+{3014, 3020, 0, NULL, NULL}
+}; /* compose_tab_46 */
+static int hash_compose_tab_47[2] =
+{0,-1}; /* hash_compose_tab_47 */
+static CompEntry compose_tab_47[] = {
+{3142, 3144, 0, NULL, NULL}
+}; /* compose_tab_47 */
+static int hash_compose_tab_48[2] =
+{0,-1}; /* hash_compose_tab_48 */
+static CompEntry compose_tab_48[] = {
+{3270, 3274, 0, NULL, NULL}
+}; /* compose_tab_48 */
+static int hash_compose_tab_49_1[2] =
+{0,-1}; /* hash_compose_tab_49_1 */
+static CompEntry compose_tab_49_1[] = {
+{3270, 3275, 0, NULL, NULL}
+}; /* compose_tab_49_1 */
+static int hash_compose_tab_49[6] =
+{2,-1,1,-1,-1,0}; /* hash_compose_tab_49 */
+static CompEntry compose_tab_49[] = {
+{3263, 3264, 0, NULL, NULL},
+{3266, 0, 1, compose_tab_49_1, hash_compose_tab_49_1},
+{3270, 3271, 0, NULL, NULL}
+}; /* compose_tab_49 */
+static int hash_compose_tab_50[2] =
+{0,-1}; /* hash_compose_tab_50 */
+static CompEntry compose_tab_50[] = {
+{3270, 3272, 0, NULL, NULL}
+}; /* compose_tab_50 */
+static int hash_compose_tab_51[4] =
+{-1,-1,0,1}; /* hash_compose_tab_51 */
+static CompEntry compose_tab_51[] = {
+{3398, 3402, 0, NULL, NULL},
+{3399, 3403, 0, NULL, NULL}
+}; /* compose_tab_51 */
+static int hash_compose_tab_52[2] =
+{0,-1}; /* hash_compose_tab_52 */
+static CompEntry compose_tab_52[] = {
+{3398, 3404, 0, NULL, NULL}
+}; /* compose_tab_52 */
+static int hash_compose_tab_53[2] =
+{-1,0}; /* hash_compose_tab_53 */
+static CompEntry compose_tab_53[] = {
+{3661, 3635, 0, NULL, NULL}
+}; /* compose_tab_53 */
+static int hash_compose_tab_54[2] =
+{-1,0}; /* hash_compose_tab_54 */
+static CompEntry compose_tab_54[] = {
+{3789, 3763, 0, NULL, NULL}
+}; /* compose_tab_54 */
+static int hash_compose_tab_55_2[4] =
+{-1,-1,0,1}; /* hash_compose_tab_55_2 */
+static CompEntry compose_tab_55_2[] = {
+{4018, 3959, 0, NULL, NULL},
+{4019, 3961, 0, NULL, NULL}
+}; /* compose_tab_55_2 */
+static int hash_compose_tab_55[6] =
+{0,-1,1,2,-1,-1}; /* hash_compose_tab_55 */
+static CompEntry compose_tab_55[] = {
+{3954, 3955, 0, NULL, NULL},
+{3956, 3957, 0, NULL, NULL},
+{3968, 0, 2, compose_tab_55_2, hash_compose_tab_55_2}
+}; /* compose_tab_55 */
+static int hash_compose_tab_56[4] =
+{-1,-1,0,1}; /* hash_compose_tab_56 */
+static CompEntry compose_tab_56[] = {
+{4018, 3958, 0, NULL, NULL},
+{4019, 3960, 0, NULL, NULL}
+}; /* compose_tab_56 */
+static int hash_compose_tab_57[4] =
+{0,1,-1,-1}; /* hash_compose_tab_57 */
+static CompEntry compose_tab_57[] = {
+{3904, 3945, 0, NULL, NULL},
+{3984, 4025, 0, NULL, NULL}
+}; /* compose_tab_57 */
+static int hash_compose_tab_58[20] =
+{-1,2,7,-1,-1,-1,0,3,5,8,-1,4,9,-1,-1,-1,1,6,-1,-1}; /* hash_compose_tab_58 */
+static CompEntry compose_tab_58[] = {
+{3906, 3907, 0, NULL, NULL},
+{3916, 3917, 0, NULL, NULL},
+{3921, 3922, 0, NULL, NULL},
+{3926, 3927, 0, NULL, NULL},
+{3931, 3932, 0, NULL, NULL},
+{3986, 3987, 0, NULL, NULL},
+{3996, 3997, 0, NULL, NULL},
+{4001, 4002, 0, NULL, NULL},
+{4006, 4007, 0, NULL, NULL},
+{4011, 4012, 0, NULL, NULL}
+}; /* compose_tab_58 */
+static int hash_compose_tab_59[96] =
+{33,12,34,-1,13,35,14,36,15,37,-1,-1,-1,-1,-1,16,38,-1,17,39,-1,18,40,-1,19,
+ 41,-1,20,42,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,43,44,45,
+ 46,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,21,47,-1,-1,-1,-1,-1,-1,-1,0,22,-1,-1,-1,1,
+ 23,2,24,3,25,4,26,5,27,6,28,7,29,8,30,9,31,10,32,11}; /* hash_compose_tab_59 */
+static CompEntry compose_tab_59[] = {
+{12358, 12436, 0, NULL, NULL},
+{12363, 12364, 0, NULL, NULL},
+{12365, 12366, 0, NULL, NULL},
+{12367, 12368, 0, NULL, NULL},
+{12369, 12370, 0, NULL, NULL},
+{12371, 12372, 0, NULL, NULL},
+{12373, 12374, 0, NULL, NULL},
+{12375, 12376, 0, NULL, NULL},
+{12377, 12378, 0, NULL, NULL},
+{12379, 12380, 0, NULL, NULL},
+{12381, 12382, 0, NULL, NULL},
+{12383, 12384, 0, NULL, NULL},
+{12385, 12386, 0, NULL, NULL},
+{12388, 12389, 0, NULL, NULL},
+{12390, 12391, 0, NULL, NULL},
+{12392, 12393, 0, NULL, NULL},
+{12399, 12400, 0, NULL, NULL},
+{12402, 12403, 0, NULL, NULL},
+{12405, 12406, 0, NULL, NULL},
+{12408, 12409, 0, NULL, NULL},
+{12411, 12412, 0, NULL, NULL},
+{12445, 12446, 0, NULL, NULL},
+{12454, 12532, 0, NULL, NULL},
+{12459, 12460, 0, NULL, NULL},
+{12461, 12462, 0, NULL, NULL},
+{12463, 12464, 0, NULL, NULL},
+{12465, 12466, 0, NULL, NULL},
+{12467, 12468, 0, NULL, NULL},
+{12469, 12470, 0, NULL, NULL},
+{12471, 12472, 0, NULL, NULL},
+{12473, 12474, 0, NULL, NULL},
+{12475, 12476, 0, NULL, NULL},
+{12477, 12478, 0, NULL, NULL},
+{12479, 12480, 0, NULL, NULL},
+{12481, 12482, 0, NULL, NULL},
+{12484, 12485, 0, NULL, NULL},
+{12486, 12487, 0, NULL, NULL},
+{12488, 12489, 0, NULL, NULL},
+{12495, 12496, 0, NULL, NULL},
+{12498, 12499, 0, NULL, NULL},
+{12501, 12502, 0, NULL, NULL},
+{12504, 12505, 0, NULL, NULL},
+{12507, 12508, 0, NULL, NULL},
+{12527, 12535, 0, NULL, NULL},
+{12528, 12536, 0, NULL, NULL},
+{12529, 12537, 0, NULL, NULL},
+{12530, 12538, 0, NULL, NULL},
+{12541, 12542, 0, NULL, NULL}
+}; /* compose_tab_59 */
+static int hash_compose_tab_60[20] =
+{-1,7,1,-1,8,2,-1,9,3,-1,-1,4,-1,-1,-1,5,-1,-1,6,0}; /* hash_compose_tab_60 */
+static CompEntry compose_tab_60[] = {
+{12399, 12401, 0, NULL, NULL},
+{12402, 12404, 0, NULL, NULL},
+{12405, 12407, 0, NULL, NULL},
+{12408, 12410, 0, NULL, NULL},
+{12411, 12413, 0, NULL, NULL},
+{12495, 12497, 0, NULL, NULL},
+{12498, 12500, 0, NULL, NULL},
+{12501, 12503, 0, NULL, NULL},
+{12504, 12506, 0, NULL, NULL},
+{12507, 12509, 0, NULL, NULL}
+}; /* compose_tab_60 */
+static int hash_compose_tab[122] =
+{30,31,52,60,32,-1,-1,33,-1,34,35,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,-1,5,6,7,8,9,10,11,12,36,13,37,14,
+ 38,15,16,55,40,-1,-1,-1,-1,17,56,-1,-1,-1,-1,-1,41,18,19,20,42,21,22,-1,45,
+ 39,-1,23,24,-1,25,26,-1,-1,-1,-1,-1,-1,-1,-1,48,-1,43,44,51,53,-1,-1,27,46,
+ 54,28,-1,-1,47,-1,-1,-1,-1,49,50,-1,-1,57,-1,58,59,29}; /* hash_compose_tab */
+static CompEntry compose_tab[] = {
+{768, 0, 39, compose_tab_0, hash_compose_tab_0},
+{769, 0, 70, compose_tab_1, hash_compose_tab_1},
+{770, 0, 27, compose_tab_2, hash_compose_tab_2},
+{771, 0, 19, compose_tab_3, hash_compose_tab_3},
+{772, 0, 28, compose_tab_4, hash_compose_tab_4},
+{774, 0, 30, compose_tab_5, hash_compose_tab_5},
+{775, 0, 40, compose_tab_6, hash_compose_tab_6},
+{776, 0, 50, compose_tab_7, hash_compose_tab_7},
+{777, 0, 15, compose_tab_8, hash_compose_tab_8},
+{778, 0, 6, compose_tab_9, hash_compose_tab_9},
+{779, 0, 6, compose_tab_10, hash_compose_tab_10},
+{780, 0, 34, compose_tab_11, hash_compose_tab_11},
+{781, 0, 17, compose_tab_12, hash_compose_tab_12},
+{783, 0, 14, compose_tab_13, hash_compose_tab_13},
+{785, 0, 12, compose_tab_14, hash_compose_tab_14},
+{787, 0, 15, compose_tab_15, hash_compose_tab_15},
+{788, 0, 17, compose_tab_16, hash_compose_tab_16},
+{795, 0, 4, compose_tab_17, hash_compose_tab_17},
+{803, 0, 39, compose_tab_18, hash_compose_tab_18},
+{804, 0, 2, compose_tab_19, hash_compose_tab_19},
+{805, 0, 2, compose_tab_20, hash_compose_tab_20},
+{807, 0, 20, compose_tab_21, hash_compose_tab_21},
+{808, 0, 10, compose_tab_22, hash_compose_tab_22},
+{813, 0, 12, compose_tab_23, hash_compose_tab_23},
+{814, 0, 2, compose_tab_24, hash_compose_tab_24},
+{816, 0, 6, compose_tab_25, hash_compose_tab_25},
+{817, 0, 17, compose_tab_26, hash_compose_tab_26},
+{834, 0, 12, compose_tab_27, hash_compose_tab_27},
+{837, 0, 6, compose_tab_28, hash_compose_tab_28},
+{1463, 0, 2, compose_tab_29, hash_compose_tab_29},
+{1464, 0, 1, compose_tab_30, hash_compose_tab_30},
+{1465, 0, 1, compose_tab_31, hash_compose_tab_31},
+{1468, 0, 22, compose_tab_32, hash_compose_tab_32},
+{1471, 0, 3, compose_tab_33, hash_compose_tab_33},
+{1473, 0, 2, compose_tab_34, hash_compose_tab_34},
+{1474, 0, 2, compose_tab_35, hash_compose_tab_35},
+{2364, 0, 11, compose_tab_36, hash_compose_tab_36},
+{2492, 0, 4, compose_tab_37, hash_compose_tab_37},
+{2494, 0, 1, compose_tab_38, hash_compose_tab_38},
+{2519, 0, 1, compose_tab_39, hash_compose_tab_39},
+{2620, 0, 5, compose_tab_40, hash_compose_tab_40},
+{2876, 0, 3, compose_tab_41, hash_compose_tab_41},
+{2878, 0, 1, compose_tab_42, hash_compose_tab_42},
+{2902, 0, 1, compose_tab_43, hash_compose_tab_43},
+{2903, 0, 1, compose_tab_44, hash_compose_tab_44},
+{3006, 0, 2, compose_tab_45, hash_compose_tab_45},
+{3031, 0, 2, compose_tab_46, hash_compose_tab_46},
+{3158, 0, 1, compose_tab_47, hash_compose_tab_47},
+{3266, 0, 1, compose_tab_48, hash_compose_tab_48},
+{3285, 0, 3, compose_tab_49, hash_compose_tab_49},
+{3286, 0, 1, compose_tab_50, hash_compose_tab_50},
+{3390, 0, 2, compose_tab_51, hash_compose_tab_51},
+{3415, 0, 1, compose_tab_52, hash_compose_tab_52},
+{3634, 0, 1, compose_tab_53, hash_compose_tab_53},
+{3762, 0, 1, compose_tab_54, hash_compose_tab_54},
+{3953, 0, 3, compose_tab_55, hash_compose_tab_55},
+{3968, 0, 2, compose_tab_56, hash_compose_tab_56},
+{4021, 0, 2, compose_tab_57, hash_compose_tab_57},
+{4023, 0, 10, compose_tab_58, hash_compose_tab_58},
+{12441, 0, 48, compose_tab_59, hash_compose_tab_59},
+{12442, 0, 10, compose_tab_60, hash_compose_tab_60}
+}; /* compose_tab */
+#define COMP_CANDIDATE_MAP_OFFSET 24
+static Uint32 comp_candidate_map[] = {
+ 0x081ABFDFU,
+ 0x000361B8U,
+ 0x00000024U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x93800000U,
+ 0x00000006U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x10000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x50000000U,
+ 0x00800000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x10000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x50000000U,
+ 0x00C00000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x40000000U,
+ 0x00800000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00400000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00600004U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x40000000U,
+ 0x00800000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00040000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00040000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00020000U,
+ 0x00000001U,
+ 0x00A00000U
+};
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 89c6625550..524db2a2eb 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1596,6 +1596,19 @@ Sint erts_binary_set_loop_limit(Sint limit);
/* erl_unicode.c */
void erts_init_unicode(void);
Sint erts_unicode_set_loop_limit(Sint limit);
+
+void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) ;
+Sint erts_native_filename_need(Eterm ioterm, int encoding);
+void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars);
+int erts_analyze_utf8(byte *source, Uint size,
+ byte **err_pos, Uint *num_chars, int *left);
+char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int allow_empty);
+
+#define ERTS_UTF8_OK 0
+#define ERTS_UTF8_INCOMPLETE 1
+#define ERTS_UTF8_ERROR 2
+#define ERTS_UTF8_ANALYZE_MORE 3
+
/* erl_trace.c */
void erts_init_trace(void);
void erts_trace_check_exiting(Eterm exiting);
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 0d15272aa8..27c5f99320 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1253,6 +1253,22 @@ char* win32_errorstr(int);
#endif
+/************************************************************************
+ * Find out the native filename encoding of the process (look at locale of
+ * Unix processes and just do UTF16 on windows
+ ************************************************************************/
+#define ERL_FILENAME_UNKNOWN 0
+#define ERL_FILENAME_LATIN1 1
+#define ERL_FILENAME_UTF8 2
+#define ERL_FILENAME_UTF8_MAC 3
+#define ERL_FILENAME_WIN_WCHAR 4
+
+int erts_get_native_filename_encoding(void);
+/* The set function is only to be used by erl_init! */
+void erts_set_user_requested_filename_encoding(int encoding);
+int erts_get_user_requested_filename_encoding(void);
+
+void erts_init_sys_common_misc(void);
#endif