aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator
diff options
context:
space:
mode:
authorPatrik Nyblom <[email protected]>2010-11-09 18:08:57 +0100
committerPatrik Nyblom <[email protected]>2010-11-30 16:32:29 +0100
commit9622ab2132e2501ee5769357a914dcc6635e515c (patch)
tree77a4f13c833d325025e6bd7cb614798727828402 /erts/emulator
parent4161de166d46b6c9c25d5ac813c94a025c7a029d (diff)
downloadotp-9622ab2132e2501ee5769357a914dcc6635e515c.tar.gz
otp-9622ab2132e2501ee5769357a914dcc6635e515c.tar.bz2
otp-9622ab2132e2501ee5769357a914dcc6635e515c.zip
Convert filenames read on MacOSX to canonical form
Diffstat (limited to 'erts/emulator')
-rw-r--r--erts/emulator/beam/bif.tab1
-rw-r--r--erts/emulator/beam/erl_unicode.c416
-rw-r--r--erts/emulator/beam/erl_unicode_normalize.h1687
-rw-r--r--erts/emulator/beam/global.h11
-rw-r--r--erts/emulator/beam/sys.h3
-rw-r--r--erts/emulator/internal_doc/dec.dat942
-rw-r--r--erts/emulator/internal_doc/dec.erl237
-rw-r--r--erts/emulator/sys/common/erl_sys_common_misc.c18
8 files changed, 3229 insertions, 86 deletions
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 3de5e63a63..60b4b1946b 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -799,6 +799,7 @@ bif erlang:nif_error/2
#
bif prim_file:internal_name2native/1
bif prim_file:internal_native2name/1
+bif prim_file:internal_normalize_utf8/1
bif file:native_name_encoding/0
#
# Obsolete
diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c
index 4f26f078ce..1e729cfc2e 100644
--- a/erts/emulator/beam/erl_unicode.c
+++ b/erts/emulator/beam/erl_unicode.c
@@ -30,6 +30,8 @@
#include "big.h"
#include "erl_unicode.h"
+#include "erl_unicode_normalize.h"
+
typedef struct _restart_context {
byte *bytes;
@@ -54,13 +56,6 @@ static BIF_RETTYPE finalize_list_to_list(Process *p,
Uint num_resulting_chars,
int state, int left,
Eterm tail);
-static int analyze_utf8(byte *source, Uint size,
- byte **err_pos, Uint *num_chars, int *left);
-#define UTF8_OK 0
-#define UTF8_INCOMPLETE 1
-#define UTF8_ERROR 2
-#define UTF8_ANALYZE_MORE 3
-
static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3);
static BIF_RETTYPE characters_to_list_trap_1(BIF_ALIST_3);
static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3);
@@ -970,11 +965,11 @@ static int is_valid_utf8(Eterm orig_bin)
bytes = erts_get_aligned_binary_bytes(orig_bin, &temp_alloc);
}
size = binary_size(orig_bin);
- ret = analyze_utf8(bytes,
+ ret = erts_analyze_utf8(bytes,
size,
&endpos,&numchar,NULL);
erts_free_aligned_binary_bytes(temp_alloc);
- return (ret == UTF8_OK);
+ return (ret == ERTS_UTF8_OK);
}
BIF_RETTYPE unicode_characters_to_binary_2(BIF_ALIST_2)
@@ -1084,14 +1079,14 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char
hp += 2;
rest_term = CONS(hp,leftover_bin,rest_term);
}
- BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, UTF8_ERROR, left, NIL));
+ BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, ERTS_UTF8_ERROR, left, NIL));
} else if (rest_term == NIL && num_leftovers != 0) {
Eterm leftover_bin = new_binary(p, leftover, num_leftovers);
if (check_leftovers(leftover,num_leftovers) != 0) {
- BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_ERROR,
+ BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_ERROR,
left, NIL));
} else {
- BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_INCOMPLETE,
+ BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_INCOMPLETE,
left, NIL));
}
} else { /* All OK */
@@ -1107,11 +1102,11 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char
rc.num_processed_bytes = 0; /* not used */
rc.num_bytes_to_process = pos;
rc.num_resulting_chars = characters;
- rc.state = UTF8_OK; /* not used */
+ rc.state = ERTS_UTF8_OK; /* not used */
BIF_TRAP3(&characters_to_list_trap_1_exp, p, make_magic_bin_for_restart(p,&rc),
rest_term, latin1);
} else { /* Success */
- BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, UTF8_OK, left, NIL));
+ BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, ERTS_UTF8_OK, left, NIL));
}
}
}
@@ -1205,7 +1200,7 @@ BIF_RETTYPE unicode_characters_to_list_2(BIF_ALIST_2)
* When input to characters_to_list is a plain binary and the format is 'unicode', we do
* a faster analyze and size count with this function.
*/
-static int analyze_utf8(byte *source, Uint size,
+int erts_analyze_utf8(byte *source, Uint size,
byte **err_pos, Uint *num_chars, int *left)
{
*err_pos = source;
@@ -1216,60 +1211,60 @@ static int analyze_utf8(byte *source, Uint size,
--size;
} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
if (size < 2) {
- return UTF8_INCOMPLETE;
+ return ERTS_UTF8_INCOMPLETE;
}
if (((source[1] & ((byte) 0xC0)) != 0x80) ||
((*source) < 0xC2) /* overlong */) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
source += 2;
size -= 2;
} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
if (size < 3) {
- return UTF8_INCOMPLETE;
+ return ERTS_UTF8_INCOMPLETE;
}
if (((source[1] & ((byte) 0xC0)) != 0x80) ||
((source[2] & ((byte) 0xC0)) != 0x80) ||
(((*source) == 0xE0) && (source[1] < 0xA0)) /* overlong */ ) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
if ((((*source) & ((byte) 0xF)) == 0xD) &&
((source[1] & 0x20) != 0)) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
if (((*source) == 0xEF) && (source[1] == 0xBF) &&
((source[2] == 0xBE) || (source[2] == 0xBF))) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
source += 3;
size -= 3;
} else if (((*source) & ((byte) 0xF8)) == 0xF0) {
if (size < 4) {
- return UTF8_INCOMPLETE;
+ return ERTS_UTF8_INCOMPLETE;
}
if (((source[1] & ((byte) 0xC0)) != 0x80) ||
((source[2] & ((byte) 0xC0)) != 0x80) ||
((source[3] & ((byte) 0xC0)) != 0x80) ||
(((*source) == 0xF0) && (source[1] < 0x90)) /* overlong */) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
if ((((*source) & ((byte)0x7)) > 0x4U) ||
((((*source) & ((byte)0x7)) == 0x4U) &&
((source[1] & ((byte)0x3F)) > 0xFU))) {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
source += 4;
size -= 4;
} else {
- return UTF8_ERROR;
+ return ERTS_UTF8_ERROR;
}
++(*num_chars);
*err_pos = source;
if (left && --(*left) <= 0) {
- return UTF8_ANALYZE_MORE;
+ return ERTS_UTF8_ANALYZE_MORE;
}
}
- return UTF8_OK;
+ return ERTS_UTF8_OK;
}
/*
@@ -1304,7 +1299,7 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz,
} else if (((*source) & ((byte) 0xE0)) == 0xC0) {
unipoint =
(((Uint) ((*source) & ((byte) 0x1F))) << 6) |
- ((Uint) (source[1] & ((byte) 0x3F)));
+ ((Uint) (source[1] & ((byte) 0x3F)));
} else if (((*source) & ((byte) 0xF0)) == 0xE0) {
unipoint =
(((Uint) ((*source) & ((byte) 0xF))) << 12) |
@@ -1330,6 +1325,216 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz,
return ret;
}
+static int is_candidate(Uint cp)
+{
+ int index,pos;
+ if (cp < 768) return 0;
+ if (cp > 4023) {
+ if (cp == 12441 || cp == 12442) return 1;
+ return 0;
+ }
+ index = cp / 32 - COMP_CANDIDATE_MAP_OFFSET;
+ pos = cp % 32;
+ return !!(comp_candidate_map[index] & (1UL << pos));
+}
+
+static int hashsearch(int *htab, int htab_size, CompEntry *cv, Uint16 c)
+{
+ int bucket = c % htab_size;
+ while (htab[bucket] != -1 && cv[htab[bucket]].c != c)
+ bucket = (bucket + 1) % htab_size;
+ return htab[bucket];
+}
+
+#define TRANSLATE_NO 0
+#define TRANSLATE_MAYBE -1
+
+/* The s array is reversed */
+static int translate(Uint16 *s, int slen, Uint16 *res)
+{
+ /* Go backwards through buffer and match against tree */
+ int pos = 0;
+ CompEntry *cv = compose_tab;
+ int *hc = hash_compose_tab;
+ int cvs = compose_tab_size;
+ int x;
+ while (pos < slen) {
+ x = hashsearch(hc,cvs*HASH_SIZE_FACTOR,cv,s[pos]);
+ if (x < 0) {
+ return TRANSLATE_NO;
+ }
+ if (cv[x].res) {
+ *res = cv[x].res;
+ return pos;
+ }
+ cvs = cv[x].num_subs;
+ hc = cv[x].hash;
+ cv = cv[x].subs;
+ ++pos;
+ }
+ return TRANSLATE_MAYBE;
+}
+
+static void handle_first_norm(Uint16 *savepoints, int *numpointsp, Uint unipoint)
+{
+ /*erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) *numpointsp);*/
+ *numpointsp = 1;
+ savepoints[0] = (Uint16) unipoint;
+}
+
+static void cleanup_norm(Eterm **hpp, Uint16 *savepoints, int numpoints, Eterm *retp)
+{
+ Eterm *hp = *hpp;
+ int res,i;
+ Uint16 newpoint;
+ Eterm ret = *retp;
+
+ ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+ hp += 2;
+
+ for (i = 1;i < numpoints;) {
+ if(!is_candidate(savepoints[i]) ||
+ ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) {
+ ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+ hp += 2;
+ ++i;
+ } else {
+ ret = CONS(hp,make_small((Uint) newpoint),ret);
+ hp += 2;
+ i += res;
+ }
+ }
+ *retp = ret;
+}
+
+static void handle_potential_norm(Eterm **hpp, Uint16 *savepoints, int *numpointsp, Uint unipoint, Eterm *retp)
+{
+ Eterm *hp = *hpp;
+ int numpoints = *numpointsp;
+ int res,i;
+ Uint16 newpoint;
+ Eterm ret = *retp;
+
+ /* erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) numpoints);*/
+ if ((unipoint >> 16) == 0) { /* otherwise we're done here */
+ savepoints[numpoints++] = (Uint16) unipoint;
+ res = translate(savepoints,numpoints,&newpoint);
+ if (res == TRANSLATE_NO) {
+ ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+ hp += 2;
+ for (i = 1;i < numpoints;) {
+ if(!is_candidate(savepoints[i]) ||
+ ((res = translate(savepoints+i,numpoints - i, &newpoint)) == 0)) {
+ ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+ hp += 2;
+ ++i;
+ } else if (res > 0) {
+ ret = CONS(hp,make_small((Uint) newpoint),ret);
+ hp += 2;
+ i += res;
+ } else { /* res < 0 */
+ /* A "maybe", means we are not done yet */
+ int j = 0;
+ while (i < numpoints) {
+ savepoints[j++] = savepoints[i++];
+ }
+ numpoints = j;
+ goto breakaway;
+ }
+ }
+ numpoints = 0;
+ breakaway:
+ ;
+ } else if (res > 0) {
+ numpoints = 0;
+ ret = CONS(hp,make_small((Uint) newpoint),ret);
+ hp += 2;
+ } /* < 0 means go on */
+ } else {
+ /* Unconditional rollup, this character is larger than 16 bit */
+ ret = CONS(hp,make_small((Uint) savepoints[0]),ret);
+ hp += 2;
+
+ for (i = 1;i < numpoints;) {
+ if(!is_candidate(savepoints[i]) ||
+ ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) {
+ ret = CONS(hp,make_small((Uint) savepoints[i]),ret);
+ hp += 2;
+ ++i;
+ } else {
+ ret = CONS(hp,make_small((Uint) newpoint),ret);
+ hp += 2;
+ i += res;
+ }
+ }
+ ret = CONS(hp,make_small(unipoint),ret);
+ hp += 2;
+ numpoints = 0;
+ }
+ *hpp = hp;
+ *numpointsp = numpoints;
+ *retp = ret;
+}
+
+static Eterm do_utf8_to_list_normalize(Process *p, Uint num, byte *bytes, Uint sz)
+{
+ Eterm *hp,*hp_end;
+ Eterm ret;
+ byte *source;
+ Uint unipoint;
+ Uint16 savepoints[4];
+ int numpoints = 0;
+
+ ASSERT(num > 0);
+
+ hp = HAlloc(p,num * 2); /* May be to much */
+ hp_end = hp + num * 2;
+ ret = NIL;
+ source = bytes + sz;
+ while(--source >= bytes) {
+ if (((*source) & ((byte) 0x80)) == 0) {
+ unipoint = (Uint) *source;
+ } else if (((*source) & ((byte) 0xE0)) == 0xC0) {
+ unipoint =
+ (((Uint) ((*source) & ((byte) 0x1F))) << 6) |
+ ((Uint) (source[1] & ((byte) 0x3F)));
+ } else if (((*source) & ((byte) 0xF0)) == 0xE0) {
+ unipoint =
+ (((Uint) ((*source) & ((byte) 0xF))) << 12) |
+ (((Uint) (source[1] & ((byte) 0x3F))) << 6) |
+ ((Uint) (source[2] & ((byte) 0x3F)));
+ } else if (((*source) & ((byte) 0xF8)) == 0xF0) {
+ unipoint =
+ (((Uint) ((*source) & ((byte) 0x7))) << 18) |
+ (((Uint) (source[1] & ((byte) 0x3F))) << 12) |
+ (((Uint) (source[2] & ((byte) 0x3F))) << 6) |
+ ((Uint) (source[3] & ((byte) 0x3F)));
+ } else {
+ /* ignore 2#10XXXXXX */
+ continue;
+ }
+ if (numpoints) {
+ handle_potential_norm(&hp,savepoints,&numpoints,unipoint,&ret);
+ continue;
+ }
+ /* We are not building up any normalizations yet, look that we shouldn't start... */
+ if (is_candidate(unipoint)) {
+ handle_first_norm(savepoints,&numpoints,unipoint);
+ continue;
+ }
+ ret = CONS(hp,make_small(unipoint),ret);
+ hp += 2;
+ }
+ /* so, we'we looped to the beginning, do we have anything saved? */
+ if (numpoints) {
+ cleanup_norm(&hp,savepoints,numpoints,&ret);
+ }
+ if (hp_end != hp) {
+ HRelease(p,hp_end,hp);
+ }
+ return ret;
+}
+
/*
* The last step of characters_to_list, build a list from the buffer 'bytes' (created in the same way
* as for characters_to_utf8). All sizes are known in advance and most data will be held in a
@@ -1378,10 +1583,10 @@ static BIF_RETTYPE finalize_list_to_list(Process *p,
*/
free_restart(bytes);
- if (state == UTF8_INCOMPLETE) {
+ if (state == ERTS_UTF8_INCOMPLETE) {
hp = HAlloc(p,4);
ret = TUPLE3(hp,am_incomplete,converted,rest);
- } else if (state == UTF8_ERROR) {
+ } else if (state == ERTS_UTF8_ERROR) {
hp = HAlloc(p,4);
ret = TUPLE3(hp,am_error,converted,rest);
} else {
@@ -1408,7 +1613,7 @@ static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3)
/*
* Hooks into the process of decoding a binary depending on state.
- * If last_state is UTF8_ANALYZE_MORE, num_bytes_to_process
+ * If last_state is ERTS_UTF8_ANALYZE_MORE, num_bytes_to_process
* and num_resulting_chars will grow
* until we're done analyzing the binary. Then we'll eat
* the bytes to process, lowering num_bytes_to_process and num_resulting_chars,
@@ -1465,14 +1670,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
left = allowed_iterations(p);
- if (state == UTF8_ANALYZE_MORE) {
- state = analyze_utf8(bytes + num_bytes_to_process,
+ if (state == ERTS_UTF8_ANALYZE_MORE) {
+ state = erts_analyze_utf8(bytes + num_bytes_to_process,
size - num_bytes_to_process,
&endpos,&numchar,&left);
cost_to_proc(p,numchar);
num_resulting_chars += numchar;
num_bytes_to_process = endpos - bytes;
- if (state == UTF8_ANALYZE_MORE) {
+ if (state == ERTS_UTF8_ANALYZE_MORE) {
Eterm epos = erts_make_integer(num_bytes_to_process,p);
Eterm enumchar = erts_make_integer(num_resulting_chars,p);
erts_free_aligned_binary_bytes(temp_alloc);
@@ -1528,7 +1733,7 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
ErlSubBin *sb;
Eterm orig;
Uint offset;
- ASSERT(state != UTF8_OK);
+ ASSERT(state != ERTS_UTF8_OK);
hp = HAlloc(p, ERL_SUB_BIN_SIZE);
sb = (ErlSubBin *) hp;
ERTS_GET_REAL_BIN(orig_bin, orig, offset, bitoffs, bitsize);
@@ -1544,14 +1749,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p,
/* Done */
- if (state == UTF8_INCOMPLETE) {
+ if (state == ERTS_UTF8_INCOMPLETE) {
if (check_leftovers(bytes + num_bytes_to_process + num_processed_bytes,
b_sz) != 0) {
goto error_return;
}
hp = HAlloc(p,4);
ret = TUPLE3(hp,am_incomplete,converted,rest);
- } else if (state == UTF8_ERROR) {
+ } else if (state == ERTS_UTF8_ERROR) {
error_return:
hp = HAlloc(p,4);
ret = TUPLE3(hp,am_error,converted,rest);
@@ -1589,7 +1794,7 @@ static BIF_RETTYPE characters_to_list_trap_3(BIF_ALIST_3)
0U, /* nothing processed yet */
num_bytes_to_process,
num_resulting_chars,
- UTF8_ANALYZE_MORE, /* always this state here */
+ ERTS_UTF8_ANALYZE_MORE, /* always this state here */
NIL); /* Nothing built -> no tail yet */
}
@@ -1642,7 +1847,7 @@ static BIF_RETTYPE utf8_to_list(BIF_ALIST_1)
BIF_ERROR(BIF_P,BADARG);
}
return do_bif_utf8_to_list(BIF_P, BIF_ARG_1, 0U, 0U, 0U,
- UTF8_ANALYZE_MORE,NIL);
+ ERTS_UTF8_ANALYZE_MORE,NIL);
}
@@ -1728,8 +1933,8 @@ binary_to_atom(Process* p, Eterm bin, Eterm enc, int must_exist)
Uint n;
int reds_left = bin_size+1; /* Number of reductions left. */
- if (analyze_utf8(bytes, bin_size, &err_pos,
- &n, &reds_left) == UTF8_OK) {
+ if (erts_analyze_utf8(bytes, bin_size, &err_pos,
+ &n, &reds_left) == ERTS_UTF8_OK) {
/*
* Correct UTF-8 encoding, but too many characters to
* fit in an atom.
@@ -1818,7 +2023,7 @@ BIF_RETTYPE binary_to_existing_atom_2(BIF_ALIST_2)
* Simpler non-interruptable routines for UTF-8 and
* Windowish UTF-16 (restricted)
**********************************************************/
-static Sint simple_char_need(Eterm ioterm, int encoding)
+Sint erts_native_filename_need(Eterm ioterm, int encoding)
{
Eterm *objp;
Eterm obj;
@@ -1833,6 +2038,7 @@ static Sint simple_char_need(Eterm ioterm, int encoding)
case ERL_FILENAME_LATIN1:
need = ap->len;
break;
+ case ERL_FILENAME_UTF8_MAC:
case ERL_FILENAME_UTF8:
for (i = 0; i < ap->len; i++) {
need += (ap->name[i] >= 0x80) ? 2 : 1;
@@ -1882,6 +2088,7 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */
}
need += 1;
break;
+ case ERL_FILENAME_UTF8_MAC:
case ERL_FILENAME_UTF8:
if (x < 0x80) {
need +=1;
@@ -1956,7 +2163,7 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */
return need;
}
-static void simple_put_chars(Eterm ioterm, int encoding, byte *p)
+void erts_native_filename_put(Eterm ioterm, int encoding, byte *p)
{
Eterm *objp;
Eterm obj;
@@ -1972,6 +2179,7 @@ static void simple_put_chars(Eterm ioterm, int encoding, byte *p)
*p++ = ap->name[i];
}
break;
+ case ERL_FILENAME_UTF8_MAC:
case ERL_FILENAME_UTF8:
for (i = 0; i < ap->len; i++) {
if(ap->name[i] < 0x80) {
@@ -2024,6 +2232,7 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */
ASSERT( x < 256);
*p++ = (byte) x;
break;
+ case ERL_FILENAME_UTF8_MAC:
case ERL_FILENAME_UTF8:
if (x < 0x80) {
*p++ = (byte) x;
@@ -2102,7 +2311,39 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */
DESTROY_ESTACK(stack);
return;
}
-
+void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars)
+{
+ Uint unipoint;
+
+ while (num_chars--) {
+ if (((*bytes) & ((byte) 0x80)) == 0) {
+ unipoint = (Uint) *bytes;
+ ++bytes;
+ } else if (((*bytes) & ((byte) 0xE0)) == 0xC0) {
+ unipoint =
+ (((Uint) ((*bytes) & ((byte) 0x1F))) << 6) |
+ ((Uint) (bytes[1] & ((byte) 0x3F)));
+ bytes += 2;
+ } else if (((*bytes) & ((byte) 0xF0)) == 0xE0) {
+ unipoint =
+ (((Uint) ((*bytes) & ((byte) 0xF))) << 12) |
+ (((Uint) (bytes[1] & ((byte) 0x3F))) << 6) |
+ ((Uint) (bytes[2] & ((byte) 0x3F)));
+ bytes +=3;
+ } else if (((*bytes) & ((byte) 0xF8)) == 0xF0) {
+ unipoint =
+ (((Uint) ((*bytes) & ((byte) 0x7))) << 18) |
+ (((Uint) (bytes[1] & ((byte) 0x3F))) << 12) |
+ (((Uint) (bytes[2] & ((byte) 0x3F))) << 6) |
+ ((Uint) (bytes[3] & ((byte) 0x3F)));
+ bytes += 4;
+ } else {
+ erl_exit(1,"Internal unicode error in prim_file:internal_name2native/1");
+ }
+ *target++ = (byte) (unipoint & 0xFF);
+ *target++ = (byte) ((unipoint >> 8) & 0xFF);
+ }
+}
/*
* This internal bif converts a filename to whatever format is suitable for the file driver
@@ -2120,7 +2361,6 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
byte *bytes;
byte *err_pos;
Uint size,num_chars;
- Uint unipoint;
/* Uninterpreted encoding except if windows widechar, in case we convert from
utf8 to win_wchar */
size = binary_size(BIF_ARG_1);
@@ -2137,7 +2377,7 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
/* In a wchar world, the emulator flags only affect how
binaries are interpreted when sent from the user. */
/* Determine real length and create a new binary */
- if (analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != UTF8_OK ||
+ if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK ||
erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) {
/* What to do now? Maybe latin1, so just take byte for byte instead */
bin_term = new_binary(BIF_P, 0, (size+1)*2);
@@ -2154,43 +2394,16 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
/* OK, UTF8 ok, number of characters is in num_chars */
bin_term = new_binary(BIF_P, 0, (num_chars+1)*2);
bin_p = binary_bytes(bin_term);
- while (num_chars--) {
- if (((*bytes) & ((byte) 0x80)) == 0) {
- unipoint = (Uint) *bytes;
- ++bytes;
- } else if (((*bytes) & ((byte) 0xE0)) == 0xC0) {
- unipoint =
- (((Uint) ((*bytes) & ((byte) 0x1F))) << 6) |
- ((Uint) (bytes[1] & ((byte) 0x3F)));
- bytes += 2;
- } else if (((*bytes) & ((byte) 0xF0)) == 0xE0) {
- unipoint =
- (((Uint) ((*bytes) & ((byte) 0xF))) << 12) |
- (((Uint) (bytes[1] & ((byte) 0x3F))) << 6) |
- ((Uint) (bytes[2] & ((byte) 0x3F)));
- bytes +=3;
- } else if (((*bytes) & ((byte) 0xF8)) == 0xF0) {
- unipoint =
- (((Uint) ((*bytes) & ((byte) 0x7))) << 18) |
- (((Uint) (bytes[1] & ((byte) 0x3F))) << 12) |
- (((Uint) (bytes[2] & ((byte) 0x3F))) << 6) |
- ((Uint) (bytes[3] & ((byte) 0x3F)));
- bytes += 4;
- } else {
- erl_exit(1,"Internal unicode error in file:name2native/1");
- }
- *bin_p++ = (byte) (unipoint & 0xFF);
- *bin_p++ = (byte) ((unipoint >> 8) & 0xFF);
- }
+ erts_copy_utf8_to_utf16_little(bin_p, bytes, num_chars);
/* zero termination */
- *bin_p++ = 0;
- *bin_p++ = 0;
+ bin_p[num_chars*2] = 0;
+ bin_p[num_chars*2+1] = 0;
erts_free_aligned_binary_bytes(temp_alloc);
BIF_RET(bin_term);
} /* binary */
- if ((need = simple_char_need(BIF_ARG_1,encoding)) < 0) {
+ if ((need = erts_native_filename_need(BIF_ARG_1,encoding)) < 0) {
BIF_ERROR(BIF_P,BADARG);
}
if (encoding == ERL_FILENAME_WIN_WCHAR) {
@@ -2201,7 +2414,7 @@ BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1)
bin_term = new_binary(BIF_P, 0, need);
bin_p = binary_bytes(bin_term);
- simple_put_chars(BIF_ARG_1,encoding,bin_p);
+ erts_native_filename_put(BIF_ARG_1,encoding,bin_p);
bin_p[need-1] = 0;
if (encoding == ERL_FILENAME_WIN_WCHAR) {
bin_p[need-2] = 0;
@@ -2223,6 +2436,7 @@ BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1)
Uint num_built; /* characters */
Uint num_eaten; /* bytes */
Eterm ret;
+ int mac = 0;
if (is_not_binary(BIF_ARG_1)) {
BIF_ERROR(BIF_P,BADARG);
@@ -2241,15 +2455,21 @@ BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1)
bytes = binary_bytes(real_bin)+offset;
BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs));
+ case ERL_FILENAME_UTF8_MAC:
+ mac = 1;
case ERL_FILENAME_UTF8:
bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
- if (analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != UTF8_OK) {
+ if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) {
erts_free_aligned_binary_bytes(temp_alloc);
goto noconvert;
}
num_built = 0;
num_eaten = 0;
- ret = do_utf8_to_list(BIF_P, num_chars, bytes, size, num_chars, &num_built, &num_eaten, NIL);
+ if (mac) {
+ ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size);
+ } else {
+ ret = do_utf8_to_list(BIF_P, num_chars, bytes, size, num_chars, &num_built, &num_eaten, NIL);
+ }
erts_free_aligned_binary_bytes(temp_alloc);
BIF_RET(ret);
case ERL_FILENAME_WIN_WCHAR:
@@ -2267,9 +2487,9 @@ BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1)
while (size > 0) {
Uint x = ((Uint) *bytes--) << 8;
x |= ((Uint) *bytes--);
+ size -= 2;
ret = CONS(hp,make_small(x),ret);
hp += 2;
- size -= 2;
}
erts_free_aligned_binary_bytes(temp_alloc);
BIF_RET(ret);
@@ -2280,11 +2500,45 @@ BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1)
BIF_RET(BIF_ARG_1);
}
+BIF_RETTYPE prim_file_internal_normalize_utf8_1(BIF_ALIST_1)
+{
+ Eterm real_bin;
+ Uint offset;
+ Uint size,num_chars;
+ Uint bitsize;
+ Uint bitoffs;
+ Eterm ret;
+ byte *temp_alloc = NULL;
+ byte *bytes;
+ byte *err_pos;
+
+ if (is_not_binary(BIF_ARG_1)) {
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ size = binary_size(BIF_ARG_1);
+ ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize);
+ if (bitsize != 0) {
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ if (size == 0) {
+ BIF_RET(NIL);
+ }
+ bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc);
+ if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) {
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_ERROR(BIF_P,BADARG);
+ }
+ ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size);
+ erts_free_aligned_binary_bytes(temp_alloc);
+ BIF_RET(ret);
+}
+
BIF_RETTYPE file_native_name_encoding_0(BIF_ALIST_0)
{
switch (erts_get_native_filename_encoding()) {
case ERL_FILENAME_LATIN1:
BIF_RET(am_latin1);
+ case ERL_FILENAME_UTF8_MAC:
case ERL_FILENAME_UTF8:
BIF_RET(am_utf8);
case ERL_FILENAME_WIN_WCHAR:
diff --git a/erts/emulator/beam/erl_unicode_normalize.h b/erts/emulator/beam/erl_unicode_normalize.h
new file mode 100644
index 0000000000..fb0a111ca2
--- /dev/null
+++ b/erts/emulator/beam/erl_unicode_normalize.h
@@ -0,0 +1,1687 @@
+/*
+* %CopyrightBegin%
+*
+* Copyright Ericsson AB 1999-2010. All Rights Reserved.
+*
+* The contents of this file are subject to the Erlang Public License,
+* Version 1.1, (the "License"); you may not use this file except in
+* compliance with the License. You should have received a copy of the
+* Erlang Public License along with this software. If not, it can be
+* retrieved online at http://www.erlang.org/.
+*
+* Software distributed under the License is distributed on an "AS IS"
+* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+* the License for the specific language governing rights and limitations
+* under the License.
+*
+* %CopyrightEnd%
+*/
+/*
+* This file is automatically generated by dec.erl, do not edit manually
+*/
+#define HASH_SIZE_FACTOR 2
+typedef struct _compose_entry {
+ Uint16 c;
+ Uint16 res;
+ Uint16 num_subs;
+ struct _compose_entry *subs;
+ int *hash;
+} CompEntry;
+
+static int compose_tab_size = 61;
+static int hash_compose_tab_0_15[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_0_15 */
+static CompEntry compose_tab_0_15[] = {
+{65, 7846, 0, NULL, NULL},
+{69, 7872, 0, NULL, NULL},
+{79, 7890, 0, NULL, NULL},
+{97, 7847, 0, NULL, NULL},
+{101, 7873, 0, NULL, NULL},
+{111, 7891, 0, NULL, NULL}
+}; /* compose_tab_0_15 */
+static int hash_compose_tab_0_16[8] =
+{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_0_16 */
+static CompEntry compose_tab_0_16[] = {
+{69, 7700, 0, NULL, NULL},
+{79, 7760, 0, NULL, NULL},
+{101, 7701, 0, NULL, NULL},
+{111, 7761, 0, NULL, NULL}
+}; /* compose_tab_0_16 */
+static int hash_compose_tab_0_17[4] =
+{-1,0,1,-1}; /* hash_compose_tab_0_17 */
+static CompEntry compose_tab_0_17[] = {
+{65, 7856, 0, NULL, NULL},
+{97, 7857, 0, NULL, NULL}
+}; /* compose_tab_0_17 */
+static int hash_compose_tab_0_18[8] =
+{-1,2,-1,-1,-1,0,1,3}; /* hash_compose_tab_0_18 */
+static CompEntry compose_tab_0_18[] = {
+{85, 475, 0, NULL, NULL},
+{117, 476, 0, NULL, NULL},
+{953, 8146, 0, NULL, NULL},
+{965, 8162, 0, NULL, NULL}
+}; /* compose_tab_0_18 */
+static int hash_compose_tab_0_19_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_19_0 */
+static CompEntry compose_tab_0_19_0[] = {
+{913, 8074, 0, NULL, NULL},
+{919, 8090, 0, NULL, NULL},
+{937, 8106, 0, NULL, NULL},
+{945, 8066, 0, NULL, NULL},
+{951, 8082, 0, NULL, NULL},
+{969, 8098, 0, NULL, NULL}
+}; /* compose_tab_0_19_0 */
+static int hash_compose_tab_0_19[28] =
+{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_0_19 */
+static CompEntry compose_tab_0_19[] = {
+{837, 0, 6, compose_tab_0_19_0, hash_compose_tab_0_19_0},
+{913, 7946, 0, NULL, NULL},
+{917, 7962, 0, NULL, NULL},
+{919, 7978, 0, NULL, NULL},
+{921, 7994, 0, NULL, NULL},
+{927, 8010, 0, NULL, NULL},
+{937, 8042, 0, NULL, NULL},
+{945, 7938, 0, NULL, NULL},
+{949, 7954, 0, NULL, NULL},
+{951, 7970, 0, NULL, NULL},
+{953, 7986, 0, NULL, NULL},
+{959, 8002, 0, NULL, NULL},
+{965, 8018, 0, NULL, NULL},
+{969, 8034, 0, NULL, NULL}
+}; /* compose_tab_0_19 */
+static int hash_compose_tab_0_20_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_20_0 */
+static CompEntry compose_tab_0_20_0[] = {
+{913, 8075, 0, NULL, NULL},
+{919, 8091, 0, NULL, NULL},
+{937, 8107, 0, NULL, NULL},
+{945, 8067, 0, NULL, NULL},
+{951, 8083, 0, NULL, NULL},
+{969, 8099, 0, NULL, NULL}
+}; /* compose_tab_0_20_0 */
+static int hash_compose_tab_0_20[30] =
+{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5,
+ 12}; /* hash_compose_tab_0_20 */
+static CompEntry compose_tab_0_20[] = {
+{837, 0, 6, compose_tab_0_20_0, hash_compose_tab_0_20_0},
+{913, 7947, 0, NULL, NULL},
+{917, 7963, 0, NULL, NULL},
+{919, 7979, 0, NULL, NULL},
+{921, 7995, 0, NULL, NULL},
+{927, 8011, 0, NULL, NULL},
+{933, 8027, 0, NULL, NULL},
+{937, 8043, 0, NULL, NULL},
+{945, 7939, 0, NULL, NULL},
+{949, 7955, 0, NULL, NULL},
+{951, 7971, 0, NULL, NULL},
+{953, 7987, 0, NULL, NULL},
+{959, 8003, 0, NULL, NULL},
+{965, 8019, 0, NULL, NULL},
+{969, 8035, 0, NULL, NULL}
+}; /* compose_tab_0_20 */
+static int hash_compose_tab_0_21[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_0_21 */
+static CompEntry compose_tab_0_21[] = {
+{79, 7900, 0, NULL, NULL},
+{85, 7914, 0, NULL, NULL},
+{111, 7901, 0, NULL, NULL},
+{117, 7915, 0, NULL, NULL}
+}; /* compose_tab_0_21 */
+static int hash_compose_tab_0_22[6] =
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_0_22 */
+static CompEntry compose_tab_0_22[] = {
+{945, 8114, 0, NULL, NULL},
+{951, 8130, 0, NULL, NULL},
+{969, 8178, 0, NULL, NULL}
+}; /* compose_tab_0_22 */
+static int hash_compose_tab_0[78] =
+{38,3,29,-1,-1,-1,-1,4,19,5,20,6,14,30,31,21,32,33,37,7,-1,-1,-1,8,34,-1,-1,9,
+ -1,35,-1,-1,-1,10,36,-1,-1,-1,-1,11,-1,12,-1,13,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,23,-1,22,-1,24,-1,25,-1,26,-1,0,-1,-1,15,1,16,27,17,2,18,28,-1,-1}; /* hash_compose_tab_0 */
+static CompEntry compose_tab_0[] = {
+{65, 192, 0, NULL, NULL},
+{69, 200, 0, NULL, NULL},
+{73, 204, 0, NULL, NULL},
+{79, 210, 0, NULL, NULL},
+{85, 217, 0, NULL, NULL},
+{87, 7808, 0, NULL, NULL},
+{89, 7922, 0, NULL, NULL},
+{97, 224, 0, NULL, NULL},
+{101, 232, 0, NULL, NULL},
+{105, 236, 0, NULL, NULL},
+{111, 242, 0, NULL, NULL},
+{117, 249, 0, NULL, NULL},
+{119, 7809, 0, NULL, NULL},
+{121, 7923, 0, NULL, NULL},
+{168, 8173, 0, NULL, NULL},
+{770, 0, 6, compose_tab_0_15, hash_compose_tab_0_15},
+{772, 0, 4, compose_tab_0_16, hash_compose_tab_0_16},
+{774, 0, 2, compose_tab_0_17, hash_compose_tab_0_17},
+{776, 0, 4, compose_tab_0_18, hash_compose_tab_0_18},
+{787, 0, 14, compose_tab_0_19, hash_compose_tab_0_19},
+{788, 0, 15, compose_tab_0_20, hash_compose_tab_0_20},
+{795, 0, 4, compose_tab_0_21, hash_compose_tab_0_21},
+{837, 0, 3, compose_tab_0_22, hash_compose_tab_0_22},
+{913, 8122, 0, NULL, NULL},
+{917, 8136, 0, NULL, NULL},
+{919, 8138, 0, NULL, NULL},
+{921, 8154, 0, NULL, NULL},
+{927, 8184, 0, NULL, NULL},
+{933, 8170, 0, NULL, NULL},
+{937, 8186, 0, NULL, NULL},
+{945, 8048, 0, NULL, NULL},
+{949, 8050, 0, NULL, NULL},
+{951, 8052, 0, NULL, NULL},
+{953, 8054, 0, NULL, NULL},
+{959, 8056, 0, NULL, NULL},
+{965, 8058, 0, NULL, NULL},
+{969, 8060, 0, NULL, NULL},
+{8127, 8141, 0, NULL, NULL},
+{8190, 8157, 0, NULL, NULL}
+}; /* compose_tab_0 */
+static int hash_compose_tab_1_39[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_1_39 */
+static CompEntry compose_tab_1_39[] = {
+{65, 7844, 0, NULL, NULL},
+{69, 7870, 0, NULL, NULL},
+{79, 7888, 0, NULL, NULL},
+{97, 7845, 0, NULL, NULL},
+{101, 7871, 0, NULL, NULL},
+{111, 7889, 0, NULL, NULL}
+}; /* compose_tab_1_39 */
+static int hash_compose_tab_1_40[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_40 */
+static CompEntry compose_tab_1_40[] = {
+{79, 7756, 0, NULL, NULL},
+{85, 7800, 0, NULL, NULL},
+{111, 7757, 0, NULL, NULL},
+{117, 7801, 0, NULL, NULL}
+}; /* compose_tab_1_40 */
+static int hash_compose_tab_1_41[8] =
+{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_1_41 */
+static CompEntry compose_tab_1_41[] = {
+{69, 7702, 0, NULL, NULL},
+{79, 7762, 0, NULL, NULL},
+{101, 7703, 0, NULL, NULL},
+{111, 7763, 0, NULL, NULL}
+}; /* compose_tab_1_41 */
+static int hash_compose_tab_1_42[4] =
+{-1,0,1,-1}; /* hash_compose_tab_1_42 */
+static CompEntry compose_tab_1_42[] = {
+{65, 7854, 0, NULL, NULL},
+{97, 7855, 0, NULL, NULL}
+}; /* compose_tab_1_42 */
+static int hash_compose_tab_1_43[12] =
+{-1,0,1,-1,-1,4,5,-1,-1,2,3,-1}; /* hash_compose_tab_1_43 */
+static CompEntry compose_tab_1_43[] = {
+{73, 7726, 0, NULL, NULL},
+{85, 471, 0, NULL, NULL},
+{105, 7727, 0, NULL, NULL},
+{117, 472, 0, NULL, NULL},
+{953, 8147, 0, NULL, NULL},
+{965, 8163, 0, NULL, NULL}
+}; /* compose_tab_1_43 */
+static int hash_compose_tab_1_44[4] =
+{-1,0,1,-1}; /* hash_compose_tab_1_44 */
+static CompEntry compose_tab_1_44[] = {
+{65, 506, 0, NULL, NULL},
+{97, 507, 0, NULL, NULL}
+}; /* compose_tab_1_44 */
+static int hash_compose_tab_1_45_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_45_0 */
+static CompEntry compose_tab_1_45_0[] = {
+{913, 8076, 0, NULL, NULL},
+{919, 8092, 0, NULL, NULL},
+{937, 8108, 0, NULL, NULL},
+{945, 8068, 0, NULL, NULL},
+{951, 8084, 0, NULL, NULL},
+{969, 8100, 0, NULL, NULL}
+}; /* compose_tab_1_45_0 */
+static int hash_compose_tab_1_45[28] =
+{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_1_45 */
+static CompEntry compose_tab_1_45[] = {
+{837, 0, 6, compose_tab_1_45_0, hash_compose_tab_1_45_0},
+{913, 7948, 0, NULL, NULL},
+{917, 7964, 0, NULL, NULL},
+{919, 7980, 0, NULL, NULL},
+{921, 7996, 0, NULL, NULL},
+{927, 8012, 0, NULL, NULL},
+{937, 8044, 0, NULL, NULL},
+{945, 7940, 0, NULL, NULL},
+{949, 7956, 0, NULL, NULL},
+{951, 7972, 0, NULL, NULL},
+{953, 7988, 0, NULL, NULL},
+{959, 8004, 0, NULL, NULL},
+{965, 8020, 0, NULL, NULL},
+{969, 8036, 0, NULL, NULL}
+}; /* compose_tab_1_45 */
+static int hash_compose_tab_1_46_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_46_0 */
+static CompEntry compose_tab_1_46_0[] = {
+{913, 8077, 0, NULL, NULL},
+{919, 8093, 0, NULL, NULL},
+{937, 8109, 0, NULL, NULL},
+{945, 8069, 0, NULL, NULL},
+{951, 8085, 0, NULL, NULL},
+{969, 8101, 0, NULL, NULL}
+}; /* compose_tab_1_46_0 */
+static int hash_compose_tab_1_46[30] =
+{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5,
+ 12}; /* hash_compose_tab_1_46 */
+static CompEntry compose_tab_1_46[] = {
+{837, 0, 6, compose_tab_1_46_0, hash_compose_tab_1_46_0},
+{913, 7949, 0, NULL, NULL},
+{917, 7965, 0, NULL, NULL},
+{919, 7981, 0, NULL, NULL},
+{921, 7997, 0, NULL, NULL},
+{927, 8013, 0, NULL, NULL},
+{933, 8029, 0, NULL, NULL},
+{937, 8045, 0, NULL, NULL},
+{945, 7941, 0, NULL, NULL},
+{949, 7957, 0, NULL, NULL},
+{951, 7973, 0, NULL, NULL},
+{953, 7989, 0, NULL, NULL},
+{959, 8005, 0, NULL, NULL},
+{965, 8021, 0, NULL, NULL},
+{969, 8037, 0, NULL, NULL}
+}; /* compose_tab_1_46 */
+static int hash_compose_tab_1_47[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_47 */
+static CompEntry compose_tab_1_47[] = {
+{79, 7898, 0, NULL, NULL},
+{85, 7912, 0, NULL, NULL},
+{111, 7899, 0, NULL, NULL},
+{117, 7913, 0, NULL, NULL}
+}; /* compose_tab_1_47 */
+static int hash_compose_tab_1_48[4] =
+{1,-1,-1,0}; /* hash_compose_tab_1_48 */
+static CompEntry compose_tab_1_48[] = {
+{67, 7688, 0, NULL, NULL},
+{99, 7689, 0, NULL, NULL}
+}; /* compose_tab_1_48 */
+static int hash_compose_tab_1_49[6] =
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_1_49 */
+static CompEntry compose_tab_1_49[] = {
+{945, 8116, 0, NULL, NULL},
+{951, 8132, 0, NULL, NULL},
+{959, 8180, 0, NULL, NULL}
+}; /* compose_tab_1_49 */
+static int hash_compose_tab_1[140] =
+{-1,-1,-1,-1,-1,-1,-1,68,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,34,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,35,-1,-1,-1,-1,64,-1,0,-1,1,-1,2,39,3,40,4,41,5,6,7,
+ 8,9,10,36,11,12,42,13,43,14,44,15,16,37,45,46,50,47,51,17,52,18,53,19,54,20,
+ 55,21,56,22,23,24,25,26,27,38,28,29,48,30,57,31,58,32,33,59,60,61,62,65,66,
+ 63,67,69,-1,-1,-1,-1,-1,49,-1,-1}; /* hash_compose_tab_1 */
+static CompEntry compose_tab_1[] = {
+{65, 193, 0, NULL, NULL},
+{67, 262, 0, NULL, NULL},
+{69, 201, 0, NULL, NULL},
+{71, 500, 0, NULL, NULL},
+{73, 205, 0, NULL, NULL},
+{75, 7728, 0, NULL, NULL},
+{76, 313, 0, NULL, NULL},
+{77, 7742, 0, NULL, NULL},
+{78, 323, 0, NULL, NULL},
+{79, 211, 0, NULL, NULL},
+{80, 7764, 0, NULL, NULL},
+{82, 340, 0, NULL, NULL},
+{83, 346, 0, NULL, NULL},
+{85, 218, 0, NULL, NULL},
+{87, 7810, 0, NULL, NULL},
+{89, 221, 0, NULL, NULL},
+{90, 377, 0, NULL, NULL},
+{97, 225, 0, NULL, NULL},
+{99, 263, 0, NULL, NULL},
+{101, 233, 0, NULL, NULL},
+{103, 501, 0, NULL, NULL},
+{105, 237, 0, NULL, NULL},
+{107, 7729, 0, NULL, NULL},
+{108, 314, 0, NULL, NULL},
+{109, 7743, 0, NULL, NULL},
+{110, 324, 0, NULL, NULL},
+{111, 243, 0, NULL, NULL},
+{112, 7765, 0, NULL, NULL},
+{114, 341, 0, NULL, NULL},
+{115, 347, 0, NULL, NULL},
+{117, 250, 0, NULL, NULL},
+{119, 7811, 0, NULL, NULL},
+{121, 253, 0, NULL, NULL},
+{122, 378, 0, NULL, NULL},
+{168, 8174, 0, NULL, NULL},
+{198, 508, 0, NULL, NULL},
+{216, 510, 0, NULL, NULL},
+{230, 509, 0, NULL, NULL},
+{248, 511, 0, NULL, NULL},
+{770, 0, 6, compose_tab_1_39, hash_compose_tab_1_39},
+{771, 0, 4, compose_tab_1_40, hash_compose_tab_1_40},
+{772, 0, 4, compose_tab_1_41, hash_compose_tab_1_41},
+{774, 0, 2, compose_tab_1_42, hash_compose_tab_1_42},
+{776, 0, 6, compose_tab_1_43, hash_compose_tab_1_43},
+{778, 0, 2, compose_tab_1_44, hash_compose_tab_1_44},
+{787, 0, 14, compose_tab_1_45, hash_compose_tab_1_45},
+{788, 0, 15, compose_tab_1_46, hash_compose_tab_1_46},
+{795, 0, 4, compose_tab_1_47, hash_compose_tab_1_47},
+{807, 0, 2, compose_tab_1_48, hash_compose_tab_1_48},
+{837, 0, 3, compose_tab_1_49, hash_compose_tab_1_49},
+{913, 8123, 0, NULL, NULL},
+{917, 8137, 0, NULL, NULL},
+{919, 8139, 0, NULL, NULL},
+{921, 8155, 0, NULL, NULL},
+{927, 8185, 0, NULL, NULL},
+{933, 8171, 0, NULL, NULL},
+{937, 8187, 0, NULL, NULL},
+{945, 8049, 0, NULL, NULL},
+{949, 8051, 0, NULL, NULL},
+{951, 8053, 0, NULL, NULL},
+{953, 8055, 0, NULL, NULL},
+{959, 8057, 0, NULL, NULL},
+{965, 8059, 0, NULL, NULL},
+{969, 8061, 0, NULL, NULL},
+{1043, 1027, 0, NULL, NULL},
+{1050, 1036, 0, NULL, NULL},
+{1075, 1107, 0, NULL, NULL},
+{1082, 1116, 0, NULL, NULL},
+{8127, 8142, 0, NULL, NULL},
+{8190, 8158, 0, NULL, NULL}
+}; /* compose_tab_1 */
+static int hash_compose_tab_2_26[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_2_26 */
+static CompEntry compose_tab_2_26[] = {
+{65, 7852, 0, NULL, NULL},
+{69, 7878, 0, NULL, NULL},
+{79, 7896, 0, NULL, NULL},
+{97, 7853, 0, NULL, NULL},
+{101, 7879, 0, NULL, NULL},
+{111, 7897, 0, NULL, NULL}
+}; /* compose_tab_2_26 */
+static int hash_compose_tab_2[54] =
+{-1,-1,-1,20,-1,-1,-1,21,-1,22,-1,0,23,1,24,2,25,3,4,5,6,-1,-1,-1,-1,7,-1,-1,
+ -1,8,-1,9,-1,10,-1,11,12,-1,-1,-1,-1,-1,-1,13,-1,14,-1,15,26,16,17,18,19,-1}; /* hash_compose_tab_2 */
+static CompEntry compose_tab_2[] = {
+{65, 194, 0, NULL, NULL},
+{67, 264, 0, NULL, NULL},
+{69, 202, 0, NULL, NULL},
+{71, 284, 0, NULL, NULL},
+{72, 292, 0, NULL, NULL},
+{73, 206, 0, NULL, NULL},
+{74, 308, 0, NULL, NULL},
+{79, 212, 0, NULL, NULL},
+{83, 348, 0, NULL, NULL},
+{85, 219, 0, NULL, NULL},
+{87, 372, 0, NULL, NULL},
+{89, 374, 0, NULL, NULL},
+{90, 7824, 0, NULL, NULL},
+{97, 226, 0, NULL, NULL},
+{99, 265, 0, NULL, NULL},
+{101, 234, 0, NULL, NULL},
+{103, 285, 0, NULL, NULL},
+{104, 293, 0, NULL, NULL},
+{105, 238, 0, NULL, NULL},
+{106, 309, 0, NULL, NULL},
+{111, 244, 0, NULL, NULL},
+{115, 349, 0, NULL, NULL},
+{117, 251, 0, NULL, NULL},
+{119, 373, 0, NULL, NULL},
+{121, 375, 0, NULL, NULL},
+{122, 7825, 0, NULL, NULL},
+{803, 0, 6, compose_tab_2_26, hash_compose_tab_2_26}
+}; /* compose_tab_2 */
+static int hash_compose_tab_3_16[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_3_16 */
+static CompEntry compose_tab_3_16[] = {
+{65, 7850, 0, NULL, NULL},
+{69, 7876, 0, NULL, NULL},
+{79, 7894, 0, NULL, NULL},
+{97, 7851, 0, NULL, NULL},
+{101, 7877, 0, NULL, NULL},
+{111, 7895, 0, NULL, NULL}
+}; /* compose_tab_3_16 */
+static int hash_compose_tab_3_17[4] =
+{-1,0,1,-1}; /* hash_compose_tab_3_17 */
+static CompEntry compose_tab_3_17[] = {
+{65, 7860, 0, NULL, NULL},
+{97, 7861, 0, NULL, NULL}
+}; /* compose_tab_3_17 */
+static int hash_compose_tab_3_18[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_3_18 */
+static CompEntry compose_tab_3_18[] = {
+{79, 7904, 0, NULL, NULL},
+{85, 7918, 0, NULL, NULL},
+{111, 7905, 0, NULL, NULL},
+{117, 7919, 0, NULL, NULL}
+}; /* compose_tab_3_18 */
+static int hash_compose_tab_3[38] =
+{-1,-1,3,4,13,14,-1,15,-1,5,6,16,-1,7,17,-1,-1,-1,-1,-1,-1,8,-1,-1,-1,9,-1,0,
+ -1,10,-1,1,-1,-1,11,2,12,18}; /* hash_compose_tab_3 */
+static CompEntry compose_tab_3[] = {
+{65, 195, 0, NULL, NULL},
+{69, 7868, 0, NULL, NULL},
+{73, 296, 0, NULL, NULL},
+{78, 209, 0, NULL, NULL},
+{79, 213, 0, NULL, NULL},
+{85, 360, 0, NULL, NULL},
+{86, 7804, 0, NULL, NULL},
+{89, 7928, 0, NULL, NULL},
+{97, 227, 0, NULL, NULL},
+{101, 7869, 0, NULL, NULL},
+{105, 297, 0, NULL, NULL},
+{110, 241, 0, NULL, NULL},
+{111, 245, 0, NULL, NULL},
+{117, 361, 0, NULL, NULL},
+{118, 7805, 0, NULL, NULL},
+{121, 7929, 0, NULL, NULL},
+{770, 0, 6, compose_tab_3_16, hash_compose_tab_3_16},
+{774, 0, 2, compose_tab_3_17, hash_compose_tab_3_17},
+{795, 0, 4, compose_tab_3_18, hash_compose_tab_3_18}
+}; /* compose_tab_3 */
+static int hash_compose_tab_4_14[4] =
+{-1,0,1,-1}; /* hash_compose_tab_4_14 */
+static CompEntry compose_tab_4_14[] = {
+{65, 480, 0, NULL, NULL},
+{97, 481, 0, NULL, NULL}
+}; /* compose_tab_4_14 */
+static int hash_compose_tab_4_15[8] =
+{-1,0,2,-1,-1,1,3,-1}; /* hash_compose_tab_4_15 */
+static CompEntry compose_tab_4_15[] = {
+{65, 478, 0, NULL, NULL},
+{85, 469, 0, NULL, NULL},
+{97, 479, 0, NULL, NULL},
+{117, 470, 0, NULL, NULL}
+}; /* compose_tab_4_15 */
+static int hash_compose_tab_4_16[8] =
+{-1,-1,1,3,0,2,-1,-1}; /* hash_compose_tab_4_16 */
+static CompEntry compose_tab_4_16[] = {
+{76, 7736, 0, NULL, NULL},
+{82, 7772, 0, NULL, NULL},
+{108, 7737, 0, NULL, NULL},
+{114, 7773, 0, NULL, NULL}
+}; /* compose_tab_4_16 */
+static int hash_compose_tab_4_17[4] =
+{1,-1,-1,0}; /* hash_compose_tab_4_17 */
+static CompEntry compose_tab_4_17[] = {
+{79, 492, 0, NULL, NULL},
+{111, 493, 0, NULL, NULL}
+}; /* compose_tab_4_17 */
+static int hash_compose_tab_4[56] =
+{-1,22,-1,-1,-1,11,13,-1,-1,0,-1,-1,-1,1,23,2,26,3,18,16,-1,-1,-1,4,17,19,-1,
+ 27,-1,5,12,-1,-1,-1,-1,-1,-1,20,-1,-1,24,6,-1,-1,-1,7,-1,8,14,9,15,21,25,-1,
+ -1,10}; /* hash_compose_tab_4 */
+static CompEntry compose_tab_4[] = {
+{65, 256, 0, NULL, NULL},
+{69, 274, 0, NULL, NULL},
+{71, 7712, 0, NULL, NULL},
+{73, 298, 0, NULL, NULL},
+{79, 332, 0, NULL, NULL},
+{85, 362, 0, NULL, NULL},
+{97, 257, 0, NULL, NULL},
+{101, 275, 0, NULL, NULL},
+{103, 7713, 0, NULL, NULL},
+{105, 299, 0, NULL, NULL},
+{111, 333, 0, NULL, NULL},
+{117, 363, 0, NULL, NULL},
+{198, 482, 0, NULL, NULL},
+{230, 483, 0, NULL, NULL},
+{775, 0, 2, compose_tab_4_14, hash_compose_tab_4_14},
+{776, 0, 4, compose_tab_4_15, hash_compose_tab_4_15},
+{803, 0, 4, compose_tab_4_16, hash_compose_tab_4_16},
+{808, 0, 2, compose_tab_4_17, hash_compose_tab_4_17},
+{913, 8121, 0, NULL, NULL},
+{921, 8153, 0, NULL, NULL},
+{933, 8169, 0, NULL, NULL},
+{945, 8113, 0, NULL, NULL},
+{953, 8145, 0, NULL, NULL},
+{965, 8161, 0, NULL, NULL},
+{1048, 1250, 0, NULL, NULL},
+{1059, 1262, 0, NULL, NULL},
+{1080, 1251, 0, NULL, NULL},
+{1091, 1263, 0, NULL, NULL}
+}; /* compose_tab_4 */
+static int hash_compose_tab_5_12[4] =
+{-1,0,1,-1}; /* hash_compose_tab_5_12 */
+static CompEntry compose_tab_5_12[] = {
+{65, 7862, 0, NULL, NULL},
+{97, 7863, 0, NULL, NULL}
+}; /* compose_tab_5_12 */
+static int hash_compose_tab_5_13[4] =
+{-1,0,1,-1}; /* hash_compose_tab_5_13 */
+static CompEntry compose_tab_5_13[] = {
+{69, 7708, 0, NULL, NULL},
+{101, 7709, 0, NULL, NULL}
+}; /* compose_tab_5_13 */
+static int hash_compose_tab_5[60] =
+{28,-1,-1,-1,-1,0,19,-1,-1,1,-1,2,29,3,14,-1,-1,-1,-1,4,20,15,-1,12,-1,5,21,
+ 13,22,23,-1,-1,-1,16,-1,-1,-1,6,-1,24,-1,7,-1,8,-1,9,17,-1,-1,-1,-1,10,25,18,
+ -1,-1,-1,11,26,27}; /* hash_compose_tab_5 */
+static CompEntry compose_tab_5[] = {
+{65, 258, 0, NULL, NULL},
+{69, 276, 0, NULL, NULL},
+{71, 286, 0, NULL, NULL},
+{73, 300, 0, NULL, NULL},
+{79, 334, 0, NULL, NULL},
+{85, 364, 0, NULL, NULL},
+{97, 259, 0, NULL, NULL},
+{101, 277, 0, NULL, NULL},
+{103, 287, 0, NULL, NULL},
+{105, 301, 0, NULL, NULL},
+{111, 335, 0, NULL, NULL},
+{117, 365, 0, NULL, NULL},
+{803, 0, 2, compose_tab_5_12, hash_compose_tab_5_12},
+{807, 0, 2, compose_tab_5_13, hash_compose_tab_5_13},
+{913, 8120, 0, NULL, NULL},
+{921, 8152, 0, NULL, NULL},
+{933, 8168, 0, NULL, NULL},
+{945, 8112, 0, NULL, NULL},
+{953, 8144, 0, NULL, NULL},
+{965, 8160, 0, NULL, NULL},
+{1040, 1232, 0, NULL, NULL},
+{1045, 1238, 0, NULL, NULL},
+{1046, 1217, 0, NULL, NULL},
+{1048, 1049, 0, NULL, NULL},
+{1059, 1038, 0, NULL, NULL},
+{1072, 1233, 0, NULL, NULL},
+{1077, 1239, 0, NULL, NULL},
+{1078, 1218, 0, NULL, NULL},
+{1080, 1081, 0, NULL, NULL},
+{1091, 1118, 0, NULL, NULL}
+}; /* compose_tab_5 */
+static int hash_compose_tab_6_36[4] =
+{1,-1,-1,0}; /* hash_compose_tab_6_36 */
+static CompEntry compose_tab_6_36[] = {
+{83, 7780, 0, NULL, NULL},
+{115, 7781, 0, NULL, NULL}
+}; /* compose_tab_6_36 */
+static int hash_compose_tab_6_38[4] =
+{1,-1,-1,0}; /* hash_compose_tab_6_38 */
+static CompEntry compose_tab_6_38[] = {
+{83, 7782, 0, NULL, NULL},
+{115, 7783, 0, NULL, NULL}
+}; /* compose_tab_6_38 */
+static int hash_compose_tab_6_39[4] =
+{1,-1,-1,0}; /* hash_compose_tab_6_39 */
+static CompEntry compose_tab_6_39[] = {
+{83, 7784, 0, NULL, NULL},
+{115, 7785, 0, NULL, NULL}
+}; /* compose_tab_6_39 */
+static int hash_compose_tab_6[80] =
+{10,-1,11,12,13,39,-1,14,15,16,17,-1,-1,-1,-1,-1,-1,-1,18,19,20,21,22,23,24,
+ -1,-1,-1,-1,25,26,-1,27,-1,28,29,30,-1,-1,31,32,33,34,-1,-1,-1,-1,-1,-1,36,
+ -1,-1,-1,-1,37,-1,-1,-1,-1,-1,38,-1,-1,35,-1,-1,0,1,2,3,4,5,6,7,-1,-1,-1,8,9,
+ -1}; /* hash_compose_tab_6 */
+static CompEntry compose_tab_6[] = {
+{66, 7682, 0, NULL, NULL},
+{67, 266, 0, NULL, NULL},
+{68, 7690, 0, NULL, NULL},
+{69, 278, 0, NULL, NULL},
+{70, 7710, 0, NULL, NULL},
+{71, 288, 0, NULL, NULL},
+{72, 7714, 0, NULL, NULL},
+{73, 304, 0, NULL, NULL},
+{77, 7744, 0, NULL, NULL},
+{78, 7748, 0, NULL, NULL},
+{80, 7766, 0, NULL, NULL},
+{82, 7768, 0, NULL, NULL},
+{83, 7776, 0, NULL, NULL},
+{84, 7786, 0, NULL, NULL},
+{87, 7814, 0, NULL, NULL},
+{88, 7818, 0, NULL, NULL},
+{89, 7822, 0, NULL, NULL},
+{90, 379, 0, NULL, NULL},
+{98, 7683, 0, NULL, NULL},
+{99, 267, 0, NULL, NULL},
+{100, 7691, 0, NULL, NULL},
+{101, 279, 0, NULL, NULL},
+{102, 7711, 0, NULL, NULL},
+{103, 289, 0, NULL, NULL},
+{104, 7715, 0, NULL, NULL},
+{109, 7745, 0, NULL, NULL},
+{110, 7749, 0, NULL, NULL},
+{112, 7767, 0, NULL, NULL},
+{114, 7769, 0, NULL, NULL},
+{115, 7777, 0, NULL, NULL},
+{116, 7787, 0, NULL, NULL},
+{119, 7815, 0, NULL, NULL},
+{120, 7819, 0, NULL, NULL},
+{121, 7823, 0, NULL, NULL},
+{122, 380, 0, NULL, NULL},
+{383, 7835, 0, NULL, NULL},
+{769, 0, 2, compose_tab_6_36, hash_compose_tab_6_36},
+{774, 784, 0, NULL, NULL},
+{780, 0, 2, compose_tab_6_38, hash_compose_tab_6_38},
+{803, 0, 2, compose_tab_6_39, hash_compose_tab_6_39}
+}; /* compose_tab_6 */
+static int hash_compose_tab_7_23[4] =
+{1,-1,-1,0}; /* hash_compose_tab_7_23 */
+static CompEntry compose_tab_7_23[] = {
+{79, 7758, 0, NULL, NULL},
+{111, 7759, 0, NULL, NULL}
+}; /* compose_tab_7_23 */
+static int hash_compose_tab_7_24[4] =
+{-1,0,1,-1}; /* hash_compose_tab_7_24 */
+static CompEntry compose_tab_7_24[] = {
+{85, 7802, 0, NULL, NULL},
+{117, 7803, 0, NULL, NULL}
+}; /* compose_tab_7_24 */
+static int hash_compose_tab_7[100] =
+{48,10,21,-1,11,12,-1,-1,-1,-1,49,13,-1,-1,-1,20,14,15,-1,16,17,18,25,-1,-1,
+ -1,-1,-1,-1,22,30,-1,-1,26,-1,-1,-1,-1,-1,-1,31,-1,-1,-1,-1,32,33,34,35,-1,
+ -1,-1,-1,27,36,-1,-1,-1,-1,37,-1,-1,-1,38,-1,0,28,39,-1,1,-1,23,2,3,24,40,-1,
+ 41,29,4,42,43,44,-1,-1,5,45,6,7,8,-1,46,-1,-1,-1,47,-1,9,-1,19}; /* hash_compose_tab_7 */
+static CompEntry compose_tab_7[] = {
+{65, 196, 0, NULL, NULL},
+{69, 203, 0, NULL, NULL},
+{72, 7718, 0, NULL, NULL},
+{73, 207, 0, NULL, NULL},
+{79, 214, 0, NULL, NULL},
+{85, 220, 0, NULL, NULL},
+{87, 7812, 0, NULL, NULL},
+{88, 7820, 0, NULL, NULL},
+{89, 376, 0, NULL, NULL},
+{97, 228, 0, NULL, NULL},
+{101, 235, 0, NULL, NULL},
+{104, 7719, 0, NULL, NULL},
+{105, 239, 0, NULL, NULL},
+{111, 246, 0, NULL, NULL},
+{116, 7831, 0, NULL, NULL},
+{117, 252, 0, NULL, NULL},
+{119, 7813, 0, NULL, NULL},
+{120, 7821, 0, NULL, NULL},
+{121, 255, 0, NULL, NULL},
+{399, 1242, 0, NULL, NULL},
+{415, 1258, 0, NULL, NULL},
+{601, 1243, 0, NULL, NULL},
+{629, 1259, 0, NULL, NULL},
+{771, 0, 2, compose_tab_7_23, hash_compose_tab_7_23},
+{772, 0, 2, compose_tab_7_24, hash_compose_tab_7_24},
+{921, 938, 0, NULL, NULL},
+{933, 939, 0, NULL, NULL},
+{953, 970, 0, NULL, NULL},
+{965, 971, 0, NULL, NULL},
+{978, 980, 0, NULL, NULL},
+{1030, 1031, 0, NULL, NULL},
+{1040, 1234, 0, NULL, NULL},
+{1045, 1025, 0, NULL, NULL},
+{1046, 1244, 0, NULL, NULL},
+{1047, 1246, 0, NULL, NULL},
+{1048, 1252, 0, NULL, NULL},
+{1054, 1254, 0, NULL, NULL},
+{1059, 1264, 0, NULL, NULL},
+{1063, 1268, 0, NULL, NULL},
+{1067, 1272, 0, NULL, NULL},
+{1072, 1235, 0, NULL, NULL},
+{1077, 1105, 0, NULL, NULL},
+{1078, 1245, 0, NULL, NULL},
+{1079, 1247, 0, NULL, NULL},
+{1080, 1253, 0, NULL, NULL},
+{1086, 1255, 0, NULL, NULL},
+{1091, 1265, 0, NULL, NULL},
+{1095, 1269, 0, NULL, NULL},
+{1099, 1273, 0, NULL, NULL},
+{1110, 1111, 0, NULL, NULL}
+}; /* compose_tab_7 */
+static int hash_compose_tab_8_12[12] =
+{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_8_12 */
+static CompEntry compose_tab_8_12[] = {
+{65, 7848, 0, NULL, NULL},
+{69, 7874, 0, NULL, NULL},
+{79, 7892, 0, NULL, NULL},
+{97, 7849, 0, NULL, NULL},
+{101, 7875, 0, NULL, NULL},
+{111, 7893, 0, NULL, NULL}
+}; /* compose_tab_8_12 */
+static int hash_compose_tab_8_13[4] =
+{-1,0,1,-1}; /* hash_compose_tab_8_13 */
+static CompEntry compose_tab_8_13[] = {
+{65, 7858, 0, NULL, NULL},
+{97, 7859, 0, NULL, NULL}
+}; /* compose_tab_8_13 */
+static int hash_compose_tab_8_14[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_8_14 */
+static CompEntry compose_tab_8_14[] = {
+{79, 7902, 0, NULL, NULL},
+{85, 7916, 0, NULL, NULL},
+{111, 7903, 0, NULL, NULL},
+{117, 7917, 0, NULL, NULL}
+}; /* compose_tab_8_14 */
+static int hash_compose_tab_8[30] =
+{-1,11,-1,-1,-1,0,-1,6,-1,1,-1,7,-1,2,-1,8,14,-1,-1,3,12,9,-1,-1,13,4,-1,10,
+ -1,5}; /* hash_compose_tab_8 */
+static CompEntry compose_tab_8[] = {
+{65, 7842, 0, NULL, NULL},
+{69, 7866, 0, NULL, NULL},
+{73, 7880, 0, NULL, NULL},
+{79, 7886, 0, NULL, NULL},
+{85, 7910, 0, NULL, NULL},
+{89, 7926, 0, NULL, NULL},
+{97, 7843, 0, NULL, NULL},
+{101, 7867, 0, NULL, NULL},
+{105, 7881, 0, NULL, NULL},
+{111, 7887, 0, NULL, NULL},
+{117, 7911, 0, NULL, NULL},
+{121, 7927, 0, NULL, NULL},
+{770, 0, 6, compose_tab_8_12, hash_compose_tab_8_12},
+{774, 0, 2, compose_tab_8_13, hash_compose_tab_8_13},
+{795, 0, 4, compose_tab_8_14, hash_compose_tab_8_14}
+}; /* compose_tab_8 */
+static int hash_compose_tab_9[12] =
+{-1,1,2,5,-1,0,-1,-1,-1,3,-1,4}; /* hash_compose_tab_9 */
+static CompEntry compose_tab_9[] = {
+{65, 197, 0, NULL, NULL},
+{85, 366, 0, NULL, NULL},
+{97, 229, 0, NULL, NULL},
+{117, 367, 0, NULL, NULL},
+{119, 7832, 0, NULL, NULL},
+{121, 7833, 0, NULL, NULL}
+}; /* compose_tab_9 */
+static int hash_compose_tab_10[12] =
+{-1,1,-1,2,4,-1,-1,0,-1,3,-1,5}; /* hash_compose_tab_10 */
+static CompEntry compose_tab_10[] = {
+{79, 336, 0, NULL, NULL},
+{85, 368, 0, NULL, NULL},
+{111, 337, 0, NULL, NULL},
+{117, 369, 0, NULL, NULL},
+{1059, 1266, 0, NULL, NULL},
+{1091, 1267, 0, NULL, NULL}
+}; /* compose_tab_10 */
+static int hash_compose_tab_11_33[4] =
+{-1,0,1,-1}; /* hash_compose_tab_11_33 */
+static CompEntry compose_tab_11_33[] = {
+{85, 473, 0, NULL, NULL},
+{117, 474, 0, NULL, NULL}
+}; /* compose_tab_11_33 */
+static int hash_compose_tab_11[68] =
+{2,3,-1,4,-1,5,-1,6,7,-1,8,9,-1,-1,10,11,12,13,-1,-1,-1,-1,14,-1,-1,-1,-1,-1,
+ 33,15,-1,16,17,18,31,19,-1,20,21,22,23,-1,24,25,-1,-1,26,27,28,29,32,-1,-1,
+ -1,30,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,-1,1}; /* hash_compose_tab_11 */
+static CompEntry compose_tab_11[] = {
+{65, 461, 0, NULL, NULL},
+{67, 268, 0, NULL, NULL},
+{68, 270, 0, NULL, NULL},
+{69, 282, 0, NULL, NULL},
+{71, 486, 0, NULL, NULL},
+{73, 463, 0, NULL, NULL},
+{75, 488, 0, NULL, NULL},
+{76, 317, 0, NULL, NULL},
+{78, 327, 0, NULL, NULL},
+{79, 465, 0, NULL, NULL},
+{82, 344, 0, NULL, NULL},
+{83, 352, 0, NULL, NULL},
+{84, 356, 0, NULL, NULL},
+{85, 467, 0, NULL, NULL},
+{90, 381, 0, NULL, NULL},
+{97, 462, 0, NULL, NULL},
+{99, 269, 0, NULL, NULL},
+{100, 271, 0, NULL, NULL},
+{101, 283, 0, NULL, NULL},
+{103, 487, 0, NULL, NULL},
+{105, 464, 0, NULL, NULL},
+{106, 496, 0, NULL, NULL},
+{107, 489, 0, NULL, NULL},
+{108, 318, 0, NULL, NULL},
+{110, 328, 0, NULL, NULL},
+{111, 466, 0, NULL, NULL},
+{114, 345, 0, NULL, NULL},
+{115, 353, 0, NULL, NULL},
+{116, 357, 0, NULL, NULL},
+{117, 468, 0, NULL, NULL},
+{122, 382, 0, NULL, NULL},
+{439, 494, 0, NULL, NULL},
+{658, 495, 0, NULL, NULL},
+{776, 0, 2, compose_tab_11_33, hash_compose_tab_11_33}
+}; /* compose_tab_11 */
+static int hash_compose_tab_12_1[4] =
+{-1,0,1,-1}; /* hash_compose_tab_12_1 */
+static CompEntry compose_tab_12_1[] = {
+{953, 912, 0, NULL, NULL},
+{965, 944, 0, NULL, NULL}
+}; /* compose_tab_12_1 */
+static int hash_compose_tab_12[34] =
+{11,4,12,5,-1,-1,-1,13,-1,6,-1,-1,-1,14,-1,7,-1,15,-1,8,-1,-1,-1,-1,-1,-1,16,
+ 9,1,2,-1,10,0,3}; /* hash_compose_tab_12 */
+static CompEntry compose_tab_12[] = {
+{168, 901, 0, NULL, NULL},
+{776, 0, 2, compose_tab_12_1, hash_compose_tab_12_1},
+{913, 902, 0, NULL, NULL},
+{917, 904, 0, NULL, NULL},
+{919, 905, 0, NULL, NULL},
+{921, 906, 0, NULL, NULL},
+{927, 908, 0, NULL, NULL},
+{933, 910, 0, NULL, NULL},
+{937, 911, 0, NULL, NULL},
+{945, 940, 0, NULL, NULL},
+{949, 941, 0, NULL, NULL},
+{951, 942, 0, NULL, NULL},
+{953, 943, 0, NULL, NULL},
+{959, 972, 0, NULL, NULL},
+{965, 973, 0, NULL, NULL},
+{969, 974, 0, NULL, NULL},
+{978, 979, 0, NULL, NULL}
+}; /* compose_tab_12 */
+static int hash_compose_tab_13[28] =
+{-1,5,10,-1,-1,11,-1,-1,-1,0,-1,-1,-1,1,6,-1,-1,2,7,-1,12,8,13,3,-1,-1,4,9}; /* hash_compose_tab_13 */
+static CompEntry compose_tab_13[] = {
+{65, 512, 0, NULL, NULL},
+{69, 516, 0, NULL, NULL},
+{73, 520, 0, NULL, NULL},
+{79, 524, 0, NULL, NULL},
+{82, 528, 0, NULL, NULL},
+{85, 532, 0, NULL, NULL},
+{97, 513, 0, NULL, NULL},
+{101, 517, 0, NULL, NULL},
+{105, 521, 0, NULL, NULL},
+{111, 525, 0, NULL, NULL},
+{114, 529, 0, NULL, NULL},
+{117, 533, 0, NULL, NULL},
+{1140, 1142, 0, NULL, NULL},
+{1141, 1143, 0, NULL, NULL}
+}; /* compose_tab_13 */
+static int hash_compose_tab_14[24] =
+{-1,2,6,-1,-1,7,-1,3,-1,8,4,-1,-1,5,-1,9,-1,0,10,-1,-1,1,11,-1}; /* hash_compose_tab_14 */
+static CompEntry compose_tab_14[] = {
+{65, 514, 0, NULL, NULL},
+{69, 518, 0, NULL, NULL},
+{73, 522, 0, NULL, NULL},
+{79, 526, 0, NULL, NULL},
+{82, 530, 0, NULL, NULL},
+{85, 534, 0, NULL, NULL},
+{97, 515, 0, NULL, NULL},
+{101, 519, 0, NULL, NULL},
+{105, 523, 0, NULL, NULL},
+{111, 527, 0, NULL, NULL},
+{114, 531, 0, NULL, NULL},
+{117, 535, 0, NULL, NULL}
+}; /* compose_tab_14 */
+static int hash_compose_tab_15_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_15_0 */
+static CompEntry compose_tab_15_0[] = {
+{913, 8072, 0, NULL, NULL},
+{919, 8088, 0, NULL, NULL},
+{937, 8104, 0, NULL, NULL},
+{945, 8064, 0, NULL, NULL},
+{951, 8080, 0, NULL, NULL},
+{969, 8096, 0, NULL, NULL}
+}; /* compose_tab_15_0 */
+static int hash_compose_tab_15[30] =
+{-1,12,-1,-1,-1,13,-1,6,-1,14,-1,-1,-1,1,-1,7,-1,2,-1,3,8,4,9,10,-1,-1,-1,0,5,
+ 11}; /* hash_compose_tab_15 */
+static CompEntry compose_tab_15[] = {
+{837, 0, 6, compose_tab_15_0, hash_compose_tab_15_0},
+{913, 7944, 0, NULL, NULL},
+{917, 7960, 0, NULL, NULL},
+{919, 7976, 0, NULL, NULL},
+{921, 7992, 0, NULL, NULL},
+{927, 8008, 0, NULL, NULL},
+{937, 8040, 0, NULL, NULL},
+{945, 7936, 0, NULL, NULL},
+{949, 7952, 0, NULL, NULL},
+{951, 7968, 0, NULL, NULL},
+{953, 7984, 0, NULL, NULL},
+{959, 8000, 0, NULL, NULL},
+{961, 8164, 0, NULL, NULL},
+{965, 8016, 0, NULL, NULL},
+{969, 8032, 0, NULL, NULL}
+}; /* compose_tab_15 */
+static int hash_compose_tab_16_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_16_0 */
+static CompEntry compose_tab_16_0[] = {
+{913, 8073, 0, NULL, NULL},
+{919, 8089, 0, NULL, NULL},
+{937, 8105, 0, NULL, NULL},
+{945, 8065, 0, NULL, NULL},
+{951, 8081, 0, NULL, NULL},
+{969, 8097, 0, NULL, NULL}
+}; /* compose_tab_16_0 */
+static int hash_compose_tab_16[34] =
+{11,3,12,4,-1,-1,-1,13,-1,5,14,6,-1,15,-1,7,-1,16,-1,8,-1,0,-1,-1,-1,-1,-1,9,
+ -1,1,-1,10,-1,2}; /* hash_compose_tab_16 */
+static CompEntry compose_tab_16[] = {
+{837, 0, 6, compose_tab_16_0, hash_compose_tab_16_0},
+{913, 7945, 0, NULL, NULL},
+{917, 7961, 0, NULL, NULL},
+{919, 7977, 0, NULL, NULL},
+{921, 7993, 0, NULL, NULL},
+{927, 8009, 0, NULL, NULL},
+{929, 8172, 0, NULL, NULL},
+{933, 8025, 0, NULL, NULL},
+{937, 8041, 0, NULL, NULL},
+{945, 7937, 0, NULL, NULL},
+{949, 7953, 0, NULL, NULL},
+{951, 7969, 0, NULL, NULL},
+{953, 7985, 0, NULL, NULL},
+{959, 8001, 0, NULL, NULL},
+{961, 8165, 0, NULL, NULL},
+{965, 8017, 0, NULL, NULL},
+{969, 8033, 0, NULL, NULL}
+}; /* compose_tab_16 */
+static int hash_compose_tab_17[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_17 */
+static CompEntry compose_tab_17[] = {
+{79, 416, 0, NULL, NULL},
+{85, 431, 0, NULL, NULL},
+{111, 417, 0, NULL, NULL},
+{117, 432, 0, NULL, NULL}
+}; /* compose_tab_17 */
+static int hash_compose_tab_18_38[8] =
+{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_18_38 */
+static CompEntry compose_tab_18_38[] = {
+{79, 7906, 0, NULL, NULL},
+{85, 7920, 0, NULL, NULL},
+{111, 7907, 0, NULL, NULL},
+{117, 7921, 0, NULL, NULL}
+}; /* compose_tab_18_38 */
+static int hash_compose_tab_18[78] =
+{9,10,-1,-1,11,12,13,14,15,16,-1,17,18,-1,-1,38,-1,-1,-1,19,20,-1,21,22,-1,-1,
+ 23,24,-1,25,26,27,28,29,-1,-1,30,31,32,33,34,35,-1,36,37,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,-1,2,3,-1,-1,4,5,-1,6,7,8}; /* hash_compose_tab_18 */
+static CompEntry compose_tab_18[] = {
+{65, 7840, 0, NULL, NULL},
+{66, 7684, 0, NULL, NULL},
+{68, 7692, 0, NULL, NULL},
+{69, 7864, 0, NULL, NULL},
+{72, 7716, 0, NULL, NULL},
+{73, 7882, 0, NULL, NULL},
+{75, 7730, 0, NULL, NULL},
+{76, 7734, 0, NULL, NULL},
+{77, 7746, 0, NULL, NULL},
+{78, 7750, 0, NULL, NULL},
+{79, 7884, 0, NULL, NULL},
+{82, 7770, 0, NULL, NULL},
+{83, 7778, 0, NULL, NULL},
+{84, 7788, 0, NULL, NULL},
+{85, 7908, 0, NULL, NULL},
+{86, 7806, 0, NULL, NULL},
+{87, 7816, 0, NULL, NULL},
+{89, 7924, 0, NULL, NULL},
+{90, 7826, 0, NULL, NULL},
+{97, 7841, 0, NULL, NULL},
+{98, 7685, 0, NULL, NULL},
+{100, 7693, 0, NULL, NULL},
+{101, 7865, 0, NULL, NULL},
+{104, 7717, 0, NULL, NULL},
+{105, 7883, 0, NULL, NULL},
+{107, 7731, 0, NULL, NULL},
+{108, 7735, 0, NULL, NULL},
+{109, 7747, 0, NULL, NULL},
+{110, 7751, 0, NULL, NULL},
+{111, 7885, 0, NULL, NULL},
+{114, 7771, 0, NULL, NULL},
+{115, 7779, 0, NULL, NULL},
+{116, 7789, 0, NULL, NULL},
+{117, 7909, 0, NULL, NULL},
+{118, 7807, 0, NULL, NULL},
+{119, 7817, 0, NULL, NULL},
+{121, 7925, 0, NULL, NULL},
+{122, 7827, 0, NULL, NULL},
+{795, 0, 4, compose_tab_18_38, hash_compose_tab_18_38}
+}; /* compose_tab_18 */
+static int hash_compose_tab_19[4] =
+{-1,0,1,-1}; /* hash_compose_tab_19 */
+static CompEntry compose_tab_19[] = {
+{85, 7794, 0, NULL, NULL},
+{117, 7795, 0, NULL, NULL}
+}; /* compose_tab_19 */
+static int hash_compose_tab_20[4] =
+{-1,0,1,-1}; /* hash_compose_tab_20 */
+static CompEntry compose_tab_20[] = {
+{65, 7680, 0, NULL, NULL},
+{97, 7681, 0, NULL, NULL}
+}; /* compose_tab_20 */
+static int hash_compose_tab_21[40] =
+{-1,-1,7,8,9,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,10,11,-1,-1,12,13,-1,
+ -1,0,1,14,15,2,3,16,17,4,5,18,6,19}; /* hash_compose_tab_21 */
+static CompEntry compose_tab_21[] = {
+{67, 199, 0, NULL, NULL},
+{68, 7696, 0, NULL, NULL},
+{71, 290, 0, NULL, NULL},
+{72, 7720, 0, NULL, NULL},
+{75, 310, 0, NULL, NULL},
+{76, 315, 0, NULL, NULL},
+{78, 325, 0, NULL, NULL},
+{82, 342, 0, NULL, NULL},
+{83, 350, 0, NULL, NULL},
+{84, 354, 0, NULL, NULL},
+{99, 231, 0, NULL, NULL},
+{100, 7697, 0, NULL, NULL},
+{103, 291, 0, NULL, NULL},
+{104, 7721, 0, NULL, NULL},
+{107, 311, 0, NULL, NULL},
+{108, 316, 0, NULL, NULL},
+{110, 326, 0, NULL, NULL},
+{114, 343, 0, NULL, NULL},
+{115, 351, 0, NULL, NULL},
+{116, 355, 0, NULL, NULL}
+}; /* compose_tab_21 */
+static int hash_compose_tab_22[20] =
+{-1,6,-1,-1,-1,0,4,7,-1,1,-1,8,-1,2,-1,-1,-1,5,9,3}; /* hash_compose_tab_22 */
+static CompEntry compose_tab_22[] = {
+{65, 260, 0, NULL, NULL},
+{69, 280, 0, NULL, NULL},
+{73, 302, 0, NULL, NULL},
+{79, 490, 0, NULL, NULL},
+{85, 370, 0, NULL, NULL},
+{97, 261, 0, NULL, NULL},
+{101, 281, 0, NULL, NULL},
+{105, 303, 0, NULL, NULL},
+{111, 491, 0, NULL, NULL},
+{117, 371, 0, NULL, NULL}
+}; /* compose_tab_22 */
+static int hash_compose_tab_23[24] =
+{-1,-1,-1,-1,2,6,3,7,-1,-1,-1,-1,4,5,8,9,-1,-1,-1,-1,0,1,10,11}; /* hash_compose_tab_23 */
+static CompEntry compose_tab_23[] = {
+{68, 7698, 0, NULL, NULL},
+{69, 7704, 0, NULL, NULL},
+{76, 7740, 0, NULL, NULL},
+{78, 7754, 0, NULL, NULL},
+{84, 7792, 0, NULL, NULL},
+{85, 7798, 0, NULL, NULL},
+{100, 7699, 0, NULL, NULL},
+{101, 7705, 0, NULL, NULL},
+{108, 7741, 0, NULL, NULL},
+{110, 7755, 0, NULL, NULL},
+{116, 7793, 0, NULL, NULL},
+{117, 7799, 0, NULL, NULL}
+}; /* compose_tab_23 */
+static int hash_compose_tab_24[4] =
+{0,1,-1,-1}; /* hash_compose_tab_24 */
+static CompEntry compose_tab_24[] = {
+{72, 7722, 0, NULL, NULL},
+{104, 7723, 0, NULL, NULL}
+}; /* compose_tab_24 */
+static int hash_compose_tab_25[12] =
+{-1,1,2,-1,-1,3,-1,-1,-1,0,4,5}; /* hash_compose_tab_25 */
+static CompEntry compose_tab_25[] = {
+{69, 7706, 0, NULL, NULL},
+{73, 7724, 0, NULL, NULL},
+{85, 7796, 0, NULL, NULL},
+{101, 7707, 0, NULL, NULL},
+{105, 7725, 0, NULL, NULL},
+{117, 7797, 0, NULL, NULL}
+}; /* compose_tab_25 */
+static int hash_compose_tab_26[34] =
+{1,-1,10,-1,-1,11,12,2,3,13,4,-1,14,-1,5,15,6,-1,-1,-1,16,-1,7,-1,-1,-1,-1,-1,
+ -1,-1,8,-1,0,9}; /* hash_compose_tab_26 */
+static CompEntry compose_tab_26[] = {
+{66, 7686, 0, NULL, NULL},
+{68, 7694, 0, NULL, NULL},
+{75, 7732, 0, NULL, NULL},
+{76, 7738, 0, NULL, NULL},
+{78, 7752, 0, NULL, NULL},
+{82, 7774, 0, NULL, NULL},
+{84, 7790, 0, NULL, NULL},
+{90, 7828, 0, NULL, NULL},
+{98, 7687, 0, NULL, NULL},
+{100, 7695, 0, NULL, NULL},
+{104, 7830, 0, NULL, NULL},
+{107, 7733, 0, NULL, NULL},
+{108, 7739, 0, NULL, NULL},
+{110, 7753, 0, NULL, NULL},
+{114, 7775, 0, NULL, NULL},
+{116, 7791, 0, NULL, NULL},
+{122, 7829, 0, NULL, NULL}
+}; /* compose_tab_26 */
+static int hash_compose_tab_27_1[4] =
+{-1,0,1,-1}; /* hash_compose_tab_27_1 */
+static CompEntry compose_tab_27_1[] = {
+{953, 8151, 0, NULL, NULL},
+{965, 8167, 0, NULL, NULL}
+}; /* compose_tab_27_1 */
+static int hash_compose_tab_27_2_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_2_0 */
+static CompEntry compose_tab_27_2_0[] = {
+{913, 8078, 0, NULL, NULL},
+{919, 8094, 0, NULL, NULL},
+{937, 8110, 0, NULL, NULL},
+{945, 8070, 0, NULL, NULL},
+{951, 8086, 0, NULL, NULL},
+{969, 8102, 0, NULL, NULL}
+}; /* compose_tab_27_2_0 */
+static int hash_compose_tab_27_2[20] =
+{-1,3,-1,-1,-1,5,8,-1,-1,9,-1,6,-1,1,7,-1,-1,0,4,2}; /* hash_compose_tab_27_2 */
+static CompEntry compose_tab_27_2[] = {
+{837, 0, 6, compose_tab_27_2_0, hash_compose_tab_27_2_0},
+{913, 7950, 0, NULL, NULL},
+{919, 7982, 0, NULL, NULL},
+{921, 7998, 0, NULL, NULL},
+{937, 8046, 0, NULL, NULL},
+{945, 7942, 0, NULL, NULL},
+{951, 7974, 0, NULL, NULL},
+{953, 7990, 0, NULL, NULL},
+{965, 8022, 0, NULL, NULL},
+{969, 8038, 0, NULL, NULL}
+}; /* compose_tab_27_2 */
+static int hash_compose_tab_27_3_0[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_3_0 */
+static CompEntry compose_tab_27_3_0[] = {
+{913, 8079, 0, NULL, NULL},
+{919, 8095, 0, NULL, NULL},
+{937, 8111, 0, NULL, NULL},
+{945, 8071, 0, NULL, NULL},
+{951, 8087, 0, NULL, NULL},
+{969, 8103, 0, NULL, NULL}
+}; /* compose_tab_27_3_0 */
+static int hash_compose_tab_27_3[22] =
+{-1,0,10,-1,-1,7,-1,8,-1,4,-1,1,-1,5,-1,-1,-1,2,-1,3,9,6}; /* hash_compose_tab_27_3 */
+static CompEntry compose_tab_27_3[] = {
+{837, 0, 6, compose_tab_27_3_0, hash_compose_tab_27_3_0},
+{913, 7951, 0, NULL, NULL},
+{919, 7983, 0, NULL, NULL},
+{921, 7999, 0, NULL, NULL},
+{933, 8031, 0, NULL, NULL},
+{937, 8047, 0, NULL, NULL},
+{945, 7943, 0, NULL, NULL},
+{951, 7975, 0, NULL, NULL},
+{953, 7991, 0, NULL, NULL},
+{965, 8023, 0, NULL, NULL},
+{969, 8039, 0, NULL, NULL}
+}; /* compose_tab_27_3 */
+static int hash_compose_tab_27_4[6] =
+{-1,-1,-1,0,1,2}; /* hash_compose_tab_27_4 */
+static CompEntry compose_tab_27_4[] = {
+{945, 8119, 0, NULL, NULL},
+{951, 8135, 0, NULL, NULL},
+{969, 8183, 0, NULL, NULL}
+}; /* compose_tab_27_4 */
+static int hash_compose_tab_27[24] =
+{0,-1,-1,-1,-1,8,11,-1,1,5,9,-1,-1,-1,-1,6,10,7,-1,2,3,4,-1,-1}; /* hash_compose_tab_27 */
+static CompEntry compose_tab_27[] = {
+{168, 8129, 0, NULL, NULL},
+{776, 0, 2, compose_tab_27_1, hash_compose_tab_27_1},
+{787, 0, 10, compose_tab_27_2, hash_compose_tab_27_2},
+{788, 0, 11, compose_tab_27_3, hash_compose_tab_27_3},
+{837, 0, 3, compose_tab_27_4, hash_compose_tab_27_4},
+{945, 8118, 0, NULL, NULL},
+{951, 8134, 0, NULL, NULL},
+{953, 8150, 0, NULL, NULL},
+{965, 8166, 0, NULL, NULL},
+{969, 8182, 0, NULL, NULL},
+{8127, 8143, 0, NULL, NULL},
+{8190, 8159, 0, NULL, NULL}
+}; /* compose_tab_27 */
+static int hash_compose_tab_28[12] =
+{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_28 */
+static CompEntry compose_tab_28[] = {
+{913, 8124, 0, NULL, NULL},
+{919, 8140, 0, NULL, NULL},
+{937, 8188, 0, NULL, NULL},
+{945, 8115, 0, NULL, NULL},
+{951, 8131, 0, NULL, NULL},
+{969, 8179, 0, NULL, NULL}
+}; /* compose_tab_28 */
+static int hash_compose_tab_29[4] =
+{0,-1,1,-1}; /* hash_compose_tab_29 */
+static CompEntry compose_tab_29[] = {
+{1488, 64302, 0, NULL, NULL},
+{1522, 64287, 0, NULL, NULL}
+}; /* compose_tab_29 */
+static int hash_compose_tab_30[2] =
+{0,-1}; /* hash_compose_tab_30 */
+static CompEntry compose_tab_30[] = {
+{1488, 64303, 0, NULL, NULL}
+}; /* compose_tab_30 */
+static int hash_compose_tab_31[2] =
+{-1,0}; /* hash_compose_tab_31 */
+static CompEntry compose_tab_31[] = {
+{1493, 64331, 0, NULL, NULL}
+}; /* compose_tab_31 */
+static int hash_compose_tab_32[44] =
+{7,8,9,10,11,-1,12,-1,13,14,-1,15,16,-1,17,18,19,20,21,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,5,6,-1}; /* hash_compose_tab_32 */
+static CompEntry compose_tab_32[] = {
+{1488, 64304, 0, NULL, NULL},
+{1489, 64305, 0, NULL, NULL},
+{1490, 64306, 0, NULL, NULL},
+{1491, 64307, 0, NULL, NULL},
+{1492, 64308, 0, NULL, NULL},
+{1493, 64309, 0, NULL, NULL},
+{1494, 64310, 0, NULL, NULL},
+{1496, 64312, 0, NULL, NULL},
+{1497, 64313, 0, NULL, NULL},
+{1498, 64314, 0, NULL, NULL},
+{1499, 64315, 0, NULL, NULL},
+{1500, 64316, 0, NULL, NULL},
+{1502, 64318, 0, NULL, NULL},
+{1504, 64320, 0, NULL, NULL},
+{1505, 64321, 0, NULL, NULL},
+{1507, 64323, 0, NULL, NULL},
+{1508, 64324, 0, NULL, NULL},
+{1510, 64326, 0, NULL, NULL},
+{1511, 64327, 0, NULL, NULL},
+{1512, 64328, 0, NULL, NULL},
+{1513, 64329, 0, NULL, NULL},
+{1514, 64330, 0, NULL, NULL}
+}; /* compose_tab_32 */
+static int hash_compose_tab_33[6] =
+{-1,0,2,-1,-1,1}; /* hash_compose_tab_33 */
+static CompEntry compose_tab_33[] = {
+{1489, 64332, 0, NULL, NULL},
+{1499, 64333, 0, NULL, NULL},
+{1508, 64334, 0, NULL, NULL}
+}; /* compose_tab_33 */
+static int hash_compose_tab_34_0[2] =
+{-1,0}; /* hash_compose_tab_34_0 */
+static CompEntry compose_tab_34_0[] = {
+{1513, 64300, 0, NULL, NULL}
+}; /* compose_tab_34_0 */
+static int hash_compose_tab_34[4] =
+{0,1,-1,-1}; /* hash_compose_tab_34 */
+static CompEntry compose_tab_34[] = {
+{1468, 0, 1, compose_tab_34_0, hash_compose_tab_34_0},
+{1513, 64298, 0, NULL, NULL}
+}; /* compose_tab_34 */
+static int hash_compose_tab_35_0[2] =
+{-1,0}; /* hash_compose_tab_35_0 */
+static CompEntry compose_tab_35_0[] = {
+{1513, 64301, 0, NULL, NULL}
+}; /* compose_tab_35_0 */
+static int hash_compose_tab_35[4] =
+{0,1,-1,-1}; /* hash_compose_tab_35 */
+static CompEntry compose_tab_35[] = {
+{1468, 0, 1, compose_tab_35_0, hash_compose_tab_35_0},
+{1513, 64299, 0, NULL, NULL}
+}; /* compose_tab_35 */
+static int hash_compose_tab_36[22] =
+{3,10,-1,-1,-1,4,5,-1,-1,-1,-1,-1,6,-1,-1,0,1,2,7,8,9,-1}; /* hash_compose_tab_36 */
+static CompEntry compose_tab_36[] = {
+{2325, 2392, 0, NULL, NULL},
+{2326, 2393, 0, NULL, NULL},
+{2327, 2394, 0, NULL, NULL},
+{2332, 2395, 0, NULL, NULL},
+{2337, 2396, 0, NULL, NULL},
+{2338, 2397, 0, NULL, NULL},
+{2344, 2345, 0, NULL, NULL},
+{2347, 2398, 0, NULL, NULL},
+{2351, 2399, 0, NULL, NULL},
+{2352, 2353, 0, NULL, NULL},
+{2355, 2356, 0, NULL, NULL}
+}; /* compose_tab_36 */
+static int hash_compose_tab_37[8] =
+{-1,0,1,-1,2,-1,-1,3}; /* hash_compose_tab_37 */
+static CompEntry compose_tab_37[] = {
+{2465, 2524, 0, NULL, NULL},
+{2466, 2525, 0, NULL, NULL},
+{2476, 2480, 0, NULL, NULL},
+{2479, 2527, 0, NULL, NULL}
+}; /* compose_tab_37 */
+static int hash_compose_tab_38[2] =
+{-1,0}; /* hash_compose_tab_38 */
+static CompEntry compose_tab_38[] = {
+{2503, 2507, 0, NULL, NULL}
+}; /* compose_tab_38 */
+static int hash_compose_tab_39[2] =
+{-1,0}; /* hash_compose_tab_39 */
+static CompEntry compose_tab_39[] = {
+{2503, 2508, 0, NULL, NULL}
+}; /* compose_tab_39 */
+static int hash_compose_tab_40[10] =
+{-1,-1,0,1,3,4,-1,-1,2,-1}; /* hash_compose_tab_40 */
+static CompEntry compose_tab_40[] = {
+{2582, 2649, 0, NULL, NULL},
+{2583, 2650, 0, NULL, NULL},
+{2588, 2651, 0, NULL, NULL},
+{2593, 2652, 0, NULL, NULL},
+{2603, 2654, 0, NULL, NULL}
+}; /* compose_tab_40 */
+static int hash_compose_tab_41[6] =
+{1,2,-1,-1,-1,0}; /* hash_compose_tab_41 */
+static CompEntry compose_tab_41[] = {
+{2849, 2908, 0, NULL, NULL},
+{2850, 2909, 0, NULL, NULL},
+{2863, 2911, 0, NULL, NULL}
+}; /* compose_tab_41 */
+static int hash_compose_tab_42[2] =
+{-1,0}; /* hash_compose_tab_42 */
+static CompEntry compose_tab_42[] = {
+{2887, 2891, 0, NULL, NULL}
+}; /* compose_tab_42 */
+static int hash_compose_tab_43[2] =
+{-1,0}; /* hash_compose_tab_43 */
+static CompEntry compose_tab_43[] = {
+{2887, 2888, 0, NULL, NULL}
+}; /* compose_tab_43 */
+static int hash_compose_tab_44[2] =
+{-1,0}; /* hash_compose_tab_44 */
+static CompEntry compose_tab_44[] = {
+{2887, 2892, 0, NULL, NULL}
+}; /* compose_tab_44 */
+static int hash_compose_tab_45[4] =
+{-1,-1,0,1}; /* hash_compose_tab_45 */
+static CompEntry compose_tab_45[] = {
+{3014, 3018, 0, NULL, NULL},
+{3015, 3019, 0, NULL, NULL}
+}; /* compose_tab_45 */
+static int hash_compose_tab_46[4] =
+{-1,-1,0,1}; /* hash_compose_tab_46 */
+static CompEntry compose_tab_46[] = {
+{2962, 2964, 0, NULL, NULL},
+{3014, 3020, 0, NULL, NULL}
+}; /* compose_tab_46 */
+static int hash_compose_tab_47[2] =
+{0,-1}; /* hash_compose_tab_47 */
+static CompEntry compose_tab_47[] = {
+{3142, 3144, 0, NULL, NULL}
+}; /* compose_tab_47 */
+static int hash_compose_tab_48[2] =
+{0,-1}; /* hash_compose_tab_48 */
+static CompEntry compose_tab_48[] = {
+{3270, 3274, 0, NULL, NULL}
+}; /* compose_tab_48 */
+static int hash_compose_tab_49_1[2] =
+{0,-1}; /* hash_compose_tab_49_1 */
+static CompEntry compose_tab_49_1[] = {
+{3270, 3275, 0, NULL, NULL}
+}; /* compose_tab_49_1 */
+static int hash_compose_tab_49[6] =
+{2,-1,1,-1,-1,0}; /* hash_compose_tab_49 */
+static CompEntry compose_tab_49[] = {
+{3263, 3264, 0, NULL, NULL},
+{3266, 0, 1, compose_tab_49_1, hash_compose_tab_49_1},
+{3270, 3271, 0, NULL, NULL}
+}; /* compose_tab_49 */
+static int hash_compose_tab_50[2] =
+{0,-1}; /* hash_compose_tab_50 */
+static CompEntry compose_tab_50[] = {
+{3270, 3272, 0, NULL, NULL}
+}; /* compose_tab_50 */
+static int hash_compose_tab_51[4] =
+{-1,-1,0,1}; /* hash_compose_tab_51 */
+static CompEntry compose_tab_51[] = {
+{3398, 3402, 0, NULL, NULL},
+{3399, 3403, 0, NULL, NULL}
+}; /* compose_tab_51 */
+static int hash_compose_tab_52[2] =
+{0,-1}; /* hash_compose_tab_52 */
+static CompEntry compose_tab_52[] = {
+{3398, 3404, 0, NULL, NULL}
+}; /* compose_tab_52 */
+static int hash_compose_tab_53[2] =
+{-1,0}; /* hash_compose_tab_53 */
+static CompEntry compose_tab_53[] = {
+{3661, 3635, 0, NULL, NULL}
+}; /* compose_tab_53 */
+static int hash_compose_tab_54[2] =
+{-1,0}; /* hash_compose_tab_54 */
+static CompEntry compose_tab_54[] = {
+{3789, 3763, 0, NULL, NULL}
+}; /* compose_tab_54 */
+static int hash_compose_tab_55_2[4] =
+{-1,-1,0,1}; /* hash_compose_tab_55_2 */
+static CompEntry compose_tab_55_2[] = {
+{4018, 3959, 0, NULL, NULL},
+{4019, 3961, 0, NULL, NULL}
+}; /* compose_tab_55_2 */
+static int hash_compose_tab_55[6] =
+{0,-1,1,2,-1,-1}; /* hash_compose_tab_55 */
+static CompEntry compose_tab_55[] = {
+{3954, 3955, 0, NULL, NULL},
+{3956, 3957, 0, NULL, NULL},
+{3968, 0, 2, compose_tab_55_2, hash_compose_tab_55_2}
+}; /* compose_tab_55 */
+static int hash_compose_tab_56[4] =
+{-1,-1,0,1}; /* hash_compose_tab_56 */
+static CompEntry compose_tab_56[] = {
+{4018, 3958, 0, NULL, NULL},
+{4019, 3960, 0, NULL, NULL}
+}; /* compose_tab_56 */
+static int hash_compose_tab_57[4] =
+{0,1,-1,-1}; /* hash_compose_tab_57 */
+static CompEntry compose_tab_57[] = {
+{3904, 3945, 0, NULL, NULL},
+{3984, 4025, 0, NULL, NULL}
+}; /* compose_tab_57 */
+static int hash_compose_tab_58[20] =
+{-1,2,7,-1,-1,-1,0,3,5,8,-1,4,9,-1,-1,-1,1,6,-1,-1}; /* hash_compose_tab_58 */
+static CompEntry compose_tab_58[] = {
+{3906, 3907, 0, NULL, NULL},
+{3916, 3917, 0, NULL, NULL},
+{3921, 3922, 0, NULL, NULL},
+{3926, 3927, 0, NULL, NULL},
+{3931, 3932, 0, NULL, NULL},
+{3986, 3987, 0, NULL, NULL},
+{3996, 3997, 0, NULL, NULL},
+{4001, 4002, 0, NULL, NULL},
+{4006, 4007, 0, NULL, NULL},
+{4011, 4012, 0, NULL, NULL}
+}; /* compose_tab_58 */
+static int hash_compose_tab_59[96] =
+{33,12,34,-1,13,35,14,36,15,37,-1,-1,-1,-1,-1,16,38,-1,17,39,-1,18,40,-1,19,
+ 41,-1,20,42,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,43,44,45,
+ 46,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,21,47,-1,-1,-1,-1,-1,-1,-1,0,22,-1,-1,-1,1,
+ 23,2,24,3,25,4,26,5,27,6,28,7,29,8,30,9,31,10,32,11}; /* hash_compose_tab_59 */
+static CompEntry compose_tab_59[] = {
+{12358, 12436, 0, NULL, NULL},
+{12363, 12364, 0, NULL, NULL},
+{12365, 12366, 0, NULL, NULL},
+{12367, 12368, 0, NULL, NULL},
+{12369, 12370, 0, NULL, NULL},
+{12371, 12372, 0, NULL, NULL},
+{12373, 12374, 0, NULL, NULL},
+{12375, 12376, 0, NULL, NULL},
+{12377, 12378, 0, NULL, NULL},
+{12379, 12380, 0, NULL, NULL},
+{12381, 12382, 0, NULL, NULL},
+{12383, 12384, 0, NULL, NULL},
+{12385, 12386, 0, NULL, NULL},
+{12388, 12389, 0, NULL, NULL},
+{12390, 12391, 0, NULL, NULL},
+{12392, 12393, 0, NULL, NULL},
+{12399, 12400, 0, NULL, NULL},
+{12402, 12403, 0, NULL, NULL},
+{12405, 12406, 0, NULL, NULL},
+{12408, 12409, 0, NULL, NULL},
+{12411, 12412, 0, NULL, NULL},
+{12445, 12446, 0, NULL, NULL},
+{12454, 12532, 0, NULL, NULL},
+{12459, 12460, 0, NULL, NULL},
+{12461, 12462, 0, NULL, NULL},
+{12463, 12464, 0, NULL, NULL},
+{12465, 12466, 0, NULL, NULL},
+{12467, 12468, 0, NULL, NULL},
+{12469, 12470, 0, NULL, NULL},
+{12471, 12472, 0, NULL, NULL},
+{12473, 12474, 0, NULL, NULL},
+{12475, 12476, 0, NULL, NULL},
+{12477, 12478, 0, NULL, NULL},
+{12479, 12480, 0, NULL, NULL},
+{12481, 12482, 0, NULL, NULL},
+{12484, 12485, 0, NULL, NULL},
+{12486, 12487, 0, NULL, NULL},
+{12488, 12489, 0, NULL, NULL},
+{12495, 12496, 0, NULL, NULL},
+{12498, 12499, 0, NULL, NULL},
+{12501, 12502, 0, NULL, NULL},
+{12504, 12505, 0, NULL, NULL},
+{12507, 12508, 0, NULL, NULL},
+{12527, 12535, 0, NULL, NULL},
+{12528, 12536, 0, NULL, NULL},
+{12529, 12537, 0, NULL, NULL},
+{12530, 12538, 0, NULL, NULL},
+{12541, 12542, 0, NULL, NULL}
+}; /* compose_tab_59 */
+static int hash_compose_tab_60[20] =
+{-1,7,1,-1,8,2,-1,9,3,-1,-1,4,-1,-1,-1,5,-1,-1,6,0}; /* hash_compose_tab_60 */
+static CompEntry compose_tab_60[] = {
+{12399, 12401, 0, NULL, NULL},
+{12402, 12404, 0, NULL, NULL},
+{12405, 12407, 0, NULL, NULL},
+{12408, 12410, 0, NULL, NULL},
+{12411, 12413, 0, NULL, NULL},
+{12495, 12497, 0, NULL, NULL},
+{12498, 12500, 0, NULL, NULL},
+{12501, 12503, 0, NULL, NULL},
+{12504, 12506, 0, NULL, NULL},
+{12507, 12509, 0, NULL, NULL}
+}; /* compose_tab_60 */
+static int hash_compose_tab[122] =
+{30,31,52,60,32,-1,-1,33,-1,34,35,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,
+ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,-1,5,6,7,8,9,10,11,12,36,13,37,14,
+ 38,15,16,55,40,-1,-1,-1,-1,17,56,-1,-1,-1,-1,-1,41,18,19,20,42,21,22,-1,45,
+ 39,-1,23,24,-1,25,26,-1,-1,-1,-1,-1,-1,-1,-1,48,-1,43,44,51,53,-1,-1,27,46,
+ 54,28,-1,-1,47,-1,-1,-1,-1,49,50,-1,-1,57,-1,58,59,29}; /* hash_compose_tab */
+static CompEntry compose_tab[] = {
+{768, 0, 39, compose_tab_0, hash_compose_tab_0},
+{769, 0, 70, compose_tab_1, hash_compose_tab_1},
+{770, 0, 27, compose_tab_2, hash_compose_tab_2},
+{771, 0, 19, compose_tab_3, hash_compose_tab_3},
+{772, 0, 28, compose_tab_4, hash_compose_tab_4},
+{774, 0, 30, compose_tab_5, hash_compose_tab_5},
+{775, 0, 40, compose_tab_6, hash_compose_tab_6},
+{776, 0, 50, compose_tab_7, hash_compose_tab_7},
+{777, 0, 15, compose_tab_8, hash_compose_tab_8},
+{778, 0, 6, compose_tab_9, hash_compose_tab_9},
+{779, 0, 6, compose_tab_10, hash_compose_tab_10},
+{780, 0, 34, compose_tab_11, hash_compose_tab_11},
+{781, 0, 17, compose_tab_12, hash_compose_tab_12},
+{783, 0, 14, compose_tab_13, hash_compose_tab_13},
+{785, 0, 12, compose_tab_14, hash_compose_tab_14},
+{787, 0, 15, compose_tab_15, hash_compose_tab_15},
+{788, 0, 17, compose_tab_16, hash_compose_tab_16},
+{795, 0, 4, compose_tab_17, hash_compose_tab_17},
+{803, 0, 39, compose_tab_18, hash_compose_tab_18},
+{804, 0, 2, compose_tab_19, hash_compose_tab_19},
+{805, 0, 2, compose_tab_20, hash_compose_tab_20},
+{807, 0, 20, compose_tab_21, hash_compose_tab_21},
+{808, 0, 10, compose_tab_22, hash_compose_tab_22},
+{813, 0, 12, compose_tab_23, hash_compose_tab_23},
+{814, 0, 2, compose_tab_24, hash_compose_tab_24},
+{816, 0, 6, compose_tab_25, hash_compose_tab_25},
+{817, 0, 17, compose_tab_26, hash_compose_tab_26},
+{834, 0, 12, compose_tab_27, hash_compose_tab_27},
+{837, 0, 6, compose_tab_28, hash_compose_tab_28},
+{1463, 0, 2, compose_tab_29, hash_compose_tab_29},
+{1464, 0, 1, compose_tab_30, hash_compose_tab_30},
+{1465, 0, 1, compose_tab_31, hash_compose_tab_31},
+{1468, 0, 22, compose_tab_32, hash_compose_tab_32},
+{1471, 0, 3, compose_tab_33, hash_compose_tab_33},
+{1473, 0, 2, compose_tab_34, hash_compose_tab_34},
+{1474, 0, 2, compose_tab_35, hash_compose_tab_35},
+{2364, 0, 11, compose_tab_36, hash_compose_tab_36},
+{2492, 0, 4, compose_tab_37, hash_compose_tab_37},
+{2494, 0, 1, compose_tab_38, hash_compose_tab_38},
+{2519, 0, 1, compose_tab_39, hash_compose_tab_39},
+{2620, 0, 5, compose_tab_40, hash_compose_tab_40},
+{2876, 0, 3, compose_tab_41, hash_compose_tab_41},
+{2878, 0, 1, compose_tab_42, hash_compose_tab_42},
+{2902, 0, 1, compose_tab_43, hash_compose_tab_43},
+{2903, 0, 1, compose_tab_44, hash_compose_tab_44},
+{3006, 0, 2, compose_tab_45, hash_compose_tab_45},
+{3031, 0, 2, compose_tab_46, hash_compose_tab_46},
+{3158, 0, 1, compose_tab_47, hash_compose_tab_47},
+{3266, 0, 1, compose_tab_48, hash_compose_tab_48},
+{3285, 0, 3, compose_tab_49, hash_compose_tab_49},
+{3286, 0, 1, compose_tab_50, hash_compose_tab_50},
+{3390, 0, 2, compose_tab_51, hash_compose_tab_51},
+{3415, 0, 1, compose_tab_52, hash_compose_tab_52},
+{3634, 0, 1, compose_tab_53, hash_compose_tab_53},
+{3762, 0, 1, compose_tab_54, hash_compose_tab_54},
+{3953, 0, 3, compose_tab_55, hash_compose_tab_55},
+{3968, 0, 2, compose_tab_56, hash_compose_tab_56},
+{4021, 0, 2, compose_tab_57, hash_compose_tab_57},
+{4023, 0, 10, compose_tab_58, hash_compose_tab_58},
+{12441, 0, 48, compose_tab_59, hash_compose_tab_59},
+{12442, 0, 10, compose_tab_60, hash_compose_tab_60}
+}; /* compose_tab */
+#define COMP_CANDIDATE_MAP_OFFSET 24
+static Uint32 comp_candidate_map[] = {
+ 0x081ABFDFU,
+ 0x000361B8U,
+ 0x00000024U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x93800000U,
+ 0x00000006U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x10000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x50000000U,
+ 0x00800000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x10000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x50000000U,
+ 0x00C00000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x40000000U,
+ 0x00800000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00400000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00600004U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x40000000U,
+ 0x00800000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00040000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00040000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00000000U,
+ 0x00020000U,
+ 0x00000001U,
+ 0x00A00000U
+};
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 89c6625550..6e86de3ebf 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1596,6 +1596,17 @@ Sint erts_binary_set_loop_limit(Sint limit);
/* erl_unicode.c */
void erts_init_unicode(void);
Sint erts_unicode_set_loop_limit(Sint limit);
+
+void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) ;
+Sint erts_native_filename_need(Eterm ioterm, int encoding);
+void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars);
+int erts_analyze_utf8(byte *source, Uint size,
+ byte **err_pos, Uint *num_chars, int *left);
+#define ERTS_UTF8_OK 0
+#define ERTS_UTF8_INCOMPLETE 1
+#define ERTS_UTF8_ERROR 2
+#define ERTS_UTF8_ANALYZE_MORE 3
+
/* erl_trace.c */
void erts_init_trace(void);
void erts_trace_check_exiting(Eterm exiting);
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index 57f2b2f16c..27c5f99320 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -1260,7 +1260,8 @@ char* win32_errorstr(int);
#define ERL_FILENAME_UNKNOWN 0
#define ERL_FILENAME_LATIN1 1
#define ERL_FILENAME_UTF8 2
-#define ERL_FILENAME_WIN_WCHAR 3
+#define ERL_FILENAME_UTF8_MAC 3
+#define ERL_FILENAME_WIN_WCHAR 4
int erts_get_native_filename_encoding(void);
/* The set function is only to be used by erl_init! */
diff --git a/erts/emulator/internal_doc/dec.dat b/erts/emulator/internal_doc/dec.dat
new file mode 100644
index 0000000000..771ef51baa
--- /dev/null
+++ b/erts/emulator/internal_doc/dec.dat
@@ -0,0 +1,942 @@
+{[59],894}.
+{[96],8175}.
+{[180],8189}.
+{[183],903}.
+{[198],1236}.
+{[230],1237}.
+{[399],1240}.
+{[415],1256}.
+{[439],1248}.
+{[601],1241}.
+{[629],1257}.
+{[658],1249}.
+{[697],884}.
+{[768],832}.
+{[768,65],192}.
+{[768,69],200}.
+{[768,73],204}.
+{[768,79],210}.
+{[768,85],217}.
+{[768,87],7808}.
+{[768,89],7922}.
+{[768,97],224}.
+{[768,101],232}.
+{[768,105],236}.
+{[768,111],242}.
+{[768,117],249}.
+{[768,119],7809}.
+{[768,121],7923}.
+{[768,168],8173}.
+{[768,770,65],7846}.
+{[768,770,69],7872}.
+{[768,770,79],7890}.
+{[768,770,97],7847}.
+{[768,770,101],7873}.
+{[768,770,111],7891}.
+{[768,772,69],7700}.
+{[768,772,79],7760}.
+{[768,772,101],7701}.
+{[768,772,111],7761}.
+{[768,774,65],7856}.
+{[768,774,97],7857}.
+{[768,776,85],475}.
+{[768,776,117],476}.
+{[768,776,953],8146}.
+{[768,776,965],8162}.
+{[768,787,837,913],8074}.
+{[768,787,837,919],8090}.
+{[768,787,837,937],8106}.
+{[768,787,837,945],8066}.
+{[768,787,837,951],8082}.
+{[768,787,837,969],8098}.
+{[768,787,913],7946}.
+{[768,787,917],7962}.
+{[768,787,919],7978}.
+{[768,787,921],7994}.
+{[768,787,927],8010}.
+{[768,787,937],8042}.
+{[768,787,945],7938}.
+{[768,787,949],7954}.
+{[768,787,951],7970}.
+{[768,787,953],7986}.
+{[768,787,959],8002}.
+{[768,787,965],8018}.
+{[768,787,969],8034}.
+{[768,788,837,913],8075}.
+{[768,788,837,919],8091}.
+{[768,788,837,937],8107}.
+{[768,788,837,945],8067}.
+{[768,788,837,951],8083}.
+{[768,788,837,969],8099}.
+{[768,788,913],7947}.
+{[768,788,917],7963}.
+{[768,788,919],7979}.
+{[768,788,921],7995}.
+{[768,788,927],8011}.
+{[768,788,933],8027}.
+{[768,788,937],8043}.
+{[768,788,945],7939}.
+{[768,788,949],7955}.
+{[768,788,951],7971}.
+{[768,788,953],7987}.
+{[768,788,959],8003}.
+{[768,788,965],8019}.
+{[768,788,969],8035}.
+{[768,795,79],7900}.
+{[768,795,85],7914}.
+{[768,795,111],7901}.
+{[768,795,117],7915}.
+{[768,837,945],8114}.
+{[768,837,951],8130}.
+{[768,837,969],8178}.
+{[768,913],8122}.
+{[768,917],8136}.
+{[768,919],8138}.
+{[768,921],8154}.
+{[768,927],8184}.
+{[768,933],8170}.
+{[768,937],8186}.
+{[768,945],8048}.
+{[768,949],8050}.
+{[768,951],8052}.
+{[768,953],8054}.
+{[768,959],8056}.
+{[768,965],8058}.
+{[768,969],8060}.
+{[768,8127],8141}.
+{[768,8190],8157}.
+{[769],833}.
+{[769,65],193}.
+{[769,67],262}.
+{[769,69],201}.
+{[769,71],500}.
+{[769,73],205}.
+{[769,75],7728}.
+{[769,76],313}.
+{[769,77],7742}.
+{[769,78],323}.
+{[769,79],211}.
+{[769,80],7764}.
+{[769,82],340}.
+{[769,83],346}.
+{[769,85],218}.
+{[769,87],7810}.
+{[769,89],221}.
+{[769,90],377}.
+{[769,97],225}.
+{[769,99],263}.
+{[769,101],233}.
+{[769,103],501}.
+{[769,105],237}.
+{[769,107],7729}.
+{[769,108],314}.
+{[769,109],7743}.
+{[769,110],324}.
+{[769,111],243}.
+{[769,112],7765}.
+{[769,114],341}.
+{[769,115],347}.
+{[769,117],250}.
+{[769,119],7811}.
+{[769,121],253}.
+{[769,122],378}.
+{[769,168],8174}.
+{[769,198],508}.
+{[769,216],510}.
+{[769,230],509}.
+{[769,248],511}.
+{[769,770,65],7844}.
+{[769,770,69],7870}.
+{[769,770,79],7888}.
+{[769,770,97],7845}.
+{[769,770,101],7871}.
+{[769,770,111],7889}.
+{[769,771,79],7756}.
+{[769,771,85],7800}.
+{[769,771,111],7757}.
+{[769,771,117],7801}.
+{[769,772,69],7702}.
+{[769,772,79],7762}.
+{[769,772,101],7703}.
+{[769,772,111],7763}.
+{[769,774,65],7854}.
+{[769,774,97],7855}.
+{[769,776,73],7726}.
+{[769,776,85],471}.
+{[769,776,105],7727}.
+{[769,776,117],472}.
+{[769,776,953],8147}.
+{[769,776,965],8163}.
+{[769,778,65],506}.
+{[769,778,97],507}.
+{[769,787,837,913],8076}.
+{[769,787,837,919],8092}.
+{[769,787,837,937],8108}.
+{[769,787,837,945],8068}.
+{[769,787,837,951],8084}.
+{[769,787,837,969],8100}.
+{[769,787,913],7948}.
+{[769,787,917],7964}.
+{[769,787,919],7980}.
+{[769,787,921],7996}.
+{[769,787,927],8012}.
+{[769,787,937],8044}.
+{[769,787,945],7940}.
+{[769,787,949],7956}.
+{[769,787,951],7972}.
+{[769,787,953],7988}.
+{[769,787,959],8004}.
+{[769,787,965],8020}.
+{[769,787,969],8036}.
+{[769,788,837,913],8077}.
+{[769,788,837,919],8093}.
+{[769,788,837,937],8109}.
+{[769,788,837,945],8069}.
+{[769,788,837,951],8085}.
+{[769,788,837,969],8101}.
+{[769,788,913],7949}.
+{[769,788,917],7965}.
+{[769,788,919],7981}.
+{[769,788,921],7997}.
+{[769,788,927],8013}.
+{[769,788,933],8029}.
+{[769,788,937],8045}.
+{[769,788,945],7941}.
+{[769,788,949],7957}.
+{[769,788,951],7973}.
+{[769,788,953],7989}.
+{[769,788,959],8005}.
+{[769,788,965],8021}.
+{[769,788,969],8037}.
+{[769,795,79],7898}.
+{[769,795,85],7912}.
+{[769,795,111],7899}.
+{[769,795,117],7913}.
+{[769,807,67],7688}.
+{[769,807,99],7689}.
+{[769,837,945],8116}.
+{[769,837,951],8132}.
+{[769,837,959],8180}.
+{[769,913],8123}.
+{[769,917],8137}.
+{[769,919],8139}.
+{[769,921],8155}.
+{[769,927],8185}.
+{[769,933],8171}.
+{[769,937],8187}.
+{[769,945],8049}.
+{[769,949],8051}.
+{[769,951],8053}.
+{[769,953],8055}.
+{[769,959],8057}.
+{[769,965],8059}.
+{[769,969],8061}.
+{[769,1043],1027}.
+{[769,1050],1036}.
+{[769,1075],1107}.
+{[769,1082],1116}.
+{[769,8127],8142}.
+{[769,8190],8158}.
+{[770,65],194}.
+{[770,67],264}.
+{[770,69],202}.
+{[770,71],284}.
+{[770,72],292}.
+{[770,73],206}.
+{[770,74],308}.
+{[770,79],212}.
+{[770,83],348}.
+{[770,85],219}.
+{[770,87],372}.
+{[770,89],374}.
+{[770,90],7824}.
+{[770,97],226}.
+{[770,99],265}.
+{[770,101],234}.
+{[770,103],285}.
+{[770,104],293}.
+{[770,105],238}.
+{[770,106],309}.
+{[770,111],244}.
+{[770,115],349}.
+{[770,117],251}.
+{[770,119],373}.
+{[770,121],375}.
+{[770,122],7825}.
+{[770,803,65],7852}.
+{[770,803,69],7878}.
+{[770,803,79],7896}.
+{[770,803,97],7853}.
+{[770,803,101],7879}.
+{[770,803,111],7897}.
+{[771,65],195}.
+{[771,69],7868}.
+{[771,73],296}.
+{[771,78],209}.
+{[771,79],213}.
+{[771,85],360}.
+{[771,86],7804}.
+{[771,89],7928}.
+{[771,97],227}.
+{[771,101],7869}.
+{[771,105],297}.
+{[771,110],241}.
+{[771,111],245}.
+{[771,117],361}.
+{[771,118],7805}.
+{[771,121],7929}.
+{[771,770,65],7850}.
+{[771,770,69],7876}.
+{[771,770,79],7894}.
+{[771,770,97],7851}.
+{[771,770,101],7877}.
+{[771,770,111],7895}.
+{[771,774,65],7860}.
+{[771,774,97],7861}.
+{[771,795,79],7904}.
+{[771,795,85],7918}.
+{[771,795,111],7905}.
+{[771,795,117],7919}.
+{[772,65],256}.
+{[772,69],274}.
+{[772,71],7712}.
+{[772,73],298}.
+{[772,79],332}.
+{[772,85],362}.
+{[772,97],257}.
+{[772,101],275}.
+{[772,103],7713}.
+{[772,105],299}.
+{[772,111],333}.
+{[772,117],363}.
+{[772,198],482}.
+{[772,230],483}.
+{[772,775,65],480}.
+{[772,775,97],481}.
+{[772,776,65],478}.
+{[772,776,85],469}.
+{[772,776,97],479}.
+{[772,776,117],470}.
+{[772,803,76],7736}.
+{[772,803,82],7772}.
+{[772,803,108],7737}.
+{[772,803,114],7773}.
+{[772,808,79],492}.
+{[772,808,111],493}.
+{[772,913],8121}.
+{[772,921],8153}.
+{[772,933],8169}.
+{[772,945],8113}.
+{[772,953],8145}.
+{[772,965],8161}.
+{[772,1048],1250}.
+{[772,1059],1262}.
+{[772,1080],1251}.
+{[772,1091],1263}.
+{[774,65],258}.
+{[774,69],276}.
+{[774,71],286}.
+{[774,73],300}.
+{[774,79],334}.
+{[774,85],364}.
+{[774,97],259}.
+{[774,101],277}.
+{[774,103],287}.
+{[774,105],301}.
+{[774,111],335}.
+{[774,117],365}.
+{[774,803,65],7862}.
+{[774,803,97],7863}.
+{[774,807,69],7708}.
+{[774,807,101],7709}.
+{[774,913],8120}.
+{[774,921],8152}.
+{[774,933],8168}.
+{[774,945],8112}.
+{[774,953],8144}.
+{[774,965],8160}.
+{[774,1040],1232}.
+{[774,1045],1238}.
+{[774,1046],1217}.
+{[774,1048],1049}.
+{[774,1059],1038}.
+{[774,1072],1233}.
+{[774,1077],1239}.
+{[774,1078],1218}.
+{[774,1080],1081}.
+{[774,1091],1118}.
+{[775,66],7682}.
+{[775,67],266}.
+{[775,68],7690}.
+{[775,69],278}.
+{[775,70],7710}.
+{[775,71],288}.
+{[775,72],7714}.
+{[775,73],304}.
+{[775,77],7744}.
+{[775,78],7748}.
+{[775,80],7766}.
+{[775,82],7768}.
+{[775,83],7776}.
+{[775,84],7786}.
+{[775,87],7814}.
+{[775,88],7818}.
+{[775,89],7822}.
+{[775,90],379}.
+{[775,98],7683}.
+{[775,99],267}.
+{[775,100],7691}.
+{[775,101],279}.
+{[775,102],7711}.
+{[775,103],289}.
+{[775,104],7715}.
+{[775,109],7745}.
+{[775,110],7749}.
+{[775,112],7767}.
+{[775,114],7769}.
+{[775,115],7777}.
+{[775,116],7787}.
+{[775,119],7815}.
+{[775,120],7819}.
+{[775,121],7823}.
+{[775,122],380}.
+{[775,383],7835}.
+{[775,769,83],7780}.
+{[775,769,115],7781}.
+{[775,774],784}.
+{[775,780,83],7782}.
+{[775,780,115],7783}.
+{[775,803,83],7784}.
+{[775,803,115],7785}.
+{[776,65],196}.
+{[776,69],203}.
+{[776,72],7718}.
+{[776,73],207}.
+{[776,79],214}.
+{[776,85],220}.
+{[776,87],7812}.
+{[776,88],7820}.
+{[776,89],376}.
+{[776,97],228}.
+{[776,101],235}.
+{[776,104],7719}.
+{[776,105],239}.
+{[776,111],246}.
+{[776,116],7831}.
+{[776,117],252}.
+{[776,119],7813}.
+{[776,120],7821}.
+{[776,121],255}.
+{[776,399],1242}.
+{[776,415],1258}.
+{[776,601],1243}.
+{[776,629],1259}.
+{[776,771,79],7758}.
+{[776,771,111],7759}.
+{[776,772,85],7802}.
+{[776,772,117],7803}.
+{[776,921],938}.
+{[776,933],939}.
+{[776,953],970}.
+{[776,965],971}.
+{[776,978],980}.
+{[776,1030],1031}.
+{[776,1040],1234}.
+{[776,1045],1025}.
+{[776,1046],1244}.
+{[776,1047],1246}.
+{[776,1048],1252}.
+{[776,1054],1254}.
+{[776,1059],1264}.
+{[776,1063],1268}.
+{[776,1067],1272}.
+{[776,1072],1235}.
+{[776,1077],1105}.
+{[776,1078],1245}.
+{[776,1079],1247}.
+{[776,1080],1253}.
+{[776,1086],1255}.
+{[776,1091],1265}.
+{[776,1095],1269}.
+{[776,1099],1273}.
+{[776,1110],1111}.
+{[777,65],7842}.
+{[777,69],7866}.
+{[777,73],7880}.
+{[777,79],7886}.
+{[777,85],7910}.
+{[777,89],7926}.
+{[777,97],7843}.
+{[777,101],7867}.
+{[777,105],7881}.
+{[777,111],7887}.
+{[777,117],7911}.
+{[777,121],7927}.
+{[777,770,65],7848}.
+{[777,770,69],7874}.
+{[777,770,79],7892}.
+{[777,770,97],7849}.
+{[777,770,101],7875}.
+{[777,770,111],7893}.
+{[777,774,65],7858}.
+{[777,774,97],7859}.
+{[777,795,79],7902}.
+{[777,795,85],7916}.
+{[777,795,111],7903}.
+{[777,795,117],7917}.
+{[778,65],197}.
+{[778,85],366}.
+{[778,97],229}.
+{[778,117],367}.
+{[778,119],7832}.
+{[778,121],7833}.
+{[779,79],336}.
+{[779,85],368}.
+{[779,111],337}.
+{[779,117],369}.
+{[779,1059],1266}.
+{[779,1091],1267}.
+{[780,65],461}.
+{[780,67],268}.
+{[780,68],270}.
+{[780,69],282}.
+{[780,71],486}.
+{[780,73],463}.
+{[780,75],488}.
+{[780,76],317}.
+{[780,78],327}.
+{[780,79],465}.
+{[780,82],344}.
+{[780,83],352}.
+{[780,84],356}.
+{[780,85],467}.
+{[780,90],381}.
+{[780,97],462}.
+{[780,99],269}.
+{[780,100],271}.
+{[780,101],283}.
+{[780,103],487}.
+{[780,105],464}.
+{[780,106],496}.
+{[780,107],489}.
+{[780,108],318}.
+{[780,110],328}.
+{[780,111],466}.
+{[780,114],345}.
+{[780,115],353}.
+{[780,116],357}.
+{[780,117],468}.
+{[780,122],382}.
+{[780,439],494}.
+{[780,658],495}.
+{[780,776,85],473}.
+{[780,776,117],474}.
+{[781,168],901}.
+{[781,776],836}.
+{[781,776,953],912}.
+{[781,776,965],944}.
+{[781,913],902}.
+{[781,917],904}.
+{[781,919],905}.
+{[781,921],906}.
+{[781,927],908}.
+{[781,933],910}.
+{[781,937],911}.
+{[781,945],940}.
+{[781,949],941}.
+{[781,951],942}.
+{[781,953],943}.
+{[781,959],972}.
+{[781,965],973}.
+{[781,969],974}.
+{[781,978],979}.
+{[783,65],512}.
+{[783,69],516}.
+{[783,73],520}.
+{[783,79],524}.
+{[783,82],528}.
+{[783,85],532}.
+{[783,97],513}.
+{[783,101],517}.
+{[783,105],521}.
+{[783,111],525}.
+{[783,114],529}.
+{[783,117],533}.
+{[783,1140],1142}.
+{[783,1141],1143}.
+{[785,65],514}.
+{[785,69],518}.
+{[785,73],522}.
+{[785,79],526}.
+{[785,82],530}.
+{[785,85],534}.
+{[785,97],515}.
+{[785,101],519}.
+{[785,105],523}.
+{[785,111],527}.
+{[785,114],531}.
+{[785,117],535}.
+{[787],835}.
+{[787,837,913],8072}.
+{[787,837,919],8088}.
+{[787,837,937],8104}.
+{[787,837,945],8064}.
+{[787,837,951],8080}.
+{[787,837,969],8096}.
+{[787,913],7944}.
+{[787,917],7960}.
+{[787,919],7976}.
+{[787,921],7992}.
+{[787,927],8008}.
+{[787,937],8040}.
+{[787,945],7936}.
+{[787,949],7952}.
+{[787,951],7968}.
+{[787,953],7984}.
+{[787,959],8000}.
+{[787,961],8164}.
+{[787,965],8016}.
+{[787,969],8032}.
+{[788,837,913],8073}.
+{[788,837,919],8089}.
+{[788,837,937],8105}.
+{[788,837,945],8065}.
+{[788,837,951],8081}.
+{[788,837,969],8097}.
+{[788,913],7945}.
+{[788,917],7961}.
+{[788,919],7977}.
+{[788,921],7993}.
+{[788,927],8009}.
+{[788,929],8172}.
+{[788,933],8025}.
+{[788,937],8041}.
+{[788,945],7937}.
+{[788,949],7953}.
+{[788,951],7969}.
+{[788,953],7985}.
+{[788,959],8001}.
+{[788,961],8165}.
+{[788,965],8017}.
+{[788,969],8033}.
+{[795,79],416}.
+{[795,85],431}.
+{[795,111],417}.
+{[795,117],432}.
+{[803,65],7840}.
+{[803,66],7684}.
+{[803,68],7692}.
+{[803,69],7864}.
+{[803,72],7716}.
+{[803,73],7882}.
+{[803,75],7730}.
+{[803,76],7734}.
+{[803,77],7746}.
+{[803,78],7750}.
+{[803,79],7884}.
+{[803,82],7770}.
+{[803,83],7778}.
+{[803,84],7788}.
+{[803,85],7908}.
+{[803,86],7806}.
+{[803,87],7816}.
+{[803,89],7924}.
+{[803,90],7826}.
+{[803,97],7841}.
+{[803,98],7685}.
+{[803,100],7693}.
+{[803,101],7865}.
+{[803,104],7717}.
+{[803,105],7883}.
+{[803,107],7731}.
+{[803,108],7735}.
+{[803,109],7747}.
+{[803,110],7751}.
+{[803,111],7885}.
+{[803,114],7771}.
+{[803,115],7779}.
+{[803,116],7789}.
+{[803,117],7909}.
+{[803,118],7807}.
+{[803,119],7817}.
+{[803,121],7925}.
+{[803,122],7827}.
+{[803,795,79],7906}.
+{[803,795,85],7920}.
+{[803,795,111],7907}.
+{[803,795,117],7921}.
+{[804,85],7794}.
+{[804,117],7795}.
+{[805,65],7680}.
+{[805,97],7681}.
+{[807,67],199}.
+{[807,68],7696}.
+{[807,71],290}.
+{[807,72],7720}.
+{[807,75],310}.
+{[807,76],315}.
+{[807,78],325}.
+{[807,82],342}.
+{[807,83],350}.
+{[807,84],354}.
+{[807,99],231}.
+{[807,100],7697}.
+{[807,103],291}.
+{[807,104],7721}.
+{[807,107],311}.
+{[807,108],316}.
+{[807,110],326}.
+{[807,114],343}.
+{[807,115],351}.
+{[807,116],355}.
+{[808,65],260}.
+{[808,69],280}.
+{[808,73],302}.
+{[808,79],490}.
+{[808,85],370}.
+{[808,97],261}.
+{[808,101],281}.
+{[808,105],303}.
+{[808,111],491}.
+{[808,117],371}.
+{[813,68],7698}.
+{[813,69],7704}.
+{[813,76],7740}.
+{[813,78],7754}.
+{[813,84],7792}.
+{[813,85],7798}.
+{[813,100],7699}.
+{[813,101],7705}.
+{[813,108],7741}.
+{[813,110],7755}.
+{[813,116],7793}.
+{[813,117],7799}.
+{[814,72],7722}.
+{[814,104],7723}.
+{[816,69],7706}.
+{[816,73],7724}.
+{[816,85],7796}.
+{[816,101],7707}.
+{[816,105],7725}.
+{[816,117],7797}.
+{[817,66],7686}.
+{[817,68],7694}.
+{[817,75],7732}.
+{[817,76],7738}.
+{[817,78],7752}.
+{[817,82],7774}.
+{[817,84],7790}.
+{[817,90],7828}.
+{[817,98],7687}.
+{[817,100],7695}.
+{[817,104],7830}.
+{[817,107],7733}.
+{[817,108],7739}.
+{[817,110],7753}.
+{[817,114],7775}.
+{[817,116],7791}.
+{[817,122],7829}.
+{[834,168],8129}.
+{[834,776,953],8151}.
+{[834,776,965],8167}.
+{[834,787,837,913],8078}.
+{[834,787,837,919],8094}.
+{[834,787,837,937],8110}.
+{[834,787,837,945],8070}.
+{[834,787,837,951],8086}.
+{[834,787,837,969],8102}.
+{[834,787,913],7950}.
+{[834,787,919],7982}.
+{[834,787,921],7998}.
+{[834,787,937],8046}.
+{[834,787,945],7942}.
+{[834,787,951],7974}.
+{[834,787,953],7990}.
+{[834,787,965],8022}.
+{[834,787,969],8038}.
+{[834,788,837,913],8079}.
+{[834,788,837,919],8095}.
+{[834,788,837,937],8111}.
+{[834,788,837,945],8071}.
+{[834,788,837,951],8087}.
+{[834,788,837,969],8103}.
+{[834,788,913],7951}.
+{[834,788,919],7983}.
+{[834,788,921],7999}.
+{[834,788,933],8031}.
+{[834,788,937],8047}.
+{[834,788,945],7943}.
+{[834,788,951],7975}.
+{[834,788,953],7991}.
+{[834,788,965],8023}.
+{[834,788,969],8039}.
+{[834,837,945],8119}.
+{[834,837,951],8135}.
+{[834,837,969],8183}.
+{[834,945],8118}.
+{[834,951],8134}.
+{[834,953],8150}.
+{[834,965],8166}.
+{[834,969],8182}.
+{[834,8127],8143}.
+{[834,8190],8159}.
+{[837,913],8124}.
+{[837,919],8140}.
+{[837,937],8188}.
+{[837,945],8115}.
+{[837,951],8131}.
+{[837,969],8179}.
+{[953],8126}.
+{[1463,1488],64302}.
+{[1463,1522],64287}.
+{[1464,1488],64303}.
+{[1465,1493],64331}.
+{[1468,1488],64304}.
+{[1468,1489],64305}.
+{[1468,1490],64306}.
+{[1468,1491],64307}.
+{[1468,1492],64308}.
+{[1468,1493],64309}.
+{[1468,1494],64310}.
+{[1468,1496],64312}.
+{[1468,1497],64313}.
+{[1468,1498],64314}.
+{[1468,1499],64315}.
+{[1468,1500],64316}.
+{[1468,1502],64318}.
+{[1468,1504],64320}.
+{[1468,1505],64321}.
+{[1468,1507],64323}.
+{[1468,1508],64324}.
+{[1468,1510],64326}.
+{[1468,1511],64327}.
+{[1468,1512],64328}.
+{[1468,1513],64329}.
+{[1468,1514],64330}.
+{[1471,1489],64332}.
+{[1471,1499],64333}.
+{[1471,1508],64334}.
+{[1473,1468,1513],64300}.
+{[1473,1513],64298}.
+{[1474,1468,1513],64301}.
+{[1474,1513],64299}.
+{[2364,2325],2392}.
+{[2364,2326],2393}.
+{[2364,2327],2394}.
+{[2364,2332],2395}.
+{[2364,2337],2396}.
+{[2364,2338],2397}.
+{[2364,2344],2345}.
+{[2364,2347],2398}.
+{[2364,2351],2399}.
+{[2364,2352],2353}.
+{[2364,2355],2356}.
+{[2492,2465],2524}.
+{[2492,2466],2525}.
+{[2492,2476],2480}.
+{[2492,2479],2527}.
+{[2494,2503],2507}.
+{[2519,2503],2508}.
+{[2620,2582],2649}.
+{[2620,2583],2650}.
+{[2620,2588],2651}.
+{[2620,2593],2652}.
+{[2620,2603],2654}.
+{[2876,2849],2908}.
+{[2876,2850],2909}.
+{[2876,2863],2911}.
+{[2878,2887],2891}.
+{[2902,2887],2888}.
+{[2903,2887],2892}.
+{[3006,3014],3018}.
+{[3006,3015],3019}.
+{[3031,2962],2964}.
+{[3031,3014],3020}.
+{[3158,3142],3144}.
+{[3266,3270],3274}.
+{[3285,3263],3264}.
+{[3285,3266,3270],3275}.
+{[3285,3270],3271}.
+{[3286,3270],3272}.
+{[3390,3398],3402}.
+{[3390,3399],3403}.
+{[3415,3398],3404}.
+{[3634,3661],3635}.
+{[3762,3789],3763}.
+{[3953,3954],3955}.
+{[3953,3956],3957}.
+{[3953,3968],3969}.
+{[3953,3968,4018],3959}.
+{[3953,3968,4019],3961}.
+{[3968,4018],3958}.
+{[3968,4019],3960}.
+{[4021,3904],3945}.
+{[4021,3984],4025}.
+{[4023,3906],3907}.
+{[4023,3916],3917}.
+{[4023,3921],3922}.
+{[4023,3926],3927}.
+{[4023,3931],3932}.
+{[4023,3986],3987}.
+{[4023,3996],3997}.
+{[4023,4001],4002}.
+{[4023,4006],4007}.
+{[4023,4011],4012}.
+{[12441,12358],12436}.
+{[12441,12363],12364}.
+{[12441,12365],12366}.
+{[12441,12367],12368}.
+{[12441,12369],12370}.
+{[12441,12371],12372}.
+{[12441,12373],12374}.
+{[12441,12375],12376}.
+{[12441,12377],12378}.
+{[12441,12379],12380}.
+{[12441,12381],12382}.
+{[12441,12383],12384}.
+{[12441,12385],12386}.
+{[12441,12388],12389}.
+{[12441,12390],12391}.
+{[12441,12392],12393}.
+{[12441,12399],12400}.
+{[12441,12402],12403}.
+{[12441,12405],12406}.
+{[12441,12408],12409}.
+{[12441,12411],12412}.
+{[12441,12445],12446}.
+{[12441,12454],12532}.
+{[12441,12459],12460}.
+{[12441,12461],12462}.
+{[12441,12463],12464}.
+{[12441,12465],12466}.
+{[12441,12467],12468}.
+{[12441,12469],12470}.
+{[12441,12471],12472}.
+{[12441,12473],12474}.
+{[12441,12475],12476}.
+{[12441,12477],12478}.
+{[12441,12479],12480}.
+{[12441,12481],12482}.
+{[12441,12484],12485}.
+{[12441,12486],12487}.
+{[12441,12488],12489}.
+{[12441,12495],12496}.
+{[12441,12498],12499}.
+{[12441,12501],12502}.
+{[12441,12504],12505}.
+{[12441,12507],12508}.
+{[12441,12527],12535}.
+{[12441,12528],12536}.
+{[12441,12529],12537}.
+{[12441,12530],12538}.
+{[12441,12541],12542}.
+{[12442,12399],12401}.
+{[12442,12402],12404}.
+{[12442,12405],12407}.
+{[12442,12408],12410}.
+{[12442,12411],12413}.
+{[12442,12495],12497}.
+{[12442,12498],12500}.
+{[12442,12501],12503}.
+{[12442,12504],12506}.
+{[12442,12507],12509}.
diff --git a/erts/emulator/internal_doc/dec.erl b/erts/emulator/internal_doc/dec.erl
new file mode 100644
index 0000000000..0315f2a52d
--- /dev/null
+++ b/erts/emulator/internal_doc/dec.erl
@@ -0,0 +1,237 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2000-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%% This program is used to generate a header file with data for
+%% normalizing denormalized unicode.
+
+%% The C header is generated from a text file containing tuples in the
+%% following format:
+%% {RevList,Translation}
+%% Where 'RevList' is a reversed list of the denormalized repressentation of
+%% the character 'Translation'. An example would be the swedish character
+%% '�', which would be represented in the file as:
+%% {[776,111],246}, as the denormalized representation of codepoint 246
+%% is [111,776] (i.e an 'o' followed by the "double dot accent character 776),
+%% while '�' instead is represented as {[776,97],228}, as the denormalized
+%% form would be [97,776] (same accent but an 'a' instead).
+%% The datafile is generated from the table on Apple's developer connection
+%% http://developer.apple.com/library/mac/#technotes/tn/tn1150table.html
+%% The generating is done whenever new data is present (i.e. dec.dat has
+%% to be changed) and not for every build. The product (the C header) is copied
+%% to $ERL_TOP/erts/beam after generation and checked in.
+%% The program and the data file is included for reference.
+
+-module(dec).
+
+-compile(export_all).
+
+-define(HASH_SIZE_FACTOR,2).
+-define(BIG_PREFIX_SIZE,392).
+
+-define(INPUT_FILE_NAME,"dec.dat").
+-define(OUTPUT_FILE_NAME,"erl_unicode_normalize.h").
+
+read(FName) ->
+ {ok,L} = file:consult(FName),
+ [{A,B} || {A,B} <- L,
+ length(A) > 1% , hd(A) < 769
+ ].
+
+dec() ->
+ L = read(?INPUT_FILE_NAME),
+ G = group(L),
+ {ok,Out} = file:open(?OUTPUT_FILE_NAME,[write]),
+ io:format
+ (Out,
+ "/*~n"
+ "* %CopyrightBegin%~n"
+ "*~n"
+ "* Copyright Ericsson AB 1999-2010. All Rights Reserved.~n"
+ "*~n"
+ "* The contents of this file are subject to the Erlang Public License,~n"
+ "* Version 1.1, (the \"License\"); you may not use this file except in~n"
+ "* compliance with the License. You should have received a copy of the~n"
+ "* Erlang Public License along with this software. If not, it can be~n"
+ "* retrieved online at http://www.erlang.org/.~n"
+ "*~n"
+ "* Software distributed under the License is distributed on an "
+ "\"AS IS\"~n"
+ "* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See~n"
+ "* the License for the specific language governing rights and "
+ "limitations~n"
+ "* under the License.~n"
+ "*~n"
+ "* %CopyrightEnd%~n"
+ "*/~n"
+ "/*~n"
+ "* This file is automatically generated by ~p.erl, "
+ "do not edit manually~n"
+ "*/~n",
+ [?MODULE]),
+
+ io:format(Out,
+ "#define HASH_SIZE_FACTOR ~w~n"
+ "typedef struct _compose_entry {~n"
+ " Uint16 c;~n"
+ " Uint16 res;~n"
+ " Uint16 num_subs;~n"
+ " struct _compose_entry *subs;~n"
+ " int *hash;~n"
+ "} CompEntry;~n~n"
+ "static int compose_tab_size = ~p;~n",
+ [?HASH_SIZE_FACTOR,length(G)]),
+ d(Out,G,[],0),
+ PreTab = tuple_to_list(make_prefix_table(G,erlang:make_tuple(102,0))),
+ dump_prefixes(Out,PreTab),
+%% Using this cuts down on the searching in the
+%% actual implementation, but wastes memory with little real gain..
+%% LL = lists:flatten([PartList || {PartList,_} <- L]),
+%% BigPreTab = tuple_to_list(
+%% make_big_prefixes(LL,
+%% erlang:make_tuple(?BIG_PREFIX_SIZE,0))),
+%% dump_big_prefixes(Out,BigPreTab),
+ file:close(Out),
+ ok.
+
+
+
+d(Out,List,D,C) ->
+ d_sub(Out,List,D,C),
+ d_top_hash(Out,List,D,C),
+ d_top(Out,List,D,C).
+d_sub(_Out,[],_D,_C) ->
+ ok;
+d_sub(Out,[{_CP,[],_Res}|T],D,C) ->
+ d_sub(Out,T,D,C+1);
+d_sub(Out,[{_CP,Subs,_Res0}|T],D,C) ->
+ d(Out,Subs,[C|D],0),
+ d_sub(Out,T,D,C+1).
+d_top(Out,L,D,C) ->
+ io:format(Out,"static CompEntry ~s[] = {~n",[format_depth(D)]),
+ d_top_1(Out,L,D,C),
+ io:format(Out,"}; /* ~s */ ~n",[format_depth(D)]).
+
+d_top_1(_Out,[],_D,_C) ->
+ ok;
+d_top_1(Out,[{CP,[],Res}|T],D,C) ->
+ io:format(Out,
+ "{~w, ~w, 0, NULL, NULL}",[CP,Res]),
+ if
+ T =:= [] ->
+ io:format(Out,"~n",[]);
+ true ->
+ io:format(Out,",~n",[])
+ end,
+ d_top_1(Out,T,D,C+1);
+d_top_1(Out,[{CP,Subs,_Res}|T],D,C) ->
+ io:format(Out,
+ "{~w, 0, ~w, ~s, ~s}",[CP,length(Subs),
+ format_depth([C|D]),
+ "hash_"++format_depth([C|D])]),
+ if
+ T =:= [] ->
+ io:format(Out,"~n",[]);
+ true ->
+ io:format(Out,",~n",[])
+ end,
+ d_top_1(Out,T,D,C+1).
+
+
+d_top_hash(Out,List,D,_C) ->
+ HSize = length(List)*?HASH_SIZE_FACTOR,
+ io:format(Out,"static int ~s[~p] = ~n",["hash_"++format_depth(D),HSize]),
+ Tup = d_top_hash_1(List,0,erlang:make_tuple(HSize,-1),HSize),
+ io:format(Out,"~p; /* ~s */ ~n",[Tup,"hash_"++format_depth(D)]).
+
+d_top_hash_1([],_,Hash,_HSize) ->
+ Hash;
+d_top_hash_1([{CP,_,_}|T],Index,Hash,HSize) ->
+ Bucket = hash_search(Hash,HSize,CP rem HSize),
+ d_top_hash_1(T,Index+1,erlang:setelement(Bucket+1,Hash,Index),HSize).
+
+hash_search(Hash,_HSize,Bucket) when element(Bucket+1,Hash) =:= -1 ->
+ Bucket;
+hash_search(Hash,HSize,Bucket) ->
+ hash_search(Hash,HSize,(Bucket + 1) rem HSize).
+
+format_depth(D) ->
+ lists:reverse(tl(lists:reverse(lists:flatten(["compose_tab_",[ integer_to_list(X) ++ "_" || X <- lists:reverse(D) ]])))).
+
+
+
+
+make_prefix_table([],Table) ->
+ Table;
+make_prefix_table([{C,_,_}|T],Table) when C =< 4023 ->
+ Index = (C div 32) + 1 - 24,
+ Pos = C rem 32,
+ X = element(Index,Table),
+ Y = X bor (1 bsl Pos),
+ NewTab = setelement(Index,Table,Y),
+ make_prefix_table(T,NewTab);
+make_prefix_table([_|T],Tab) ->
+ make_prefix_table(T,Tab).
+
+dump_prefixes(Out,L) ->
+ io:format(Out,"#define COMP_CANDIDATE_MAP_OFFSET 24~n",[]),
+ io:format(Out,"static Uint32 comp_candidate_map[] = {~n",[]),
+ dump_prefixes_1(Out,L).
+dump_prefixes_1(Out,[H]) ->
+ io:format(Out," 0x~8.16.0BU~n",[H]),
+ io:format(Out,"};~n",[]);
+dump_prefixes_1(Out,[H|T]) ->
+ io:format(Out," 0x~8.16.0BU,~n",[H]),
+ dump_prefixes_1(Out,T).
+
+%% make_big_prefixes([],Table) ->
+%% Table;
+%% make_big_prefixes([C|T],Table) ->
+%% Index = (C div 32) + 1,
+%% Pos = C rem 32,
+%% X = element(Index,Table),
+%% Y = X bor (1 bsl Pos),
+%% NewTab = setelement(Index,Table,Y),
+%% make_big_prefixes(T,NewTab).
+
+%% dump_big_prefixes(Out,L) ->
+%% io:format(Out,"#define BIG_COMP_CANDIDATE_SIZE ~w~n", [?BIG_PREFIX_SIZE]),
+%% io:format(Out,"static Uint32 big_comp_candidate_map[] = {~n",[]),
+%% dump_prefixes_1(Out,L).
+
+pick([],_,Acc) ->
+ {lists:reverse(Acc),[]};
+pick([{[H|TT],N}|T],H,Acc) ->
+ pick(T,H,[{TT,N}|Acc]);
+pick([{[H|_],_}|_]=L,M,Acc) when H =/= M ->
+ {lists:reverse(Acc),L}.
+
+
+group([]) ->
+ [];
+group([{[H],N}|T]) ->
+ {Part,Rest} = pick(T,H,[]),
+ [{H,group(Part),N}| group(Rest)];
+group([{[H|_],_}|_]=L) ->
+ {Part,Rest} = pick(L,H,[]),
+ [{H,group(Part),0}| group(Rest)].
+
+
+
+
+
diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c
index 7338ffc963..461e763f03 100644
--- a/erts/emulator/sys/common/erl_sys_common_misc.c
+++ b/erts/emulator/sys/common/erl_sys_common_misc.c
@@ -17,10 +17,11 @@
* %CopyrightEnd%
*/
+
+
/*
- * Description: Check I/O
- *
- * Author: Rickard Green
+ * Darwin needs conversion!
+ * http://developer.apple.com/library/mac/#qa/qa2001/qa1235.html
*/
#ifdef HAVE_CONFIG_H
@@ -30,6 +31,10 @@
#include "sys.h"
#include "global.h"
+#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__)
+#define __DARWIN__ 1
+#endif
+
#if !defined(__WIN32__)
#include <locale.h>
#if !defined(HAVE_SETLOCALE) || !defined(HAVE_NL_LANGINFO) || !defined(HAVE_LANGINFO_H)
@@ -42,7 +47,7 @@
/* Written once and only once */
static int filename_encoding = ERL_FILENAME_UNKNOWN;
-#if defined(__WIN32__)
+#if defined(__WIN32__) || defined(__DARWIN__)
static int user_filename_encoding = ERL_FILENAME_UTF8; /* Default unicode on windows */
#else
static int user_filename_encoding = ERL_FILENAME_LATIN1;
@@ -88,6 +93,11 @@ void erts_init_sys_common_misc(void)
}
# endif
}
+# if defined(__DARWIN__)
+ if (filename_encoding == ERL_FILENAME_UTF8) {
+ filename_encoding = ERL_FILENAME_UTF8_MAC;
+ }
+# endif
#endif
}