From f1c69ee583dcd1f525562cf6adc382b8464b1578 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 13 Jun 2017 15:51:42 +0200 Subject: Revert "erts: Do not generate atoms on old latin1 external format" This reverts commit 65b04e233e09e3cc2e0fda3c28e155b95c5a4baf. --- erts/emulator/beam/dist.h | 1 - erts/emulator/beam/external.c | 39 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 4 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 93a651b24d..3e17645997 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -53,7 +53,6 @@ | DFLAG_EXPORT_PTR_TAG \ | DFLAG_BIT_BINARIES \ | DFLAG_MAP_TAG \ - | DFLAG_UTF8_ATOMS \ | DFLAG_BIG_CREATION) /* opcodes used in distribution messages */ diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 1190d90b8e..06213daa67 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2090,6 +2090,7 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) { int iix; int len; + int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS); ASSERT(is_atom(atom)); @@ -2118,8 +2119,8 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) if (iix < 0) { Atom *a = atom_tab(atom_val(atom)); len = a->len; - { - if (len > 255) { + if (utf8_atoms || a->latin1_chars < 0) { + if (len > 255) { *ep++ = ATOM_UTF8_EXT; put_int16(len, ep); ep += 2; @@ -2131,6 +2132,32 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) } sys_memcpy((char *) ep, (char *) a->name, len); } + else { + if (a->latin1_chars <= 255 && (dflags & DFLAG_SMALL_ATOM_TAGS)) { + *ep++ = SMALL_ATOM_EXT; + if (len == a->latin1_chars) { + sys_memcpy(ep+1, a->name, len); + } + else { + len = erts_utf8_to_latin1(ep+1, a->name, len); + ASSERT(len == a->latin1_chars); + } + put_int8(len, ep); + ep++; + } + else { + *ep++ = ATOM_EXT; + if (len == a->latin1_chars) { + sys_memcpy(ep+2, a->name, len); + } + else { + len = erts_utf8_to_latin1(ep+2, a->name, len); + ASSERT(len == a->latin1_chars); + } + put_int16(len, ep); + ep += 2; + } + } ep += len; return ep; } @@ -4053,13 +4080,19 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, else { Atom *a = atom_tab(atom_val(obj)); int alen; - { + if ((dflags & DFLAG_UTF8_ATOMS) || a->latin1_chars < 0) { alen = a->len; result += 1 + 1 + alen; if (alen > 255) { result++; /* ATOM_UTF8_EXT (not small) */ } } + else { + alen = a->latin1_chars; + result += 1 + 1 + alen; + if (alen > 255 || !(dflags & DFLAG_SMALL_ATOM_TAGS)) + result++; /* ATOM_EXT (not small) */ + } insert_acache_map(acmp, obj, dflags); } break; -- cgit v1.2.3 From bdc0f3504fb13f777a7cc826caa5fd10dc6fc291 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 14 Jun 2017 13:23:34 +0200 Subject: Introduce minor vsn 2 in term_to_binary/2 --- erts/doc/src/erlang.xml | 42 ++++++++++++++++++++++++++++++------------ erts/emulator/beam/external.c | 4 +++- 2 files changed, 33 insertions(+), 13 deletions(-) (limited to 'erts') diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index d9cc5ef936..687ff38cbf 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -8860,18 +8860,36 @@ hello

Option {minor_version, Version} can be used to control some encoding details. This option was introduced in Erlang/OTP R11B-4. - The valid values for Version are - 0 and 1.

-

As from Erlang/OTP 17.0, {minor_version, 1} is the default. It - forces any floats in the term to be encoded in a more - space-efficient and exact way (namely in the 64-bit IEEE format, - rather than converted to a textual representation).

-

As from Erlang/OTP R11B-4, binary_to_term/1 can decode this - representation.

-

{minor_version, 0} means that floats are encoded - using a textual representation. This option is useful to - ensure that releases before Erlang/OTP R11B-4 can decode resulting - binary.

+ The valid values for Version are:

+ + 0 + +

Floats are encoded using a textual representation. + This option is useful to ensure that releases before Erlang/OTP + R11B-4 can decode resulting binary.

+

This version encode atoms that can be represented by a + latin1 string using latin1 encoding while only atoms that + cannot be represented by latin1 are encoded using utf8.

+ + 1 + +

This is as of Erlang/OTP 17.0 the default. It forces any floats + in the term to be encoded in a more space-efficient and exact way + (namely in the 64-bit IEEE format, rather than converted to a + textual representation). As from Erlang/OTP R11B-4, + binary_to_term/1 can decode this representation.

+

This version encode atoms that can be represented by a + latin1 string using latin1 encoding while only atoms that + cannot be represented by latin1 are encoded using utf8.

+ + 2 + +

Drops usage of the latin1 atom encoding and unconditionally + use utf8 encoding for all atoms. This will be changed to the + default in a future major release of Erlang/OTP. Erlang/OTP + systems as of R16B can decode this representation.

+ +

See also binary_to_term/1.

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 06213daa67..95275847f8 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1129,8 +1129,10 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) case 0: flags = TERM_TO_BINARY_DFLAGS & ~DFLAG_NEW_FLOATS; break; - case 1: + case 1: /* Current default... */ flags = TERM_TO_BINARY_DFLAGS; + case 2: + flags = TERM_TO_BINARY_DFLAGS | DFLAG_UTF8_ATOMS; break; default: goto error; -- cgit v1.2.3 From 376008a4416d36517e295e8bf49abc8f63c6c172 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 13 Jun 2017 15:59:32 +0200 Subject: Update preloaded modules --- erts/preloaded/ebin/erl_prim_loader.beam | Bin 54324 -> 54832 bytes erts/preloaded/ebin/erl_tracer.beam | Bin 2140 -> 2184 bytes erts/preloaded/ebin/erlang.beam | Bin 105164 -> 106168 bytes erts/preloaded/ebin/erts_code_purger.beam | Bin 11280 -> 11380 bytes .../ebin/erts_dirty_process_code_checker.beam | Bin 2052 -> 2104 bytes erts/preloaded/ebin/erts_internal.beam | Bin 10964 -> 11064 bytes .../ebin/erts_literal_area_collector.beam | Bin 3232 -> 3288 bytes erts/preloaded/ebin/init.beam | Bin 49928 -> 50300 bytes erts/preloaded/ebin/otp_ring0.beam | Bin 1388 -> 1424 bytes erts/preloaded/ebin/prim_eval.beam | Bin 1412 -> 1496 bytes erts/preloaded/ebin/prim_file.beam | Bin 43600 -> 43980 bytes erts/preloaded/ebin/prim_inet.beam | Bin 75448 -> 76036 bytes erts/preloaded/ebin/prim_zip.beam | Bin 22772 -> 22988 bytes erts/preloaded/ebin/zlib.beam | Bin 14164 -> 14280 bytes 14 files changed, 0 insertions(+), 0 deletions(-) (limited to 'erts') diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam index 922e0eb009..5095367fea 100644 Binary files a/erts/preloaded/ebin/erl_prim_loader.beam and b/erts/preloaded/ebin/erl_prim_loader.beam differ diff --git a/erts/preloaded/ebin/erl_tracer.beam b/erts/preloaded/ebin/erl_tracer.beam index 578de5ffcb..9cb0ee6119 100644 Binary files a/erts/preloaded/ebin/erl_tracer.beam and b/erts/preloaded/ebin/erl_tracer.beam differ diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam index 63518ed6e1..2efc6816e8 100644 Binary files a/erts/preloaded/ebin/erlang.beam and b/erts/preloaded/ebin/erlang.beam differ diff --git a/erts/preloaded/ebin/erts_code_purger.beam b/erts/preloaded/ebin/erts_code_purger.beam index 58218c715a..203e08c512 100644 Binary files a/erts/preloaded/ebin/erts_code_purger.beam and b/erts/preloaded/ebin/erts_code_purger.beam differ diff --git a/erts/preloaded/ebin/erts_dirty_process_code_checker.beam b/erts/preloaded/ebin/erts_dirty_process_code_checker.beam index af764347eb..39b6ae9c4a 100644 Binary files a/erts/preloaded/ebin/erts_dirty_process_code_checker.beam and b/erts/preloaded/ebin/erts_dirty_process_code_checker.beam differ diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam index 072dc68712..e53ddea4c3 100644 Binary files a/erts/preloaded/ebin/erts_internal.beam and b/erts/preloaded/ebin/erts_internal.beam differ diff --git a/erts/preloaded/ebin/erts_literal_area_collector.beam b/erts/preloaded/ebin/erts_literal_area_collector.beam index fb1179ddae..96d8be2d69 100644 Binary files a/erts/preloaded/ebin/erts_literal_area_collector.beam and b/erts/preloaded/ebin/erts_literal_area_collector.beam differ diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam index a87fcbf0fa..d20e76ba9c 100644 Binary files a/erts/preloaded/ebin/init.beam and b/erts/preloaded/ebin/init.beam differ diff --git a/erts/preloaded/ebin/otp_ring0.beam b/erts/preloaded/ebin/otp_ring0.beam index 1071606f07..cc98612499 100644 Binary files a/erts/preloaded/ebin/otp_ring0.beam and b/erts/preloaded/ebin/otp_ring0.beam differ diff --git a/erts/preloaded/ebin/prim_eval.beam b/erts/preloaded/ebin/prim_eval.beam index 752df348ff..0e429a93d5 100644 Binary files a/erts/preloaded/ebin/prim_eval.beam and b/erts/preloaded/ebin/prim_eval.beam differ diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam index 50cb86d841..ed2a8767c1 100644 Binary files a/erts/preloaded/ebin/prim_file.beam and b/erts/preloaded/ebin/prim_file.beam differ diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam index 16bf38fcd7..ef64e3fcbf 100644 Binary files a/erts/preloaded/ebin/prim_inet.beam and b/erts/preloaded/ebin/prim_inet.beam differ diff --git a/erts/preloaded/ebin/prim_zip.beam b/erts/preloaded/ebin/prim_zip.beam index bf6fc60752..9203fe632b 100644 Binary files a/erts/preloaded/ebin/prim_zip.beam and b/erts/preloaded/ebin/prim_zip.beam differ diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam index 6554c5324a..c29359aa34 100644 Binary files a/erts/preloaded/ebin/zlib.beam and b/erts/preloaded/ebin/zlib.beam differ -- cgit v1.2.3