From 5cce8836ccffbaf71879b2efbb5788278e7f7e90 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 8 Mar 2017 14:41:54 +0100 Subject: erts: Fix faulty ASSERT for failed dec_term when a recursive call to dec_term() has already done erts_factory_undo and set factory->hp to NULL. --- erts/emulator/beam/external.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts') diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 205a7711ec..186b250c7e 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -4010,7 +4010,7 @@ error: factory->hp = hp; /* the largest must be the freshest */ } } - else ASSERT(factory->hp == hp); + else ASSERT(!factory->hp || factory->hp == hp); error_hamt: erts_factory_undo(factory); -- cgit v1.2.3 From 65b04e233e09e3cc2e0fda3c28e155b95c5a4baf Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 7 Mar 2017 17:04:40 +0100 Subject: erts: Do not generate atoms on old latin1 external format term_to_binary will always generate utf8 atoms ATOM_UTF8_EXT and SMALL_ATOM_UTF8_EXT. Old latin1 atoms, ATOM_EXT and SMALL_ATOM_EXT, are still decoded. --- erts/emulator/beam/dist.h | 1 + erts/emulator/beam/external.c | 39 +++------------------------------------ 2 files changed, 4 insertions(+), 36 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 8f6be1061a..6ed36a478e 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -53,6 +53,7 @@ | DFLAG_EXPORT_PTR_TAG \ | DFLAG_BIT_BINARIES \ | DFLAG_MAP_TAG \ + | DFLAG_UTF8_ATOMS \ | DFLAG_BIG_CREATION) /* opcodes used in distribution messages */ diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 186b250c7e..285ae4ac78 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2093,7 +2093,6 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) { int iix; int len; - int utf8_atoms = (int) (dflags & DFLAG_UTF8_ATOMS); ASSERT(is_atom(atom)); @@ -2122,8 +2121,8 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) if (iix < 0) { Atom *a = atom_tab(atom_val(atom)); len = a->len; - if (utf8_atoms || a->latin1_chars < 0) { - if (len > 255) { + { + if (len > 255) { *ep++ = ATOM_UTF8_EXT; put_int16(len, ep); ep += 2; @@ -2135,32 +2134,6 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) } sys_memcpy((char *) ep, (char *) a->name, len); } - else { - if (a->latin1_chars <= 255 && (dflags & DFLAG_SMALL_ATOM_TAGS)) { - *ep++ = SMALL_ATOM_EXT; - if (len == a->latin1_chars) { - sys_memcpy(ep+1, a->name, len); - } - else { - len = erts_utf8_to_latin1(ep+1, a->name, len); - ASSERT(len == a->latin1_chars); - } - put_int8(len, ep); - ep++; - } - else { - *ep++ = ATOM_EXT; - if (len == a->latin1_chars) { - sys_memcpy(ep+2, a->name, len); - } - else { - len = erts_utf8_to_latin1(ep+2, a->name, len); - ASSERT(len == a->latin1_chars); - } - put_int16(len, ep); - ep += 2; - } - } ep += len; return ep; } @@ -4085,19 +4058,13 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, else { Atom *a = atom_tab(atom_val(obj)); int alen; - if ((dflags & DFLAG_UTF8_ATOMS) || a->latin1_chars < 0) { + { alen = a->len; result += 1 + 1 + alen; if (alen > 255) { result++; /* ATOM_UTF8_EXT (not small) */ } } - else { - alen = a->latin1_chars; - result += 1 + 1 + alen; - if (alen > 255 || !(dflags & DFLAG_SMALL_ATOM_TAGS)) - result++; /* ATOM_EXT (not small) */ - } insert_acache_map(acmp, obj, dflags); } break; -- cgit v1.2.3 From 1b7b4f7398765188815f697444e42029454dcd3d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Wed, 8 Mar 2017 20:34:55 +0100 Subject: erts: Mark latin1 atom encoding as deprecated which means tags ATOM_EXT and SMALL_ATOM_EXT. --- erts/doc/src/erl_ext_dist.xml | 156 ++++++++++++++++++++---------------------- 1 file changed, 76 insertions(+), 80 deletions(-) (limited to 'erts') diff --git a/erts/doc/src/erl_ext_dist.xml b/erts/doc/src/erl_ext_dist.xml index a436a9ca74..da2dc94e5b 100644 --- a/erts/doc/src/erl_ext_dist.xml +++ b/erts/doc/src/erl_ext_dist.xml @@ -51,7 +51,7 @@ term into the external format. To convert binary data encoding to a term, the BIF - erlang:binary_to_term/1c> is used. + erlang:binary_to_term/1 is used.

The distribution does this implicitly when sending messages across @@ -119,22 +119,18 @@ Compressed Data Format when Expanded -

As from ERTS 9.0 (OTP 20), UTF-8 encoded atoms may contain any Unicode - character. Although the support for UTF-8 encoded atoms in the external - format is available since ERTS 5.10 (OTP R16), passing atoms that cannot - be encoded in Latin-1 is an error in versions earlier than - Erlang/OTP 20, and the behavior is undefined.

-

When distribution flag - DFLAG_UTF8_ATOMS has been exchanged between both nodes - in the - distribution handshake, all atoms in the distribution header - are encoded in UTF-8, otherwise in Latin-1. The two - new tags ATOM_UTF8_EXT - and - SMALL_ATOM_UTF8_EXT - are only used if the distribution flag DFLAG_UTF8_ATOMS has - been exchanged between nodes, or if an atom containing characters - that cannot be encoded in Latin-1 is encountered.

+

As from ERTS 9.0 (OTP 20), atoms may contain any Unicode + characters and are always encoded using the UTF-8 external formats + ATOM_UTF8_EXT + or SMALL_ATOM_UTF8_EXT. + The old Latin-1 formats ATOM_EXT + and SMALL_ATOM_EXT + are deprecated and are only kept for backward + compatibility when decoding terms encoded by older nodes.

+

Support for UTF-8 encoded atoms in the external format has been + available since ERTS 5.10 (OTP R16). This abillity allows such old nodes + to decode, store and encode any Unicode atoms received from a new OTP 20 + node.

The maximum number of allowed characters in an atom is 255. In the UTF-8 case, each character can need 4 bytes to be encoded.

@@ -389,28 +385,6 @@

-
- - ATOM_EXT - - - 1 - 2 - Len - - - 100 - Len - AtomName - - ATOM_EXT
-

- An atom is stored with a 2 byte unsigned length in big-endian order, - followed by Len numbers of 8-bit Latin-1 characters that forms - the AtomName. The maximum allowed value for Len is 255. -

-
-
REFERENCE_EXT @@ -432,8 +406,8 @@ Encodes a reference object (an object generated with erlang:make_ref/0). The Node term is an encoded atom, that is, - ATOM_EXT, - SMALL_ATOM_EXT, or + ATOM_UTF8_EXT, + SMALL_ATOM_UTF8_EXT, or ATOM_CACHE_REF. The ID field contains a big-endian unsigned integer, but is to be regarded as uninterpreted data, @@ -771,39 +745,6 @@

-
- - SMALL_ATOM_EXT - - - 1 - 1 - Len - - - 115 - Len - AtomName - - SMALL_ATOM_EXT
-

- An atom is stored with a 1 byte unsigned length, - followed by Len numbers of 8-bit Latin-1 characters that - forms the AtomName. Longer atoms can be represented - by ATOM_EXT. -

- -

- SMALL_ATOM_EXT was introduced in ERTS 5.7.2 and - require an exchange of distribution flag - - DFLAG_SMALL_ATOM_TAGS in the - - distribution handshake. -

- -
-
FUN_EXT @@ -838,8 +779,8 @@ Module

Encoded as an atom, using - ATOM_EXT, - SMALL_ATOM_EXT, + ATOM_UTF8_EXT, + SMALL_ATOM_UTF8_EXT, or ATOM_CACHE_REF. This is the module that the fun is implemented in. @@ -933,8 +874,8 @@ Module

Encoded as an atom, using - ATOM_EXT, - SMALL_ATOM_EXT, + ATOM_UTF8_EXT, + SMALL_ATOM_UTF8_EXT, or ATOM_CACHE_REF. Is the module that the fun is implemented in. @@ -996,8 +937,8 @@

Module and Function are atoms - (encoded using ATOM_EXT, - SMALL_ATOM_EXT, or + (encoded using ATOM_UTF8_EXT, + SMALL_ATOM_UTF8_EXT, or ATOM_CACHE_REF).

@@ -1109,6 +1050,61 @@ in the beginning of this section.

+ +
+ + ATOM_EXT (deprecated) + + + 1 + 2 + Len + + + 100 + Len + AtomName + + ATOM_EXT
+

+ An atom is stored with a 2 byte unsigned length in big-endian order, + followed by Len numbers of 8-bit Latin-1 characters that forms + the AtomName. The maximum allowed value for Len is 255. +

+
+ +
+ + SMALL_ATOM_EXT (deprecated) + + + 1 + 1 + Len + + + 115 + Len + AtomName + + SMALL_ATOM_EXT
+

+ An atom is stored with a 1 byte unsigned length, + followed by Len numbers of 8-bit Latin-1 characters that + forms the AtomName. +

+ +

+ SMALL_ATOM_EXT was introduced in ERTS 5.7.2 and + require an exchange of distribution flag + + DFLAG_SMALL_ATOM_TAGS in the + + distribution handshake. +

+ +
+ -- cgit v1.2.3 From b4b8529371f4c157ab741f30a8aeeccf16c102bc Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 14 Mar 2017 12:07:23 +0100 Subject: erts: Remove fun_r13_SUITE --- erts/emulator/test/Makefile | 1 - erts/emulator/test/fun_r13_SUITE.erl | 74 ------------------------------------ 2 files changed, 75 deletions(-) delete mode 100644 erts/emulator/test/fun_r13_SUITE.erl (limited to 'erts') diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index 5478932b13..927a9366b9 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -66,7 +66,6 @@ MODULES= \ exception_SUITE \ float_SUITE \ fun_SUITE \ - fun_r13_SUITE \ gc_SUITE \ guard_SUITE \ hash_SUITE \ diff --git a/erts/emulator/test/fun_r13_SUITE.erl b/erts/emulator/test/fun_r13_SUITE.erl deleted file mode 100644 index a45ed08b9d..0000000000 --- a/erts/emulator/test/fun_r13_SUITE.erl +++ /dev/null @@ -1,74 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2007-2016. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - --module(fun_r13_SUITE). --compile(r13). - --export([all/0, suite/0, - dist_old_release/1]). - --include_lib("common_test/include/ct.hrl"). - -suite() -> - [{ct_hooks,[ts_install_cth]}, - {timetrap, {minutes, 1}}]. - -all() -> - [dist_old_release]. - -dist_old_release(Config) when is_list(Config) -> - case test_server:is_release_available("r12b") of - true -> do_dist_old(Config); - false -> {skip,"No R12B found"} - end. - -do_dist_old(Config) when is_list(Config) -> - Pa = filename:dirname(code:which(?MODULE)), - Name = fun_dist_r12, - {ok,Node} = test_server:start_node(Name, peer, - [{args,"-pa "++Pa}, - {erl,[{release,"r12b"}]}]), - - Pid = spawn_link(Node, - fun() -> - receive - Fun when is_function(Fun) -> - R12BFun = fun(H) -> cons(H, [b,c]) end, - Fun(Fun, R12BFun) - end - end), - Self = self(), - Fun = fun(F, R12BFun) -> - {pid,Self} = erlang:fun_info(F, pid), - {module,?MODULE} = erlang:fun_info(F, module), - Self ! {ok,F,R12BFun} - end, - Pid ! Fun, - receive - {ok,Fun,R12BFun} -> - [a,b,c] = R12BFun(a); - Other -> - ct:fail({bad_message,Other}) - end, - true = test_server:stop_node(Node), - ok. - -cons(H, T) -> - [H|T]. -- cgit v1.2.3