diff options
28 files changed, 379 insertions, 94 deletions
diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index 6bb1109415..05b519fe7d 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -813,6 +813,29 @@ typedef enum { </item> </taglist> </item> + <tag><marker id="ErlNifHash"/><c>ErlNifHash</c></tag> + <item> + <p>An enumeration of the supported hash types that can be generated + using <seealso marker="#enif_hash"><c>enif_hash</c></seealso>. + </p> + <taglist> + <tag><c>ERL_NIF_INTERNAL_HASH</c></tag> + <item> + <p>Non-portable hash function that only guarantees the same hash + for the same term within one Erlang VM instance.</p> + <p>It takes 32-bit salt values and generates hashes within <c>0..2^32-1</c>.</p> + </item> + <tag><c>ERL_NIF_PHASH2</c></tag> + <item> + <p>Portable hash function that gives the same hash for the + same Erlang term regardless of machine architecture and ERTS version.</p> + <p><em>It ignores salt values</em> and generates hashes within <c>0..2^27-1</c>.</p> + <p>Slower than <c>ERL_NIF_INTERNAL_HASH.</c> + It corresponds to <seealso marker="erlang#phash2-1"><c>erlang:phash2/1</c></seealso>. + </p> + </item> + </taglist> + </item> </taglist> </section> @@ -1387,6 +1410,19 @@ typedef enum { </func> <func> + <name> + <ret>ErlNifUInt64</ret> + <nametext>enif_hash(ErlNifHash type, ERL_NIF_TERM term, ErlNifUInt64 salt)</nametext> + </name> + <fsummary>Hash terms.</fsummary> + <desc> + <p>Hashes <c>term</c> according to the specified + <seealso marker="#ErlNifHash"><c>ErlNifHash</c></seealso> <c>type</c>.</p> + <p>Ranges of taken salt (if any) and returned value depend on the hash type.</p> + </desc> + </func> + + <func> <name><ret>int</ret><nametext>enif_inspect_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, ErlNifBinary* bin)</nametext></name> <fsummary>Inspect the content of a binary.</fsummary> diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 3b86a4dcdf..152d144234 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -32,40 +32,52 @@ <p>This document describes the changes made to the ERTS application.</p> -<section><title>Erts 8.3.2</title> +<section><title>Erts 8.3.3</title> <section><title>Fixed Bugs and Malfunctions</title> <list> <item> <p> - The <c>+Bi</c> command line argument of <c>erl</c> - erroneously caused <c>SIGTERM</c> to be ignored by the VM - as well as of all its child processes. This bug was - introduced in erts version 8.3.</p> + Fixed memory segment cache used for multiblock carriers. + Huge (> 2GB) memory segments could cause a VM crash. + Creation of such huge memory segments used for multiblock + carriers is however very uncommon.</p> <p> - Own Id: OTP-14358 Aux Id: OTP-14085 </p> + Own Id: OTP-14360 Aux Id: ERL-401, PR-1417 </p> + </item> + <item> + <p> + Fix release note for OTP-14290 in ERTS version 8.3.1. It + was erroneously placed under "Known Bugs and Problems".</p> + <p> + Own Id: OTP-14363 Aux Id: OTP-14290 </p> </item> </list> </section> </section> -<section><title>Erts 8.3.1</title> +<section><title>Erts 8.3.2</title> <section><title>Fixed Bugs and Malfunctions</title> <list> <item> <p> - Trying to open a directory with file:read_file/1 on Unix - leaked a file descriptor. This bug has now been fixed.</p> + The <c>+Bi</c> command line argument of <c>erl</c> + erroneously caused <c>SIGTERM</c> to be ignored by the VM + as well as of all its child processes. This bug was + introduced in erts version 8.3.</p> <p> - Own Id: OTP-14308 Aux Id: ERL-383 </p> + Own Id: OTP-14358 Aux Id: OTP-14085 </p> </item> </list> </section> +</section> - <section><title>Known Bugs and Problems</title> +<section><title>Erts 8.3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> <list> <item> <p> @@ -76,6 +88,13 @@ <p> Own Id: OTP-14290</p> </item> + <item> + <p> + Trying to open a directory with file:read_file/1 on Unix + leaked a file descriptor. This bug has now been fixed.</p> + <p> + Own Id: OTP-14308 Aux Id: ERL-383 </p> + </item> </list> </section> diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 3a70c6036b..1a680b127c 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -3950,7 +3950,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups)); } else if (ERTS_IS_ATOM_STR("internal_hash", tp[1])) { - Uint hash = (Uint) make_internal_hash(tp[2]); + Uint hash = (Uint) make_internal_hash(tp[2], 0); Uint hsz = 0; Eterm* hp; erts_bld_uint(NULL, &hsz, hash); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 80c4824eeb..9009c00833 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -189,7 +189,7 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix, /* optimised version of make_hash (normal case? atomic key) */ #define MAKE_HASH(term) \ ((is_atom(term) ? (atom_tab(atom_val(term))->slot.bucket.hvalue) : \ - make_internal_hash(term)) % MAX_HASH) + make_internal_hash(term, 0)) % MAX_HASH) #ifdef ERTS_SMP # define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1) diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index 61a841f7f0..f7d0413685 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -57,7 +57,7 @@ typedef struct flatmap_s { #define hashmap_size(x) (((hashmap_head_t*) hashmap_val(x))->size) -#define hashmap_make_hash(Key) make_internal_hash(Key) +#define hashmap_make_hash(Key) make_internal_hash(Key, 0) #define hashmap_restore_hash(Heap,Lvl,Key) \ (((Lvl) < 8) ? hashmap_make_hash(Key) >> (4*(Lvl)) : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), (Key))) >> (4*((Lvl) & 7))) diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 872b58d1ef..e041fd7b83 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -55,6 +55,7 @@ #include "dtrace-wrapper.h" #include "erl_process.h" #include "erl_bif_unique.h" +#include "erl_utils.h" #undef ERTS_WANT_NFUNC_SCHED_INTERNALS__ #define ERTS_WANT_NFUNC_SCHED_INTERNALS__ #include "erl_nfunc_sched.h" @@ -1210,6 +1211,22 @@ int enif_compare(Eterm lhs, Eterm rhs) return result; } +ErlNifUInt64 enif_hash(ErlNifHash type, Eterm term, ErlNifUInt64 salt) +{ + switch (type) { + case ERL_NIF_INTERNAL_HASH: + return make_internal_hash(term, (Uint32) salt); + case ERL_NIF_PHASH2: + /* It appears that make_hash2 doesn't always react to seasoning + * as well as it should. Therefore, let's make it ignore the salt + * value and declare salted uses of phash2 as unsupported. + */ + return make_hash2(term) & ((1 << 27) - 1); + default: + return 0; + } +} + int enif_get_tuple(ErlNifEnv* env, Eterm tpl, int* arity, const Eterm** array) { Eterm* ptr; diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index ac45f3ac81..5a81f5fbbb 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -236,6 +236,11 @@ typedef enum { ERL_NIF_BIN2TERM_SAFE = 0x20000000 } ErlNifBinaryToTerm; +typedef enum { + ERL_NIF_INTERNAL_HASH = 1, + ERL_NIF_PHASH2 = 2 +} ErlNifHash; + /* * Return values from enif_thread_type(). Negative values * reserved for specific types of non-scheduler threads. diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index 01d9e386ed..b34058f303 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -180,6 +180,7 @@ ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type_x,(ErlNifEnv*, ERL_NIF_API_FUNC_DECL(int, enif_monitor_process,(ErlNifEnv*,void* obj,const ErlNifPid*,ErlDrvMonitor *monitor)); ERL_NIF_API_FUNC_DECL(int, enif_demonitor_process,(ErlNifEnv*,void* obj,const ErlDrvMonitor *monitor)); ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const ErlNifMonitor*)); +ERL_NIF_API_FUNC_DECL(ErlNifUInt64,enif_hash,(ErlNifHash type, ERL_NIF_TERM term, ErlNifUInt64 salt)); /* ** ADD NEW ENTRIES HERE (before this comment) !!! @@ -342,6 +343,7 @@ ERL_NIF_API_FUNC_DECL(int, enif_compare_monitors,(const ErlNifMonitor*,const Erl # define enif_monitor_process ERL_NIF_API_FUNC_MACRO(enif_monitor_process) # define enif_demonitor_process ERL_NIF_API_FUNC_MACRO(enif_demonitor_process) # define enif_compare_monitors ERL_NIF_API_FUNC_MACRO(enif_compare_monitors) +# define enif_hash ERL_NIF_API_FUNC_MACRO(enif_hash) /* ** ADD NEW ENTRIES HERE (before this comment) diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 7cfdf20341..01e240c65d 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -57,7 +57,7 @@ ((is_small(Term)) ? (Uint32) unsigned_val(Term) : \ ((is_atom(Term)) ? \ (Uint32) atom_val(Term) : \ - make_internal_hash(Term))) + make_internal_hash(Term, 0))) #define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm)) diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 870f1f142d..a5fc3a2477 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -3229,7 +3229,7 @@ static int tracer_cmp_fun(void* a, void* b) static HashValue tracer_hash_fun(void* obj) { - return make_internal_hash(((ErtsTracerNif*)obj)->module); + return make_internal_hash(((ErtsTracerNif*)obj)->module, 0); } static void *tracer_alloc_fun(void* tmpl) diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 47289a0af1..75d7e47239 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -120,7 +120,7 @@ int erts_is_builtin(Eterm, Eterm, int); Uint32 block_hash(byte *, unsigned, Uint32); Uint32 make_hash2(Eterm); Uint32 make_hash(Eterm); -Uint32 make_internal_hash(Eterm); +Uint32 make_internal_hash(Eterm, Uint32 salt); void erts_save_emu_args(int argc, char **argv); Eterm erts_get_emu_args(struct process *c_p); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 9263798a28..96a7bfe8ac 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1578,16 +1578,13 @@ do { /* Lightweight mixing of constant (type info) */ \ } while (0) Uint32 -make_internal_hash(Eterm term) +make_internal_hash(Eterm term, Uint32 salt) { Uint32 hash; - Uint32 hash_xor_pairs; - - ERTS_UNDEF(hash_xor_pairs, 0); /* Optimization. Simple cases before declaration of estack. */ if (primary_tag(term) == TAG_PRIMARY_IMMED1) { - hash = 0; + hash = salt; #if ERTS_SIZEOF_ETERM == 8 UINT32_HASH_2((Uint32)term, (Uint32)(term >> 32), HCONST); #elif ERTS_SIZEOF_ETERM == 4 @@ -1601,7 +1598,7 @@ make_internal_hash(Eterm term) Eterm tmp; DECLARE_ESTACK(s); - hash = 0; + hash = salt; for (;;) { switch (primary_tag(term)) { case TAG_PRIMARY_LIST: @@ -1664,6 +1661,11 @@ make_internal_hash(Eterm term) Eterm* ptr = boxed_val(term) + 1; Uint size; int i; + + /* + * We rely on key-value iteration order being constant + * for identical maps (in this VM instance). + */ switch (hdr & _HEADER_MAP_SUBTAG_MASK) { case HAMT_SUBTAG_HEAD_FLATMAP: { @@ -1675,26 +1677,10 @@ make_internal_hash(Eterm term) if (size == 0) goto pop_next; - /* We want a hash function that is *independent* of - * the order in which keys and values are encountered. - * We therefore calculate context independent hashes for all . - * key-value pairs and then xor them together. - * - * We *do* need to use an initial seed for each pair, i.e. the - * hash value, so the hash value is reset for each pair with 'hash'. - * If we don't, no additional entropy is given to the system and the - * hash collision resolution in maps:from_list/1 would fail. - */ - ESTACK_PUSH(s, hash_xor_pairs); - ESTACK_PUSH(s, hash); - ESTACK_PUSH(s, HASH_MAP_TAIL); for (i = size - 1; i >= 0; i--) { - ESTACK_PUSH(s, hash); /* initial seed for all pairs */ - ESTACK_PUSH(s, HASH_MAP_PAIR); ESTACK_PUSH(s, vs[i]); ESTACK_PUSH(s, ks[i]); } - hash_xor_pairs = 0; goto pop_next; } case HAMT_SUBTAG_HEAD_ARRAY: @@ -1703,10 +1689,6 @@ make_internal_hash(Eterm term) UINT32_HASH(size, HCONST_16); if (size == 0) goto pop_next; - ESTACK_PUSH(s, hash_xor_pairs); - ESTACK_PUSH(s, hash); - ESTACK_PUSH(s, HASH_MAP_TAIL); - hash_xor_pairs = 0; } switch (hdr & _HEADER_MAP_SUBTAG_MASK) { case HAMT_SUBTAG_HEAD_ARRAY: @@ -1722,8 +1704,6 @@ make_internal_hash(Eterm term) while (i) { if (is_list(*ptr)) { Eterm* cons = list_val(*ptr); - ESTACK_PUSH(s, hash); /* initial seed for all pairs */ - ESTACK_PUSH(s, HASH_MAP_PAIR); ESTACK_PUSH(s, CDR(cons)); ESTACK_PUSH(s, CAR(cons)); } @@ -1739,7 +1719,7 @@ make_internal_hash(Eterm term) case EXPORT_SUBTAG: { Export* ep = *((Export **) (export_val(term) + 1)); - /* Assumes Export entries never moves */ + /* Assumes Export entries never move */ POINTER_HASH(ep, HCONST_14); goto pop_next; } @@ -1894,7 +1874,7 @@ make_internal_hash(Eterm term) goto pop_next; } default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_internal_hash(0x%X, %lu)\n", term, salt); } } break; @@ -1907,7 +1887,7 @@ make_internal_hash(Eterm term) goto pop_next; default: - erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_internal_hash(0x%X, %lu)\n", term, salt); pop_next: if (ESTACK_ISEMPTY(s)) { @@ -1919,17 +1899,6 @@ make_internal_hash(Eterm term) term = ESTACK_POP(s); switch (term) { - case HASH_MAP_TAIL: { - hash = (Uint32) ESTACK_POP(s); - UINT32_HASH(hash_xor_pairs, HCONST_19); - hash_xor_pairs = (Uint32) ESTACK_POP(s); - goto pop_next; - } - case HASH_MAP_PAIR: - hash_xor_pairs ^= hash; - hash = (Uint32) ESTACK_POP(s); /* initial seed for all pairs */ - goto pop_next; - case HASH_CDR: CONST_HASH(HCONST_18); /* Hash CDR i cons cell */ goto pop_next; diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index d1895f3793..1e99078906 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -83,12 +83,13 @@ static const int debruijn[32] = { 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 }; -#define LOG2(X) (debruijn[((Uint32)(((X) & -(X)) * 0x077CB531U)) >> 27]) +#define LSB(X) (debruijn[((Uint32)(((X) & -(X)) * 0x077CB531U)) >> 27]) #define CACHE_AREAS (32 - MSEG_ALIGN_BITS) /* FIXME: segment sizes > 2 GB result in bogus negative indices */ -#define SIZE_TO_CACHE_AREA_IDX(S) (LOG2((S)) - MSEG_ALIGN_BITS) +/* NOTE: using LSB instead of proper log2 only works if S is a power of 2 */ +#define SIZE_TO_CACHE_AREA_IDX(S) (LSB((S)) - MSEG_ALIGN_BITS) #define MAX_CACHE_SIZE (30) #define MSEG_FLG_IS_2POW(X) ((X) & ERTS_MSEG_FLG_2POW) diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 693db42e58..8ad11d3bf3 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -56,7 +56,10 @@ nif_is_process_alive/1, nif_is_port_alive/1, nif_term_to_binary/1, nif_binary_to_term/1, nif_port_command/1, - nif_snprintf/1 + nif_snprintf/1, + nif_internal_hash/1, + nif_internal_hash_salted/1, + nif_phash2/1 ]). -export([many_args_100/100]). @@ -90,7 +93,10 @@ all() -> nif_is_process_alive, nif_is_port_alive, nif_term_to_binary, nif_binary_to_term, nif_port_command, - nif_snprintf]. + nif_snprintf, + nif_internal_hash, + nif_internal_hash_salted, + nif_phash2]. groups() -> [{G, [], api_repeaters()} || G <- api_groups()] @@ -2610,6 +2616,154 @@ nif_snprintf(Config) -> <<"{{hello,world,-33},",0>> = format_term_nif(20,{{hello,world, -33}, 3.14, self()}), ok. +nif_internal_hash(Config) -> + ensure_lib_loaded(Config), + HashValueBitSize = nif_hash_result_bitsize(internal), + Terms = unique([random_term() || _ <- lists:seq(1, 5000)]), + HashValues = [hash_nif(internal, Term, 0) || Term <- Terms], + test_bit_distribution_fitness(HashValues, HashValueBitSize, 0.05). + +nif_internal_hash_salted(Config) -> + ensure_lib_loaded(Config), + test_salted_nif_hash(internal). + +nif_phash2(Config) -> + ensure_lib_loaded(Config), + HashValueBitSize = nif_hash_result_bitsize(phash2), + Terms = unique([random_term() || _ <- lists:seq(1, 5000)]), + HashValues = + lists:map( + fun (Term) -> + HashValue = erlang:phash2(Term), + Salt = random_uint32(), % phash2 should ignore salt + NifHashValue = hash_nif(phash2, Term, Salt), + (HashValue =:= NifHashValue + orelse ct:fail("Expected: ~p\nActual: ~p", + [HashValue, NifHashValue])), + HashValue + end, + Terms), + test_bit_distribution_fitness(HashValues, HashValueBitSize, 0.05). + +test_salted_nif_hash(HashType) -> + HashValueBitSize = nif_hash_result_bitsize(HashType), + Terms = unique([random_term() || _ <- lists:seq(1, 5000)]), + Salts = unique([random_uint32() || _ <- lists:seq(1, 100)]), + {HashValuesPerSalt, HashValuesPerTerm} = + lists:mapfoldl( + fun (Salt, Acc) -> + {HashValues, NewAcc} = + lists:mapfoldl( + fun (Term, AccB) -> + HashValue = hash_nif(HashType, Term, Salt), + NewAccB = dict:append(Term, HashValue, AccB), + {HashValue, NewAccB} + end, + Acc, + Terms), + {{Salt, HashValues}, NewAcc} + end, + dict:new(), + Salts), + + % Test per-salt hash distribution of different terms + lists:foreach( + fun ({_Salt, HashValues}) -> + test_bit_distribution_fitness(HashValues, HashValueBitSize, 0.05) + end, + HashValuesPerSalt), + + % Test per-term hash distribution of different salts + dict:fold( + fun (_Term, HashValues, Acc) -> + % Be more tolerant of relative deviation, + % as there's fewer hash values here. + test_bit_distribution_fitness(HashValues, HashValueBitSize, 0.30), + Acc + end, + ok, + HashValuesPerTerm). + +test_bit_distribution_fitness(Integers, BitSize, MaxRelativeDeviation) -> + MaxInteger = (1 bsl BitSize) - 1, + OnesPerBit = + lists:foldl( + fun (Integer, Acc) when Integer >= 0, Integer =< MaxInteger -> + lists:foldl( + fun (BitIndex, AccB) -> + BitValue = (Integer band (1 bsl BitIndex)) bsr BitIndex, + orddict:update_counter(BitIndex, BitValue, AccB) + end, + Acc, + lists:seq(0, BitSize - 1)) + end, + orddict:new(), + Integers), + + ExpectedNrOfOnes = length(Integers) div 2, + FailureText = + orddict:fold( + fun (BitIndex, NrOfOnes, Acc) -> + RelativeDeviation = abs(NrOfOnes - ExpectedNrOfOnes) / length(Integers), + case RelativeDeviation >= MaxRelativeDeviation of + false -> Acc; + true -> + [Acc, + io_lib:format( + "Unreasonable deviation on number of set bits (i=~p): " + "expected ~p, got ~p (relative dev. ~.3f)~n", + [BitIndex, ExpectedNrOfOnes, NrOfOnes, RelativeDeviation])] + end + end, + [], + OnesPerBit), + + (FailureText =:= [] orelse ct:fail(FailureText)). + +nif_hash_result_bitsize(internal) -> 32; +nif_hash_result_bitsize(phash2) -> 27. + +unique(List) -> + lists:usort(List). + +random_uint32() -> + rand:uniform(1 bsl 32) - 1. + +random_term() -> + case rand:uniform(6) of + 1 -> rand:uniform(1 bsl 27) - 1; % small + 2 -> (1 bsl 27) + rand:uniform(1 bsl 128); % big + 3 -> random_sign() * (rand:uniform() * (1 bsl 53)); % float + 4 -> random_binary(); + 5 -> random_pid(); + 6 -> + Length = rand:uniform(10), + List = [random_term() || _ <- lists:seq(1, Length)], + case rand:uniform(2) of + 1 -> + List; + 2 -> + list_to_tuple(List) + end + end. + +random_sign() -> + case rand:uniform(2) of + 1 -> -1.0; + 2 -> 1.0 + end. + +random_binary() -> + list_to_binary(random_bytes(rand:uniform(32) - 1)). + +random_bytes(0) -> + []; +random_bytes(N) when N > 0 -> + [rand:uniform(256) - 1 | random_bytes(N - 1)]. + +random_pid() -> + Processes = erlang:processes(), + lists:nth(rand:uniform(length(Processes)), Processes). %% The NIFs: lib_version() -> undefined. @@ -2621,6 +2775,7 @@ type_test() -> ?nif_stub. tuple_2_list(_) -> ?nif_stub. is_identical(_,_) -> ?nif_stub. compare(_,_) -> ?nif_stub. +hash_nif(_Type, _Term, _Salt) -> ?nif_stub. many_args_100(_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_) -> ?nif_stub. clone_bin(_) -> ?nif_stub. make_sub_bin(_,_,_) -> ?nif_stub. diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index 8fe5ee809a..a255c9f096 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -687,6 +687,31 @@ static ERL_NIF_TERM compare(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) return enif_make_int(env, enif_compare(argv[0],argv[1])); } +static ERL_NIF_TERM hash_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + if (argc != 3) { + return enif_make_badarg(env); + } + + ErlNifHash type; + if (enif_is_identical(argv[0], enif_make_atom(env, "internal"))) { + type = ERL_NIF_INTERNAL_HASH; + } + else if (enif_is_identical(argv[0], enif_make_atom(env, "phash2"))) { + type = ERL_NIF_PHASH2; + } + else { + return enif_make_badarg(env); + } + + ErlNifUInt64 salt; + if (! enif_get_uint64(env, argv[2], &salt)) { + return enif_make_badarg(env); + } + + return enif_make_uint64(env, enif_hash(type, argv[1], salt)); +} + static ERL_NIF_TERM many_args_100(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { int i, k; @@ -2864,6 +2889,7 @@ static ErlNifFunc nif_funcs[] = {"tuple_2_list", 1, tuple_2_list}, {"is_identical",2,is_identical}, {"compare",2,compare}, + {"hash_nif",3,hash_nif}, {"many_args_100", 100, many_args_100}, {"clone_bin", 1, clone_bin}, {"make_sub_bin", 3, make_sub_bin}, diff --git a/erts/lib_src/win/ethread.c b/erts/lib_src/win/ethread.c index ba77b3dc88..98fcd2d451 100644 --- a/erts/lib_src/win/ethread.c +++ b/erts/lib_src/win/ethread.c @@ -203,6 +203,8 @@ ethr_init(ethr_init_data *id) if (!ethr_not_inited__) return EINVAL; + ethr_not_inited__ = 0; + #ifdef _WIN32_WINNT os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); GetVersionEx(&os_version); @@ -238,8 +240,6 @@ ethr_init(ethr_init_data *id) if (erts_get_cpu_configured(ethr_cpu_info__) == 1) child_wait_spin_count = 0; - ethr_not_inited__ = 0; - return 0; error: diff --git a/lib/dialyzer/doc/src/notes.xml b/lib/dialyzer/doc/src/notes.xml index cd4ec4c068..0dbee1ce34 100644 --- a/lib/dialyzer/doc/src/notes.xml +++ b/lib/dialyzer/doc/src/notes.xml @@ -32,6 +32,22 @@ <p>This document describes the changes made to the Dialyzer application.</p> +<section><title>Dialyzer 3.1.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Report unknown types properly. A bug was introduced + in Erlang/OTP 19.3, where warnings about unknown types + were simply discarded. </p> + <p> + Own Id: OTP-14368</p> + </item> + </list> + </section> + +</section> + <section><title>Dialyzer 3.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl index 051b700231..29aa25b98e 100644 --- a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl +++ b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl @@ -149,7 +149,7 @@ analysis_start(Parent, Analysis, LegalWarnings) -> dialyzer_codeserver:finalize_exported_types(MergedExpTypes, TmpCServer1), erlang:garbage_collect(), % reduce heap size ?timing(State#analysis_state.timing_server, "remote", - contracts_and_records(TmpCServer2)) + contracts_and_records(TmpCServer2, Parent)) catch throw:{error, _ErrorMsg} = Error -> exit(Error) end, @@ -173,15 +173,14 @@ analysis_start(Parent, Analysis, LegalWarnings) -> codeserver = Codeserver0} = State2, {Codeserver, MiniPlt3} = move_data(Codeserver0, MiniPlt2), dialyzer_callgraph:dispose_race_server(NewCallgraph), - rcv_and_send_ext_types(Parent), %% Since the PLT is never used, a dummy is sent: DummyPlt = dialyzer_plt:new(), send_codeserver_plt(Parent, Codeserver, DummyPlt), MiniPlt4 = dialyzer_plt:delete_list(MiniPlt3, NonExportsList), send_analysis_done(Parent, MiniPlt4, DocPlt). -contracts_and_records(CodeServer) -> - Fun = contrs_and_recs(CodeServer), +contracts_and_records(CodeServer, Parent) -> + Fun = contrs_and_recs(CodeServer, Parent), {Pid, Ref} = erlang:spawn_monitor(Fun), dialyzer_codeserver:give_away(CodeServer, Pid), Pid ! {self(), go}, @@ -189,16 +188,17 @@ contracts_and_records(CodeServer) -> Return end. --spec contrs_and_recs(dialyzer_codeserver:codeserver()) -> +-spec contrs_and_recs(dialyzer_codeserver:codeserver(), pid()) -> fun(() -> no_return()). -contrs_and_recs(TmpCServer2) -> +contrs_and_recs(TmpCServer2, Parent) -> fun() -> - Parent = receive {Pid, go} -> Pid end, + Caller = receive {Pid, go} -> Pid end, TmpCServer3 = dialyzer_utils:process_record_remote_types(TmpCServer2), TmpServer4 = dialyzer_contracts:process_contract_remote_types(TmpCServer3), - dialyzer_codeserver:give_away(TmpServer4, Parent), + dialyzer_codeserver:give_away(TmpServer4, Caller), + rcv_and_send_ext_types(Caller, Parent), exit(TmpServer4) end. @@ -525,13 +525,13 @@ default_includes(Dir) -> %% Handle Messages %%------------------------------------------------------------------- -rcv_and_send_ext_types(Parent) -> +rcv_and_send_ext_types(SendTo, Parent) -> Self = self(), Self ! {Self, done}, case rcv_ext_types(Self, []) of [] -> ok; ExtTypes -> - Parent ! {Self, ext_types, ExtTypes}, + Parent ! {SendTo, ext_types, ExtTypes}, ok end. diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl index e0497159b3..75d6e3bc65 100644 --- a/lib/dialyzer/src/dialyzer_utils.erl +++ b/lib/dialyzer/src/dialyzer_utils.erl @@ -212,7 +212,7 @@ keep_compile_option(_) -> true. -type type_table() :: erl_types:type_table(). --spec get_record_and_type_info(cerl:module()) -> +-spec get_record_and_type_info(cerl:c_module()) -> {'ok', type_table()} | {'error', string()}. get_record_and_type_info(Core) -> @@ -478,7 +478,7 @@ merge_types(CServer, Plt) -> -type spec_map() :: dialyzer_codeserver:contracts(). -type callback_map() :: dialyzer_codeserver:contracts(). --spec get_spec_info(module(), cerl:module(), type_table()) -> +-spec get_spec_info(module(), cerl:c_module(), type_table()) -> {'ok', spec_map(), callback_map()} | {'error', string()}. get_spec_info(ModName, Core, RecordsMap) -> @@ -555,7 +555,7 @@ get_core_line([L | _As]) when is_integer(L) -> L; get_core_line([_ | As]) -> get_core_line(As); get_core_line([]) -> undefined. --spec get_fun_meta_info(module(), cerl:module(), [dial_warn_tag()]) -> +-spec get_fun_meta_info(module(), cerl:c_module(), [dial_warn_tag()]) -> dialyzer_codeserver:fun_meta_info() | {'error', string()}. get_fun_meta_info(M, Core, LegalWarnings) -> @@ -591,7 +591,7 @@ process_options([{{_M, _F, _A}=MFA, Opts}|Left], Warnings) -> end; process_options([], _Warnings) -> []. --spec get_nowarn_unused_function(module(), cerl:module(), [fa()]) -> +-spec get_nowarn_unused_function(module(), cerl:c_module(), [fa()]) -> [{mfa(), 'no_unused'}]. get_nowarn_unused_function(M, Core, Functions) -> @@ -607,7 +607,7 @@ get_nowarn_unused_function(M, Core, Functions) -> end, [{{M, F, A}, no_unused} || {F, A} <- FAs]. --spec get_func_suppressions(module(), cerl:module(), [fa()]) -> +-spec get_func_suppressions(module(), cerl:c_module(), [fa()]) -> [{mfa(), 'nowarn_function' | dial_warn_tag()}]. get_func_suppressions(M, Core, Functions) -> @@ -621,7 +621,7 @@ get_func_suppressions(M, Core, Functions) -> lists:foreach(Fun, TagsFAs), [{{M, F, A}, W} || {{Warnings, _L, _File}, {F, A}} <- TagsFAs, W <- Warnings]. --spec get_options(cerl:module(), [dial_warn_tag()]) -> +-spec get_options(cerl:c_module(), [dial_warn_tag()]) -> ordsets:ordset(dial_warn_tag()). get_options(Core, LegalWarnings) -> @@ -735,7 +735,7 @@ format_sig(Type, RecDict) -> flat_format(Fmt, Lst) -> lists:flatten(io_lib:format(Fmt, Lst)). --spec get_options_with_tag(atom(), cerl:module()) -> [term()]. +-spec get_options_with_tag(atom(), cerl:c_module()) -> [term()]. get_options_with_tag(Tag, Core) -> [O || {Key, Value} <- cerl:module_attrs(Core), diff --git a/lib/dialyzer/test/options2_SUITE_data/dialyzer_options b/lib/dialyzer/test/options2_SUITE_data/dialyzer_options index 5db2e50d23..be57e2de72 100644 --- a/lib/dialyzer/test/options2_SUITE_data/dialyzer_options +++ b/lib/dialyzer/test/options2_SUITE_data/dialyzer_options @@ -1 +1 @@ -{dialyzer_options, [{defines, [{'vsn', 4}]}, {warnings, [no_return]}]}. +{dialyzer_options, [{defines, [{'vsn', 4}]}, {warnings, [unknown, no_return]}]}. diff --git a/lib/dialyzer/test/options2_SUITE_data/results/unknown_type b/lib/dialyzer/test/options2_SUITE_data/results/unknown_type new file mode 100644 index 0000000000..d308c5a810 --- /dev/null +++ b/lib/dialyzer/test/options2_SUITE_data/results/unknown_type @@ -0,0 +1,2 @@ + +:0: Unknown type unknown:type/0
\ No newline at end of file diff --git a/lib/dialyzer/test/options2_SUITE_data/src/unknown_type.erl b/lib/dialyzer/test/options2_SUITE_data/src/unknown_type.erl new file mode 100644 index 0000000000..7a891c0831 --- /dev/null +++ b/lib/dialyzer/test/options2_SUITE_data/src/unknown_type.erl @@ -0,0 +1,7 @@ +-module(unknown_type). + +-export([t/0]). + +-spec t() -> unknown:type(). +t() -> + a. diff --git a/lib/hipe/regalloc/hipe_range_split.erl b/lib/hipe/regalloc/hipe_range_split.erl index 39b086d9f7..385df695f2 100644 --- a/lib/hipe/regalloc/hipe_range_split.erl +++ b/lib/hipe/regalloc/hipe_range_split.erl @@ -270,8 +270,8 @@ defbutlast(L, Defs) -> maps:get(L, Defs). -spec defseti_new() -> defseti(). -spec defseti_union(defseti(), defseti()) -> defseti(). --spec defseti_add_ordset(ordset:ordset(temp()), defseti()) -> defseti(). --spec defseti_from_ordset(ordset:ordset(temp())) -> defseti(). +-spec defseti_add_ordset(ordsets:ordset(temp()), defseti()) -> defseti(). +-spec defseti_from_ordset(ordsets:ordset(temp())) -> defseti(). -spec defseti_finalise(defseti()) -> defsetf(). -spec defsetf_member(temp(), defsetf()) -> boolean(). -spec defsetf_intersect_ordset(ordsets:ordset(temp()), defsetf()) @@ -415,7 +415,7 @@ rdefset_finalise(Ord) -> {arr, bitarr_from_bitord(Ord)}. %% rdefsetf_top() -> top. rdefsetf_empty() -> {arr, bitarr_new()}. --spec rdefsetf_add_ordset(ordset:ordset(temp()), rdefsetf()) -> rdefsetf(). +-spec rdefsetf_add_ordset(ordsets:ordset(temp()), rdefsetf()) -> rdefsetf(). rdefsetf_add_ordset(_, top) -> top; rdefsetf_add_ordset(OS, {arr, Arr}) -> {arr, lists:foldl(fun bitarr_set/2, Arr, OS)}. diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 2aa48cd50a..ef01136978 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -33,7 +33,22 @@ <file>notes.xml</file> </header> - <section><title>Inets 6.3.7</title> + <section><title>Inets 6.3.8</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Added missing release note for inets-6.3.7</p> + <p> + Own Id: OTP-14383</p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 6.3.7</title> <section><title>Fixed Bugs and Malfunctions</title> <list> @@ -44,6 +59,13 @@ <p> Own Id: OTP-14242</p> </item> + + <item> + <p>Make default port, 80 and 443, implicit in automatic redirection. + </p> + <p> Own Id: OTP-14301 + </p> + </item> </list> </section> diff --git a/lib/ssl/src/ssl.appup.src b/lib/ssl/src/ssl.appup.src index 2eda9d9491..bfdd0c205b 100644 --- a/lib/ssl/src/ssl.appup.src +++ b/lib/ssl/src/ssl.appup.src @@ -1,7 +1,6 @@ %% -*- erlang -*- {"%VSN%", [ - {<<"8.1.1">>, [{load_module, tls_connection, soft_purge, soft_purge, []}]}, {<<"8\\..*">>, [{restart_application, ssl}]}, {<<"7\\..*">>, [{restart_application, ssl}]}, {<<"6\\..*">>, [{restart_application, ssl}]}, @@ -10,7 +9,6 @@ {<<"3\\..*">>, [{restart_application, ssl}]} ], [ - {<<"8.1.1">>, [{load_module, tls_connection, soft_purge, soft_purge, []}]}, {<<"8\\..*">>, [{restart_application, ssl}]}, {<<"7\\..*">>, [{restart_application, ssl}]}, {<<"6\\..*">>, [{restart_application, ssl}]}, diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index 82184f5c74..25b2a2bec0 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 8.1.2 +SSL_VSN = 8.2 diff --git a/lib/wx/c_src/wxe_impl.cpp b/lib/wx/c_src/wxe_impl.cpp index 90cd35455a..18b03af947 100644 --- a/lib/wx/c_src/wxe_impl.cpp +++ b/lib/wx/c_src/wxe_impl.cpp @@ -267,6 +267,8 @@ int WxeApp::dispatch_cmds() return more; } +#define BREAK_BATCH 200 + int WxeApp::dispatch(wxeFifo * batch) { int ping = 0; @@ -279,7 +281,11 @@ int WxeApp::dispatch(wxeFifo * batch) erl_drv_mutex_unlock(wxe_batch_locker_m); switch(event->op) { case WXE_BATCH_END: - if(blevel>0) blevel--; + if(blevel>0) { + blevel--; + if(blevel==0) + wait += BREAK_BATCH*100; + } break; case WXE_BATCH_BEGIN: blevel++; @@ -311,10 +317,13 @@ int WxeApp::dispatch(wxeFifo * batch) erl_drv_mutex_lock(wxe_batch_locker_m); batch->Cleanup(); } - if(blevel <= 0 || wait > 3) { + if(blevel <= 0 || wait > BREAK_BATCH) { erl_drv_mutex_unlock(wxe_batch_locker_m); - if(blevel > 0) return 1; // We are still in a batch but we can let wx check for events - else return 0; + if(blevel > 0) { + return 1; // We are still in a batch but we can let wx check for events + } else { + return 0; + } } // sleep until something happens // fprintf(stderr, "%s:%d sleep %d %d %d\r\n", __FILE__, __LINE__, batch->m_n, blevel, wait);fflush(stderr); diff --git a/otp_versions.table b/otp_versions.table index f0aa904cae..ea9eb73887 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,4 @@ +OTP-19.3.3 : dialyzer-3.1.1 erts-8.3.3 inets-6.3.8 # asn1-4.0.4 common_test-1.14 compiler-7.0.4 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.4 debugger-4.2.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 jinterface-1.7.1 kernel-5.2 megaco-3.18.1 mnesia-4.14.3 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssh-4.4.2 ssl-8.1.2 stdlib-3.3 syntax_tools-2.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.13 : OTP-19.3.2 : erts-8.3.2 # asn1-4.0.4 common_test-1.14 compiler-7.0.4 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.4 debugger-4.2.1 dialyzer-3.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 inets-6.3.7 jinterface-1.7.1 kernel-5.2 megaco-3.18.1 mnesia-4.14.3 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssh-4.4.2 ssl-8.1.2 stdlib-3.3 syntax_tools-2.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.13 : OTP-19.3.1 : crypto-3.7.4 erts-8.3.1 inets-6.3.7 ssh-4.4.2 ssl-8.1.2 # asn1-4.0.4 common_test-1.14 compiler-7.0.4 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 debugger-4.2.1 dialyzer-3.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 jinterface-1.7.1 kernel-5.2 megaco-3.18.1 mnesia-4.14.3 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 stdlib-3.3 syntax_tools-2.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.13 : OTP-19.3 : common_test-1.14 compiler-7.0.4 crypto-3.7.3 dialyzer-3.1 diameter-1.12.2 erl_interface-3.9.3 erts-8.3 hipe-3.15.4 inets-6.3.6 kernel-5.2 observer-2.3.1 os_mon-2.4.2 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssh-4.4.1 ssl-8.1.1 stdlib-3.3 tools-2.9.1 typer-0.9.12 xmerl-1.3.13 # asn1-4.0.4 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 debugger-4.2.1 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 et-1.6 eunit-2.3.2 gs-1.6.2 ic-4.4.2 jinterface-1.7.1 megaco-3.18.1 mnesia-4.14.3 odbc-2.12 orber-3.8.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 syntax_tools-2.1.1 wx-1.8 : |