diff options
Diffstat (limited to 'erts')
50 files changed, 1389 insertions, 512 deletions
diff --git a/erts/doc/src/Makefile b/erts/doc/src/Makefile index bc01919da1..bb96293947 100644 --- a/erts/doc/src/Makefile +++ b/erts/doc/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2018. All Rights Reserved. +# Copyright Ericsson AB 1997-2019. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -47,6 +47,20 @@ XML_REF1_FILES = epmd.xml \ run_erl.xml \ start.xml +ifeq ($(USE_ESOCK), yes) +XML_REF3_ESOCK_EFILES = socket.xml +XML_CHAPTER_ESOCK_EFILES = socket_usage.xml +ESOCK_USE_SOCKET_XML=<xi:include href="socket.xml"\/> +ESOCK_USE_SOCKET_SPECS_XML=<xi:include href="../specs/specs_socket.xml"/> +ESOCK_USE_SOCKET_USAGE_XML=<xi:include href="socket_usage.xml"/> +else +XML_REF3_ESOCK_EFILES = +XML_CHAPTER_ESOCK_EFILES = +ESOCK_USE_SOCKET_XML = +ESOCK_USE_SOCKET_SPECS_XML = +ESOCK_USE_SOCKET_USAGE_XML = +endif + XML_REF3_EFILES = \ erl_prim_loader.xml \ erlang.xml \ @@ -56,8 +70,7 @@ XML_REF3_EFILES = \ atomics.xml \ counters.xml \ zlib.xml \ - socket.xml \ - net.xml + $(XML_REF3_ESOCK_EFILES) XML_REF3_FILES = \ $(XML_REF3_EFILES) \ @@ -94,7 +107,7 @@ XML_CHAPTER_FILES = \ driver.xml \ absform.xml \ inet_cfg.xml \ - socket_usage.xml \ + $(XML_CHAPTER_ESOCK_EFILES) \ erl_ext_dist.xml \ erl_dist_protocol.xml \ communication.xml \ @@ -159,7 +172,7 @@ $(HTMLDIR)/%.gif: %.gif $(XML_FIGURE_DIR)/%.png: ../../emulator/internal_doc/figures/%.png $(INSTALL_DATA) $< $@ -docs: figures man pdf html $(INFO_FILE) +docs: part ref_man specs figures man pdf html $(INFO_FILE) $(TOP_PDF_FILE): $(XML_FILES) @@ -169,6 +182,10 @@ html: gifs $(HTML_REF_MAN_FILE) man: $(MAN1_FILES) $(MAN3_FILES) +ref_man: ref_man.xml +part: part.xml +specs: specs.xml + gifs: $(GIF_FILES:%=$(HTMLDIR)/%) $(INFO_FILE): $(INFO_FILE_SRC) $(ERL_TOP)/make/$(TARGET)/otp.mk @@ -196,6 +213,19 @@ $(SPECDIR)/specs_%.xml: $(XMLDIR)/%.xml: ../../emulator/internal_doc/%.md $(ERL_TOP)/make/emd2exml $(ERL_TOP)/make/emd2exml $< $@ +ref_man.xml: ref_man.xml.src + ($(PERL) -p -e 's?%ESOCK_USE_SOCKET_XML%?$(ESOCK_USE_SOCKET_XML)?' \ + $<) > $@ + +part.xml: part.xml.src + ($(PERL) -p -e 's?%ESOCK_USE_SOCKET_USAGE_XML%?$(ESOCK_USE_SOCKET_USAGE_XML)?' \ + $<) > $@ + +specs.xml: specs.xml.src + ($(PERL) -p -e 's?%ESOCK_USE_SOCKET_SPECS_XML%?$(ESOCK_USE_SOCKET_SPECS_XML)?' \ + $<) > $@ + + # ---------------------------------------------------- # Release Target # ---------------------------------------------------- diff --git a/erts/doc/src/net.xml b/erts/doc/src/net.xml deleted file mode 100644 index 6fbc37076c..0000000000 --- a/erts/doc/src/net.xml +++ /dev/null @@ -1,129 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!DOCTYPE erlref SYSTEM "erlref.dtd"> - -<erlref> - <header> - <copyright> - <year>2018</year><year>2018</year> - <holder>Ericsson AB. All Rights Reserved.</holder> - </copyright> - <legalnotice> - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - </legalnotice> - - <title>net</title> - <prepared></prepared> - <docno></docno> - <date></date> - <rev></rev> - <file>net.xml</file> - </header> - <module since="OTP 22.0">net</module> - <modulesummary>Network interface.</modulesummary> - <description> - <p>This module provides an API for the network interface.</p> - <note> - <p>There is currently <em>no</em> support for Windows. </p> - </note> - </description> - - <datatypes> - <datatype> - <name name="address_info"/> - </datatype> - <datatype> - <name name="name_info"/> - </datatype> - <datatype> - <name name="name_info_flags"/> - </datatype> - <datatype> - <name name="name_info_flag"/> - </datatype> - <datatype> - <name name="name_info_flag_ext"/> - </datatype> - <datatype> - <name name="network_interface_name"/> - </datatype> - <datatype> - <name name="network_interface_index"/> - </datatype> - </datatypes> - - <funcs> - <func> - <name name="gethostname" arity="0"/> - <fsummary>Get hostname.</fsummary> - <desc> - <p>Returns the name of the current host.</p> - </desc> - </func> - - <func> - <name name="getnameinfo" arity="1" since="OTP 22.0"/> - <name name="getnameinfo" arity="2" since="OTP 22.0"/> - <fsummary>Address-to-name transaltion.</fsummary> - <desc> - <p>Address-to-name translation in a protocol-independant manner.</p> - <p>This function is the inverse of - <seealso marker="#getaddrinfo/1"><c>getaddrinfo</c></seealso>. - It converts a socket address to a corresponding host and service.</p> - </desc> - </func> - - <func> - <name name="getaddrinfo" arity="1" since="OTP 22.0"/> - <name name="getaddrinfo" arity="2" clause_i="1" since="OTP 22.0"/> - <name name="getaddrinfo" arity="2" clause_i="2" since="OTP 22.0"/> - <name name="getaddrinfo" arity="2" clause_i="3" since="OTP 22.0"/> - <fsummary>Network address and service transation.</fsummary> - <desc> - <p>Network address and service translation.</p> - <p>This function is the inverse of - <seealso marker="#getnameinfo/1"><c>getnameinfo</c></seealso>. - It converts host and service to a corresponding socket address.</p> - <p>One of the <c>Host</c> and <c>Service</c> may be <c>undefined</c> - but <em>not</em> both.</p> - </desc> - </func> - - <func> - <name name="if_name2index" arity="1" since="OTP 22.0"/> - <fsummary>Mappings between network interface names and indexes.</fsummary> - <desc> - <p>Mappings between network interface names and indexes.</p> - </desc> - </func> - - <func> - <name name="if_index2name" arity="1" since="OTP 22.0"/> - <fsummary>Mappings between network interface index and names.</fsummary> - <desc> - <p>Mappings between network interface index and names.</p> - </desc> - </func> - - <func> - <name name="if_names" arity="0" since="OTP 22.0"/> - <fsummary>Get network interface names and indexes.</fsummary> - <desc> - <p>Get network interface names and indexes.</p> - </desc> - </func> - - </funcs> - -</erlref> - diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index cfa952f01c..5ca387ffd8 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -31,6 +31,54 @@ </header> <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 10.4.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + An invalid value test caused the socket:setopt(Socket, + ip, add_membership, ip_mreq()) to fail with badarg. The + same for drop_membership.</p> + <p> + Own Id: OTP-15908 Aux Id: ERL-980 </p> + </item> + <item> + <p> + Fixed bug causing VM crash when doing textual dump of a + process containing an unhandled monitor down signal. + Textual process dumps can be done with + <c>erlang:system_info(procs)</c>, trace feature + <c>process_dump</c>, Erlang shell break menu and a + crashdump. Bug exist since OTP 21.0.</p> + <p> + Own Id: OTP-15909 Aux Id: ERL-979 </p> + </item> + <item> + <p><c>lists:subtract/2</c> would produce incorrect + results for some inputs on 64-bit platforms.</p> + <p> + Own Id: OTP-15938 Aux Id: ERL-986 </p> + </item> + <item> + <p>Fixed a bug in the loader that was similar to + <c>OTP-15938</c>, yielding incorrect code for some inputs + on 64-bit platforms.</p> + <p> + Own Id: OTP-15939</p> + </item> + <item> + <p> + Fixed bug causing scheduler threads in rare cases to + block spinnning indefinitely. Bug exists since OTP 21.0.</p> + <p> + Own Id: OTP-15941 Aux Id: PR-2313 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 10.4.3</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -592,6 +640,46 @@ </section> +<section><title>Erts 10.3.5.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed bug causing VM crash when doing textual dump of a + process containing an unhandled monitor down signal. + Textual process dumps can be done with + <c>erlang:system_info(procs)</c>, trace feature + <c>process_dump</c>, Erlang shell break menu and a + crashdump. Bug exist since OTP 21.0.</p> + <p> + Own Id: OTP-15909 Aux Id: ERL-979 </p> + </item> + <item> + <p><c>lists:subtract/2</c> would produce incorrect + results for some inputs on 64-bit platforms.</p> + <p> + Own Id: OTP-15938 Aux Id: ERL-986 </p> + </item> + <item> + <p>Fixed a bug in the loader that was similar to + <c>OTP-15938</c>, yielding incorrect code for some inputs + on 64-bit platforms.</p> + <p> + Own Id: OTP-15939</p> + </item> + <item> + <p> + Fixed bug causing scheduler threads in rare cases to + block spinnning indefinitely. Bug exists since OTP 21.0.</p> + <p> + Own Id: OTP-15941 Aux Id: PR-2313 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 10.3.5.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/doc/src/part.xml b/erts/doc/src/part.xml.src index f0b8a00b90..9b20beffad 100644 --- a/erts/doc/src/part.xml +++ b/erts/doc/src/part.xml.src @@ -4,7 +4,7 @@ <part xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>1996</year><year>2018</year> + <year>1996</year><year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -42,7 +42,7 @@ <xi:include href="tty.xml"/> <xi:include href="driver.xml"/> <xi:include href="inet_cfg.xml"/> - <xi:include href="socket_usage.xml"/> + %ESOCK_USE_SOCKET_USAGE_XML% <xi:include href="erl_ext_dist.xml"/> <xi:include href="erl_dist_protocol.xml"/> </part> diff --git a/erts/doc/src/ref_man.xml b/erts/doc/src/ref_man.xml.src index 80cdcf9145..7dd003763c 100644 --- a/erts/doc/src/ref_man.xml +++ b/erts/doc/src/ref_man.xml.src @@ -4,7 +4,7 @@ <application xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>1996</year><year>2018</year> + <year>1996</year><year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -46,10 +46,9 @@ <xi:include href="erts_alloc.xml"/> <xi:include href="escript.xml"/> <xi:include href="init.xml"/> - <xi:include href="net.xml"/> <xi:include href="persistent_term.xml"/> <xi:include href="run_erl.xml"/> - <xi:include href="socket.xml"/> + %ESOCK_USE_SOCKET_XML% <xi:include href="start.xml"/> <xi:include href="start_erl.xml"/> <xi:include href="werl.xml"/> diff --git a/erts/doc/src/specs.xml b/erts/doc/src/specs.xml.src index 68fab5edf1..54224c15f5 100644 --- a/erts/doc/src/specs.xml +++ b/erts/doc/src/specs.xml.src @@ -5,8 +5,7 @@ <xi:include href="../specs/specs_erl_tracer.xml"/> <xi:include href="../specs/specs_init.xml"/> <xi:include href="../specs/specs_persistent_term.xml"/> - <xi:include href="../specs/specs_socket.xml"/> - <xi:include href="../specs/specs_net.xml"/> + %ESOCK_USE_SOCKET_SPECS_XML% <xi:include href="../specs/specs_zlib.xml"/> <xi:include href="../specs/specs_atomics.xml"/> <xi:include href="../specs/specs_counters.xml"/> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index a9f3bb8e89..ba5ba8abef 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -636,10 +636,9 @@ GENERATE += $(TTF_DIR)/driver_tab.c ifeq ($(USE_ESOCK), yes) ESOCK_PRELOAD_BEAM = \ $(ERL_TOP)/erts/preloaded/ebin/socket.beam \ - $(ERL_TOP)/erts/preloaded/ebin/net.beam + $(ERL_TOP)/erts/preloaded/ebin/prim_net.beam else -ESOCK_PRELOAD_BEAM = \ - $(ERL_TOP)/erts/preloaded/ebin/net.beam +ESOCK_PRELOAD_BEAM = endif PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ @@ -850,7 +849,7 @@ ifeq ($(USE_ESOCK), yes) ESOCK_NIF_OBJS = \ $(OBJDIR)/socket_nif.o \ - $(OBJDIR)/net_nif.o + $(OBJDIR)/prim_net_nif.o ifneq ($(TARGET), win32) # These are *currently* only needed for non-win32, diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 941c3ebbbe..35f2ea6688 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -4612,7 +4612,15 @@ typedef struct SortGenOpArg { static int genopargtermcompare(SortGenOpArg* a, SortGenOpArg* b) { - return CMP_TERM(a->term, b->term); + Sint res = CMP_TERM(a->term, b->term); + + if (res < 0) { + return -1; + } else if (res > 0) { + return 1; + } + + return 0; } static int diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index db9c258cb7..602db106b1 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -413,6 +413,7 @@ bif re:compile/1 bif re:compile/2 bif re:run/2 bif re:run/3 +bif re:internal_run/4 # # Bifs in lists module. diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index d8501ea6ac..4537e3e569 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -80,7 +80,7 @@ dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) byte *extp = edep->data->extp; Eterm msg; Sint ctl_len; - Sint size = ctl_len = erts_decode_dist_ext_size(edep, 0); + Sint size = ctl_len = erts_decode_dist_ext_size(edep, 0, 0); if (size < 0) { erts_fprintf(dbg_file, "DIST MSG DEBUG: erts_decode_dist_ext_size(%s) failed:\n", @@ -1462,7 +1462,7 @@ int erts_net_message(Port *prt, #endif goto data_error; case ERTS_PREP_DIST_EXT_SUCCESS: - ctl_len = erts_decode_dist_ext_size(&ede, 1); + ctl_len = erts_decode_dist_ext_size(&ede, 1, 0); if (ctl_len < 0) { #ifdef ERTS_DIST_MSG_DBG erts_fprintf(dbg_file, "DIST MSG DEBUG: erts_decode_dist_ext_size(CTL) failed:\n"); @@ -1543,39 +1543,6 @@ int erts_net_message(Port *prt, edep = erts_get_dist_ext(&seq->hfrag); ede_hfrag = &seq->hfrag; - /* If the sequence consisted of more than 1 fragment we create one large - binary out of all of the fragments. This because erts_decode_ext - cannot handle a segmented buffer. - TODO: Move this copy to as late as possible, preferably in in the - erts_decode_dist_ext in the receiving process. - */ - if (edep->data->frag_id > 1) { - Uint sz = 0; - Binary *bin; - int i; - byte *ep; - - for (i = 0; i < edep->data->frag_id; i++) - sz += edep->data[i].ext_endp - edep->data[i].extp; - - bin = erts_bin_nrml_alloc(sz); - ep = (byte*)bin->orig_bytes; - - for (i = 0; i < edep->data->frag_id; i++) { - sys_memcpy(ep, edep->data[i].extp, edep->data[i].ext_endp - edep->data[i].extp); - ep += edep->data[i].ext_endp - edep->data[i].extp; - erts_bin_release(edep->data[i].binp); - edep->data[i].binp = NULL; - edep->data[i].extp = NULL; - edep->data[i].ext_endp = NULL; - } - - edep->data->frag_id = 1; - edep->data->extp = (byte*)bin->orig_bytes; - edep->data->ext_endp = ep; - edep->data->binp = bin; - } - break; } default: diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index b23fa77f5f..fa2edfef1e 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -413,12 +413,25 @@ typedef struct { #define ERTS_RBT_GET_LEFT(T) ((T)->left) #define ERTS_RBT_SET_LEFT(T, L) ((T)->left = (L)) #define ERTS_RBT_GET_KEY(T) ((T)->key) -#define ERTS_RBT_CMP_KEYS(KX, KY) CMP_TERM(KX, KY) +#define ERTS_RBT_CMP_KEYS(KX, KY) subtract_term_cmp((KX), (KY)) #define ERTS_RBT_WANT_LOOKUP_INSERT #define ERTS_RBT_WANT_LOOKUP #define ERTS_RBT_WANT_DELETE #define ERTS_RBT_UNDEF +/* erl_rbtree expects comparisons to return an int */ +static int subtract_term_cmp(Eterm a, Eterm b) { + Sint res = CMP_TERM(a, b); + + if (res < 0) { + return -1; + } else if (res > 0) { + return 1; + } + + return 0; +} + #include "erl_rbtree.h" static int subtract_continue(Process *p, ErtsSubtractContext *context); diff --git a/erts/emulator/beam/erl_bif_re.c b/erts/emulator/beam/erl_bif_re.c index e0b9202fe7..b3bf1c7ee3 100644 --- a/erts/emulator/beam/erl_bif_re.c +++ b/erts/emulator/beam/erl_bif_re.c @@ -46,7 +46,7 @@ static Export *urun_trap_exportp = NULL; static Export *ucompile_trap_exportp = NULL; static BIF_RETTYPE re_exec_trap(BIF_ALIST_3); -static BIF_RETTYPE re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3); +static BIF_RETTYPE re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, int first); static void *erts_erts_pcre_malloc(size_t size) { return erts_alloc(ERTS_ALC_T_RE_HEAP,size); @@ -1094,7 +1094,7 @@ build_capture(Eterm capture_spec[CAPSPEC_SIZE], const pcre *code) * The actual re:run/2,3 BIFs */ static BIF_RETTYPE -re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) +re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3, int first) { const pcre *code_tmp; RestartContext restart; @@ -1120,6 +1120,14 @@ re_run(Process *p, Eterm arg1, Eterm arg2, Eterm arg3) < 0) { BIF_ERROR(p,BADARG); } + if (!first) { + /* + * 'first' is false when re:grun() previously has called re:internal_run() + * with the same subject; i.e., no need to do yet another validation of + * the subject regarding utf8 encoding... + */ + options |= PCRE_NO_UTF8_CHECK; + } is_list_cap = ((pflags & PARSE_FLAG_CAPTURE_OPT) && (capture[CAPSPEC_TYPE] == am_list)); @@ -1360,15 +1368,28 @@ handle_iolist: } BIF_RETTYPE +re_internal_run_4(BIF_ALIST_4) +{ + int first; + if (BIF_ARG_4 == am_false) + first = 0; + else if (BIF_ARG_4 == am_true) + first = !0; + else + BIF_ERROR(BIF_P,BADARG); + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, first); +} + +BIF_RETTYPE re_run_3(BIF_ALIST_3) { - return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, !0); } BIF_RETTYPE re_run_2(BIF_ALIST_2) { - return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, NIL); + return re_run(BIF_P,BIF_ARG_1, BIF_ARG_2, NIL, !0); } /* @@ -1407,6 +1428,7 @@ static BIF_RETTYPE re_exec_trap(BIF_ALIST_3) loop_count = 0xFFFFFFFF; #endif rc = erts_pcre_exec(NULL, &(restartp->extra), NULL, 0, 0, 0, NULL, 0); + ASSERT(loop_count != 0xFFFFFFFF); BUMP_REDS(BIF_P, loop_count / LOOP_FACTOR); if (rc == PCRE_ERROR_LOOP_LIMIT) { diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 67a73e4d57..7ab8034606 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -151,6 +151,7 @@ static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); static void sweep_off_heap(Process *p, int fullsweep); static void offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size); static void offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size); +static void offset_heap_ptr_nstack(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size); static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size, Eterm* objv, int nobj); static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size); @@ -1054,9 +1055,10 @@ erts_garbage_collect_hibernate(Process* p) n_htop = tmp_n_htop; \ } while(0) + /* * offset_nstack() can ignore the descriptor-based traversal the other - * nstack procedures use and simply call offset_heap_ptr() instead. + * nstack procedures use and do a simpler word by word traversal instead. * This relies on two facts: * 1. The only live non-Erlang terms on an nstack are return addresses, * and they will be skipped thanks to the low/high range check. @@ -1071,14 +1073,51 @@ static ERTS_INLINE void offset_nstack(Process* p, Sint offs, { if (p->hipe.nstack) { ASSERT(p->hipe.nsp && p->hipe.nstend); - offset_heap_ptr(hipe_nstack_start(p), hipe_nstack_used(p), - offs, area, area_size); + offset_heap_ptr_nstack(hipe_nstack_start(p), hipe_nstack_used(p), + offs, area, area_size); } else { ASSERT(!p->hipe.nsp && !p->hipe.nstend); } } +/* + * This is the same as offset_heap_ptr() + * + * Except for VALGRIND. It allows benign offsetting of undefined (dead) words + * on the nstack while also retaining them as undefined. This suppresses + * valgrinds "Conditional jump or move depends on uninitialised value(s)". + */ +static void +offset_heap_ptr_nstack(Eterm* hp, Uint sz, Sint offs, + char* area, Uint area_size) +{ + while (sz--) { + Eterm val = *hp; +#ifdef VALGRIND + Eterm val_vbits; + VALGRIND_GET_VBITS(&val, &val_vbits, sizeof(val)); + VALGRIND_MAKE_MEM_DEFINED(&val, sizeof(val)); +#endif + switch (primary_tag(val)) { + case TAG_PRIMARY_LIST: + case TAG_PRIMARY_BOXED: + if (ErtsInArea(ptr_val(val), area, area_size)) { +#ifdef VALGRIND + VALGRIND_SET_VBITS(&val, val_vbits, sizeof(val)); +#endif + *hp = offset_ptr(val, offs); + } + hp++; + break; + default: + hp++; + break; + } + } +} + + #else /* !HIPE */ #define fullsweep_nstack(p,n_htop) (n_htop) diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 6645341512..1bebf6efe2 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -527,7 +527,7 @@ erts_msg_attached_data_size_aux(ErtsMessage *msg) if (edep->heap_size < 0) { - sz = erts_decode_dist_ext_size(edep, 1); + sz = erts_decode_dist_ext_size(edep, 1, 1); if (sz < 0) { /* Bad external * We leave the message intact in this case as it's not worth the trouble diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c index f58a606d57..fb900ca7ba 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.c +++ b/erts/emulator/beam/erl_proc_sig_queue.c @@ -3028,7 +3028,7 @@ erts_proc_sig_decode_dist(Process *proc, ErtsProcLocks proc_locks, if (edep->heap_size >= 0) need = edep->heap_size; else { - need = erts_decode_dist_ext_size(edep, 1); + need = erts_decode_dist_ext_size(edep, 1, 1); if (need < 0) { /* bad signal; remove it... */ return 0; @@ -4051,6 +4051,7 @@ erts_proc_sig_signal_size(ErtsSignal *sig) case ERTS_MON_TYPE_DIST_PROC: case ERTS_MON_TYPE_NODE: size = erts_monitor_size((ErtsMonitor *) sig); + break; default: ERTS_INTERNAL_ERROR("Unexpected sig type"); break; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 1f6adb98ef..de0564292d 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -8568,9 +8568,6 @@ erts_start_schedulers(void) { ethr_tid tid; int res = 0; - Uint actual; - Uint wanted = erts_no_schedulers; - Uint wanted_no_schedulers = erts_no_schedulers; char name[16]; ethr_thr_opts opts = ETHR_THR_OPTS_DEFAULT_INITER; int ix; @@ -8584,40 +8581,34 @@ erts_start_schedulers(void) erts_snprintf(opts.name, 16, "runq_supervisor"); erts_atomic_init_nob(&runq_supervisor_sleeping, 0); if (0 != ethr_event_init(&runq_supervision_event)) - erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision event\n"); + erts_exit(ERTS_ABORT_EXIT, "Failed to create run-queue supervision event\n"); res = ethr_thr_create(&runq_supervisor_tid, runq_supervisor, NULL, &opts); if (0 != res) - erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread, " + erts_exit(ERTS_ABORT_EXIT, "Failed to create run-queue supervision thread, " "error = %d\n", res); } opts.suggested_stack_size = erts_sched_thread_suggested_stack_size; - if (wanted < 1) - wanted = 1; - if (wanted > ERTS_MAX_NO_OF_SCHEDULERS) { - wanted = ERTS_MAX_NO_OF_SCHEDULERS; - res = ENOTSUP; - } - - for (actual = 0; actual < wanted; actual++) { - ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual); - - ASSERT(actual == esdp->no - 1); - - erts_snprintf(opts.name, 16, "%lu_scheduler", actual + 1); + ASSERT(erts_no_schedulers > 0 && erts_no_schedulers <= ERTS_MAX_NO_OF_SCHEDULERS); + for (ix = 0; ix < erts_no_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); + ASSERT(ix == esdp->no - 1); + erts_snprintf(opts.name, 16, "%lu_scheduler", ix + 1); res = ethr_thr_create(&esdp->tid, sched_thread_func, (void*)esdp, &opts); - if (res != 0) { - break; + erts_exit(ERTS_ABORT_EXIT, "Failed to create scheduler thread %d, error = %d\n", ix, res); } } - erts_no_schedulers = actual; + + /* Probably not needed as thread create will imply a memory barrier, + but we do one just to be safe. */ + ERTS_THR_MEMORY_BARRIER; { for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { @@ -8626,7 +8617,7 @@ erts_start_schedulers(void) opts.suggested_stack_size = erts_dcpu_sched_thread_suggested_stack_size; res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts); if (res != 0) - erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d, error = %d\n", ix, res); + erts_exit(ERTS_ABORT_EXIT, "Failed to create dirty cpu scheduler thread %d, error = %d\n", ix, res); } for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); @@ -8634,40 +8625,22 @@ erts_start_schedulers(void) opts.suggested_stack_size = erts_dio_sched_thread_suggested_stack_size; res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts); if (res != 0) - erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d, error = %d\n", ix, res); + erts_exit(ERTS_ABORT_EXIT, "Failed to create dirty io scheduler thread %d, error = %d\n", ix, res); } } - ERTS_THR_MEMORY_BARRIER; - erts_snprintf(opts.name, 16, "aux"); res = ethr_thr_create(&tid, aux_thread, NULL, &opts); if (res != 0) - erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread, error = %d\n", res); + erts_exit(ERTS_ABORT_EXIT, "Failed to create aux thread, error = %d\n", res); for (ix = 0; ix < erts_no_poll_threads; ix++) { erts_snprintf(opts.name, 16, "%d_poller", ix); res = ethr_thr_create(&tid, poll_thread, (void*)(UWord)ix, &opts); if (res != 0) - erts_exit(ERTS_ERROR_EXIT, "Failed to create poll thread\n"); - } - - if (actual < 1) - erts_exit(ERTS_ERROR_EXIT, - "Failed to create any scheduler-threads: %s (%d)\n", - erl_errno_id(res), - res); - if (res != 0) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - ASSERT(actual != wanted_no_schedulers); - erts_dsprintf(dsbufp, - "Failed to create %beu scheduler-threads (%s:%d); " - "only %beu scheduler-thread%s created.\n", - wanted_no_schedulers, erl_errno_id(res), res, - actual, actual == 1 ? " was" : "s were"); - erts_send_error_to_logger_nogl(dsbufp); + erts_exit(ERTS_ABORT_EXIT, "Failed to create poll thread\n"); } } @@ -12097,6 +12070,7 @@ erts_proc_exit_handle_dist_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) ErtsHeapFactory factory; Sint reds_consumed = 0; + ASSERT(c_p->flags & F_DISABLE_GC); ASSERT(erts_monitor_is_target(mon) && mon->type == ERTS_MON_TYPE_DIST_PROC); mdp = erts_monitor_to_data(mon); @@ -12144,7 +12118,6 @@ erts_proc_exit_handle_dist_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) switch (code) { case ERTS_DSIG_SEND_CONTINUE: case ERTS_DSIG_SEND_YIELD: - erts_set_gc_state(c_p, 0); ctxt->dist_state = erts_dsend_export_trap_context(c_p, &ctx); reds_consumed = reds; /* force yield */ break; @@ -12152,7 +12125,6 @@ erts_proc_exit_handle_dist_monitor(ErtsMonitor *mon, void *vctxt, Sint reds) break; case ERTS_DSIG_SEND_TOO_LRG: erts_kill_dist_connection(dep, dist->connection_id); - erts_set_gc_state(c_p, 1); break; default: ASSERT(! "Invalid dsig send exit monitor result"); @@ -12356,6 +12328,7 @@ erts_proc_exit_handle_dist_link(ErtsLink *lnk, void *vctxt, Sint reds) ErtsHeapFactory factory; Sint reds_consumed = 0; + ASSERT(c_p->flags & F_DISABLE_GC); ASSERT(lnk->type == ERTS_LNK_TYPE_DIST_PROC); dlnk = erts_link_to_other(lnk, &ldp); dist = ((ErtsLinkDataExtended *) ldp)->dist; @@ -12395,7 +12368,6 @@ erts_proc_exit_handle_dist_link(ErtsLink *lnk, void *vctxt, Sint reds) switch (code) { case ERTS_DSIG_SEND_YIELD: case ERTS_DSIG_SEND_CONTINUE: - erts_set_gc_state(c_p, 0); ctxt->dist_state = erts_dsend_export_trap_context(c_p, &ctx); reds_consumed = reds; /* force yield */ break; @@ -12403,7 +12375,6 @@ erts_proc_exit_handle_dist_link(ErtsLink *lnk, void *vctxt, Sint reds) break; case ERTS_DSIG_SEND_TOO_LRG: erts_kill_dist_connection(dep, dist->connection_id); - erts_set_gc_state(c_p, 1); break; default: ASSERT(! "Invalid dsig send exit monitor result"); @@ -12951,6 +12922,8 @@ restart: yield_allowed = 0; #endif + /* Enable GC again, through strictly not needed it puts + the process in a consistent state. */ erts_set_gc_state(p, 1); /* Set state to not active as we don't want this process diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 0d6b512f78..745a2a482c 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -2353,6 +2353,8 @@ erts_try_change_runq_proc(Process *p, ErtsRunQueue *rq) old_rqint); if (act_rqint == old_rqint) return !0; + + old_rqint = act_rqint; } } diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index ec67ab2aed..ce61cdf040 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1062,11 +1062,38 @@ bad_dist_ext(ErtsDistExternal *edep) } Sint -erts_decode_dist_ext_size(ErtsDistExternal *edep, int kill_connection) +erts_decode_dist_ext_size(ErtsDistExternal *edep, int kill_connection, int payload) { Sint res; byte *ep; + if (edep->data->frag_id > 1 && payload) { + Uint sz = 0; + Binary *bin; + int i; + byte *ep; + + for (i = 0; i < edep->data->frag_id; i++) + sz += edep->data[i].ext_endp - edep->data[i].extp; + + bin = erts_bin_nrml_alloc(sz); + ep = (byte*)bin->orig_bytes; + + for (i = 0; i < edep->data->frag_id; i++) { + sys_memcpy(ep, edep->data[i].extp, edep->data[i].ext_endp - edep->data[i].extp); + ep += edep->data[i].ext_endp - edep->data[i].extp; + erts_bin_release(edep->data[i].binp); + edep->data[i].binp = NULL; + edep->data[i].extp = NULL; + edep->data[i].ext_endp = NULL; + } + + edep->data->frag_id = 1; + edep->data->extp = (byte*)bin->orig_bytes; + edep->data->ext_endp = ep; + edep->data->binp = bin; + } + if (edep->data->extp >= edep->data->ext_endp) goto fail; #ifndef ERTS_DEBUG_USE_DIST_SEP @@ -1164,6 +1191,7 @@ Eterm erts_decode_ext(ErtsHeapFactory* factory, byte **ext, Uint32 flags) if (flags) { ASSERT(flags == ERTS_DIST_EXT_BTT_SAFE); ede.flags = flags; /* a dummy struct just for the flags */ + ede.data = NULL; edep = &ede; } else { edep = NULL; @@ -1233,8 +1261,10 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) ede.data->extp = binary_bytes(real_bin)+offset; ede.data->ext_endp = ede.data->extp + size; + ede.data->frag_id = 1; + ede.data->binp = NULL; - hsz = erts_decode_dist_ext_size(&ede, 1); + hsz = erts_decode_dist_ext_size(&ede, 1, 1); if (hsz < 0) goto badarg; @@ -1765,6 +1795,7 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Eterm bin, B2TContext *ctx) case B2TDecodeBinary: { ErtsDistExternal fakedep; fakedep.flags = ctx->flags; + fakedep.data = NULL; dec_term(&fakedep, NULL, NULL, NULL, ctx); break; } @@ -3762,6 +3793,30 @@ dec_term_atom_common: hp += heap_bin_size(n); sys_memcpy(hb->data, ep, n); *objp = make_binary(hb); + } else if (edep && edep->data && edep->data->binp && + n > (edep->data->binp->orig_size / 4)) { + /* If we decode a refc binary from a distribution data + entry we know that it is a refc binary to begin with + so we just increment it and use the reference. This + means that the entire distribution data entry will + remain until this binary is de-allocated so we only + do it if a substantial part (> 25%) of the data + is a binary. */ + ProcBin* pb = (ProcBin *) hp; + Binary* bptr = edep->data->binp; + erts_refc_inc(&bptr->intern.refc, 1); + pb->thing_word = HEADER_PROC_BIN; + pb->size = n; + pb->next = factory->off_heap->first; + factory->off_heap->first = (struct erl_off_heap_header*)pb; + pb->val = bptr; + pb->bytes = (byte*) ep; + ERTS_ASSERT((byte*)(bptr->orig_bytes) < ep && + ep+n <= (byte*)(bptr->orig_bytes+bptr->orig_size)); + pb->flags = 0; + OH_OVERHEAD(factory->off_heap, pb->size / sizeof(Eterm)); + hp += PROC_BIN_SIZE; + *objp = make_binary(pb); } else { Binary* dbin = erts_bin_nrml_alloc(n); diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index b556c9076c..e362a6c81f 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -199,7 +199,7 @@ typedef enum { ErtsPrepDistExtRes erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint, struct binary *, DistEntry *, Uint32, ErtsAtomCache *); -Sint erts_decode_dist_ext_size(ErtsDistExternal *, int); +Sint erts_decode_dist_ext_size(ErtsDistExternal *, int, int); Eterm erts_decode_dist_ext(ErtsHeapFactory*, ErtsDistExternal *, int); Sint erts_decode_ext_size(byte*, Uint); diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 311c5fdd6a..66ff8d8450 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -12733,7 +12733,7 @@ static void packet_inet_command(ErlDrvData e, char* buf, ErlDrvSizeT len) len -= 4; ptr += 4; if (len < anc_len) goto return_einval; - if (anc_len == 0 && !!0/*XXX-short-circuit-for-testing*/) { + if (anc_len == 0) { /* Empty ancillary data */ /* Now "ptr" is the user data ptr, "len" is data length: */ inet_output_count(desc, len); @@ -12772,10 +12772,7 @@ static void packet_inet_command(ErlDrvData e, char* buf, ErlDrvSizeT len) if (compile_ancillary_data(&mhdr, ptr, anc_len) != 0) { goto return_einval; } - if (mhdr.msg_controllen == 0) { - /* XXX Testing - only possible for anc_len == 0 */ - mhdr.msg_control = NULL; - } + ASSERT(mhdr.msg_controllen != 0); len -= anc_len; ptr += anc_len; /* Now "ptr" is the user data ptr, "len" is data length: */ diff --git a/erts/emulator/nifs/common/net_nif.c b/erts/emulator/nifs/common/prim_net_nif.c index 8a69052935..11a8ff724e 100644 --- a/erts/emulator/nifs/common/net_nif.c +++ b/erts/emulator/nifs/common/prim_net_nif.c @@ -1653,4 +1653,4 @@ LOCAL_ERROR_REASON_ATOMS return !net; } -ERL_NIF_INIT(net, net_funcs, on_load, NULL, NULL, NULL) +ERL_NIF_INIT(prim_net, net_funcs, on_load, NULL, NULL, NULL) diff --git a/erts/emulator/nifs/common/socket_nif.c b/erts/emulator/nifs/common/socket_nif.c index 0145cae31b..8b0d7ce4a9 100644 --- a/erts/emulator/nifs/common/socket_nif.c +++ b/erts/emulator/nifs/common/socket_nif.c @@ -9172,35 +9172,67 @@ ERL_NIF_TERM nsetopt_lvl_ip_update_membership(ErlNifEnv* env, #endif // It must be a map - if (!IS_MAP(env, eVal)) + if (!IS_MAP(env, eVal)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "value *not* a map\r\n") ); return enif_make_badarg(env); + } // It must have atleast two attributes - if (!enif_get_map_size(env, eVal, &sz) || (sz >= 2)) + if (!enif_get_map_size(env, eVal, &sz) || (sz < 2)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "invalid map value: %T\r\n", eVal) ); return enif_make_badarg(env); + } - if (!GET_MAP_VAL(env, eVal, atom_multiaddr, &eMultiAddr)) + if (!GET_MAP_VAL(env, eVal, atom_multiaddr, &eMultiAddr)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "failed get multiaddr (map) attribute\r\n") ); return enif_make_badarg(env); + } - if (!GET_MAP_VAL(env, eVal, atom_interface, &eInterface)) + if (!GET_MAP_VAL(env, eVal, atom_interface, &eInterface)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "failed get interface (map) attribute\r\n") ); return enif_make_badarg(env); + } if ((xres = esock_decode_ip4_address(env, eMultiAddr, - &mreq.imr_multiaddr)) != NULL) + &mreq.imr_multiaddr)) != NULL) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "failed decode multiaddr %T: %s\r\n", eMultiAddr, xres) ); return esock_make_error_str(env, xres); + } if ((xres = esock_decode_ip4_address(env, eInterface, - &mreq.imr_interface)) != NULL) + &mreq.imr_interface)) != NULL) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "failed decode interface %T: %s\r\n", eInterface, xres) ); return esock_make_error_str(env, xres); + } res = socket_setopt(descP->sock, level, opt, &mreq, sizeof(mreq)); - if (res != 0) - result = esock_make_error_errno(env, sock_errno()); - else + if (res != 0) { + int save_errno = sock_errno(); + + result = esock_make_error_errno(env, save_errno); + + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "failed setopt: %T (%d)\r\n", result, save_errno) ); + + } else { result = esock_atom_ok; + } return result; } @@ -9804,33 +9836,65 @@ ERL_NIF_TERM nsetopt_lvl_ipv6_update_membership(ErlNifEnv* env, #endif // It must be a map - if (!IS_MAP(env, eVal)) + if (!IS_MAP(env, eVal)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ipv6_update_membership -> " + "value *not* a map\r\n") ); return enif_make_badarg(env); + } // It must have atleast two attributes - if (!enif_get_map_size(env, eVal, &sz) || (sz >= 2)) + if (!enif_get_map_size(env, eVal, &sz) || (sz < 2)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ipv6_update_membership -> " + "invalid map value: %T\r\n", eVal) ); return enif_make_badarg(env); + } - if (!GET_MAP_VAL(env, eVal, atom_multiaddr, &eMultiAddr)) + if (!GET_MAP_VAL(env, eVal, atom_multiaddr, &eMultiAddr)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ipv6_update_membership -> " + "failed get multiaddr (map) attribute\r\n") ); return enif_make_badarg(env); + } - if (!GET_MAP_VAL(env, eVal, atom_interface, &eInterface)) + if (!GET_MAP_VAL(env, eVal, atom_interface, &eInterface)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ipv6_update_membership -> " + "failed get interface (map) attribute\r\n") ); return enif_make_badarg(env); + } if ((xres = esock_decode_ip6_address(env, eMultiAddr, - &mreq.ipv6mr_multiaddr)) != NULL) + &mreq.ipv6mr_multiaddr)) != NULL) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ipv6_update_membership -> " + "failed decode multiaddr %T: %s\r\n", eMultiAddr, xres) ); return esock_make_error_str(env, xres); + } - if (!GET_UINT(env, eInterface, &mreq.ipv6mr_interface)) + if (!GET_UINT(env, eInterface, &mreq.ipv6mr_interface)) { + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ip_update_membership -> " + "failed decode interface %T: %s\r\n", eInterface, xres) ); return esock_make_error(env, esock_atom_einval); + } res = socket_setopt(descP->sock, level, opt, &mreq, sizeof(mreq)); - if (res != 0) - result = esock_make_error_errno(env, sock_errno()); - else + if (res != 0) { + int save_errno = sock_errno(); + + result = esock_make_error_errno(env, save_errno); + + SSDBG( descP, + ("SOCKET", "nsetopt_lvl_ipv6_update_membership -> " + "failed setopt: %T (%d)\r\n", result, save_errno) ); + + } else { result = esock_atom_ok; + } return result; } diff --git a/erts/emulator/pcre/LICENCE b/erts/emulator/pcre/LICENCE index f6ef7fd766..760a6666b6 100644 --- a/erts/emulator/pcre/LICENCE +++ b/erts/emulator/pcre/LICENCE @@ -25,7 +25,7 @@ Email domain: cam.ac.uk University of Cambridge Computing Service, Cambridge, England. -Copyright (c) 1997-2018 University of Cambridge +Copyright (c) 1997-2019 University of Cambridge All rights reserved. @@ -34,9 +34,9 @@ PCRE JUST-IN-TIME COMPILATION SUPPORT Written by: Zoltan Herczeg Email local part: hzmester -Emain domain: freemail.hu +Email domain: freemail.hu -Copyright(c) 2010-2018 Zoltan Herczeg +Copyright(c) 2010-2019 Zoltan Herczeg All rights reserved. @@ -45,9 +45,9 @@ STACK-LESS JUST-IN-TIME COMPILER Written by: Zoltan Herczeg Email local part: hzmester -Emain domain: freemail.hu +Email domain: freemail.hu -Copyright(c) 2009-2018 Zoltan Herczeg +Copyright(c) 2009-2019 Zoltan Herczeg All rights reserved. diff --git a/erts/emulator/pcre/pcre-8.42.tar.bz2 b/erts/emulator/pcre/pcre-8.42.tar.bz2 Binary files differdeleted file mode 100644 index 61bfa38970..0000000000 --- a/erts/emulator/pcre/pcre-8.42.tar.bz2 +++ /dev/null diff --git a/erts/emulator/pcre/pcre-8.43.tar.bz2 b/erts/emulator/pcre/pcre-8.43.tar.bz2 Binary files differnew file mode 100644 index 0000000000..e20c601f71 --- /dev/null +++ b/erts/emulator/pcre/pcre-8.43.tar.bz2 diff --git a/erts/emulator/pcre/pcre.h b/erts/emulator/pcre/pcre.h index 3563791223..49c9fc6dc8 100644 --- a/erts/emulator/pcre/pcre.h +++ b/erts/emulator/pcre/pcre.h @@ -43,9 +43,9 @@ POSSIBILITY OF SUCH DAMAGE. /* The current PCRE version information. */ #define PCRE_MAJOR 8 -#define PCRE_MINOR 42 +#define PCRE_MINOR 43 #define PCRE_PRERELEASE -#define PCRE_DATE 2018-03-20 +#define PCRE_DATE 2019-02-23 /* When an application links to a PCRE DLL in Windows, the symbols that are imported have to be identified as such. When building PCRE, the appropriate @@ -240,6 +240,9 @@ with J. */ #define PCRE_UTF8_ERR20 20 #define PCRE_UTF8_ERR21 21 #define PCRE_UTF8_ERR22 22 /* Unused (was non-character) */ +#if defined(ERLANG_INTEGRATION) +#define PCRE_UTF8_YIELD 23 +#endif /* Specific error codes for UTF-16 validity checks */ diff --git a/erts/emulator/pcre/pcre_compile.c b/erts/emulator/pcre/pcre_compile.c index ae7f6e2a2a..6ac222b27e 100644 --- a/erts/emulator/pcre/pcre_compile.c +++ b/erts/emulator/pcre/pcre_compile.c @@ -6,7 +6,7 @@ and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel - Copyright (c) 1997-2016 University of Cambridge + Copyright (c) 1997-2018 University of Cambridge ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without @@ -3300,7 +3300,7 @@ for(;;) if ((*xclass_flags & XCL_MAP) == 0) { /* No bits are set for characters < 256. */ - if (list[1] == 0) return TRUE; + if (list[1] == 0) return (*xclass_flags & XCL_NOT) == 0; /* Might be an empty repeat. */ continue; } @@ -7643,6 +7643,8 @@ for (;; ptr++) /* Can't determine a first byte now */ if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; + zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; continue; @@ -8683,10 +8685,18 @@ do { if (!is_anchored(scode, new_map, cd, atomcount)) return FALSE; } - /* Positive forward assertions and conditions */ + /* Positive forward assertion */ - else if (op == OP_ASSERT || op == OP_COND) + else if (op == OP_ASSERT) + { + if (!is_anchored(scode, bracket_map, cd, atomcount)) return FALSE; + } + + /* Condition; not anchored if no second branch */ + + else if (op == OP_COND) { + if (scode[GET(scode,1)] != OP_ALT) return FALSE; if (!is_anchored(scode, bracket_map, cd, atomcount)) return FALSE; } diff --git a/erts/emulator/pcre/pcre_exec.c b/erts/emulator/pcre/pcre_exec.c index 1946e97a72..55a7b377bf 100644 --- a/erts/emulator/pcre/pcre_exec.c +++ b/erts/emulator/pcre/pcre_exec.c @@ -6642,10 +6642,16 @@ typedef struct { REAL_PCRE *Xre; heapframe Xframe_zero; /* Always NO_RECURSE */ + /* for yield in valid_utf() */ + + struct PRIV(valid_utf_ystate) valid_utf_ystate; + /* Original function parameters that need be saved */ int Xstart_offset; int Xoffsetcount; int *Xoffsets; + int Xlength; + PCRE_SPTR Xsubject; } PcreExecContext; #endif @@ -6675,6 +6681,7 @@ pcre32_exec(const pcre32 *argument_re, const pcre32_extra *extra_data, #endif { #ifndef ERLANG_INTEGRATION +#define ERTS_UPDATE_CONSUMED(X, MD) int rc, ocount, arg_offset_max; int newline; BOOL using_temporary_offsets = FALSE; @@ -6736,6 +6743,8 @@ heapframe frame_zero; start_offset = exec_context->Xstart_offset; \ offsetcount = exec_context->Xoffsetcount; \ offsets = exec_context->Xoffsets; \ + length = exec_context->Xlength; \ + subject = exec_context->Xsubject; \ } while (0) #define SWAPOUT() do { \ @@ -6750,8 +6759,30 @@ heapframe frame_zero; exec_context->Xstart_offset = start_offset; \ exec_context->Xoffsetcount = offsetcount; \ exec_context->Xoffsets = offsets; \ + exec_context->Xlength = length; \ + exec_context->Xsubject = subject; \ } while (0) +#define ERTS_UPDATE_CONSUMED(X, MD) \ +do { \ + if (((X)->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) { \ + unsigned long consumed__; \ + if (!(X)->restart_data) { \ + consumed__ = 0; \ + } \ + else { \ + PcreExecContext *ctx__ = (PcreExecContext *) \ + (*(X)->restart_data); \ + consumed__ = ctx__->valid_utf_ystate.cnt; \ + ctx__->valid_utf_ystate.cnt = 0; \ + } \ + if ((MD)) { \ + match_data *md__ = (MD); \ + consumed__ += (X)->loop_limit - md__->loop_limit; \ + } \ + *((X)->loop_counter_return) = consumed__; \ + } \ +} while (0) PcreExecContext *exec_context; PcreExecContext internal_context; @@ -6776,15 +6807,21 @@ pcre_uchar req_char; /* we are restarting, every initialization is skipped and we jump directly into the loop */ exec_context = (PcreExecContext *) *(extra_data->restart_data); SWAPIN(); - + if (exec_context->valid_utf_ystate.yielded) + goto restart_valid_utf; goto RESTART_INTERRUPTED; } else { if (extra_data != NULL && (extra_data->flags & PCRE_EXTRA_LOOP_LIMIT)) { exec_context = (PcreExecContext *) (erts_pcre_malloc)(sizeof(PcreExecContext)); - *(extra_data->restart_data) = (void *) exec_context; + *(extra_data->restart_data) = (void *) exec_context; + exec_context->valid_utf_ystate.yielded = 0; /* need freeing by special routine from client */ } else { +#if defined(ERLANG_INTEGRATION) + fprintf(stderr, "Unexpected execution path\n"); + abort(); +#endif exec_context = &internal_context; } @@ -6865,9 +6902,38 @@ code for an invalid string if a results vector is available. */ if (utf && (options & PCRE_NO_UTF8_CHECK) == 0) { int erroroffset; - int errorcode = PRIV(valid_utf)((PCRE_PUCHAR)subject, length, &erroroffset); + int errorcode; + +#if !defined(ERLANG_INTEGRATION) + errorcode = PRIV(valid_utf)((PCRE_PUCHAR)subject, length); +#else + struct PRIV(valid_utf_ystate) *ystate; + + if (!extra_data || !extra_data->restart_data) { + ystate = NULL; + } + else if (!(extra_data->flags & PCRE_EXTRA_LOOP_LIMIT)) { + exec_context->valid_utf_ystate.cnt = 10; + ystate = NULL; + } + else { + exec_context->valid_utf_ystate.yielded = 0; + restart_valid_utf: + ystate = &exec_context->valid_utf_ystate; + ystate->cnt = (int) extra_data->loop_limit; + } + errorcode = PRIV(yielding_valid_utf)((PCRE_PUCHAR)subject, length, + &erroroffset, ystate); +#endif if (errorcode != 0) { +#if defined(ERLANG_INTEGRATION) + if (ystate && ystate->yielded) { + ERTS_UPDATE_CONSUMED(extra_data, NULL); + SWAPOUT(); + return PCRE_ERROR_LOOP_LIMIT; + } +#endif if (offsetcount >= 2) { offsets[0] = erroroffset; @@ -6890,6 +6956,11 @@ if (utf && (options & PCRE_NO_UTF8_CHECK) == 0) return PCRE_ERROR_BADUTF8_OFFSET; #endif } +#if defined(ERLANG_INTEGRATION) +else { + exec_context->valid_utf_ystate.cnt = 0; +} +#endif #endif /* If the pattern was successfully studied with JIT support, run the JIT @@ -6950,7 +7021,11 @@ if (extra_data != NULL) #ifdef ERLANG_INTEGRATION if ((flags & PCRE_EXTRA_LOOP_LIMIT) != 0) { - md->loop_limit = extra_data->loop_limit; + md->loop_limit = extra_data->loop_limit; + if (extra_data->restart_data) + md->loop_limit -= extra_data->loop_limit - exec_context->valid_utf_ystate.cnt; + if (md->loop_limit < 10) + md->loop_limit = 10; /* At least do something if we've come this far... */ } #endif } @@ -7266,14 +7341,8 @@ for(;;) #endif if ((start_bits[c/8] & (1 << (c&7))) != 0) { -#ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } -#endif - break; + ERTS_UPDATE_CONSUMED(extra_data, md); + break; } start_match++; } @@ -7298,13 +7367,7 @@ for(;;) (pcre_uint32)(end_subject - start_match) < study->minlength) { rc = MATCH_NOMATCH; -#ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } -#endif + ERTS_UPDATE_CONSUMED(extra_data, md); break; } @@ -7353,13 +7416,7 @@ for(;;) if (p >= end_subject) { rc = MATCH_NOMATCH; -#ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } -#endif + ERTS_UPDATE_CONSUMED(extra_data, md); break; } @@ -7390,11 +7447,7 @@ for(;;) EDEBUGF(("Calling match...")); rc = match(start_match, md->start_code, start_match, 2, md, NULL, 0); #ifdef ERLANG_INTEGRATION - if ((extra_data->flags & PCRE_EXTRA_LOOP_LIMIT) != 0) - { - *extra_data->loop_counter_return = - (extra_data->loop_limit - md->loop_limit); - } + ERTS_UPDATE_CONSUMED(extra_data, md); SWAPOUT(); while(rc == PCRE_ERROR_LOOP_LIMIT) { EDEBUGF(("Loop limit break detected")); diff --git a/erts/emulator/pcre/pcre_internal.h b/erts/emulator/pcre/pcre_internal.h index c84dcb5a38..71f473e86f 100644 --- a/erts/emulator/pcre/pcre_internal.h +++ b/erts/emulator/pcre/pcre_internal.h @@ -2756,6 +2756,17 @@ extern int PRIV(strcmp_uc_c8_utf)(const pcre_uchar *, #endif /* COMPILE_PCRE[8|16|32] */ +#if defined(ERLANG_INTEGRATION) +struct PRIV(valid_utf_ystate) { + unsigned int cnt; + int length; + int yielded; + PCRE_PUCHAR p; +}; +extern int PRIV(yielding_valid_utf)(PCRE_PUCHAR, int, int *, + struct PRIV(valid_utf_ystate) *); +#endif + extern const pcre_uchar *PRIV(find_bracket)(const pcre_uchar *, BOOL, int); extern BOOL PRIV(is_newline)(PCRE_PUCHAR, int, PCRE_PUCHAR, int *, BOOL); diff --git a/erts/emulator/pcre/pcre_jit_compile.c b/erts/emulator/pcre/pcre_jit_compile.c index 926e40f6d3..2d2288f81e 100644 --- a/erts/emulator/pcre/pcre_jit_compile.c +++ b/erts/emulator/pcre/pcre_jit_compile.c @@ -9002,7 +9002,7 @@ if (exact > 1) #ifdef SUPPORT_UTF && !common->utf #endif - ) + && type != OP_ANYNL && type != OP_EXTUNI) { OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(exact)); add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_GREATER, TMP1, 0, STR_END, 0)); diff --git a/erts/emulator/pcre/pcre_valid_utf8.c b/erts/emulator/pcre/pcre_valid_utf8.c index 516d8f4725..1dc1f9ba0c 100644 --- a/erts/emulator/pcre/pcre_valid_utf8.c +++ b/erts/emulator/pcre/pcre_valid_utf8.c @@ -107,19 +107,80 @@ Returns: = 0 if the string is a valid UTF-8 string int PRIV(valid_utf)(PCRE_PUCHAR string, int length, int *erroroffset) { + +#if defined(ERLANG_INTEGRATION) + return PRIV(yielding_valid_utf)(string, length, erroroffset, NULL); +} + +int +PRIV(yielding_valid_utf)(PCRE_PUCHAR string, int length, int *erroroffset, struct PRIV(valid_utf_ystate) *ystate) +{ +#endif + #ifdef SUPPORT_UTF register PCRE_PUCHAR p; +#if defined(ERLANG_INTEGRATION) +register long cnt; + +if (!ystate) { + cnt = -1; +} +else { + cnt = ystate->cnt; + if (ystate->yielded) { + p = ystate->p; + length = ystate->length; + if (length < 0) + goto restart_length; + else + goto restart_validate; + } +} +#endif + if (length < 0) { - for (p = string; *p != 0; p++); - length = (int)(p - string); + for (p = string; *p != 0; p++) { +#if defined(ERLANG_INTEGRATION) + if (cnt > 0 && --cnt == 0) { + /* + * Return with cnt set to amount consumed; + * i.e. same amount as at start... + */ + ystate->yielded = !0; + ystate->length = length; + ystate->p = p; + return PCRE_UTF8_YIELD; + } + restart_length: + (void) !0; +#endif + } + length = (int)(p - string); } for (p = string; length-- > 0; p++) { register pcre_uchar ab, c, d; +#if defined(ERLANG_INTEGRATION) + + if (cnt > 0 && --cnt == 0) { + /* + * Return with cnt set to amount consumed; + * i.e. same amount as at start... + */ + ystate->yielded = !0; + ystate->length = length; + ystate->p = p; + return PCRE_UTF8_YIELD; + } + + restart_validate: + +#endif + c = *p; if (c < 128) continue; /* ASCII character */ @@ -290,6 +351,14 @@ for (p = string; length-- > 0; p++) } } +#if defined(ERLANG_INTEGRATION) +if (ystate) { + /* Return with cnt set to amount consumed... */ + ystate->cnt -= cnt; + ystate->yielded = 0; +} +#endif + #else /* Not SUPPORT_UTF */ (void)(string); /* Keep picky compilers happy */ (void)(length); diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index 7885d35d9d..9dcdd60060 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -1400,6 +1400,10 @@ get_conflicting_unicode_atoms(CIX, N) -> %% The message_latency_large tests that small distribution messages are %% not blocked by other large distribution messages. Basically it tests %% that fragmentation of distribution messages works. +%% +%% Because of large problems to get reliable values from these testcases +%% they no longer fail when the latency is incorrect. However, they are +%% kept as they continue to find bugs in the distribution implementation. message_latency_large_message(Config) when is_list(Config) -> measure_latency_large_message(?FUNCTION_NAME, fun(Dropper, Payload) -> Dropper ! Payload end). @@ -1484,7 +1488,11 @@ measure_latency_large_message(Nodename, DataFun) -> case {lists:max(Times), lists:min(Times)} of {Max, Min} when Max * 0.25 > Min, BuildType =:= opt -> - ct:fail({incorrect_latency, IndexTimes}); + %% We only issue a comment for this failure as the + %% testcases proved very difficult to run successfully + %% on many platforms. + ct:comment({incorrect_latency, IndexTimes}), + ok; _ -> ok end. @@ -1503,10 +1511,7 @@ measure_latency(DataFun, Dropper, Echo, Payload) -> end end) || _ <- lists:seq(1,2)], - [receive - {monitor, _Sender, busy_dist_port, _Info} -> - ok - end || _ <- lists:seq(1,10)], + wait_for_busy_dist(2 * 60 * 1000, 10), {TS, Times} = timer:tc(fun() -> @@ -1530,6 +1535,18 @@ measure_latency(DataFun, Dropper, Echo, Payload) -> end || {Sender, Ref} <- Senders], TS. +wait_for_busy_dist(_Tmo, 0) -> + ok; +wait_for_busy_dist(Tmo, N) -> + T0 = erlang:monotonic_time(millisecond), + receive + {monitor, _Sender, busy_dist_port, _Info} -> + wait_for_busy_dist(Tmo - (erlang:monotonic_time(millisecond) - T0), N - 1) + after Tmo -> + ct:log("Timed out waiting for busy_dist, ~p left",[N]), + timeout + end. + flush() -> receive _ -> @@ -2600,7 +2617,7 @@ verify_nc(Node) -> demonitor(MonRef,[flush]), ok; {Ref, Error} -> - ct:log("~p",[Error]), + ct:log("~s",[Error]), ct:fail(failed_nc_refc_check); {'DOWN', MonRef, _, _, _} = Down -> ct:log("~p",[Down]), diff --git a/erts/emulator/test/dump_SUITE.erl b/erts/emulator/test/dump_SUITE.erl index 9f8ac42fa9..6133b82756 100644 --- a/erts/emulator/test/dump_SUITE.erl +++ b/erts/emulator/test/dump_SUITE.erl @@ -68,12 +68,14 @@ signal_abort(Config) -> {ok, Node} = start_node(Config), - _P1 = spawn(Node, ?MODULE, load, []), - _P2 = spawn(Node, ?MODULE, load, []), - _P3 = spawn(Node, ?MODULE, load, []), - _P4 = spawn(Node, ?MODULE, load, []), - _P5 = spawn(Node, ?MODULE, load, []), - _P6 = spawn(Node, ?MODULE, load, []), + SO = rpc:call(Node, erlang, system_info, [schedulers_online]), + + _P1 = spawn_opt(Node, ?MODULE, load, [], [{scheduler, (0 rem SO) + 1}]), + _P2 = spawn_opt(Node, ?MODULE, load, [], [{scheduler, (1 rem SO) + 1}]), + _P3 = spawn_opt(Node, ?MODULE, load, [], [{scheduler, (2 rem SO) + 1}]), + _P4 = spawn_opt(Node, ?MODULE, load, [], [{scheduler, (3 rem SO) + 1}]), + _P5 = spawn_opt(Node, ?MODULE, load, [], [{scheduler, (4 rem SO) + 1}]), + _P6 = spawn_opt(Node, ?MODULE, load, [], [{scheduler, (5 rem SO) + 1}]), timer:sleep(500), @@ -140,13 +142,13 @@ free_dump(Config) when is_list(Config) -> {ok, NodeA} = start_node(Config), {ok, NodeB} = start_node(Config), - Self = self(), PidA = spawn_link( NodeA, fun() -> Self ! ready, + Reason = lists:duplicate(1000000,100), receive ok -> spawn(fun() -> @@ -154,24 +156,29 @@ free_dump(Config) when is_list(Config) -> timer:sleep(5), receive M -> - io:format("~p",[M]), - erlang:halt("dump") - end + io:format("~p",[M]) +%% We may want to add this timeout here in-case no busy condition is triggered +%% after 60 * 1000 -> +%% io:format("Timeout") + end, + erlang:halt("dump") end), - exit(lists:duplicate(1000000,100)) + exit(Reason) end end), - spawn_link(NodeB, - fun() -> - [erlang:monitor(process, PidA) || _ <- lists:seq(1,10000)], - Self ! done, - receive _ -> ok end - end), + PidB = spawn_link(NodeB, + fun() -> + [erlang:monitor(process, PidA) || _ <- lists:seq(1,10000)], + Self ! done, + receive _ -> ok end + end), receive done -> ok end, true = rpc:call(NodeA, os, putenv, ["ERL_CRASH_DUMP",Dump]), - ct:pal("~p",[rpc:call(NodeA, distribution_SUITE, make_busy, [NodeB, 1000])]), + %% Make the node busy towards NodeB for 10 seconds. + BusyPid = rpc:call(NodeA, distribution_SUITE, make_busy, [NodeB,10000]), + ct:pal("~p",[BusyPid]), receive ready -> unlink(PidA), PidA ! ok end, @@ -185,6 +192,10 @@ free_dump(Config) when is_list(Config) -> file:delete(Dump), + unlink(PidB), + + rpc:call(NodeB, erlang, halt, [0]), + ok. diff --git a/erts/emulator/test/multi_load_SUITE.erl b/erts/emulator/test/multi_load_SUITE.erl index edf3205812..c79e2b6dcd 100644 --- a/erts/emulator/test/multi_load_SUITE.erl +++ b/erts/emulator/test/multi_load_SUITE.erl @@ -30,7 +30,15 @@ all() -> [many,on_load,errors]. many(_Config) -> - Ms = make_modules(100, fun many_module/1), + + N = case erlang:system_info(build_type) of + valgrind -> + 10; + _ -> + 100 + end, + + Ms = make_modules(N, fun many_module/1), io:put_chars("Light load\n" "=========="), diff --git a/erts/emulator/test/net_SUITE.erl b/erts/emulator/test/net_SUITE.erl index 6111fc76a5..c6e77a5373 100644 --- a/erts/emulator/test/net_SUITE.erl +++ b/erts/emulator/test/net_SUITE.erl @@ -20,6 +20,8 @@ %% %% This test suite is basically a "placeholder" for a proper test suite... +%% Also we should really call prim_net directly, and not net (since that does +%% not even reside here). %% %% Run the entire test suite: @@ -127,6 +129,7 @@ api_basic_cases() -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% init_per_suite(Config) -> + %% We test on the socket module for simplicity case lists:member(socket, erlang:loaded()) of true -> case os:type() of diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl index ef4635a6f5..7d2db5257c 100644 --- a/erts/emulator/test/node_container_SUITE.erl +++ b/erts/emulator/test/node_container_SUITE.erl @@ -570,7 +570,17 @@ node_controller_refc(Config) when is_list(Config) -> wait_until(fun () -> not is_process_alive(P) end), lists:foreach(fun (Proc) -> garbage_collect(Proc) end, processes()), false = get_node_references({Node,Creation}), - false = get_dist_references(Node), + wait_until(fun () -> + case get_dist_references(Node) of + false -> + true; + [{{system,thread_progress_delete_timer}, + [{system,1}]}] -> + false; + Other -> + ct:fail(Other) + end + end), false = lists:member(Node, nodes(known)), nc_refc_check(node()), erts_debug:set_internal_state(node_tab_delayed_delete, -1), %% restore original value @@ -871,7 +881,22 @@ magic_ref(Config) when is_list(Config) -> {'DOWN', Mon, process, Pid, _} -> ok end, - {Addr0, 2, true} = erts_debug:get_internal_state({magic_ref,MRef0}), + MaxTime = erlang:monotonic_time(millisecond) + 1000, + %% The DOWN signal is sent before heap is cleaned up, + %% so we might need to wait some time after the DOWN + %% signal has been received before the heap actually + %% has been cleaned up... + wait_until(fun () -> + case erts_debug:get_internal_state({magic_ref,MRef0}) of + {Addr0, 2, true} -> + true; + {Addr0, 3, true} -> + true = MaxTime >= erlang:monotonic_time(millisecond), + false; + Error -> + ct:fail(Error) + end + end), id(MRef0), id(MRef1), MRefExt = term_to_binary(erts_debug:set_internal_state(make, magic_ref)), diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 3684cde8d4..13dde12e69 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -2600,14 +2600,20 @@ garb_other_running(Config) when is_list(Config) -> no_priority_inversion(Config) when is_list(Config) -> Prio = process_flag(priority, max), - HTLs = lists:map(fun (_) -> + Master = self(), + Executing = make_ref(), + HTLs = lists:map(fun (Sched) -> spawn_opt(fun () -> + Master ! {self(), Executing}, tok_loop() end, - [{priority, high}, monitor, link]) + [{priority, high}, + {scheduler, Sched}, + monitor, + link]) end, - lists:seq(1, 2*erlang:system_info(schedulers))), - receive after 500 -> ok end, + lists:seq(1, erlang:system_info(schedulers_online))), + lists:foreach(fun ({P, _}) -> receive {P,Executing} -> ok end end, HTLs), LTL = spawn_opt(fun () -> tok_loop() end, @@ -2629,14 +2635,19 @@ no_priority_inversion(Config) when is_list(Config) -> no_priority_inversion2(Config) when is_list(Config) -> Prio = process_flag(priority, max), - MTLs = lists:map(fun (_) -> + Master = self(), + Executing = make_ref(), + MTLs = lists:map(fun (Sched) -> spawn_opt(fun () -> + Master ! {self(), Executing}, tok_loop() end, - [{priority, max}, monitor, link]) + [{priority, max}, + {scheduler, Sched}, + monitor, link]) end, - lists:seq(1, 2*erlang:system_info(schedulers))), - receive after 2000 -> ok end, + lists:seq(1, erlang:system_info(schedulers_online))), + lists:foreach(fun ({P, _}) -> receive {P,Executing} -> ok end end, MTLs), {PL, ML} = spawn_opt(fun () -> tok_loop() end, diff --git a/erts/emulator/test/socket_SUITE.erl b/erts/emulator/test/socket_SUITE.erl index d51e7fa464..410582f4a1 100644 --- a/erts/emulator/test/socket_SUITE.erl +++ b/erts/emulator/test/socket_SUITE.erl @@ -106,6 +106,7 @@ api_opt_simple_otp_options/1, api_opt_simple_otp_rcvbuf_option/1, api_opt_simple_otp_controlling_process/1, + api_opt_ip_add_drop_membership/1, %% *** API Operation Timeout *** api_to_connect_tcp4/1, @@ -589,6 +590,8 @@ groups() -> {api_basic, [], api_basic_cases()}, {api_async, [], api_async_cases()}, {api_options, [], api_options_cases()}, + {api_options_otp, [], api_options_otp_cases()}, + {api_options_ip, [], api_options_ip_cases()}, {api_op_with_timeout, [], api_op_with_timeout_cases()}, {socket_closure, [], socket_closure_cases()}, {sc_ctrl_proc_exit, [], sc_cp_exit_cases()}, @@ -710,11 +713,22 @@ api_async_cases() -> api_options_cases() -> [ + {group, api_options_otp}, + {group, api_options_ip} + ]. + +api_options_otp_cases() -> + [ api_opt_simple_otp_options, api_opt_simple_otp_rcvbuf_option, api_opt_simple_otp_controlling_process ]. +api_options_ip_cases() -> + [ + api_opt_ip_add_drop_membership + ]. + api_op_with_timeout_cases() -> [ api_to_connect_tcp4, @@ -7551,7 +7565,7 @@ api_opt_simple_otp_controlling_process(suite) -> api_opt_simple_otp_controlling_process(doc) -> []; api_opt_simple_otp_controlling_process(_Config) when is_list(_Config) -> - ?TT(?SECS(5)), + ?TT(?SECS(30)), tc_try(api_opt_simple_otp_controlling_process, fun() -> api_opt_simple_otp_controlling_process() end). @@ -7796,6 +7810,320 @@ api_opt_simple_otp_controlling_process() -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +%% Tests that the add_mambership and drop_membership ip options work. +%% We create one server and two clients. The server only send messages, +%% the clients only receives messages. +%% An UDP datagram is forbidden (RFC 1122) from having a source address +%% that is a multicast address (or a broadcast address). +%% So, the server create a socket "for sending" and the clients sockets +%% "for receiving". +%% Sending socket: Bound to the local address (and any port). +%% When sending, the dest will be the multicast address +%% and port of the receiving socket. +%% Receiving socket: Bound to the multicast address and port. +api_opt_ip_add_drop_membership(suite) -> + []; +api_opt_ip_add_drop_membership(doc) -> + ["OTP-15908 (ERL-980)"]; +api_opt_ip_add_drop_membership(_Config) when is_list(_Config) -> + ?TT(?SECS(30)), + tc_try(api_opt_ip_add_drop_membership, + fun() -> + has_ip_add_membership_support(), + has_ip_drop_membership_support(), + has_ip_multicast_support() + end, + fun() -> api_opt_ip_add_drop_membership() end). + + +api_opt_ip_add_drop_membership() -> + Set = fun(S, Key, Val) -> + socket:setopt(S, ip, Key, Val) + end, + AddMembership = fun(S, Val) -> Set(S, add_membership, Val) end, + DropMembership = fun(S, Val) -> Set(S, drop_membership, Val) end, + + ServerSeq = + [ + %% *** Wait for start order part *** + #{desc => "await start", + cmd => fun(State) -> + {Tester, MSA} = ?SEV_AWAIT_START(), + {ok, State#{tester => Tester, msa => MSA}} + end}, + #{desc => "monitor tester", + cmd => fun(#{tester := Tester} = _State) -> + _MRef = erlang:monitor(process, Tester), + ok + end}, + + %% *** Init part *** + #{desc => "which local address", + cmd => fun(#{domain := Domain} = State) -> + LSA = which_local_socket_addr(Domain), + {ok, State#{local_sa => LSA}} + end}, + #{desc => "create socket", + cmd => fun(#{domain := Domain} = State) -> + case socket:open(Domain, dgram, udp) of + {ok, Sock} -> + {ok, State#{sock => Sock}}; + {error, _} = ERROR -> + ERROR + end + end}, + #{desc => "make recv socket reuse addr", + cmd => fun(#{sock := Sock} = _State) -> + case socket:setopt(Sock, socket, reuseaddr, true) of + ok -> + ok; + {error, Reason} = ERROR -> + ?SEV_EPRINT("Failed set reuseaddr: " + "~n ~p", [Reason]), + ERROR + end + end}, + #{desc => "bind recv socket to multicast address", + cmd => fun(#{sock := Sock, msa := MSA} = State) -> + case socket:bind(Sock, MSA) of + {ok, Port} -> + ?SEV_IPRINT("bound to:" + "~n ~p", [Port]), + {ok, State#{msa => MSA#{port => Port}}}; + {error, _} = ERROR -> + ERROR + end + end}, + #{desc => "announce ready (init)", + cmd => fun(#{tester := Tester}) -> + ?SEV_ANNOUNCE_READY(Tester, init), + ok + end}, + + %% The actual test + #{desc => "await continue (add_membership)", + cmd => fun(#{tester := Tester} = _State) -> + ?SEV_AWAIT_CONTINUE(Tester, tester, add_membership) + end}, + #{desc => "add membership", + cmd => fun(#{sock := Sock, + msa := #{addr := MAddr}, + local_sa := #{addr := Addr}} = State) -> + MReq = #{multiaddr => MAddr, + interface => Addr}, + ?SEV_IPRINT("try add membership to:" + "~n ~p", [MReq]), + case AddMembership(Sock, MReq) of + ok -> + ?SEV_IPRINT("membership added"), + {ok, State#{mreq => MReq}}; + {error, Reason} = ERROR -> + ?SEV_EPRINT("Failed adding membership to: " + "~n ~p" + "~n Reason: ~p", + [MReq, Reason]), + ERROR + end + end}, + #{desc => "announce ready (add-membership)", + cmd => fun(#{tester := Tester}) -> + ?SEV_ANNOUNCE_READY(Tester, add_membership), + ok + end}, + + #{desc => "await continue (drop_membership)", + cmd => fun(#{tester := Tester} = _State) -> + ?SEV_AWAIT_CONTINUE(Tester, tester, drop_membership) + end}, + #{desc => "drop membership", + cmd => fun(#{sock := Sock, + mreq := MReq} = State) -> + ?SEV_IPRINT("try drop membership from:" + "~n ~p", [MReq]), + case DropMembership(Sock, MReq) of + ok -> + ?SEV_IPRINT("membership dropped"), + {ok, maps:remove(mreq, State)}; + {error, Reason} = ERROR -> + ?SEV_EPRINT("Failed drop membership from: " + "~n ~p" + "~n Reason: ~p", + [MReq, Reason]), + ERROR + end + end}, + #{desc => "announce ready (drop-membership)", + cmd => fun(#{tester := Tester}) -> + ?SEV_ANNOUNCE_READY(Tester, drop_membership), + ok + end}, + + + %% Termination + #{desc => "await terminate (from tester)", + cmd => fun(#{tester := Tester} = State) -> + case ?SEV_AWAIT_TERMINATE(Tester, tester) of + ok -> + {ok, maps:remove(tester, State)}; + {error, _} = ERROR -> + ERROR + end + end}, + #{desc => "close socket", + cmd => fun(#{sock := Sock} = State) -> + socket:close(Sock), + {ok, maps:remove(sock, State)} + end}, + + %% *** We are done *** + ?SEV_FINISH_NORMAL + ], + + + TesterSeq = + [ + %% *** Init part *** + #{desc => "monitor server", + cmd => fun(#{server := Server} = _State) -> + _MRef = erlang:monitor(process, Server), + ok + end}, + + %% Start the server + #{desc => "order server start", + cmd => fun(#{server := Pid, msa := MSA} = _State) -> + ?SEV_ANNOUNCE_START(Pid, MSA), + ok + end}, + #{desc => "await server ready (init)", + cmd => fun(#{server := Pid} = _State) -> + case ?SEV_AWAIT_READY(Pid, server, init) of + ok -> + ok; + {error, Reason} = ERROR -> + ?SEV_EPRINT("Start of server failed: " + "~n ~p", [Reason]), + ERROR + end + end}, + + + %% *** The actual test *** + #{desc => "order server to continue (add-membership)", + cmd => fun(#{server := Server} = _State) -> + ?SEV_ANNOUNCE_CONTINUE(Server, add_membership), + ok + end}, + #{desc => "await server ready (add-membership)", + cmd => fun(#{server := Server} = _State) -> + ?SEV_AWAIT_READY(Server, server, add_membership) + end}, + + #{desc => "order server to continue (drop-membership)", + cmd => fun(#{server := Server} = _State) -> + ?SEV_ANNOUNCE_CONTINUE(Server, drop_membership), + ok + end}, + #{desc => "await server ready (drop-membership)", + cmd => fun(#{server := Server} = _State) -> + ?SEV_AWAIT_READY(Server, server, drop_membership) + end}, + + ?SEV_SLEEP(?SECS(1)), + + %% *** Termination *** + #{desc => "order server terminate", + cmd => fun(#{server := Server} = _State) -> + ?SEV_ANNOUNCE_TERMINATE(Server), + ok + end}, + #{desc => "await server termination", + cmd => fun(#{server := Server} = State) -> + ?SEV_AWAIT_TERMINATION(Server), + {ok, maps:remove(server, State)} + end}, + + %% *** We are done *** + ?SEV_FINISH_NORMAL + ], + + + i("get multicast address"), + Domain = inet, + MAddr = which_ip_multicast_address(), + MSA = #{family => Domain, addr => MAddr}, + + i("start server evaluator"), + ServerInitState = #{domain => Domain}, + Server = ?SEV_START("server", ServerSeq, ServerInitState), + + i("start tester evaluator"), + TesterInitState = #{domain => Domain, + msa => MSA, + server => Server#ev.pid}, + Tester = ?SEV_START("tester", TesterSeq, TesterInitState), + + i("await evaluator(s)"), + ok = ?SEV_AWAIT_FINISH([Tester, Server]). + + + +which_ip_multicast_address() -> + which_multicast_address(inet). + +which_multicast_address(Domain) -> + case os:type() of + {unix, linux} -> + WhichMAddr = fun([_, _, MAddr]) -> MAddr end, + which_multicast_address2(Domain, WhichMAddr); + + {unix, sunos} -> + WhichMAddr = fun([_, MAddr, _]) -> MAddr end, + which_multicast_address2(Domain, WhichMAddr); + + Type -> + not_supported({multicast, Type}) + end. + +%% Note that the 'netstat -g' table looks different on linux and SunOS +%% Linux: IfName - RefCnt - Group +%% SunOS: IfName - Group - RefCnt + +which_multicast_address2(Domain, WhichMAddr) -> + IfName = which_local_host_ifname(Domain), + NetstatGroupsStr = os:cmd("netstat -g | grep " ++ IfName), + NetstatGroups0 = string:tokens(NetstatGroupsStr, [$\n]), + NetstatGroups = [string:tokens(G, [$ ]) || G <- NetstatGroups0], + MAddrs = [WhichMAddr(NetstatGroup) || NetstatGroup <- + NetstatGroups], + which_multicast_address3(Domain, MAddrs). + +which_multicast_address3(_Domain, []) -> + not_supported({multicast, no_valid_addrs}); +which_multicast_address3(Domain, [MAddrStr|MAddrs]) -> + %% Even on linux some of these are not actually addresses, but + %% "host names", such as all-systems.mcast.net. But both + %% address strings, such as "224.0.0.251" and host name strings + %% gets translated into an address by the inet:inet:getaddr/2. + case inet:getaddr(MAddrStr, Domain) of + {ok, MAddr} -> + MAddr; + {error, _} -> + which_multicast_address3(Domain, MAddrs) + end. + +which_local_host_ifname(Domain) -> + case which_local_host_info(Domain) of + {ok, {Name, _Addr, _Flags}} -> + Name; + {error, Reason} -> + not_supported({multicast, Reason}) + end. + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% %% API OPERATIONS WITH TIMEOUT %% @@ -16556,7 +16884,7 @@ traffic_ping_pong_send_and_receive_udp2(InitState) -> LSA = which_local_socket_addr(Domain), {ok, State#{local_sa => LSA}} end}, - #{desc => "create listen socket", + #{desc => "create socket", cmd => fun(#{domain := Domain, proto := Proto} = State) -> case socket:open(Domain, dgram, Proto) of {ok, Sock} -> @@ -24108,70 +24436,115 @@ which_local_socket_addr(local = Domain) -> %% We should really implement this using the (new) net module, %% but until that gets the necessary functionality... which_local_socket_addr(Domain) -> - case inet:getifaddrs() of - {ok, IFL} -> - Addr = which_addr(Domain, IFL), + case which_local_host_info(Domain) of + {ok, {_Name, _Flags, Addr}} -> #{family => Domain, addr => Addr}; {error, Reason} -> - ?FAIL({inet, getifaddrs, Reason}) + ?FAIL(Reason) + end. + + +%% Returns the interface (name), flags and address (not 127...) +%% of the local host. +which_local_host_info(Domain) -> + case inet:getifaddrs() of + {ok, IFL} -> + which_local_host_info(Domain, IFL); + {error, _} = ERROR -> + ERROR end. -which_addr(_Domain, []) -> +which_local_host_info(_Domain, []) -> ?FAIL(no_address); -which_addr(Domain, [{"lo" ++ _, _}|IFL]) -> - which_addr(Domain, IFL); -which_addr(Domain, [{_Name, IFO}|IFL]) -> - case which_addr2(Domain, IFO) of - {ok, Addr} -> - Addr; - {error, no_address} -> - which_addr(Domain, IFL) +which_local_host_info(Domain, [{"lo" ++ _, _}|IFL]) -> + which_local_host_info(Domain, IFL); +which_local_host_info(Domain, [{"docker" ++ _, _}|IFL]) -> + which_local_host_info(Domain, IFL); +which_local_host_info(Domain, [{"br-" ++ _, _}|IFL]) -> + which_local_host_info(Domain, IFL); +which_local_host_info(Domain, [{Name, IFO}|IFL]) -> + case which_local_host_info2(Domain, IFO) of + {ok, {Flags, Addr}} -> + {ok, {Name, Flags, Addr}}; + {error, _} -> + which_local_host_info(Domain, IFL) end; -which_addr(Domain, [_|IFL]) -> - which_addr(Domain, IFL). +which_local_host_info(Domain, [_|IFL]) -> + which_local_host_info(Domain, IFL). -which_addr2(_Domain, []) -> +which_local_host_info2(Domain, IFO) -> + case lists:keysearch(flags, 1, IFO) of + {value, {flags, Flags}} -> + which_local_host_info2(Domain, IFO, Flags); + false -> + {error, no_flags} + end. + +which_local_host_info2(_Domain, [], _Flags) -> {error, no_address}; -which_addr2(inet = _Domain, [{addr, Addr}|_IFO]) +which_local_host_info2(inet = _Domain, [{addr, Addr}|_IFO], Flags) when (size(Addr) =:= 4) andalso (element(1, Addr) =/= 127) -> - {ok, Addr}; -which_addr2(inet6 = _Domain, [{addr, Addr}|_IFO]) + {ok, {Flags, Addr}}; +which_local_host_info2(inet6 = _Domain, [{addr, Addr}|_IFO], Flags) when (size(Addr) =:= 8) andalso (element(1, Addr) =/= 0) andalso (element(1, Addr) =/= 16#fe80) -> - {ok, Addr}; -which_addr2(Domain, [_|IFO]) -> - which_addr2(Domain, IFO). - + {ok, {Flags, Addr}}; +which_local_host_info2(Domain, [_|IFO], Flags) -> + which_local_host_info2(Domain, IFO, Flags). -unlink_path(Path) -> - unlink_path(Path, fun() -> ok end, fun() -> ok end). - -unlink_path(Path, Success, Failure) when is_list(Path) andalso - is_function(Success, 0) andalso - is_function(Failure, 0) -> - ?SEV_IPRINT("try unlink path: " - "~n ~s", [Path]), - case os:cmd("unlink " ++ Path) of - "" -> - ?SEV_IPRINT("path unlinked: " - "~n Path: ~s", [Path]), - Success(); - Result -> - ?SEV_EPRINT("unlink maybe failed: " - "~n Path: ~s" - "~n Res: ~s", [Path, Result]), - Failure() - end; -unlink_path(_, _, _) -> - ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% Here are all the *general* test vase condition functions. +%% Here are all the *general* test case condition functions. + +%% We also need (be able) to figure out the the multicast address, +%% which we only support for some platforms (linux and sunos). +%% We don't do that here, but since we can only do that (find a +%% multicast address) for specific platforms, we check that we are +%% on of those platforms here. +has_ip_multicast_support() -> + case os:type() of + {unix, OsName} when (OsName =:= linux) orelse + (OsName =:= sunos) -> + case which_local_host_info(inet) of + {ok, {_Name, Flags, _Addr}} -> + case lists:member(multicast, Flags) of + true -> + ok; + false -> + not_supported(multicast) + end; + {error, Reason} -> + not_supported({multicast, Reason}) + end; + Type -> + not_supported({multicast, Type}) + end. + +has_ip_add_membership_support() -> + has_socket_option_ip_support(add_membership). + +has_ip_drop_membership_support() -> + has_socket_option_ip_support(drop_membership). + + +has_socket_option_ip_support(Opt) -> + has_socket_option_support(ip, Opt). + +has_socket_option_support(Level, Option) -> + case socket:supports(options, Level, Option) of + true -> + ok; + false -> + not_supported({options, Level, Option}) + end. + + + unix_domain_socket_host_cond() -> unix_domain_socket_host_cond(os:type(), os:version()). @@ -24211,6 +24584,34 @@ has_support_ipv6() -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +unlink_path(Path) -> + unlink_path(Path, fun() -> ok end, fun() -> ok end). + +unlink_path(Path, Success, Failure) when is_list(Path) andalso + is_function(Success, 0) andalso + is_function(Failure, 0) -> + ?SEV_IPRINT("try unlink path: " + "~n ~s", [Path]), + case os:cmd("unlink " ++ Path) of + "" -> + ?SEV_IPRINT("path unlinked: " + "~n Path: ~s", [Path]), + Success(); + Result -> + ?SEV_EPRINT("unlink maybe failed: " + "~n Path: ~s" + "~n Res: ~s", [Path, Result]), + Failure() + end; +unlink_path(_, _, _) -> + ok. + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +not_supported(What) -> + skip({not_supported, What}). + not_yet_implemented() -> skip("not yet implemented"). diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl index 55b1162cfb..b48be3dd04 100644 --- a/erts/emulator/test/system_info_SUITE.erl +++ b/erts/emulator/test/system_info_SUITE.erl @@ -37,10 +37,13 @@ -export([process_count/1, system_version/1, misc_smoke_tests/1, heap_size/1, wordsize/1, memory/1, ets_limit/1, atom_limit/1, + procs_bug/1, ets_count/1, atom_count/1, system_logger/1]). -export([init/1, handle_event/2, handle_call/2]). +-export([init_per_testcase/2, end_per_testcase/2]). + suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap, {minutes, 2}}]. @@ -48,8 +51,20 @@ suite() -> all() -> [process_count, system_version, misc_smoke_tests, ets_count, heap_size, wordsize, memory, ets_limit, atom_limit, atom_count, + procs_bug, system_logger]. + +init_per_testcase(procs_bug, Config) -> + procs_bug(init_per_testcase, Config); +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(procs_bug, Config) -> + procs_bug(end_per_testcase, Config); +end_per_testcase(_, _) -> + ok. + %%% %%% The test cases ------------------------------------------------------------- %%% @@ -654,3 +669,41 @@ handle_call(Msg, State) -> handle_event(Event, State) -> State ! {report_handler, Event}, {ok, State}. + + +%% OTP-15909: Provoke bug that would cause VM crash +%% if doing system_info(procs) when process have queued exit/down signals. +procs_bug(init_per_testcase, Config) -> + %% Use single scheduler and process prio to starve monitoring processes + %% from handling their received DOWN signals. + OldSchedOnline = erlang:system_flag(schedulers_online,1), + [{schedulers_online, OldSchedOnline} | Config]; +procs_bug(end_per_testcase, Config) -> + erlang:system_flag(schedulers_online, + proplists:get_value(schedulers_online, Config)), + ok. + +procs_bug(Config) when is_list(Config) -> + {Monee,_} = spawn_opt(fun () -> receive die -> ok end end, + [monitor,{priority,max}]), + Papa = self(), + Pids = [begin + P = spawn_opt(fun () -> + erlang:monitor(process, Monee), + Papa ! {self(),ready}, + receive "nada" -> no end + end, + [link, {priority,normal}]), + {P, ready} = receive M -> M end, + P + end + || _ <- lists:seq(1,10)], + process_flag(priority,high), + Monee ! die, + {'DOWN',_,process,Monee,normal} = receive M -> M end, + + %% This call did crash VM as Pids have pending DOWN signals. + erlang:system_info(procs), + process_flag(priority,normal), + [begin unlink(P), exit(P, kill) end || P <- Pids], + ok. diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index c793243c13..477a501876 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -1690,9 +1690,9 @@ static char **build_args_from_string(char *string) for(;;) { switch (state) { case Start: - if (!*p) + if (!*p) goto done; - if (argc >= alloced - 1) { /* Make room for extra NULL */ + if (argc >= alloced - 2) { /* Make room for extra NULL and "--" */ argv = erealloc(argv, (alloced += 10) * sizeof(char *)); } cur_s = argc + argv; @@ -1781,11 +1781,14 @@ static char **build_args_from_string(char *string) } } done: - argv[argc] = NULL; /* Sure to be large enough */ if (!argc) { efree(argv); return NULL; } + argv[argc++] = "--"; /* Add a -- separator in order + for different from different environments + to effect each other */ + argv[argc++] = NULL; /* Sure to be large enough */ return argv; #undef ENSURE } @@ -2036,11 +2039,13 @@ initial_argv_massage(int *argc, char ***argv) argv_buf ab = {0}, xab = {0}; int ix, vix, ac; char **av; + char *sep = "--"; struct { int argc; char **argv; } avv[] = {{INT_MAX, NULL}, {INT_MAX, NULL}, {INT_MAX, NULL}, - {INT_MAX, NULL}, {INT_MAX, NULL}, {INT_MAX, NULL}}; + {INT_MAX, NULL}, {INT_MAX, NULL}, + {INT_MAX, NULL}, {INT_MAX, NULL}}; /* * The environment flag containing OTP release is intentionally * undocumented and intended for OTP internal use only. @@ -2060,6 +2065,8 @@ initial_argv_massage(int *argc, char ***argv) if (*argc > 1) { avv[vix].argc = *argc - 1; avv[vix++].argv = &(*argv)[1]; + avv[vix].argc = 1; + avv[vix++].argv = &sep; } av = build_args_from_env("ERL_FLAGS"); diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index 7a5d31b9eb..59de9bdec8 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -292,26 +292,38 @@ if [ "x$GDB" = "x" ]; then valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU_NAME.log" fi fi - if [ "x$VALGRIND_MISC_FLAGS" = "x" ]; then - valgrind_misc_flags="--show-possibly-lost=no --child-silent-after-fork=yes\ - --suppressions=$ROOTDIR/erts/emulator/valgrind/suppress.standard" - else - valgrind_misc_flags="$VALGRIND_MISC_FLAGS" + # Add default flags + vgflags=$VALGRIND_MISC_FLAGS + if [ "x${vgflags#*--show-possibly-lost}" = "x$vgflags" ]; then + vgflags="$vgflags --show-possibly-lost=no" + fi + if [ "x${vgflags#*--child-silent-after-fork}" = "x$vgflags" ]; then + vgflags="$vgflags --child-silent-after-fork=yes" + fi + if [ "x${vgflags#*--suppressions}" = "x$vgflags" ]; then + if [ "x$ERL_TOP" != "x" ]; then + vgflags="$vgflags --suppressions=$ERL_TOP/erts/emulator/valgrind/suppress.standard" + else + echo "No valgrind suppression file found in \$VALGRIND_MISC_FLAGS and \$ERL_TOP not set." + fi fi if which taskset > /dev/null && test -e /proc/cpuinfo; then # We only let valgrind utilize one core with "taskset 1" as it can be very slow # on multiple cores (especially with async threads). Valgrind only run one pthread # at a time anyway so there is no point letting it utilize more than one core. # Use $sched_arg to force all schedulers online to emulate multicore. - taskset1="taskset 1" ncpu=`cat /proc/cpuinfo | grep -w processor | wc -l` + # Choose a random core in order to not collide with any other valgrind + # run on the same machine. + taskset1=$((1 << (`shuf -i 1-$ncpu -n 1` - 1) )) + taskset1="taskset $taskset1" sched_arg="-S$ncpu:$ncpu" else taskset1= sched_arg= fi - exec $taskset1 valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU_NAME $sched_arg $emu_xargs "$@" + exec $taskset1 valgrind $valgrind_xml $valgrind_log $vgflags $BINDIR/$EMU_NAME $sched_arg $emu_xargs "$@" elif [ $run_rr = yes ]; then if [ $1 = replay ]; then diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index 730f0a0c64..8b6abb5336 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -37,7 +37,7 @@ document etp-help % - GDB command toolbox for analyzing core dumps from the % Erlang emulator (BEAM). % -% Should work for 32-bit erts-5.2/R9B, ... +% Should work for 32-bit and 64-bit unix gdb % % The commands are prefixed with: % etp: Acronym for erts-term-print @@ -55,7 +55,8 @@ document etp-help % Special commands for not really terms: % etp-mfa, etp-cp, etp-disasm, % etp-msgq, etpf-msgq, -% etp-stacktrace, etp-stackdump, etpf-stackdump, etp-dictdump +% etp-stacktrace, etp-stacktrace-emu, etp-stackdump, etp-stackdump-emu, +% etpf-stackdump, etp-dictdump % etp-process-info, etp-process-memory-info % etp-port-info, etp-port-state, etp-port-sched-flags % etp-heapdump, etp-offheapdump, etpf-offheapdump, @@ -1452,30 +1453,74 @@ end define etp-stack-preamble set $etp_stack_p = ($arg0)->stop set $etp_stack_end = ($arg0)->hend + if ($arg0)->state.counter & 0x8000 + printf "%%%%%% WARNING: The process is currently running, so c_p->stop will not be correct\r\n" + printf "%%%%%% Consider using %s-emu instead\r\n", $arg1 + end printf "%% Stacktrace (%u)\n", $etp_stack_end-$etp_stack_p if ($arg0)->i != 0 - etp-1 ((Eterm)($arg0)->i) 0 - printf " (I)\n" + printf "I: " + etp ((Eterm)($arg0)->i) + end + if ($arg0)->cp != 0 + printf "cp:" + etp ((Eterm)($arg0)->cp) end +end + +define etp-stack-preamble-emu + set $etp_stack_p = E + set $etp_stack_end = ($arg0)->hend + printf "%% Stacktrace (%u)\n", $etp_stack_end-$etp_stack_p + printf "I: " + etp ((BeamInstr)I) if ($arg0)->cp != 0 - etp-1 ((Eterm)($arg0)->cp) 0 - printf " (cp)\n" + printf "cp: " + etp ((Eterm)($arg0)->cp) end end +define etp-stacktrace-1 + set $etp_stack_stop = (Eterm*)($arg0) + set $etp_stack_send = (Eterm*)($arg1) + set $etp_stack_cnt = 0 + while $etp_stack_stop < $etp_stack_send + if ($etp_stack_stop[0] & 0x3) == 0x0 + # Continuation pointer + printf "%d: ", $etp_stack_cnt + etp $etp_stack_stop[0] + end + set $etp_stack_stop++ + set $etp_stack_cnt++ + end +end + +define etp-stacktrace-emu +# Args: Process* +# +# Non-reentrant +# + etp-stack-preamble-emu ($arg0) + etp-stacktrace-1 $etp_stack_p $etp_stack_end +end + +document etp-stacktrace-emu +%--------------------------------------------------------------------------- +% etp-stacktrace-emu Process* +% +% Take an Process* and print a stactrace for the process. +% This macro assumes that the current frame is the process_main frame +% and that E is not optimized out. +%--------------------------------------------------------------------------- +end + define etp-stacktrace # Args: Process* # # Non-reentrant # - etp-stack-preamble ($arg0) - while $etp_stack_p < $etp_stack_end - if ($etp_stack_p[0] & 0x3) == 0x0 - # Continuation pointer - etp $etp_stack_p[0] - end - set $etp_stack_p++ - end + etp-stack-preamble ($arg0) "etp-stacktrace" + etp-stacktrace-1 $etp_stack_p $etp_stack_end end document etp-stacktrace @@ -1488,16 +1533,48 @@ document etp-stacktrace %--------------------------------------------------------------------------- end +define etp-stackdump-1 + # Args: Eterm *stop, Eterm *hend + # + # Non-reentrant + # + set $etp_stackdump_stop = (Eterm*)($arg0) + set $etp_stackdump_send = (Eterm*)($arg1) + set $etp_stackdump_cnt = 0 + while $etp_stackdump_stop < $etp_stackdump_send + printf "%d: ", $etp_stackdump_cnt + etp $etp_stackdump_stop[0] + set $etp_stackdump_stop++ + set $etp_stackdump_cnt++ + end +end + +define etp-stackdump-emu +# Args: Process* +# +# Non-reentrant +# + etp-stack-preamble-emu ($arg0) + etp-stackdump-1 $etp_stack_p $etp_stack_end +end + +document etp-stacktrace-emu +%--------------------------------------------------------------------------- +% etp-stacktrace-emu Process* +% +% Take an Process* and print a stactdump for the process. +% This macro assumes that the current frame is the process_main frame +% and that E is not optimized out. +%--------------------------------------------------------------------------- +end + define etp-stackdump # Args: Process* # # Non-reentrant # - etp-stack-preamble ($arg0) - while $etp_stack_p < $etp_stack_end - etp $etp_stack_p[0] - set $etp_stack_p++ - end + etp-stack-preamble ($arg0) "etp-stackdump" + etp-stackdump-1 $etp_stack_p $etp_stack_end end document etp-stackdump @@ -2959,12 +3036,21 @@ define etp-fds end end +document etp-fds +%--------------------------------------------------------------------------- +% etp-fds +% +% Print the state of the fds currently in check_io. Only works in running systems. +%--------------------------------------------------------------------------- +end + define etp-disasm-1 - set $code_ptr = ((BeamInstr*)$arg0) - set $addr = *$code_ptr + set $code_ptr = ((BeamInstr*)($arg0)) + set $addr32 = (BeamInstr)(Uint32)*$code_ptr + set $addr64 = (BeamInstr)(Uint64)*$code_ptr set $i = 0 - while $i < (sizeof(opc) / sizeof(OpEntry)) - if $addr == beam_ops[$i] + while $i < num_instructions + if $addr32 == beam_ops[$i] || $addr64 == beam_ops[$i] printf "%s %d", opc[$i].name, opc[$i].sz set $next_i = $code_ptr + opc[$i].sz set $i += 4999 @@ -2974,6 +3060,11 @@ define etp-disasm-1 end define etp-disasm + if $argc == 1 + set $code_end = $arg0 + else + set $code_end = $arg1 + end etp-cp-func-info-1 $arg0 if $etp_cp_p == 0 printf "invalid argument" @@ -2982,7 +3073,7 @@ define etp-disasm printf ": " etp-disasm-1 $arg0 printf "\r\n" - while $next_i < ((BeamInstr*)$arg1) + while $next_i < ((BeamInstr*)$code_end) set $prev_i = $next_i etp-cp-func-info-1 $next_i etp-mfa-1 $etp_cp_p $cp_cp_p_offset @@ -2998,6 +3089,16 @@ define etp-disasm end end +document etp-disasm +%--------------------------------------------------------------------------- +% etp-fds BeamInstr* (BeamInstr*) +% +% Disassemble the instructions inbetween arg0 and arg1, +% if no second argument is given only the current +% instruction is printed. +%--------------------------------------------------------------------------- +end + ############################################################################ # # Timer Wheel diff --git a/erts/preloaded/ebin/erl_init.beam b/erts/preloaded/ebin/erl_init.beam Binary files differindex 0c032e8e91..bc7639781c 100644 --- a/erts/preloaded/ebin/erl_init.beam +++ b/erts/preloaded/ebin/erl_init.beam diff --git a/erts/preloaded/ebin/net.beam b/erts/preloaded/ebin/net.beam Binary files differdeleted file mode 100644 index 88d546a3af..0000000000 --- a/erts/preloaded/ebin/net.beam +++ /dev/null diff --git a/erts/preloaded/ebin/prim_net.beam b/erts/preloaded/ebin/prim_net.beam Binary files differnew file mode 100644 index 0000000000..9d50b3210f --- /dev/null +++ b/erts/preloaded/ebin/prim_net.beam diff --git a/erts/preloaded/src/Makefile b/erts/preloaded/src/Makefile index 27d450c873..e1bb2ee5c4 100644 --- a/erts/preloaded/src/Makefile +++ b/erts/preloaded/src/Makefile @@ -36,10 +36,9 @@ include $(ERL_TOP)/lib/kernel/vsn.mk ifeq ($(USE_ESOCK), yes) PRE_LOADED_ERL_ESOCK_MODULES = \ socket \ - net + prim_net else -PRE_LOADED_ERL_ESOCK_MODULES = \ - net +PRE_LOADED_ERL_ESOCK_MODULES = endif PRE_LOADED_ERL_MODULES = \ @@ -82,9 +81,9 @@ APP_FILE= erts.app APP_SRC= $(APP_FILE).src APP_TARGET= $(STATIC_EBIN)/$(APP_FILE) ifeq ($(USE_ESOCK), yes) -APP_ESOCK_MODS= net, socket +APP_ESOCK_MODS= prim_net, socket else -APP_ESOCK_MODS= net +APP_ESOCK_MODS= endif diff --git a/erts/preloaded/src/erl_init.erl b/erts/preloaded/src/erl_init.erl index d209c4033b..dadf7dda6f 100644 --- a/erts/preloaded/src/erl_init.erl +++ b/erts/preloaded/src/erl_init.erl @@ -35,7 +35,8 @@ start(Mod, BootArgs) -> erl_tracer:on_load(), prim_buffer:on_load(), prim_file:on_load(), - conditional_load(socket, [socket, net]), % socket:on_load(), net:on_load(), + %% socket:on_load(), prim_net:on_load(), + conditional_load(socket, [socket, prim_net]), %% Proceed to the specified boot module run(Mod, boot, BootArgs). @@ -49,7 +50,9 @@ run(M, F, A) -> end. conditional_load(CondMod, Mods2Load) -> - conditional_load(CondMod, erlang:loaded(), Mods2Load). + Loaded = erlang:loaded(), + %% erlang:display({?MODULE, conditional_load, Loaded}), + conditional_load(CondMod, Loaded, Mods2Load). conditional_load(_CondMod, [], _Mods2LOad) -> ok; diff --git a/erts/preloaded/src/net.erl b/erts/preloaded/src/prim_net.erl index 13d2e3a117..107043d1aa 100644 --- a/erts/preloaded/src/net.erl +++ b/erts/preloaded/src/prim_net.erl @@ -18,7 +18,7 @@ %% %CopyrightEnd% %% --module(net). +-module(prim_net). -compile(no_native). @@ -31,22 +31,14 @@ -export([ gethostname/0, - getnameinfo/1, getnameinfo/2, - getaddrinfo/1, getaddrinfo/2, + getnameinfo/2, + getaddrinfo/2, if_name2index/1, if_index2name/1, if_names/0 ]). -%% Deprecated functions from the "old" net module --export([call/4, - cast/4, - broadcast/3, - ping/1, - relay/1, - sleep/1]). - -export_type([ address_info/0, name_info/0, @@ -59,13 +51,6 @@ network_interface_index/0 ]). --deprecated({call, 4, eventually}). --deprecated({cast, 4, eventually}). --deprecated({broadcast, 3, eventually}). --deprecated({ping, 1, eventually}). --deprecated({relay, 1, eventually}). --deprecated({sleep, 1, eventually}). - -type name_info_flags() :: [name_info_flag()|name_info_flag_ext()]. -type name_info_flag() :: namereqd | @@ -88,21 +73,6 @@ %% =========================================================================== %% -%% D E P R E C A T E D F U N C T I O N S -%% -%% =========================================================================== - -call(N,M,F,A) -> rpc:call(N,M,F,A). -cast(N,M,F,A) -> rpc:cast(N,M,F,A). -broadcast(M,F,A) -> rpc:eval_everywhere(M,F,A). -ping(Node) -> net_adm:ping(Node). -sleep(T) -> receive after T -> ok end. -relay(X) -> slave:relay(X). - - - -%% =========================================================================== -%% %% Administrative and utility API %% %% =========================================================================== @@ -117,7 +87,7 @@ on_load() -> Extra :: map(). on_load(Extra) -> - ok = erlang:load_nif(atom_to_list(?MODULE), Extra). + ok = erlang:load_nif(atom_to_list(net), Extra). -spec info() -> list(). @@ -159,14 +129,6 @@ gethostname() -> %% %% --spec getnameinfo(SockAddr) -> {ok, Info} | {error, Reason} when - SockAddr :: socket:sockaddr(), - Info :: name_info(), - Reason :: term(). - -getnameinfo(SockAddr) -> - getnameinfo(SockAddr, undefined). - -spec getnameinfo(SockAddr, Flags) -> {ok, Info} | {error, Reason} when SockAddr :: socket:sockaddr(), Flags :: name_info_flags() | undefined, @@ -178,44 +140,18 @@ getnameinfo(SockAddr, [] = _Flags) -> getnameinfo(#{family := Fam, addr := _Addr} = SockAddr, Flags) when ((Fam =:= inet) orelse (Fam =:= inet6)) andalso (is_list(Flags) orelse (Flags =:= undefined)) -> - nif_getnameinfo((catch ensure_sockaddr(SockAddr)), Flags); + nif_getnameinfo(socket:ensure_sockaddr(SockAddr), Flags); getnameinfo(#{family := Fam, path := _Path} = SockAddr, Flags) when (Fam =:= local) andalso (is_list(Flags) orelse (Flags =:= undefined)) -> nif_getnameinfo(SockAddr, Flags). -%% This function is intended to "handle" the case when the user -%% has built their (OTP) system with "--disable-esock". -%% That means the socket module does not exist. This is not really -%% a problem since the nif_getnameinfo won't work either (since -%% the nif file is not part of the system). The result of calling -%% getnameinfo will be a undef exception (erlang:nif_error(undef)). -%% -%% The only functions in this module that actually work in this case -%% (--disable-esock) is the depricated stuff (call, cast, ...). -%% -ensure_sockaddr(SockAddr) -> - try socket:ensure_sockaddr(SockAddr) - catch - error:undef:_ -> - undefined - end. - %% =========================================================================== %% %% getaddrinfo - Network address and service translation %% %% There is also a "hint" argument that we "at some point" should implement. --spec getaddrinfo(Host) -> {ok, Info} | {error, Reason} when - Host :: string(), - Info :: [address_info()], - Reason :: term(). - -getaddrinfo(Host) when is_list(Host) -> - getaddrinfo(Host, undefined). - - -spec getaddrinfo(Host, undefined) -> {ok, Info} | {error, Reason} when Host :: string(), Info :: [address_info()], diff --git a/erts/test/erlexec_SUITE.erl b/erts/test/erlexec_SUITE.erl index 602dc5ce2e..952e6da4dc 100644 --- a/erts/test/erlexec_SUITE.erl +++ b/erts/test/erlexec_SUITE.erl @@ -30,7 +30,7 @@ -export([all/0, suite/0, init_per_testcase/2, end_per_testcase/2]). -export([args_file/1, evil_args_file/1, env/1, args_file_env/1, - otp_7461/1, otp_7461_remote/1, otp_8209/1, + otp_7461/1, otp_7461_remote/1, argument_separation/1, zdbbl_dist_buf_busy_limit/1]). -include_lib("common_test/include/ct.hrl"). @@ -51,21 +51,28 @@ suite() -> all() -> [args_file, evil_args_file, env, args_file_env, - otp_7461, otp_8209, zdbbl_dist_buf_busy_limit]. + otp_7461, argument_separation, zdbbl_dist_buf_busy_limit]. %% Test that plain first argument does not -%% destroy -home switch [OTP-8209] -otp_8209(Config) when is_list(Config) -> +%% destroy -home switch [OTP-8209] or interact with environments +argument_separation(Config) when is_list(Config) -> {ok,[[PName]]} = init:get_argument(progname), SNameS = "erlexec_test_01", SName = list_to_atom(SNameS++"@"++ hd(tl(string:lexemes(atom_to_list(node()),"@")))), - Cmd = PName ++ " dummy_param -sname "++SNameS++" -setcookie "++ - atom_to_list(erlang:get_cookie()), - open_port({spawn,Cmd},[]), + Cmd = PName ++ " cmd_param -sname "++SNameS++" -setcookie "++ + atom_to_list(erlang:get_cookie()) ++ " -cmd_test", + open_port({spawn,Cmd},[{env,[{"ERL_AFLAGS","-atest"}, + {"ERL_FLAGS","env_param -test"}, + {"ERL_ZFLAGS","zenv_param"}]}]), pong = loop_ping(SName,40), + ct:log("emu_args: ~p",[rpc:call(SName,erlang,system_info,[emu_args])]), {ok,[[_]]} = rpc:call(SName,init,get_argument,[home]), - ["dummy_param"] = rpc:call(SName,init,get_plain_arguments,[]), + {ok,[[]]} = rpc:call(SName,init,get_argument,[atest]), + {ok,[[]]} = rpc:call(SName,init,get_argument,[cmd_test]), + {ok,[[]]} = rpc:call(SName,init,get_argument,[test]), + error = rpc:call(SName,init,get_argument,[unkown]), + ["cmd_param","env_param","zenv_param"] = rpc:call(SName,init,get_plain_arguments,[]), ok = cleanup_nodes(), ok. @@ -85,6 +92,7 @@ cleanup_node(SNameS,N) -> end. loop_ping(_,0) -> + flush(), pang; loop_ping(Node,N) -> case net_adm:ping(Node) of @@ -98,6 +106,14 @@ loop_ping(Node,N) -> pong end. +flush() -> + receive M -> + ct:pal("~p",[M]), + flush() + after 10 -> + ok + end. + args_file(Config) when is_list(Config) -> AFN1 = privfile("1", Config), AFN2 = privfile("2", Config), diff --git a/erts/vsn.mk b/erts/vsn.mk index 40a9685e9d..f06fd08540 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% # -VSN = 10.4.3 +VSN = 10.4.4 # Port number 4365 in 4.2 # Port number 4366 in 4.3 |