diff options
186 files changed, 4896 insertions, 1331 deletions
diff --git a/.gitignore b/.gitignore index 0e9d07757f..b90bda1763 100644 --- a/.gitignore +++ b/.gitignore @@ -139,6 +139,7 @@ JAVADOC-GENERATED /make/output.mk /make/emd2exml /make/make_emakefile +/make/install_dir_data.sh # Created by "out_build update_primary" /bootstrap/primary_compiler/ @@ -307,6 +308,7 @@ JAVADOC-GENERATED /lib/jinterface/doc/html/java /lib/jinterface/pom.xml /lib/jinterface/target +/lib/jinterface/doc/src/jdoc # kernel @@ -361,7 +363,6 @@ JAVADOC-GENERATED /system/doc/html /system/doc/xml /system/doc/top/PR.template -/system/doc/top/erlresolvelinks.js # test_server diff --git a/Makefile.in b/Makefile.in index fa7c128379..749cd27f9b 100644 --- a/Makefile.in +++ b/Makefile.in @@ -434,7 +434,7 @@ endif PATH=$(BOOT_PREFIX)"$${PATH}" \ ERL_TOP=$(ERL_TOP) $(MAKE) TESTROOT="$(RELEASE_ROOT)" DOCGEN=$(BOOTSTRAP_ROOT)/bootstrap/lib/erl_docgen $@ ifneq ($(OTP_SMALL_BUILD),true) - echo "OTP doc built" > $(ERL_TOP)/make/otp_doc_built + test -f $(ERL_TOP)/make/otp_doc_built || echo "OTP doc built" > $(ERL_TOP)/make/otp_doc_built endif xmllint: docs @@ -450,7 +450,9 @@ else $(MAKE) -C system/doc $@ endif -mod2app: +mod2app: $(ERL_TOP)/make/$(TARGET)/mod2app.xml + +$(ERL_TOP)/make/$(TARGET)/mod2app.xml: erts/doc/src/Makefile lib/*/doc/src/Makefile PATH=$(BOOT_PREFIX)"$${PATH}" escript $(BOOTSTRAP_ROOT)/bootstrap/lib/erl_docgen/priv/bin/xref_mod_app.escript -topdir $(ERL_TOP) -outfile $(ERL_TOP)/make/$(TARGET)/mod2app.xml # ---------------------------------------------------------------------- @@ -488,7 +490,7 @@ else $(make_verbose)cd lib && \ ERL_TOP=$(ERL_TOP) PATH=$(BOOT_PREFIX)"$${PATH}" \ $(MAKE) opt BUILD_ALL=true - $(V_at)echo "OTP built" > $(ERL_TOP)/make/otp_built + $(V_at)test -f $(ERL_TOP)/make/otp_built || echo "OTP built" > $(ERL_TOP)/make/otp_built endif kernel: $(make_verbose)cd lib/kernel && \ diff --git a/OTP_VERSION b/OTP_VERSION index cd23cb3bd7..0fac3b01f5 100644 --- a/OTP_VERSION +++ b/OTP_VERSION @@ -1 +1 @@ -21.2.6 +21.3.1 diff --git a/erts/autoconf/vxworks/sed.general b/erts/autoconf/vxworks/sed.general index 0e99b4dba4..d32fbdc5c0 100644 --- a/erts/autoconf/vxworks/sed.general +++ b/erts/autoconf/vxworks/sed.general @@ -103,7 +103,6 @@ s|@INSTALL_PROGRAM@|${INSTALL}| s|@INSTALL_SCRIPT@|${INSTALL}| s|@INSTALL_DATA@|${INSTALL} -m 644| s|@INSTALL_DIR@|$(INSTALL) -d| -s|@RM@|/bin/rm| s|@MKDIR@|/bin/mkdir| s|@ERLANG_OSTYPE@|vxworks| s|@vxworks_reclaim@|reclaim.h| diff --git a/erts/configure.in b/erts/configure.in index 9245e4dc90..3ba8216a19 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -792,11 +792,6 @@ AC_SUBST(LIBCARBON) _search_path=/bin:/usr/bin:/usr/local/bin:$PATH -AC_PATH_PROG(RM, rm, false, $_search_path) -if test "$ac_cv_path_RM" = false; then - AC_MSG_ERROR([No 'rm' command found]) -fi - AC_PATH_PROG(MKDIR, mkdir, false, $_search_path) if test "$ac_cv_path_MKDIR" = false; then AC_MSG_ERROR([No 'mkdir' command found]) @@ -811,9 +806,9 @@ _search_path= # Remove old configuration information. -# Next line should be placed after AC_PATH_PROG(RM, ...), but before -# first output to CONN_INFO. So this is just the right place. -$RM -f "$ERL_TOP/erts/CONF_INFO" +# Next line should be before first output to CONN_INFO. So this is +# just the right place. +rm -f "$ERL_TOP/erts/CONF_INFO" dnl Check if we should/can build a sharing-preserving emulator AC_MSG_CHECKING(if we are building a sharing-preserving emulator) @@ -851,7 +846,7 @@ fi ## Delete previous failed configure results if test -f doc/CONF_INFO; then - $RM doc/CONF_INFO + rm -f doc/CONF_INFO fi AC_CHECK_PROGS(XSLTPROC, xsltproc) @@ -3288,14 +3283,14 @@ if test "$enable_dtrace_test" = "yes" ; then AC_MSG_CHECKING([for 2-stage DTrace precompilation]) AC_TRY_COMPILE([ #include "foo-dtrace.h" ], [ERLANG_DIST_PORT_BUSY_ENABLED();], - [$RM -f $DTRACE_2STEP_TEST + [rm -f $DTRACE_2STEP_TEST dtrace -G $DTRACE_CPP $DTRACE_BITS_FLAG -Iemulator/beam -o $DTRACE_2STEP_TEST -s emulator/beam/erlang_dtrace.d conftest.$OBJEXT 2>&AS_MESSAGE_LOG_FD if test -f $DTRACE_2STEP_TEST; then - $RM $DTRACE_2STEP_TEST + rm -f $DTRACE_2STEP_TEST DTRACE_ENABLED_2STEP=yes fi], []) - $RM -f foo-dtrace.h + rm -f foo-dtrace.h AS_IF([test "x$DTRACE_ENABLED_2STEP" = "xyes"], [AC_MSG_RESULT([yes])], [AC_MSG_RESULT([no])]) @@ -3499,7 +3494,7 @@ ssl_done=yes # Default only one run # Remove all SKIP files from previous runs for a in ssl crypto ssh; do - $RM -f $ERL_TOP/lib/$a/SKIP + rm -f $ERL_TOP/lib/$a/SKIP done SSL_DYNAMIC_ONLY=$enable_dynamic_ssl @@ -4120,7 +4115,7 @@ need_java="jinterface ic/java_src" # Remove all SKIP files from previous runs for a in $need_java ; do - $RM -f $ERL_TOP/lib/$a/SKIP + rm -f $ERL_TOP/lib/$a/SKIP done if test "X$with_javac" = "Xno"; then @@ -4171,7 +4166,7 @@ dnl this deliberately does not believe that 'gcc' is a C++ compiler AC_CHECK_TOOLS(CXX, [$CCC c++ g++ CC cxx cc++ cl], false) # Remove SKIP file from previous run -$RM -f $ERL_TOP/lib/orber/SKIP +rm -f $ERL_TOP/lib/orber/SKIP if test "$CXX" = false; then echo "No C++ compiler found" > $ERL_TOP/lib/orber/SKIP @@ -4326,7 +4321,6 @@ AC_CONFIG_FILES([../make/make_emakefile:../make/make_emakefile.in], dnl dnl The ones below should be moved to their respective lib dnl -dnl ../lib/ssl/c_src/$host/Makefile:../lib/ssl/c_src/Makefile.in AC_CONFIG_FILES([ ../lib/os_mon/c_src/$host/Makefile:../lib/os_mon/c_src/Makefile.in ../lib/crypto/c_src/$host/Makefile:../lib/crypto/c_src/Makefile.in @@ -4334,4 +4328,6 @@ AC_CONFIG_FILES([ ../lib/tools/c_src/$host/Makefile:../lib/tools/c_src/Makefile.in ]) +AC_CONFIG_FILES([../make/install_dir_data.sh:../make/install_dir_data.sh.in], [chmod +x ../make/install_dir_data.sh]) + AC_OUTPUT diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 9ea1beb887..5c45f8d83b 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -31,6 +31,222 @@ </header> <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 10.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>When multiplying a number by itself, a word beyond the + number on the heap could be read (and ignored). This bug + was extremely unlikely to actually cause a real + problem.</p> + <p> + Own Id: OTP-15484</p> + </item> + <item> + <p> + Fix bug where doing <c>seq_trace:reset_trace()</c> while + another process was doing a garbage collection could + cause the run-time system to segfault.</p> + <p> + Own Id: OTP-15490</p> + </item> + <item> + <p> + Fix reading of ancillary data from packet oriented + sockets on old Linux kernel versions. Without this fix, + getting the data would cause the port to enter an + infinite loop.</p> + <p> + Own Id: OTP-15494</p> + </item> + <item> + <p> + Fix bug where crash dumping or doing + <c>erlang:system_info(procs)</c> while another process + was doing a garbage collection could cause the run-time + system to segfault.</p> + <p> + Own Id: OTP-15527</p> + </item> + <item> + <p> + Fix <c>erlang:system_info(kernel_poll)</c> to return + correct value. Before this fix, the call always returned + <c>false</c>.</p> + <p> + Own Id: OTP-15556</p> + </item> + <item> + <p> + Fix bug in <c>enif_make_map_from_arrays</c> that would + produce broken maps when number of keys were 32. Bug + exists since OTP 21.0.</p> + <p> + Own Id: OTP-15567</p> + </item> + <item> + <p> + Fix a bug in <c>binary:encode_unsigned</c> that may cause + a read of uninitialized memory.</p> + <p> + The bug existed since the function was added (OTP + R16B02).</p> + <p> + Own Id: OTP-15583 Aux Id: PR-2118 </p> + </item> + <item> + <p> + Fixed a bug that could cause <c>heart</c> to kill an + exiting node before it had time to flush all buffered + writes. If environment variable + <c>HEART_KILL_SIGNAL=SIGABRT</c> was set a superfluous + core dump could also be generated.</p> + <p> + Own Id: OTP-15599 Aux Id: ERIERL-298 </p> + </item> + <item> + <p> + Fix <c>enif_consume_timeslice</c> to be a no-op on dirty + scheduler and not crash debug compiled emulator.</p> + <p> + Own Id: OTP-15604</p> + </item> + <item> + <p> + Fixed macro redefinition warnings.</p> + <p> + Own Id: OTP-15629</p> + </item> + <item> + <p> + <c>to_erl</c> fixed to not garble terminal input beyond + 7-bit ASCII.</p> + <p> + Own Id: OTP-15650 Aux Id: ERL-854, PR-2161 </p> + </item> + <item> + <p> + Minor fixes for <c>make clean</c>.</p> + <p> + Own Id: OTP-15657</p> + </item> + <item> + <p> + Fixed a bug in all <c>ets:select*</c> and + <c>ets:match*</c> functions that could in some rare cases + lead to very poor performance.</p> + <p> + Own Id: OTP-15660 Aux Id: ERL-869 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add <c>erlang:system_flag(system_logger, Pid)</c> and + <c>erlang:system_info(system_logger)</c>. This + system_flag can be used to set the process that will + receive the logging messages generated by ERTS.</p> + <p> + Own Id: OTP-15375</p> + </item> + <item> + <p><c>integer_to_list/2</c> and + <c>integer_to_binary/2</c> are now implemented in C, + improving their performance.</p> + <p> + Own Id: OTP-15503 Aux Id: PR-2052 </p> + </item> + <item> + <p> + Improved <c>term_to_binary</c> to do more fair reduction + count and yielding when encoding large byte lists + (strings).</p> + <p> + Own Id: OTP-15514 Aux Id: ERL-774 </p> + </item> + <item> + <p> + Made internal port drivers more robust against + <c>erlang:port_control</c> with invalid arguments and + added documentation warnings about such abuse.</p> + <p> + Own Id: OTP-15555 Aux Id: ERIERL-231 </p> + </item> + <item> + <p> + Fix bug on NetBSD where the <c>exit_status</c> from a + port program would never be sent.</p> + <p> + Own Id: OTP-15558 Aux Id: ERL-725 </p> + </item> + <item> + <p>There is a new function <c>persistent:term(Key, + Default)</c> to allow specifying a default when looking + up a persistent term.</p> + <p> + Own Id: OTP-15576 Aux Id: ERL-843 </p> + </item> + <item> + <p>A transitory emulator option '<c>+ztma true</c>' has + been added to allow running existing BEAM code that + relies on "tuple calls" (dispatch on parameterized + modules) which has been compiled under OTP 20 or earlier. + This option will be removed in OTP 22, so such modules + should eventually be recompiled with the + <c>+tuple_calls</c> option.</p> + <p> + Own Id: OTP-15580 Aux Id: PR-2113 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 10.2.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixes of install/release phase in build system.</p> + <list> <item>The source tree was modified when + installing/releasing and/or applying a patch.</item> + <item>Some files were installed with wrong access + rights.</item> <item>If applying a patch (using + <c>otp_patch_apply</c>) as another user (except root) + than the user that built the source, the documentation + was not properly updated.</item> </list> + <p> + Own Id: OTP-15551</p> + </item> + <item> + <p> + Setting the <c>recbuf</c> size of an inet socket the + <c>buffer</c> is also automatically increased. Fix a bug + where the auto adjustment of inet buffer size would be + triggered even if an explicit inet buffer size had + already been set.</p> + <p> + Own Id: OTP-15651 Aux Id: ERIERL-304 </p> + </item> + <item> + <p> + Reading from UDP using active <c>true</c> or active + <c>N</c> mode has been optimized when more packets than + specified by <c>read_packets</c> are available on the + socket.</p> + <p> + Own Id: OTP-15652 Aux Id: ERIERL-304 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 10.2.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 57a9d45887..e7648f2396 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -266,7 +266,6 @@ DEXPORT = @DEXPORT@ RANLIB = @RANLIB@ STRIP = strip PERL = @PERL@ -RM = @RM@ MKDIR = @MKDIR@ USING_MINGW=@MIXED_CYGWIN_MINGW@ @@ -467,13 +466,13 @@ $(ERTS_LIB): .PHONY: clean clean: - $(RM) -f $(GENERATE) - $(RM) -rf $(TARGET)/*.c $(TARGET)/*.h $(TARGET)/*-GENERATED - $(RM) -rf $(TARGET)/*/* - $(RM) -rf obj/$(TARGET) - $(RM) -rf pcre/obj/$(TARGET) $(PCRE_GENINC) - $(RM) -rf zlib/obj/$(TARGET) - $(RM) -rf bin/$(TARGET) + $(RM) $(GENERATE) + $(RM) -r $(TARGET)/*.c $(TARGET)/*.h $(TARGET)/*-GENERATED + $(RM) -r $(TARGET)/*/* + $(RM) -r obj/$(TARGET) + $(RM) -r pcre/obj/$(TARGET) $(PCRE_GENINC) + $(RM) -r zlib/obj/$(TARGET) + $(RM) -r bin/$(TARGET) cd $(ERTS_LIB_DIR) && $(MAKE) clean .PHONY: docs diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index f6a98c85fd..16c06766fb 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -4674,6 +4674,16 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(am_notsup); #endif } + else if (ERTS_IS_ATOM_STR("mbuf", BIF_ARG_1)) { + Uint sz = size_object(BIF_ARG_2); + ErlHeapFragment* frag = new_message_buffer(sz); + Eterm *hp = frag->mem; + Eterm copy = copy_struct(BIF_ARG_2, sz, &hp, &frag->off_heap); + frag->next = BIF_P->mbuf; + BIF_P->mbuf = frag; + BIF_P->mbuf_sz += sz; + BIF_RET(copy); + } } BIF_ERROR(BIF_P, BADARG); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 752d3ae3a8..42d7909a08 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1323,11 +1323,7 @@ static int match_traverse(Process* p, DbTableHash* tb, unlock_hash_function(lck); break; } - if (iterations_left <= 0 || MBUF(p)) { - /* - * We have either reached our limit, or just created some heap fragments. - * Since many heap fragments will make the GC slower, trap and GC now. - */ + if (iterations_left <= 0) { unlock_hash_function(lck); ret_value = ctx->on_trap(ctx, slot_ix, got, &mpi.mp, ret); goto done; @@ -1433,11 +1429,7 @@ static int match_traverse_continue(Process* p, DbTableHash* tb, unlock_hash_function(lck); break; } - if (iterations_left <= 0 || MBUF(p)) { - /* - * We have either reached our limit, or just created some heap fragments. - * Since many heap fragments will make the GC slower, trap and GC now. - */ + if (iterations_left <= 0) { unlock_hash_function(lck); ret_value = ctx->on_trap(ctx, slot_ix, got, mpp, ret); goto done; diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 45e4be2426..8c5fc0acb2 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -3310,13 +3310,6 @@ static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr, if (is_value(ret)) { sc->accum = CONS(hp, ret, sc->accum); } - if (MBUF(sc->p)) { - /* - * Force a trap and GC if a heap fragment was created. Many heap fragments - * make the GC slow. - */ - sc->max = 0; - } if (--(sc->max) <= 0) { return 0; } @@ -3371,13 +3364,6 @@ static int doit_select_chunk(DbTableTree *tb, TreeDbTerm *this, void *ptr, ++(sc->got); sc->accum = CONS(hp, ret, sc->accum); } - if (MBUF(sc->p)) { - /* - * Force a trap and GC if a heap fragment was created. Many heap fragments - * make the GC slow. - */ - sc->max = 0; - } if (--(sc->max) <= 0 || sc->got == sc->chunk_size) { return 0; } diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index a231638b50..c5deed38ad 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1946,7 +1946,7 @@ do_allocate_logger_message(Eterm gleader, ErtsMonotonicTime *ts, Eterm *pid, else sz += MAP4_SZ /* metadata map w gl w pid*/; - *ts = ERTS_MONOTONIC_TO_USEC(erts_get_monotonic_time(NULL) + erts_get_time_offset()); + *ts = ERTS_MONOTONIC_TO_USEC(erts_os_system_time()); erts_bld_sint64(NULL, &sz, *ts); *bp = new_message_buffer(sz); diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index a6a5c20266..b71ce0389d 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -574,7 +574,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) driver_select(port, e, mode | (on?ERL_DRV_USE:0), on) #define sock_select(d, flags, onoff) do { \ - ASSERT(!(d)->is_ignored); \ + ASSERT(!INET_IGNORED(d)); \ (d)->event_mask = (onoff) ? \ ((d)->event_mask | (flags)) : \ ((d)->event_mask & ~(flags)); \ @@ -898,6 +898,15 @@ static size_t my_strnlen(const char *s, size_t maxlen) #define INET_CMSG_RECVTCLASS (1 << 1) /* am_recvtclass, am_tclass */ #define INET_CMSG_RECVTTL (1 << 2) /* am_recvttl, am_ttl */ +/* Inet flags */ +#define INET_FLG_BUFFER_SET (1 << 0) /* am_buffer has been set by user */ +#define INET_FLG_IS_IGNORED (1 << 1) /* If a fd is ignored by the inet_drv. + This flag should be set to true when + the fd is used outside of inet_drv. */ +#define INET_FLG_IS_IGNORED_RD (1 << 2) +#define INET_FLG_IS_IGNORED_WR (1 << 3) +#define INET_FLG_IS_IGNORED_PASS (1 << 4) + /* ** End of interface constants. **--------------------------------------------------------------------------*/ @@ -943,10 +952,11 @@ static size_t my_strnlen(const char *s, size_t maxlen) #define INET_IFNAMSIZ 16 /* INET Ignore states */ -#define INET_IGNORE_NONE 0 -#define INET_IGNORE_READ (1 << 0) -#define INET_IGNORE_WRITE (1 << 1) -#define INET_IGNORE_PASSIVE (1 << 2) +#define INET_IGNORE_CLEAR(desc) ((desc)->flags & ~(INET_IGNORE_READ|INET_IGNORE_WRITE|INET_IGNORE_PASSIVE)) +#define INET_IGNORED(desc) ((desc)->flags & INET_FLG_IS_IGNORED) +#define INET_IGNORE_READ (INET_FLG_IS_IGNORED|INET_FLG_IS_IGNORED_RD) +#define INET_IGNORE_WRITE (INET_FLG_IS_IGNORED|INET_FLG_IS_IGNORED_WR) +#define INET_IGNORE_PASSIVE (INET_FLG_IS_IGNORED|INET_FLG_IS_IGNORED_PASS) /* Max length of Erlang Term Buffer (for outputting structured terms): */ #ifdef HAVE_SCTP @@ -1128,9 +1138,7 @@ typedef struct { double send_avg; /* average packet size sent */ subs_list empty_out_q_subs; /* Empty out queue subscribers */ - int is_ignored; /* if a fd is ignored by the inet_drv. - This flag should be set to true when - the fd is used outside of inet_drv. */ + int flags; #ifdef HAVE_SETNS char *netns; /* Socket network namespace name as full file path */ @@ -4551,7 +4559,7 @@ static void desc_close(inet_descriptor* desc) * We should close the fd here, but the other driver might still * be selecting on it. */ - if (!desc->is_ignored) + if (!INET_IGNORED(desc)) driver_select(desc->port,(ErlDrvEvent)(long)desc->event, ERL_DRV_USE, 0); else @@ -6312,6 +6320,7 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) (long)desc->port, desc->s, ival)); if (ival < INET_MIN_BUFFER) ival = INET_MIN_BUFFER; desc->bufsz = ival; + desc->flags |= INET_FLG_BUFFER_SET; continue; case INET_LOPT_ACTIVE: @@ -6507,6 +6516,16 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) case INET_OPT_RCVBUF: type = SO_RCVBUF; DEBUGF(("inet_set_opts(%ld): s=%d, SO_RCVBUF=%d\r\n", (long)desc->port, desc->s, ival)); + if (!(desc->flags & INET_FLG_BUFFER_SET)) { + /* make sure we have desc->bufsz >= SO_RCVBUF */ + if (ival > (1 << 16) && desc->stype == SOCK_DGRAM && !IS_SCTP(desc)) + /* For UDP we don't want to automatically + set the buffer size to be larger than + the theoretical max MTU */ + desc->bufsz = 1 << 16; + else if (ival > desc->bufsz) + desc->bufsz = ival; + } break; case INET_OPT_LINGER: type = SO_LINGER; if (len < 4) @@ -6752,23 +6771,17 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) } DEBUGF(("inet_set_opts(%ld): s=%d returned %d\r\n", (long)desc->port, desc->s, res)); - if (type == SO_RCVBUF) { - /* make sure we have desc->bufsz >= SO_RCVBUF */ - if (ival > (1 << 16) && desc->stype == SOCK_DGRAM && !IS_SCTP(desc)) - /* For UDP we don't want to automatically - set the buffer size to be larger than - the theoretical max MTU */ - desc->bufsz = 1 << 16; - else if (ival > desc->bufsz) - desc->bufsz = ival; - } } if ( ((desc->stype == SOCK_STREAM) && IS_CONNECTED(desc)) || ((desc->stype == SOCK_DGRAM) && IS_OPEN(desc))) { - if (desc->active != old_active) + if (desc->active != old_active) { + /* Need to cancel the read_packet timer if we go from active to passive. */ + if (desc->active == INET_PASSIVE && desc->stype == SOCK_DGRAM) + driver_cancel_timer(desc->port); sock_select(desc, (FD_READ|FD_CLOSE), (desc->active>0)); + } /* XXX: UDP sockets could also trigger immediate read here NIY */ if ((desc->stype==SOCK_STREAM) && desc->active) { @@ -6910,6 +6923,7 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) if (desc->bufsz < INET_MIN_BUFFER) desc->bufsz = INET_MIN_BUFFER; + desc->flags |= INET_FLG_BUFFER_SET; res = 0; /* This does not affect the kernel buffer size */ continue; @@ -7031,6 +7045,7 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) smaller than the kernel one: */ if (desc->bufsz <= arg.ival) desc->bufsz = arg.ival; + desc->flags |= INET_FLG_BUFFER_SET; break; } case INET_OPT_SNDBUF: @@ -7041,10 +7056,6 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) arg_ptr = (char*) (&arg.ival); arg_sz = sizeof ( arg.ival); - /* Adjust the size of the user-level recv buffer, so it's not - smaller than the kernel one: */ - if (desc->bufsz <= arg.ival) - desc->bufsz = arg.ival; break; } case INET_OPT_REUSEADDR: @@ -9101,7 +9112,7 @@ static ErlDrvData inet_start(ErlDrvPort port, int size, int protocol) sys_memzero((char *)&desc->remote,sizeof(desc->remote)); - desc->is_ignored = 0; + desc->flags = 0; #ifdef HAVE_SETNS desc->netns = NULL; @@ -9503,19 +9514,19 @@ static ErlDrvSSizeT inet_ctl(inet_descriptor* desc, int cmd, char* buf, if (desc->stype != SOCK_STREAM) return ctl_error(EINVAL, rbuf, rsize); - if (*buf == 1 && !desc->is_ignored) { + if (*buf == 1 && !INET_IGNORED(desc)) { sock_select(desc, (FD_READ|FD_WRITE|FD_CLOSE|ERL_DRV_USE_NO_CALLBACK), 0); if (desc->active) - desc->is_ignored = INET_IGNORE_READ; + desc->flags |= INET_IGNORE_READ; else - desc->is_ignored = INET_IGNORE_PASSIVE; - } else if (*buf == 0 && desc->is_ignored) { + desc->flags |= INET_IGNORE_PASSIVE; + } else if (*buf == 0 && INET_IGNORED(desc)) { int flags = FD_CLOSE; - if (desc->is_ignored & INET_IGNORE_READ) + if (desc->flags & INET_IGNORE_READ) flags |= FD_READ; - if (desc->is_ignored & INET_IGNORE_WRITE) + if (desc->flags & INET_IGNORE_WRITE) flags |= FD_WRITE; - desc->is_ignored = INET_IGNORE_NONE; + desc->flags = INET_IGNORE_CLEAR(desc); if (flags != FD_CLOSE) sock_select(desc, flags, 1); } else @@ -10231,17 +10242,17 @@ static ErlDrvSSizeT tcp_inet_ctl(ErlDrvData e, unsigned int cmd, if (enq_async(INETP(desc), tbuf, TCP_REQ_RECV) < 0) return ctl_error(EALREADY, rbuf, rsize); - if (INETP(desc)->is_ignored || tcp_recv(desc, n) == 0) { + if (INET_IGNORED(INETP(desc)) || tcp_recv(desc, n) == 0) { if (timeout == 0) async_error_am(INETP(desc), am_timeout); else { if (timeout != INET_INFINITY) add_multi_timer(desc, INETP(desc)->port, 0, timeout, &tcp_inet_recv_timeout); - if (!INETP(desc)->is_ignored) + if (!INET_IGNORED(INETP(desc))) sock_select(INETP(desc),(FD_READ|FD_CLOSE),1); else - INETP(desc)->is_ignored |= INET_IGNORE_READ; + INETP(desc)->flags |= INET_IGNORE_READ; } } return ctl_reply(INET_REP_OK, tbuf, 2, rbuf, rsize); @@ -11108,7 +11119,7 @@ static int tcp_inet_input(tcp_descriptor* desc, HANDLE event) #ifdef DEBUG long port = (long) desc->inet.port; /* Used after driver_exit() */ #endif - ASSERT(!INETP(desc)->is_ignored); + ASSERT(!INET_IGNORED(INETP(desc))); DEBUGF(("tcp_inet_input(%ld) {s=%d\r\n", port, desc->inet.s)); /* XXX fprintf(stderr,"tcp_inet_input(%ld) {s=%d}\r\n",(long) desc->inet.port, desc->inet.s); */ if (desc->inet.state == INET_STATE_ACCEPTING) { @@ -11443,8 +11454,8 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev) DEBUGF(("tcp_sendv(%ld): s=%d, about to send "LLU","LLU" bytes\r\n", (long)desc->inet.port, desc->inet.s, (llu_t)h_len, (llu_t)len)); - if (INETP(desc)->is_ignored) { - INETP(desc)->is_ignored |= INET_IGNORE_WRITE; + if (INET_IGNORED(INETP(desc))) { + INETP(desc)->flags |= INET_IGNORE_WRITE; n = 0; } else if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) { driver_enqv(ix, ev, 0); @@ -11479,7 +11490,7 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev) DEBUGF(("tcp_sendv(%ld): s=%d, Send failed, queuing\r\n", (long)desc->inet.port, desc->inet.s)); driver_enqv(ix, ev, n); - if (!INETP(desc)->is_ignored) + if (!INET_IGNORED(INETP(desc))) sock_select(INETP(desc),(FD_WRITE|FD_CLOSE), 1); } return 0; @@ -11548,8 +11559,8 @@ static int tcp_send(tcp_descriptor* desc, char* ptr, ErlDrvSizeT len) DEBUGF(("tcp_send(%ld): s=%d, about to send "LLU","LLU" bytes\r\n", (long)desc->inet.port, desc->inet.s, (llu_t)h_len, (llu_t)len)); - if (INETP(desc)->is_ignored) { - INETP(desc)->is_ignored |= INET_IGNORE_WRITE; + if (INET_IGNORED(INETP(desc))) { + INETP(desc)->flags |= INET_IGNORE_WRITE; n = 0; } else if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) { sock_send(desc->inet.s, buf, 0, 0); @@ -11582,7 +11593,7 @@ static int tcp_send(tcp_descriptor* desc, char* ptr, ErlDrvSizeT len) n -= h_len; driver_enq(ix, ptr+n, len-n); } - if (!INETP(desc)->is_ignored) + if (!INET_IGNORED(INETP(desc))) sock_select(INETP(desc),(FD_WRITE|FD_CLOSE), 1); } return 0; @@ -11863,7 +11874,7 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event) int ret = 0; ErlDrvPort ix = desc->inet.port; - ASSERT(!INETP(desc)->is_ignored); + ASSERT(!INET_IGNORED(INETP(desc))); DEBUGF(("tcp_inet_output(%ld) {s=%d\r\n", (long)desc->inet.port, desc->inet.s)); if (desc->inet.state == INET_STATE_CONNECTING) { @@ -12538,9 +12549,12 @@ static void packet_inet_timeout(ErlDrvData e) { udp_descriptor * udesc = (udp_descriptor*) e; inet_descriptor * desc = INETP(udesc); - if (!(desc->active)) + if (!(desc->active)) { sock_select(desc, FD_READ, 0); - async_error_am (desc, am_timeout); + async_error_am (desc, am_timeout); + } else { + (void)packet_inet_input(udesc, desc->s); + } } @@ -12896,6 +12910,15 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event) sock_select(desc, FD_READ, 1); } #endif + + /* We set a timer on the port to trigger now. + This emulates a "yield" operation as that is + what we want to do here. We do *NOT* do a deselect + as that is expensive, instead we check if the + socket it still active when the timeout triggers + and if it is not, then we just ignore the timeout */ + driver_set_timer(desc->port, 0); + return count; } diff --git a/erts/etc/unix/to_erl.c b/erts/etc/unix/to_erl.c index afff8f7e54..ed4fe12e8b 100644 --- a/erts/etc/unix/to_erl.c +++ b/erts/etc/unix/to_erl.c @@ -245,7 +245,6 @@ int main(int argc, char **argv) tty_smode.c_iflag = 1*BRKINT |/*Signal interrupt on break.*/ 1*IGNPAR |/*Ignore characters with parity errors.*/ - 1*ISTRIP |/*Strip character.*/ 0; #if 0 diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in index 8e1f5b58c4..1da11c2d0a 100644 --- a/erts/lib_src/Makefile.in +++ b/erts/lib_src/Makefile.in @@ -29,7 +29,6 @@ CC=@CC@ LD=@LD@ AR=@AR@ RANLIB=@RANLIB@ -RM=@RM@ MKDIR=@MKDIR@ INSTALL=@INSTALL@ INSTALL_DIR=@INSTALL_DIR@ @@ -536,9 +535,9 @@ release_docs_spec: # .PHONY: clean clean: - $(RM) -rf ../lib/internal/$(TARGET)/* - $(RM) -rf ../lib/$(TARGET)/* - $(RM) -rf obj/$(TARGET)/* + $(RM) -r ../lib/internal/$(TARGET)/* + $(RM) -r ../lib/$(TARGET)/* + $(RM) -r obj/$(TARGET)/* # # Make dependencies diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam Binary files differindex 7cb34c2d58..6f5a75a2db 100644 --- a/erts/preloaded/ebin/erl_prim_loader.beam +++ b/erts/preloaded/ebin/erl_prim_loader.beam diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex 7d848ba790..d8c3a26674 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam Binary files differindex ffb349cdf3..d3d4a75a11 100644 --- a/erts/preloaded/ebin/prim_file.beam +++ b/erts/preloaded/ebin/prim_file.beam diff --git a/erts/preloaded/src/erl_prim_loader.erl b/erts/preloaded/src/erl_prim_loader.erl index fefdd34292..1605c20f2c 100644 --- a/erts/preloaded/src/erl_prim_loader.erl +++ b/erts/preloaded/src/erl_prim_loader.erl @@ -297,12 +297,13 @@ check_file_result(Func, Target, {error,Reason}) -> "Target: " ++ TargetStr ++ ". " ++ "Function: " ++ atom_to_list(Func) ++ ". " ++ Process end, - %% this is equal to calling error_logger:error_report/1 which - %% we don't want to do from code_server during system boot + %% This is equal to calling logger:error/2 which + %% we don't want to do from code_server during system boot. + %% We don't want to call logger:timestamp() either. logger ! {log,error,#{label=>{?MODULE,file_error},report=>Report}, #{pid=>self(), gl=>group_leader(), - time=>erlang:system_time(microsecond), + time=>os:system_time(microsecond), error_logger=>#{tag=>error_report, type=>std_error}}}, error diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index bdcdf72c2f..86b4f35ae5 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -483,13 +483,16 @@ do_handle_msg(Msg,State) -> {From, {ensure_loaded, _}} -> From ! {init, not_allowed}; X -> + %% This is equal to calling logger:info/3 which we don't + %% want to do from the init process, at least not during + %% system boot. We don't want to call logger:timestamp() + %% either. case whereis(user) of undefined -> - Time = erlang:system_time(microsecond), catch logger ! {log, info, "init got unexpected: ~p", [X], #{pid=>self(), gl=>self(), - time=>Time, + time=>os:system_time(microsecond), error_logger=>#{tag=>info_msg}}}; User -> User ! X, diff --git a/erts/preloaded/src/prim_file.erl b/erts/preloaded/src/prim_file.erl index 0994e2a9f4..1aa5d85c64 100644 --- a/erts/preloaded/src/prim_file.erl +++ b/erts/preloaded/src/prim_file.erl @@ -572,12 +572,13 @@ list_dir_convert([RawName | Rest], SkipInvalid, Result) -> {error, ignore} -> list_dir_convert(Rest, SkipInvalid, Result); {error, warning} -> - %% this is equal to calling error_logger:warning_msg/2 which - %% we don't want to do from code_server during system boot + %% This is equal to calling logger:warning/3 which + %% we don't want to do from code_server during system boot. + %% We don't want to call logger:timestamp() either. logger ! {log,warning,"Non-unicode filename ~p ignored\n", [RawName], #{pid=>self(), gl=>group_leader(), - time=>erlang:system_time(microsecond), + time=>os:system_time(microsecond), error_logger=>#{tag=>warning_msg}}}, list_dir_convert(Rest, SkipInvalid, Result); {error, _} -> diff --git a/erts/vsn.mk b/erts/vsn.mk index e4bdb1a8eb..fac608ed4e 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% # -VSN = 10.2.4 +VSN = 10.3 # Port number 4365 in 4.2 # Port number 4366 in 4.3 diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index 38fdc2442e..8b40173e8a 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -33,6 +33,66 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.17</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + A bug caused <c>ct:encrypt_config_file/3</c> and + <c>ct:decrypt_config_file/3</c> to fail with + <c>badmatch</c> if input parameter <c>KeyOrFile</c> was + <c>{key,string()}</c>. This is now corrected.</p> + <p> + Own Id: OTP-15540</p> + </item> + <item> + <p> + The status of a test case which failed with timetrap + timeout in <c>end_per_testcase</c> could not be modified + by returning <c>{fail,Reason}</c> from a + <c>post_end_per_testcase</c> hook function. This is now + corrected.</p> + <p> + Own Id: OTP-15584 Aux Id: ERIERL-282 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + A new variant of the <c>newline</c> option to + <c>ct_telnet:cmd/3</c> and <c>ct_telnet:send/3</c> is + added, which allows to specify a string to append as + newline indicator on a command. By default, the value is + "\n", but in some cases it is required to be "\r\n", + which this option allows.</p> + <p> + A faulty regular expression given as parameter to + <c>ct_telnet:expect/2,3</c> would earlier crash and look + like an internal error in common_test. A better error + indication is now given, but the test case will still + fail.</p> + <p> + Own Id: OTP-15229 Aux Id: ERIERL-203 </p> + </item> + <item> + <p> + Since the yang RFC allows more than one top element of + config data in an <c>edit-config</c> element, + <c>ct_netconfc:edit_config/3,4,5</c> can now take a list + of XML elements.</p> + <p> + Own Id: OTP-15298</p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.16.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index fd5d4a57aa..23eb8d9656 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.16.1 +COMMON_TEST_VSN = 1.17 diff --git a/lib/compiler/doc/src/notes.xml b/lib/compiler/doc/src/notes.xml index 02e6203137..d45dfef8f3 100644 --- a/lib/compiler/doc/src/notes.xml +++ b/lib/compiler/doc/src/notes.xml @@ -32,6 +32,38 @@ <p>This document describes the changes made to the Compiler application.</p> +<section><title>Compiler 7.3.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>An expression such as <c>(A / B) band 16#ff</c> would + crash the compiler.</p> + <p> + Own Id: OTP-15518 Aux Id: ERL-829 </p> + </item> + <item> + <p>There could be an incorrect warning when the + <c>tuple_calls</c> option was given. The generated code + would be correct. Here is an example of code that would + trigger the warning:</p> + <p><c>(list_to_atom("prefix_" ++ + atom_to_list(suffix))):doit(X)</c>.</p> + <p> + Own Id: OTP-15552 Aux Id: ERL-838 </p> + </item> + <item> + <p>Optimize (again) Dialyzer's handling of + left-associative use of <c>andalso</c> and <c>orelse</c> + in guards.</p> + <p> + Own Id: OTP-15577 Aux Id: ERL-851, PR-2141, PR-1944 </p> + </item> + </list> + </section> + +</section> + <section><title>Compiler 7.3.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/compiler/vsn.mk b/lib/compiler/vsn.mk index efedb414ad..a523627384 100644 --- a/lib/compiler/vsn.mk +++ b/lib/compiler/vsn.mk @@ -1 +1 @@ -COMPILER_VSN = 7.3.1 +COMPILER_VSN = 7.3.2 diff --git a/lib/crypto/c_src/crypto.c b/lib/crypto/c_src/crypto.c index df607732bf..194a3d30e9 100644 --- a/lib/crypto/c_src/crypto.c +++ b/lib/crypto/c_src/crypto.c @@ -177,7 +177,8 @@ && !defined(HAS_LIBRESSL) \ && defined(HAVE_EC) # define HAVE_ED_CURVE_DH -# if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1)) +# if OPENSSL_VERSION_NUMBER >= (PACKED_OPENSSL_VERSION_PLAIN(1,1,1)) \ + && !defined(FIPS_SUPPORT) # define HAVE_EDDSA # endif #endif @@ -1425,8 +1426,6 @@ static void init_algorithms_types(ErlNifEnv* env) #endif algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cbc"); algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cbc128"); - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cfb8"); - algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cfb128"); algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cbc256"); algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_ctr"); algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_ecb"); @@ -1441,6 +1440,8 @@ static void init_algorithms_types(ErlNifEnv* env) #ifdef HAVE_AES_IGE algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"aes_ige256"); #endif + algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cfb8"); + algo_cipher[algo_cipher_cnt++] = enif_make_atom(env, "aes_cfb128"); #ifndef OPENSSL_NO_DES algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"des_cbc"); algo_cipher[algo_cipher_cnt++] = enif_make_atom(env,"des_cfb"); @@ -2325,21 +2326,24 @@ static ERL_NIF_TERM block_crypt_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM return enif_raise_exception(env, atom_notsup); } - if (argv[0] == atom_aes_cfb8 - && (key.size == 24 || key.size == 32)) { - /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes? - * Fall back on low level API - */ - return aes_cfb_8_crypt(env, argc-1, argv+1); + if (argv[0] == atom_aes_cfb8) { + CHECK_NO_FIPS_MODE(); + if ((key.size == 24 || key.size == 32)) { + /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes? + * Fall back on low level API + */ + return aes_cfb_8_crypt(env, argc-1, argv+1); + } + } + else if (argv[0] == atom_aes_cfb128) { + CHECK_NO_FIPS_MODE(); + if ((key.size == 24 || key.size == 32)) { + /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes? + * Fall back on low level API + */ + return aes_cfb_128_crypt_nif(env, argc-1, argv+1); + } } - else if (argv[0] == atom_aes_cfb128 - && (key.size == 24 || key.size == 32)) { - /* Why do EVP_CIPHER_CTX_set_key_length() fail on these key sizes? - * Fall back on low level API - */ - return aes_cfb_128_crypt_nif(env, argc-1, argv+1); - } - ivec_size = EVP_CIPHER_iv_length(cipher); #ifdef HAVE_ECB_IVEC_BUG @@ -4357,8 +4361,11 @@ static int get_pkey_digest_type(ErlNifEnv *env, ERL_NIF_TERM algorithm, ERL_NIF_ *md = NULL; if (type == atom_none && algorithm == atom_rsa) return PKEY_OK; + if (algorithm == atom_eddsa) #ifdef HAVE_EDDSA - if (algorithm == atom_eddsa) return PKEY_OK; + return PKEY_OK; +#else + return PKEY_NOTSUP; #endif digp = get_digest_type(type); if (!digp) return PKEY_BADARG; diff --git a/lib/crypto/doc/src/notes.xml b/lib/crypto/doc/src/notes.xml index 0a3f68ade2..c0b302734e 100644 --- a/lib/crypto/doc/src/notes.xml +++ b/lib/crypto/doc/src/notes.xml @@ -31,6 +31,38 @@ </header> <p>This document describes the changes made to the Crypto application.</p> +<section><title>Crypto 4.4.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixes a bug that caused <c>crypto:sign</c> and + <c>crypto:verify</c> to return the error message + <c>badarg</c> instead of <c>notsup</c> in one case. That + case was when signing or verifying with eddsa keys (that + is, ed15519 or ed448), but only when FIPS was supported + and enabled.</p> + <p> + Own Id: OTP-15634</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Added a crypto benchmark test suite.</p> + <p> + Own Id: OTP-15447</p> + </item> + </list> + </section> + +</section> + <section><title>Crypto 4.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index 72cb9aabfd..bc8b124b10 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -512,17 +512,17 @@ block_encrypt(Type, Key, Ivec, PlainText) when Type =:= des_cbc; Type =:= aes_cbc256; Type =:= aes_cbc; Type =:= rc2_cbc -> - block_crypt_nif(Type, Key, Ivec, PlainText, true); + notsup_to_error(block_crypt_nif(Type, Key, Ivec, PlainText, true)); block_encrypt(Type, Key0, Ivec, PlainText) when Type =:= des3_cbc; Type =:= des_ede3 -> Key = check_des3_key(Key0), - block_crypt_nif(des_ede3_cbc, Key, Ivec, PlainText, true); + notsup_to_error(block_crypt_nif(des_ede3_cbc, Key, Ivec, PlainText, true)); block_encrypt(des3_cbf, Key0, Ivec, PlainText) -> % cfb misspelled Key = check_des3_key(Key0), - block_crypt_nif(des_ede3_cbf, Key, Ivec, PlainText, true); + notsup_to_error(block_crypt_nif(des_ede3_cbf, Key, Ivec, PlainText, true)); block_encrypt(des3_cfb, Key0, Ivec, PlainText) -> Key = check_des3_key(Key0), - block_crypt_nif(des_ede3_cfb, Key, Ivec, PlainText, true); + notsup_to_error(block_crypt_nif(des_ede3_cfb, Key, Ivec, PlainText, true)); block_encrypt(aes_ige256, Key, Ivec, PlainText) -> notsup_to_error(aes_ige_crypt_nif(Key, Ivec, PlainText, true)); block_encrypt(Type, Key, Ivec, {AAD, PlainText}) when Type =:= aes_gcm; @@ -549,17 +549,17 @@ block_decrypt(Type, Key, Ivec, Data) when Type =:= des_cbc; Type =:= aes_cfb128; Type =:= aes_cbc256; Type =:= rc2_cbc -> - block_crypt_nif(Type, Key, Ivec, Data, false); + notsup_to_error(block_crypt_nif(Type, Key, Ivec, Data, false)); block_decrypt(Type, Key0, Ivec, Data) when Type =:= des3_cbc; Type =:= des_ede3 -> Key = check_des3_key(Key0), - block_crypt_nif(des_ede3_cbc, Key, Ivec, Data, false); + notsup_to_error(block_crypt_nif(des_ede3_cbc, Key, Ivec, Data, false)); block_decrypt(des3_cbf, Key0, Ivec, Data) -> % cfb misspelled Key = check_des3_key(Key0), - block_crypt_nif(des_ede3_cbf, Key, Ivec, Data, false); + notsup_to_error(block_crypt_nif(des_ede3_cbf, Key, Ivec, Data, false)); block_decrypt(des3_cfb, Key0, Ivec, Data) -> Key = check_des3_key(Key0), - block_crypt_nif(des_ede3_cfb, Key, Ivec, Data, false); + notsup_to_error(block_crypt_nif(des_ede3_cfb, Key, Ivec, Data, false)); block_decrypt(aes_ige256, Key, Ivec, Data) -> notsup_to_error(aes_ige_crypt_nif(Key, Ivec, Data, false)); block_decrypt(Type, Key, Ivec, {AAD, Data, Tag}) when Type =:= aes_gcm; @@ -571,13 +571,13 @@ block_decrypt(Type, Key, Ivec, {AAD, Data, Tag}) when Type =:= aes_gcm; -spec block_encrypt(Type::block_cipher_without_iv(), Key::key(), PlainText::iodata()) -> binary(). block_encrypt(Type, Key, PlainText) -> - block_crypt_nif(Type, Key, PlainText, true). + notsup_to_error(block_crypt_nif(Type, Key, PlainText, true)). -spec block_decrypt(Type::block_cipher_without_iv(), Key::key(), Data::iodata()) -> binary(). block_decrypt(Type, Key, Data) -> - block_crypt_nif(Type, Key, Data, false). + notsup_to_error(block_crypt_nif(Type, Key, Data, false)). -spec next_iv(Type:: cbc_cipher(), Data) -> NextIVec when % Type :: cbc_cipher(), %des_cbc | des3_cbc | aes_cbc | aes_ige, diff --git a/lib/crypto/test/Makefile b/lib/crypto/test/Makefile index 8b320e01a9..988d95a8bc 100644 --- a/lib/crypto/test/Makefile +++ b/lib/crypto/test/Makefile @@ -7,7 +7,6 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk MODULES = \ crypto_bench_SUITE \ - blowfish_SUITE \ crypto_SUITE \ engine_SUITE diff --git a/lib/crypto/test/blowfish_SUITE.erl b/lib/crypto/test/blowfish_SUITE.erl deleted file mode 100644 index a931ebb47e..0000000000 --- a/lib/crypto/test/blowfish_SUITE.erl +++ /dev/null @@ -1,300 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2009-2018. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% %CopyrightEnd% -%% - -%% --module(blowfish_SUITE). - -%% Note: This directive should only be used in test suites. --compile(export_all). - --include_lib("common_test/include/ct.hrl"). - --define(TIMEOUT, 120000). % 2 min - --define(KEY, to_bin("0123456789ABCDEFF0E1D2C3B4A59687")). --define(IVEC, to_bin("FEDCBA9876543210")). -%% "7654321 Now is the time for " (includes trailing '\0') --define(DATA, to_bin("37363534333231204E6F77206973207468652074696D6520666F722000")). --define(DATA_PADDED, to_bin("37363534333231204E6F77206973207468652074696D6520666F722000000000")). - -%% Test server callback functions -%%-------------------------------------------------------------------- -%% Function: init_per_suite(Config) -> Config -%% Config - [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Description: Initialization before the whole suite -%% -%% Note: This function is free to add any key/value pairs to the Config -%% variable, but should NOT alter/remove any existing entries. -%%-------------------------------------------------------------------- -init_per_suite(Config) -> - case catch crypto:start() of - ok -> - catch ct:comment("~s",[element(3,hd(crypto:info_lib()))]), - catch ct:log("crypto:info_lib() -> ~p~n" - "crypto:supports() -> ~p~n" - "crypto:version() -> ~p~n" - ,[crypto:info_lib(), crypto:supports(), crypto:version()]), - Config; - _Else -> - {skip,"Could not start crypto!"} - end. - -%%-------------------------------------------------------------------- -%% Function: end_per_suite(Config) -> _ -%% Config - [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Description: Cleanup after the whole suite -%%-------------------------------------------------------------------- -end_per_suite(_Config) -> - crypto:stop(). - -%%-------------------------------------------------------------------- -%% Function: init_per_testcase(TestCase, Config) -> Config -%% Case - atom() -%% Name of the test case that is about to be run. -%% Config - [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% -%% Description: Initialization before each test case -%% -%% Note: This function is free to add any key/value pairs to the Config -%% variable, but should NOT alter/remove any existing entries. -%% Description: Initialization before each test case -%%-------------------------------------------------------------------- -init_per_testcase(_TestCase, Config0) -> - Config = lists:keydelete(watchdog, 1, Config0), - Dog = test_server:timetrap(?TIMEOUT), - [{watchdog, Dog} | Config]. - -%%-------------------------------------------------------------------- -%% Function: end_per_testcase(TestCase, Config) -> _ -%% Case - atom() -%% Name of the test case that is about to be run. -%% Config - [tuple()] -%% A list of key/value pairs, holding the test case configuration. -%% Description: Cleanup after each test case -%%-------------------------------------------------------------------- -end_per_testcase(_TestCase, Config) -> - Dog = ?config(watchdog, Config), - case Dog of - undefined -> - ok; - _ -> - test_server:timetrap_cancel(Dog) - end. - -%%-------------------------------------------------------------------- -%% Function: all(Clause) -> TestCases -%% Clause - atom() - suite | doc -%% TestCases - [Case] -%% Case - atom() -%% Name of a test case. -%% Description: Returns a list of all test cases in this test suite -%%-------------------------------------------------------------------- -suite() -> [{ct_hooks,[ts_install_cth]}]. - -all() -> -[{group, fips}, - {group, non_fips}]. - -groups() -> - [{fips, [], [no_ecb, no_cbc, no_cfb64, no_ofb64]}, - {non_fips, [], [ecb, cbc, cfb64, ofb64]}]. - -init_per_group(fips, Config) -> - case crypto:info_fips() of - enabled -> - Config; - not_enabled -> - case crypto:enable_fips_mode(true) of - true -> - enabled = crypto:info_fips(), - Config; - false -> - {skip, "Failed to enable FIPS mode"} - end; - not_supported -> - {skip, "FIPS mode not supported"} - end; -init_per_group(non_fips, Config) -> - case crypto:info_fips() of - enabled -> - true = crypto:enable_fips_mode(false), - not_enabled = crypto:info_fips(), - Config; - _NotEnabled -> - Config - end; -init_per_group(_GroupName, Config) -> - Config. - -end_per_group(_GroupName, Config) -> - Config. - - -%% Test cases start here. -%%-------------------------------------------------------------------- - -ecb_test(KeyBytes, ClearBytes, CipherBytes) -> - {Key, Clear, Cipher} = - {to_bin(KeyBytes), to_bin(ClearBytes), to_bin(CipherBytes)}, - ?line m(crypto:block_encrypt(blowfish_ecb, Key, Clear), Cipher), - true. - -ecb(doc) -> - "Test that ECB mode is OK"; -ecb(suite) -> - []; -ecb(Config) when is_list(Config) -> - true = ecb_test("0000000000000000", "0000000000000000", "4EF997456198DD78"), - true = ecb_test("FFFFFFFFFFFFFFFF", "FFFFFFFFFFFFFFFF", "51866FD5B85ECB8A"), - true = ecb_test("3000000000000000", "1000000000000001", "7D856F9A613063F2"), - true = ecb_test("1111111111111111", "1111111111111111", "2466DD878B963C9D"), - true = ecb_test("0123456789ABCDEF", "1111111111111111", "61F9C3802281B096"), - true = ecb_test("1111111111111111", "0123456789ABCDEF", "7D0CC630AFDA1EC7"), - true = ecb_test("0000000000000000", "0000000000000000", "4EF997456198DD78"), - true = ecb_test("FEDCBA9876543210", "0123456789ABCDEF", "0ACEAB0FC6A0A28D"), - true = ecb_test("7CA110454A1A6E57", "01A1D6D039776742", "59C68245EB05282B"), - true = ecb_test("0131D9619DC1376E", "5CD54CA83DEF57DA", "B1B8CC0B250F09A0"), - true = ecb_test("07A1133E4A0B2686", "0248D43806F67172", "1730E5778BEA1DA4"), - true = ecb_test("3849674C2602319E", "51454B582DDF440A", "A25E7856CF2651EB"), - true = ecb_test("04B915BA43FEB5B6", "42FD443059577FA2", "353882B109CE8F1A"), - true = ecb_test("0113B970FD34F2CE", "059B5E0851CF143A", "48F4D0884C379918"), - true = ecb_test("0170F175468FB5E6", "0756D8E0774761D2", "432193B78951FC98"), - true = ecb_test("43297FAD38E373FE", "762514B829BF486A", "13F04154D69D1AE5"), - true = ecb_test("07A7137045DA2A16", "3BDD119049372802", "2EEDDA93FFD39C79"), - true = ecb_test("04689104C2FD3B2F", "26955F6835AF609A", "D887E0393C2DA6E3"), - true = ecb_test("37D06BB516CB7546", "164D5E404F275232", "5F99D04F5B163969"), - true = ecb_test("1F08260D1AC2465E", "6B056E18759F5CCA", "4A057A3B24D3977B"), - true = ecb_test("584023641ABA6176", "004BD6EF09176062", "452031C1E4FADA8E"), - true = ecb_test("025816164629B007", "480D39006EE762F2", "7555AE39F59B87BD"), - true = ecb_test("49793EBC79B3258F", "437540C8698F3CFA", "53C55F9CB49FC019"), - true = ecb_test("4FB05E1515AB73A7", "072D43A077075292", "7A8E7BFA937E89A3"), - true = ecb_test("49E95D6D4CA229BF", "02FE55778117F12A", "CF9C5D7A4986ADB5"), - true = ecb_test("018310DC409B26D6", "1D9D5C5018F728C2", "D1ABB290658BC778"), - true = ecb_test("1C587F1C13924FEF", "305532286D6F295A", "55CB3774D13EF201"), - true = ecb_test("0101010101010101", "0123456789ABCDEF", "FA34EC4847B268B2"), - true = ecb_test("1F1F1F1F0E0E0E0E", "0123456789ABCDEF", "A790795108EA3CAE"), - true = ecb_test("E0FEE0FEF1FEF1FE", "0123456789ABCDEF", "C39E072D9FAC631D"), - true = ecb_test("0000000000000000", "FFFFFFFFFFFFFFFF", "014933E0CDAFF6E4"), - true = ecb_test("FFFFFFFFFFFFFFFF", "0000000000000000", "F21E9A77B71C49BC"), - true = ecb_test("0123456789ABCDEF", "0000000000000000", "245946885754369A"), - true = ecb_test("FEDCBA9876543210", "FFFFFFFFFFFFFFFF", "6B5C5A9C5D9E0A5A"), - ok. - -cbc(doc) -> - "Test that CBC mode is OK"; -cbc(suite) -> - []; -cbc(Config) when is_list(Config) -> - true = crypto:block_encrypt(blowfish_cbc, ?KEY, ?IVEC, ?DATA_PADDED) =:= - to_bin("6B77B4D63006DEE605B156E27403979358DEB9E7154616D959F1652BD5FF92CC"), - ok. - -cfb64(doc) -> - "Test that CFB64 mode is OK"; -cfb64(suite) -> - []; -cfb64(Config) when is_list(Config) -> - true = crypto:block_encrypt(blowfish_cfb64, ?KEY, ?IVEC, ?DATA) =:= - to_bin("E73214A2822139CAF26ECF6D2EB9E76E3DA3DE04D1517200519D57A6C3"), - ok. - -ofb64(doc) -> - "Test that OFB64 mode is OK"; -ofb64(suite) -> - []; -ofb64(Config) when is_list(Config) -> - true = crypto:block_encrypt(blowfish_ofb64, ?KEY, ?IVEC, ?DATA) =:= - to_bin("E73214A2822139CA62B343CC5B65587310DD908D0C241B2263C2CF80DA"), - ok. - -no_ecb(doc) -> - "Test that ECB mode is disabled"; -no_ecb(suite) -> - []; -no_ecb(Config) when is_list(Config) -> - notsup(fun crypto:block_encrypt/3, - [blowfish_ecb, - to_bin("0000000000000000"), - to_bin("FFFFFFFFFFFFFFFF")]). - -no_cbc(doc) -> - "Test that CBC mode is disabled"; -no_cbc(suite) -> - []; -no_cbc(Config) when is_list(Config) -> - notsup(fun crypto:block_encrypt/4, - [blowfish_cbc, ?KEY, ?IVEC, ?DATA_PADDED]). - -no_cfb64(doc) -> - "Test that CFB64 mode is disabled"; -no_cfb64(suite) -> - []; -no_cfb64(Config) when is_list(Config) -> - notsup(fun crypto:block_encrypt/4, - [blowfish_cfb64, ?KEY, ?IVEC, ?DATA]), - ok. - -no_ofb64(doc) -> - "Test that OFB64 mode is disabled"; -no_ofb64(suite) -> - []; -no_ofb64(Config) when is_list(Config) -> - notsup(fun crypto:block_encrypt/4, - [blowfish_ofb64, ?KEY, ?IVEC, ?DATA]). - -%% Helper functions - -%% Assert function fails with notsup error -notsup(Fun, Args) -> - ok = try - {error, {return, apply(Fun, Args)}} - catch - error:notsup -> - ok; - Class:Error -> - {error, {Class, Error}} - end. - - -%% Convert a hexadecimal string to a binary. --spec(to_bin(L::string()) -> binary()). -to_bin(L) -> - to_bin(L, []). - -%% @spec dehex(char()) -> integer() -%% @doc Convert a hex digit to its integer value. --spec(dehex(char()) -> integer()). -dehex(C) when C >= $0, C =< $9 -> - C - $0; -dehex(C) when C >= $a, C =< $f -> - C - $a + 10; -dehex(C) when C >= $A, C =< $F -> - C - $A + 10. - --spec(to_bin(L::string(), list()) -> binary()). -to_bin([], Acc) -> - iolist_to_binary(lists:reverse(Acc)); -to_bin([C1, C2 | Rest], Acc) -> - to_bin(Rest, [(dehex(C1) bsl 4) bor dehex(C2) | Acc]). - -m(X,X) -> ok. diff --git a/lib/crypto/test/crypto_SUITE.erl b/lib/crypto/test/crypto_SUITE.erl index 98de1d7700..cbfa96cc16 100644 --- a/lib/crypto/test/crypto_SUITE.erl +++ b/lib/crypto/test/crypto_SUITE.erl @@ -99,6 +99,8 @@ groups() -> {group, rsa}, {group, dss}, {group, ecdsa}, + {group, no_ed25519}, + {group, no_ed448}, {group, dh}, {group, ecdh}, {group, no_srp}, @@ -113,8 +115,8 @@ groups() -> {group, no_blowfish_cfb64}, {group, no_blowfish_ofb64}, {group, aes_cbc128}, - {group, aes_cfb8}, - {group, aes_cfb128}, + {group, no_aes_cfb8}, + {group, no_aes_cfb128}, {group, aes_cbc256}, {group, no_aes_ige256}, {group, no_rc2_cbc}, @@ -183,8 +185,16 @@ groups() -> {chacha20, [], [stream]}, {poly1305, [], [poly1305]}, {aes_cbc, [], [block]}, + {no_aes_cfb8,[], [no_support, no_block]}, + {no_aes_cfb128,[], [no_support, no_block]}, {no_md4, [], [no_support, no_hash]}, {no_md5, [], [no_support, no_hash, no_hmac]}, + {no_ed25519, [], [no_support, no_sign_verify + %% Does not work yet: ,public_encrypt, private_encrypt + ]}, + {no_ed448, [], [no_support, no_sign_verify + %% Does not work yet: ,public_encrypt, private_encrypt + ]}, {no_ripemd160, [], [no_support, no_hash]}, {no_srp, [], [no_support, no_generate_compute]}, {no_des_cbc, [], [no_support, no_block]}, @@ -251,7 +261,7 @@ init_per_group(fips, Config) -> enabled = crypto:info_fips(), FIPSConfig; false -> - {skip, "Failed to enable FIPS mode"} + {fail, "Failed to enable FIPS mode"} end; not_supported -> {skip, "FIPS mode not supported"} @@ -401,17 +411,6 @@ block() -> block(Config) when is_list(Config) -> Fips = proplists:get_bool(fips, Config), Type = ?config(type, Config), - %% See comment about EVP_CIPHER_CTX_set_key_length in - %% block_crypt_nif in crypto.c. - case {Fips, Type} of - {true, aes_cfb8} -> - throw({skip, "Cannot test aes_cfb8 in FIPS mode because of key length issue"}); - {true, aes_cfb128} -> - throw({skip, "Cannot test aes_cfb128 in FIPS mode because of key length issue"}); - _ -> - ok - end, - Blocks = lazy_eval(proplists:get_value(block, Config)), lists:foreach(fun block_cipher/1, Blocks), lists:foreach(fun block_cipher/1, block_iolistify(Blocks)), @@ -500,6 +499,13 @@ sign_verify(Config) when is_list(Config) -> SignVerify = proplists:get_value(sign_verify, Config), lists:foreach(fun do_sign_verify/1, SignVerify). +%%-------------------------------------------------------------------- +no_sign_verify() -> + [{doc, "Test disabled sign/verify digital signatures"}]. +no_sign_verify(Config) when is_list(Config) -> + [SignVerifyHd|_] = proplists:get_value(sign_verify, Config), + notsup(fun do_sign_verify/1, [SignVerifyHd]). + %%-------------------------------------------------------------------- public_encrypt() -> [{doc, "Test public_encrypt/decrypt "}]. diff --git a/lib/crypto/vsn.mk b/lib/crypto/vsn.mk index 6a91244715..deba17fb66 100644 --- a/lib/crypto/vsn.mk +++ b/lib/crypto/vsn.mk @@ -1 +1 @@ -CRYPTO_VSN = 4.4 +CRYPTO_VSN = 4.4.1 diff --git a/lib/debugger/doc/src/Makefile b/lib/debugger/doc/src/Makefile index 56d6085e9c..49b5a4be57 100644 --- a/lib/debugger/doc/src/Makefile +++ b/lib/debugger/doc/src/Makefile @@ -114,8 +114,7 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_docs_spec: docs $(INSTALL_DIR) "$(RELSYSDIR)/doc/pdf" $(INSTALL_DATA) $(TOP_PDF_FILE) "$(RELSYSDIR)/doc/pdf" - $(INSTALL_DIR) "$(RELSYSDIR)/doc/html" - ($(CP) -rf $(HTMLDIR) "$(RELSYSDIR)/doc") + $(INSTALL_DIR_DATA) $(HTMLDIR) "$(RELSYSDIR)/doc/html" $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" $(INSTALL_DIR) "$(RELEASE_PATH)/man/man3" $(INSTALL_DATA) $(MAN3DIR)/* "$(RELEASE_PATH)/man/man3" diff --git a/lib/dialyzer/doc/src/notes.xml b/lib/dialyzer/doc/src/notes.xml index 3cf776e566..bc422c43a0 100644 --- a/lib/dialyzer/doc/src/notes.xml +++ b/lib/dialyzer/doc/src/notes.xml @@ -32,6 +32,36 @@ <p>This document describes the changes made to the Dialyzer application.</p> +<section><title>Dialyzer 3.3.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix a bug that caused Dialyzer to crash when + analyzing a contract with a module name differing from + the analyzed module's name. The bug was introduced in + Erlang/OTP 18. </p> + <p> + Own Id: OTP-15562 Aux Id: ERL-845 </p> + </item> + <item> + <p> Fix a bug in the handling of the <c>Key</c> argument + of <c>lists:{keysearch, keyfind, keymember}</c>. </p> + <p> + Own Id: OTP-15570</p> + </item> + <item> + <p>Optimize (again) Dialyzer's handling of + left-associative use of <c>andalso</c> and <c>orelse</c> + in guards.</p> + <p> + Own Id: OTP-15577 Aux Id: ERL-851, PR-2141, PR-1944 </p> + </item> + </list> + </section> + +</section> + <section><title>Dialyzer 3.3.1</title> <section><title>Improvements and New Features</title> diff --git a/lib/dialyzer/vsn.mk b/lib/dialyzer/vsn.mk index 98ab533a58..7221993963 100644 --- a/lib/dialyzer/vsn.mk +++ b/lib/dialyzer/vsn.mk @@ -1 +1 @@ -DIALYZER_VSN = 3.3.1 +DIALYZER_VSN = 3.3.2 diff --git a/lib/diameter/doc/src/diameter.xml b/lib/diameter/doc/src/diameter.xml index 0a0194af2d..85522c99b2 100644 --- a/lib/diameter/doc/src/diameter.xml +++ b/lib/diameter/doc/src/diameter.xml @@ -1,7 +1,9 @@ <?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd" [ - <!ENTITY spawn_opt + <!ENTITY spawn_opt2 '<seealso marker="erts:erlang#spawn_opt-2">erlang:spawn_opt/2</seealso>'> + <!ENTITY spawn_opt5 + '<seealso marker="erts:erlang#spawn_opt-5">erlang:spawn_opt/5</seealso>'> <!ENTITY nodes '<seealso marker="erts:erlang#nodes-0">erlang:nodes/0</seealso>'> <!ENTITY make_ref @@ -21,7 +23,7 @@ <copyright> <year>2011</year> -<year>2017</year> +<year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -1384,12 +1386,22 @@ the same peer.</p> </item> <tag> -<marker id="spawn_opt"/><c>{spawn_opt, [term()]}</c></tag> +<marker id="spawn_opt"/><c>{spawn_opt, [term()] | {M,F,A}}</c></tag> <item> <p> -Options passed to &spawn_opt; when spawning a process for an -incoming Diameter request. -Options <c>monitor</c> and <c>link</c> are ignored.</p> +An options list passed to &spawn_opt2; to spawn a handler process for an +incoming Diameter request on the local node, or an MFA that returns +the pid of a handler process.</p> + +<p> +Options <c>monitor</c> and <c>link</c> are ignored in the list-valued +case. +An MFA is applied with an additional term prepended to its argument +list, and should return either the pid of the handler process that +invokes <c>diameter_traffic:request/1</c> on the term in order to +process the request, or the atom <c>discard</c>. +The handler process need not be local, but diameter must be started on +the remote node.</p> <p> Defaults to the empty list.</p> diff --git a/lib/diameter/doc/src/diameter_transport.xml b/lib/diameter/doc/src/diameter_transport.xml index 67fd54bc56..0a8ef321c6 100644 --- a/lib/diameter/doc/src/diameter_transport.xml +++ b/lib/diameter/doc/src/diameter_transport.xml @@ -14,7 +14,8 @@ <erlref> <header> <copyright> -<year>2011</year><year>2016</year> +<year>2011</year> +<year>2019</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -174,10 +175,13 @@ its parent.</p> <taglist> -<tag><c>{diameter, {send, &message;}}</c></tag> +<tag><c>{diameter, {send, &message; | false}}</c></tag> <item> <p> -An outbound Diameter message.</p> +An outbound Diameter message. +The atom <c>false</c> can only be received when request +acknowledgements have been requests: see the <c>ack</c> message +below.</p> </item> <tag><c>{diameter, {close, Pid}}</c></tag> @@ -246,6 +250,27 @@ A <c>LocalAddr</c> list has the same semantics as one returned from &start;.</p> </item> +<tag><c>{diameter, ack}</c></tag> +<item> +<p> +Request acknowledgements of unanswered requests. +A transport process should send this once before passing incoming +Diameter messages into diameter. +As a result, every Diameter request passed into diameter with a +<c>recv</c> message (below) will be answered with a +<c>send</c> message (above), either a &message; for the transport +process to send or the atom <c>false</c> if the request has been +discarded or otherwise not answered.</p> + +<p> +This is to allow a transport process to keep count of the number +of incoming request messages that have not yet been answered or +discarded, to allow it to regulate the amount of incoming traffic. +Both diameter_tcp and diameter_sctp request acknowledgements when a +<c>message_cb</c> is configured, turning send/recv message into +callbacks that can be used to regulate traffic.</p> +</item> + <tag><c>{diameter, {recv, &message;}}</c></tag> <item> <p> diff --git a/lib/diameter/doc/src/notes.xml b/lib/diameter/doc/src/notes.xml index cc92bd99f0..5777225ae7 100644 --- a/lib/diameter/doc/src/notes.xml +++ b/lib/diameter/doc/src/notes.xml @@ -43,6 +43,41 @@ first.</p> <!-- ===================================================================== --> +<section><title>diameter 2.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix failure of incoming answer message with faulty + Experimental-Result-Code. Failure to decode the AVP + resulted in an uncaught exception, with no no + handle_answer/error callback as a consequence.</p> + <p> + Own Id: OTP-15569 Aux Id: ERIERL-302 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add spawn_opt MFA configuration to allow a callback to + spawn a handler process for an incoming Diameter request + on an an arbitrary node. Module diameter_dist provides a + route_session/2 that can be used to distribute requests + based on Session-Id, although this module is currently + only documented in the module itself and may change.</p> + <p> + Own Id: OTP-15398</p> + </item> + </list> + </section> + +</section> + <section><title>diameter 2.1.6</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/diameter/src/base/diameter.erl b/lib/diameter/src/base/diameter.erl index b90b794611..7f172e1fa1 100644 --- a/lib/diameter/src/base/diameter.erl +++ b/lib/diameter/src/base/diameter.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -365,7 +365,7 @@ call(SvcName, App, Message) -> | {connect_timer, 'Unsigned32'()} | {watchdog_timer, 'Unsigned32'() | {module(), atom(), list()}} | {watchdog_config, [{okay|suspect, non_neg_integer()}]} - | {spawn_opt, list()}. + | {spawn_opt, list() | mfa()}. %% Options passed to start_service/2 diff --git a/lib/diameter/src/base/diameter_callback.erl b/lib/diameter/src/base/diameter_callback.erl index d04a416bef..3bcf550cd8 100644 --- a/lib/diameter/src/base/diameter_callback.erl +++ b/lib/diameter/src/base/diameter_callback.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -70,7 +70,7 @@ -module(diameter_callback). -%% Default callbacks when no aleternate is specified. +%% Default callbacks when no alternate is specified. -export([peer_up/3, peer_down/3, pick_peer/4, diff --git a/lib/diameter/src/base/diameter_dist.erl b/lib/diameter/src/base/diameter_dist.erl new file mode 100644 index 0000000000..5c29ea95a4 --- /dev/null +++ b/lib/diameter/src/base/diameter_dist.erl @@ -0,0 +1,525 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(diameter_dist). + +-behaviour(gen_server). + +%% +%% Implements callbacks that can be configured as a spawn_opt +%% transport configuration, to be able to distribute incoming Diameter +%% requests to handler processes (local or remote) in various ways. +%% + +%% spawn_opt callbacks +-export([spawn_local/2, + spawn_local/1, + route_session/2, + route_session/1]). + +%% signal availability for handling incoming requests to route_sesssion/2 +-export([attach/1, + detach/1]). + +%% consistent hashing +-export([hash/3, %% for use as default MFA in route_session/2 options map + hash/2]). %% arbitrary key/values + +-include_lib("diameter/include/diameter.hrl"). + +%% server start +-export([start_link/0]). + +%% gen_server callbacks +-export([init/1, + handle_info/2, + handle_cast/2, + handle_call/3, + code_change/3, + terminate/2]). + +-type request() :: tuple(). %% callback argument from diameter_traffic + +-define(SERVER, ?MODULE). %% server monitoring node connections + +%% Maps a node name binary to the corresponding atom. Used by +%% route_session/2 to map the optional value of a Session-Id to +%% node(). +-define(NODE_TABLE, diameter_dist_node). + +%% Maps a diameter:service_name() to a node() that has called attach/1 +%% to declare its willingness to handle incoming requests for the +%% service. Use by route_session/2 in case the optional value mapping +%% has failed. +-define(SERVICE_TABLE, diameter_dist_service). + +-define(B(A), atom_to_binary(A, utf8)). +-define(ORCOND(List), list_to_tuple(['orelse', false | List])). +-define(HASH(T), erlang:phash2(T, 16#100000000)). + +%% spawn_local/2 +%% +%% Callback that is equivalent to an options list. That is, the +%% following are equivalent when passed as options to +%% diameter:add_transport/2. +%% +%% {spawn_opt, Opts} +%% {spawn_opt, {diameter_dist, spawn_local, [Opts]}} + +-spec spawn_local(ReqT :: request(), Opts :: list()) + -> pid(). + +spawn_local(ReqT, Opts) -> + spawn_opt(diameter_traffic, request, [ReqT], Opts). + +%% spawn_local/1 + +spawn_local(ReqT) -> + spawn_local(ReqT, []). + +%% route_session/2 +%% +%% Callback that maps the Session-Id of an incoming request to a +%% handler node. +%% +%% With an options list, maps an id whose optional value is the name +%% of a connected node to the same node, to handle the case that the +%% session id has been returned from diameter:session_id/1; otherwise +%% to a node that has called diameter_dist:attach/1 using the +%% consistent hashing provided by hash/3, or to the local node() if a +%% session id could not be extracted or there are no attached nodes. A +%% handler process is spawned on the selected node using +%% erlang:spawn_opt/4. +%% +%% Different behaviour can be configured by supplying an options map +%% of the following form: +%% +%% #{search => non_neg_integer(), +%% id => [binary()], +%% default => discard | local | mfa(), +%% dispatch => list() | mfa()} +%% +%% The search member limits the number of AVPs that are examined in +%% the message (from the front), to avoid searching entire message in +%% case it's known that peers follow RFC 6733's recommendation that +%% Session-Id be placed at the head of a message. The default is to +%% search the entire message. +%% +%% The id member restricts the optional value mapping to session ids +%% whose DiamterIdentity is one of those specified. Set this to the +%% list of Diameter identities advertised by the service in question +%% (typically one) to ensure that only locally generated session ids +%% are mapped; or to the empty list to disable the mapping. +%% +%% The default member determines where to handle a message whose +%% Session-Id isn't found or whose optional value isn't mapped to the +%% name of a connected node. The atom local says the local node, an +%% MFA is invoked on Session-Id | false, the name of the diameter +%% service, and the message binary, and should return either a node() +%% or false to discard the message. Defaults to {diameter_dist, hash, []}. +%% +%% The dispatch member determines how the pid() of the request handler +%% process is retrieved. An MFA is applied to a previously selected +%% node(), and the module, function, and arguments list to apply in +%% the handler process to handle the request, the MFA being supplied +%% by diameter, and returns pid() | discard. A list is equivalent to +%% {erlang, spawn_opt, []}. Defaults to []. +%% +%% This can be used with search = 0 to route on something other than +%% Session-Id, but this is probably no simpler than just implementing +%% an own spawn_opt callback. (Except with the default dispatch possibly.) +%% +%% Note that if the peer is also implemented with OTP diameter and +%% generating session ids with diameter:session_id/1 then +%% route_session/2 can map an optional value to a local node that +%% happens to have the same name as one of the peer's nodes. This +%% could lead to an uneven distribution; for example, if the peer +%% nodes are a subset of the local nodes. In practice, it's typically +%% known if it's peers or the local node originating sessions; if the +%% former then setting id = [] disables the optional value mapping, if +%% the latter then setting default = local disables the hashing. +-spec route_session(ReqT :: request(), Opts) + -> discard + | pid() + when Opts :: pos_integer() %% aka #{search => N} + | list() %% aka #{dispatch => Opts} + | #{search => non_neg_integer(), %% limit number of examined AVPs + id => [binary()], %% restrict optional value map on DiamIdent + default => local %% handle locally + | discard + | mfa(), %% return node() | false + dispatch => list() %% spawn options + | mfa()}. %% (Node, M, F, A) -> pid() | discard + +route_session(ReqT, Opts) -> + {_, Bin} = Info = diameter_traffic:request_info(ReqT), + Sid = session_id(avps(Bin), search(Opts)), + Node = default(node_of_session_id(Sid, Opts), Sid, Opts, Info), + dispatch(Node, ReqT, dispatch(Opts)). + +%% avps/1 + +avps(<<_:20/binary, Bin/binary>>) -> + Bin; + +avps(_) -> + false. + +%% dispatch/3 + +dispatch(false, _, _) -> + discard; + +dispatch(Node, ReqT, {M,F,A}) -> + apply(M, F, [Node, diameter_traffic, request, [ReqT] | A]); + +dispatch(Node, ReqT, Opts) -> + spawn_opt(Node, diameter_traffic, request, [ReqT], Opts). + +%% route_session/1 + +route_session(ReqT) -> + route_session(ReqT, []). + +%% node_of_session_id/2 +%% +%% Return the node name encoded as optional value in a Session-Id, +%% assuming the id has been created with diameter:session_id/0. Lookup +%% the node name to ensure we don't convert arbitrary binaries to +%% atom. + +node_of_session_id([Id, _, _, Bin], #{id := Ids}) -> + lists:member(Id, Ids) andalso nodemap(Bin); + +node_of_session_id([_, _, _, Bin], _) -> + nodemap(Bin); + +node_of_session_id(_, _) -> + false. + +%% nodemap/1 + +nodemap(Bin) -> + try + ets:lookup_element(?NODE_TABLE, Bin, 2) + catch + error: badarg -> false + end. + +%% session_id/2 + +session_id(_, 0) -> %% give up + false; + +%% Session-Id = Command Code 263, V-bit = 0. +session_id(<<263:32, 0:1, _:7, Len:24, _/binary>> = Bin, _) -> + case Bin of + <<Avp:Len/binary, _/binary>> -> + <<_:8/binary, Sid/binary>> = Avp, + split(Sid); + _ -> + false + end; + +%% Jump to the next AVP. This is potentially costly for a message with +%% many AVPs and no Session-Id, which an attacker is prone to send. +%% 8.8 or RFC 6733 says that Session-Id SHOULD (but not MUST) appear +%% immediately following the Diameter Header, so there is no +%% guarantee. +session_id(<<_:40, Len:24, _/binary>> = Bin, N) -> + Pad = (4 - (Len rem 4)) rem 4, + case Bin of + <<_:Len/binary, _:Pad/binary, Rest/binary>> -> + session_id(Rest, if N == infinity -> N; true -> N-1 end); + _ -> + false + end; + +session_id(_, _) -> + false. + +%% split/1 +%% +%% Split a Session-Id at no more than three semicolons: the optional +%% value (if any) follows the third. binary:split/2 does better than +%% matching character by character, especially when the pattern is +%% compiled. + +split(Bin) -> + split(3, Bin, pattern()). + +%% split/3 + +split(0, Bin, _) -> + [Bin]; + +split(N, Bin, Pattern) -> + [H|T] = binary:split(Bin, Pattern), + [H | case T of + [] -> + T; + [Rest] -> + split(N-1, Rest, Pattern) + end]. + +%% pattern/0 +%% +%% Since this is being called in a watchdog process, compile the +%% pattern once and maintain it in the process dictionary. + +pattern() -> + case get(?MODULE) of + undefined -> + CP = binary:compile_pattern(<<$;>>), %% tuple + put(?MODULE, CP), + CP; + CP -> + CP + end. + +%% dispatch/1 + +dispatch(#{} = Opts) -> + maps:get(dispatch, Opts, []); + +dispatch(Opts) + when is_list(Opts) -> + Opts; + +dispatch(_) -> + []. + +%% search/1 +%% +%% Bound number of AVPs examined when looking for Session-Id. + +search(#{search := N}) + when is_integer(N), 0 =< N -> + N; + +search(N) + when is_integer(N), 0 =< N -> + N; + +search(_) -> + infinity. + +%% default/3 +%% +%% Choose a node when Session-Id lookup has failed. + +default(false, _, #{default := discard}, _) -> + false; + +default(false, _, #{default := local}, _) -> + node(); + +default(false, Sid, #{default := {M,F,A}}, Info) -> + {ServiceName, Bin} = Info, + apply(M, F, [Sid, ServiceName, Bin | A]); %% node() | false + +default(false, Sid, _, Info) -> %% aka {?MODULE, hash, []} + {ServiceName, Bin} = Info, + hash(Sid, ServiceName, Bin); + +default(Node, _, _, _) -> + Node. + +%% =========================================================================== + +%% hash/3 +%% +%% Consistent hashing of Session-Id to an attached node, or the local +%% node if Session-Id = false or no attached nodes. + +hash(Sid, ServiceName, _) -> + case false /= Sid andalso attached(ServiceName) of + [_|_] = Nodes -> + hash(Sid, Nodes); + _ -> + node() + end. + +%% hash/2 +%% +%% Consistent hashing on arbitrary key/values. Returns false if the +%% list is empty. + +%% No key or no values. +hash(_, []) -> + false; + +%% Not much choice. +hash(_, [Value]) -> + Value; + +%% Hash on a circle and choose the closest predecessor. +hash(Key, Values) -> + Hash = ?HASH(Key), + tl(lists:foldl(fun(V,A) -> + choose(Hash, [?HASH({Key, V}) | V], A) + end, + false, %% < list() + Values)). + +%% choose/3 + +choose(Hash, [Hash1 | _] = T, [Hash2 | _]) + when Hash1 =< Hash, Hash < Hash2 -> + T; + +choose(Hash, [Hash1 | _], [Hash2 | _] = T) + when Hash2 =< Hash, Hash < Hash1 -> + T; + +choose(_, T1, T2) -> + max(T1, T2). + +%% =========================================================================== + +%% attach/1 +%% +%% Register the local node as a handler of incoming requests for the +%% specified services when using the route_session/2 spawn_opt +%% callback. + +attach(ServiceNames) -> + abcast({attach, node(), ServiceNames}). + +%% detach/1 +%% +%% Deregister the local node as a handler of incoming requests. + +detach(ServiceNames) -> + abcast({detach, node(), ServiceNames}). + +%% abcast/1 + +abcast(T) -> + gen_server:abcast([node() | nodes()], ?SERVER, T), + ok. + +%% attached/1 + +attached(ServiceName) -> + try + ets:lookup_element(?SERVICE_TABLE, ServiceName, 2) + catch + error: badarg -> [] + end. + +%% cast/2 + +cast(Node, T) -> + gen_server:cast({?SERVER, Node}, T). + +%% attach/2 + +attach(Node, S) -> + case sets:to_list(S) of + [] -> + ok; + Services -> + cast(Node, {attach, node(), Services}) + end. + +%% =========================================================================== + +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, _Args = [], _Opts = []). + +%% init/1 +%% +%% Maintain [node() | nodes()] in a table that maps from binary-valued +%% names, so we can lookup the corresponding atoms rather than convert +%% binaries that aren't necessarily node names. + +init([]) -> + ets:new(?NODE_TABLE, [set, named_table]), + ets:new(?SERVICE_TABLE, [bag, named_table]), + ok = net_kernel:monitor_nodes(true, [{node_type, all}, nodedown_reason]), + ets:insert(?NODE_TABLE, [{?B(N), N} || N <- [node() | nodes()]]), + abcast({attach, node()}), + {ok, sets:new()}. + +%% handle_call/3 + +handle_call(_, _From, S) -> + {reply, nok, S}. + +%% handle_cast/2 + +%% Remote node is asking which services the local node wants to handle. +handle_cast({attach, Node}, S) + when Node /= node() -> + attach(Node, S), + {noreply, S}; + +%% Node wants to handle incoming requests ... +handle_cast({attach, Node, ServiceNames}, S) -> + ets:insert(?SERVICE_TABLE, [{N, Node} || N <- ServiceNames]), + {noreply, case node() of + Node -> + sets:union(S, sets:from_list(ServiceNames)); + _ -> + S + end}; + +%% ... or not. +handle_cast({detach, Node, ServiceNames}, S) -> + ets:select_delete(?SERVICE_TABLE, [{{'$1', Node}, + [?ORCOND([{'==', '$1', {const, N}} + || N <- ServiceNames])], + [true]}]), + {noreply, case node() of + Node -> + sets:subtract(S, sets:from_list(ServiceNames)); + _ -> + S + end}; + +handle_cast(_, S) -> + {noreply, S}. + +%% handle_info/2 + +handle_info({nodeup, Node, _}, S) -> + ets:insert(?NODE_TABLE, {?B(Node), Node}), + cast(Node, {attach, node()}), %% ask which services remote node handles + attach(Node, S), %% say which service local node handles + {noreply, S}; + +handle_info({nodedown, Node, _}, S) -> + ets:delete(?NODE_TABLE, ?B(Node)), + ets:select_delete(?SERVICE_TABLE, [{{'_', Node}, [], [true]}]), + {noreply, S}; + +handle_info(_, S) -> + {noreply, S}. + +%% terminate/2 + +terminate(_, _) -> + ok. + +%% code_change/3 + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. diff --git a/lib/diameter/src/base/diameter_misc_sup.erl b/lib/diameter/src/base/diameter_misc_sup.erl index 343688be23..fec5a41b5c 100644 --- a/lib/diameter/src/base/diameter_misc_sup.erl +++ b/lib/diameter/src/base/diameter_misc_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ diameter_stats, %% statistics counter management diameter_reg, %% service/property publishing diameter_peer, %% remote peer manager + diameter_dist, %% request distribution diameter_config]). %% configuration/restart %% start_link/0 diff --git a/lib/diameter/src/base/diameter_traffic.erl b/lib/diameter/src/base/diameter_traffic.erl index 2d3e4a2ac9..8423e30269 100644 --- a/lib/diameter/src/base/diameter_traffic.erl +++ b/lib/diameter/src/base/diameter_traffic.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2017. All Rights Reserved. +%% Copyright Ericsson AB 2013-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -42,8 +42,12 @@ peer_up/1, peer_down/1]). +%% towards diameter_dist +-export([request_info/1]). + %% internal -export([send/1, %% send from remote node + request/1, %% process request in handler process init/1]). %% monitor process start -include_lib("diameter/include/diameter.hrl"). @@ -232,7 +236,7 @@ incr_rc(Dir, Pkt, TPid, MsgDict, AppDict, Dict0) -> -spec receive_message(pid(), Route, #diameter_packet{}, module(), RecvData) -> pid() %% request handler | boolean() %% answer, known request or not - | discard %% request discarded by MFA + | discard %% request discarded when Route :: {Handler, RequestRef, TPid} | Ack, RecvData :: {[SpawnOpt], #recvdata{}}, @@ -252,7 +256,8 @@ receive_message(TPid, Route, Pkt, Dict0, RecvData) -> recv(true, Ack, TPid, Pkt, Dict0, T) when is_boolean(Ack) -> {Opts, RecvData} = T, - spawn_request(Ack, TPid, Pkt, Dict0, RecvData, Opts); + AppT = find_app(TPid, Pkt, RecvData), + ack(Ack, TPid, spawn_request(AppT, Opts, Ack, TPid, Pkt, Dict0, RecvData)); %% ... answer to known request ... recv(false, {Pid, Ref, TPid}, _, Pkt, Dict0, _) -> @@ -274,58 +279,73 @@ recv(false, false, TPid, Pkt, _, _) -> incr(TPid, {{unknown, 0}, recv, discarded}), false. -%% spawn_request/6 +%% spawn_request/7 + +spawn_request(false, _, _, _, _, _, _) -> %% no transport + discard; -%% An MFA should return a pid() or the atom 'discard'. The latter -%% results in an acknowledgment back to the transport process when -%% appropriate, to ensure that send/recv callbacks can count -%% outstanding requests. Acknowledgement is implicit if the +%% An MFA should return the pid() of a process in which the argument +%% fun in applied, or the atom 'discard' if the fun is not applied. +%% The latter results in an acknowledgment back to the transport +%% process when appropriate, to ensure that send/recv callbacks can +%% count outstanding requests. Acknowledgement is implicit if the %% handler process dies (in a handle_request callback for example). -spawn_request(Ack, TPid, Pkt, Dict0, RecvData, {M,F,A}) -> - ReqF = fun() -> - ack(Ack, TPid, recv_request(Ack, TPid, Pkt, Dict0, RecvData)) - end, - ack(Ack, TPid, apply(M, F, [ReqF | A])); +spawn_request(AppT, {M,F,A}, Ack, TPid, Pkt, Dict0, RecvData) -> + %% Term to pass to request/1 in an appropriate process. Module + %% diameter_dist implements callbacks. + ReqT = {Pkt, AppT, Ack, TPid, Dict0, RecvData}, + apply(M, F, [ReqT | A]); %% A spawned process acks implicitly when it dies, so there's no need %% to handle 'discard'. -spawn_request(Ack, TPid, Pkt, Dict0, RecvData, Opts) -> +spawn_request(AppT, Opts, Ack, TPid, Pkt, Dict0, RecvData) -> spawn_opt(fun() -> - recv_request(Ack, TPid, Pkt, Dict0, RecvData) + recv_request(Ack, TPid, Pkt, Dict0, RecvData, AppT) end, Opts). +%% request_info/1 +%% +%% Limited request information for diameter_dist. + +request_info({Pkt, _AppT, _Ack, _TPid, _Dict0, RecvData} = _ReqT) -> + {RecvData#recvdata.service_name, Pkt#diameter_packet.bin}. + +%% request/1 +%% +%% Called from a handler process chosen by a transport spawn_opt MFA +%% to process an incoming request. + +request({Pkt, AppT, Ack, TPid, Dict0, RecvData} = _ReqT) -> + ack(Ack, TPid, recv_request(Ack, TPid, Pkt, Dict0, RecvData, AppT)). + %% ack/3 ack(Ack, TPid, RC) -> - RC == discard andalso Ack andalso (TPid ! {send, false}), + RC == discard + andalso Ack + andalso (TPid ! {send, false}), RC. %% --------------------------------------------------------------------------- -%% recv_request/5 +%% recv_request/6 %% --------------------------------------------------------------------------- -spec recv_request(Ack :: boolean(), TPid :: pid(), #diameter_packet{}, Dict0 :: module(), - #recvdata{}) + #recvdata{}, + AppT :: {#diameter_app{}, #diameter_caps{}} + | #diameter_caps{}) %% no suitable app -> ok %% answer was sent - | discard %% or not - | false. %% no transport - -recv_request(Ack, - TPid, - #diameter_packet{header = #diameter_header{application_id = Id}} - = Pkt, - Dict0, - #recvdata{peerT = PeerT, - apps = Apps, - counters = Count} - = RecvData) -> + | discard. %% or not + +recv_request(Ack, TPid, Pkt, Dict0, RecvData, AppT) -> Ack andalso (TPid ! {handler, self()}), - case diameter_service:find_incoming_app(PeerT, TPid, Id, Apps) of + case AppT of {#diameter_app{id = Aid, dictionary = AppDict} = App, Caps} -> + Count = RecvData#recvdata.counters, Count andalso incr(recv, Pkt, TPid, AppDict), DecPkt = decode(Aid, AppDict, RecvData, Pkt), Count andalso incr_error(recv, DecPkt, TPid, AppDict), @@ -349,11 +369,20 @@ recv_request(Ack, Dict0, RecvData, DecPkt, - [[]]); - false = No -> %% transport has gone down - No + [[]]) end. +%% find_app/3 +%% +%% Lookup the application of a received Diameter request on the node +%% on which it's received. + +find_app(TPid, + #diameter_packet{header = #diameter_header{application_id = Id}}, + #recvdata{peerT = PeerT, + apps = Apps}) -> + diameter_service:find_incoming_app(PeerT, TPid, Id, Apps). + %% decode/4 decode(Id, Dict, #recvdata{codec = Opts}, Pkt) -> diff --git a/lib/diameter/src/diameter.appup.src b/lib/diameter/src/diameter.appup.src index 4e6b983bac..52263633fb 100644 --- a/lib/diameter/src/diameter.appup.src +++ b/lib/diameter/src/diameter.appup.src @@ -60,7 +60,8 @@ {"2.1.3", [{restart_application, diameter}]}, %% 20.2 {"2.1.4", [{restart_application, diameter}]}, %% 20.3 {"2.1.4.1", [{restart_application, diameter}]}, %% 20.3.8.19 - {"2.1.5", [{update, diameter_peer_fsm}]} %% 21.0 + {"2.1.5", [{restart_application, diameter}]}, %% 21.0 + {"2.1.6", [{restart_application, diameter}]} %% 21.1 ], [ {"0.9", [{restart_application, diameter}]}, @@ -102,6 +103,7 @@ {"2.1.3", [{restart_application, diameter}]}, {"2.1.4", [{restart_application, diameter}]}, {"2.1.4.1", [{restart_application, diameter}]}, - {"2.1.5", [{update, diameter_peer_fsm}]} + {"2.1.5", [{restart_application, diameter}]}, + {"2.1.6", [{restart_application, diameter}]} ] }. diff --git a/lib/diameter/src/modules.mk b/lib/diameter/src/modules.mk index bb86de016a..d16292bb88 100644 --- a/lib/diameter/src/modules.mk +++ b/lib/diameter/src/modules.mk @@ -1,7 +1,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2017. All Rights Reserved. +# Copyright Ericsson AB 2010-2019. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,6 +40,7 @@ RT_MODULES = \ base/diameter_config \ base/diameter_config_sup \ base/diameter_codec \ + base/diameter_dist \ base/diameter_gen \ base/diameter_lib \ base/diameter_misc_sup \ diff --git a/lib/diameter/test/diameter_dist_SUITE.erl b/lib/diameter/test/diameter_dist_SUITE.erl new file mode 100644 index 0000000000..b2e4c35b9a --- /dev/null +++ b/lib/diameter/test/diameter_dist_SUITE.erl @@ -0,0 +1,332 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% Tests of traffic between two Diameter nodes, the server being +%% spread across three Erlang nodes. +%% + +-module(diameter_dist_SUITE). + +-export([suite/0, + all/0]). + +%% testcases +-export([enslave/1, enslave/0, + ping/1, + start/1, + connect/1, + send/1, + stop/1, stop/0]). + +%% diameter callbacks +-export([peer_up/3, + peer_down/3, + pick_peer/4, + prepare_request/3, + prepare_retransmit/3, + handle_answer/4, + handle_error/4, + handle_request/3]). + +-export([call/1]). + +-include("diameter.hrl"). +-include("diameter_gen_base_rfc6733.hrl"). + +%% =========================================================================== + +-define(util, diameter_util). + +-define(CLIENT, 'CLIENT'). +-define(SERVER, 'SERVER'). +-define(REALM, "erlang.org"). +-define(DICT, diameter_gen_base_rfc6733). +-define(ADDR, {127,0,0,1}). + +%% Config for diameter:start_service/2. +-define(SERVICE(Host), + [{'Origin-Host', Host ++ [$.|?REALM]}, + {'Origin-Realm', ?REALM}, + {'Host-IP-Address', [?ADDR]}, + {'Vendor-Id', 12345}, + {'Product-Name', "OTP/diameter"}, + {'Auth-Application-Id', [?DICT:id()]}, + {'Origin-State-Id', origin()}, + {spawn_opt, {diameter_dist, route_session, [#{id => []}]}}, + {sequence, fun sequence/0}, + {string_decode, false}, + {application, [{dictionary, ?DICT}, + {module, ?MODULE}, + {request_errors, callback}, + {answer_errors, callback}]}]). + +-define(SUCCESS, 2001). +-define(BUSY, 3004). +-define(LOGOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_LOGOUT'). +-define(MOVED, ?'DIAMETER_BASE_TERMINATION-CAUSE_USER_MOVED'). +-define(TIMEOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_SESSION_TIMEOUT'). + +-define(L, atom_to_list). +-define(A, list_to_atom). + +%% The order here is significant and causes the server to listen +%% before the clients connect. The server listens on the first node, +%% and distributes requests to the other two. +-define(NODES, [{server0, ?SERVER}, + {server1, ?SERVER}, + {server2, ?SERVER}, + {client, ?CLIENT}]). + +%% Options to ct_slave:start/2. +-define(TIMEOUTS, [{T, 15000} || T <- [boot_timeout, + init_timeout, + start_timeout]]). + +%% =========================================================================== + +suite() -> + [{timetrap, {seconds, 60}}]. + +all() -> + [enslave, + ping, + start, + connect, + send, + stop]. + +%% =========================================================================== +%% start/stop testcases + +%% enslave/1 +%% +%% Start four slave nodes, three to implement a Diameter server, +%% one to implement a client. + +enslave() -> + [{timetrap, {seconds, 30*length(?NODES)}}]. + +enslave(Config) -> + Here = filename:dirname(code:which(?MODULE)), + Ebin = filename:join([Here, "..", "ebin"]), + Dirs = [Here, Ebin], + Nodes = [{N,S} || {M,S} <- ?NODES, N <- [slave(M, Dirs)]], + ?util:write_priv(Config, nodes, [{N,S} || {{N,ok},S} <- Nodes]), + [] = [{T,S} || {{_,E} = T, S} <- Nodes, E /= ok]. + +slave(Name, Dirs) -> + add_pathsa(Dirs, ct_slave:start(Name, ?TIMEOUTS)). + +add_pathsa(Dirs, {ok, Node}) -> + {Node, rpc:call(Node, code, add_pathsa, [Dirs])}; +add_pathsa(_, No) -> + {No, error}. + +%% ping/1 +%% +%% Ensure the server nodes are connected so that diameter_dist can attach. + +ping({S, Nodes}) -> + ?SERVER = S, + [N || {N,_} <- Nodes, + node() /= N, + pang <- [net_adm:ping(N)]]; + +ping(Config) -> + Nodes = lists:droplast(?util:read_priv(Config, nodes)), + [] = [{N,RC} || {N,S} <- Nodes, + RC <- [rpc:call(N, ?MODULE, ping, [{S,Nodes}])], + RC /= []]. + +%% start/1 +%% +%% Start diameter services. + +start(SvcName) + when is_atom(SvcName) -> + ok = diameter:start(), + ok = diameter:start_service(SvcName, ?SERVICE((?L(SvcName)))); + +start(Config) -> + Nodes = ?util:read_priv(Config, nodes), + [] = [{N,RC} || {N,S} <- Nodes, + RC <- [rpc:call(N, ?MODULE, start, [S])], + RC /= ok]. + +sequence() -> + sequence(sname()). + +sequence(client) -> + {0,32}; +sequence(Server) -> + "server" ++ N = ?L(Server), + {list_to_integer(N), 30}. + +origin() -> + origin(sname()). + +origin(client) -> + 99; +origin(Server) -> + "server" ++ N = ?L(Server), + list_to_integer(N). + +%% connect/1 +%% +%% Establish one connection from the client, terminated on the first +%% server node, the others handling requests. + +connect({?SERVER, Config, [{Node, _} | _]}) -> + if Node == node() -> %% server0 + ?util:write_priv(Config, lref, {Node, ?util:listen(?SERVER, tcp)}); + true -> + diameter_dist:attach([?SERVER]) + end, + ok; + +connect({?CLIENT, Config, _}) -> + ?util:connect(?CLIENT, tcp, ?util:read_priv(Config, lref)), + ok; + +connect(Config) -> + Nodes = ?util:read_priv(Config, nodes), + [] = [{N,RC} || {N,S} <- Nodes, + RC <- [rpc:call(N, ?MODULE, connect, [{S, Config, Nodes}])], + RC /= ok]. + +%% stop/1 +%% +%% Stop the slave nodes. + +stop() -> + [{timetrap, {seconds, 30*length(?NODES)}}]. + +stop(_Config) -> + [] = [{N,E} || {N,_} <- ?NODES, + {error, _, _} = E <- [ct_slave:stop(N)]]. + +%% =========================================================================== +%% traffic testcases + +%% send/1 +%% +%% Send 100 requests and ensure the node name sent as User-Name isn't +%% the node terminating transport. + +send(Config) -> + send(Config, 100, dict:new()). + +%% send/2 + +send(Config, 0, Dict) -> + [{Server0, _} | _] = ?util:read_priv(Config, nodes) , + Node = atom_to_binary(Server0, utf8), + {false, _} = {dict:is_key(Node, Dict), dict:to_list(Dict)}; + +send(Config, N, Dict) -> + #diameter_base_STA{'Result-Code' = ?SUCCESS, + 'User-Name' = [ServerNode]} + = send(Config, str(?LOGOUT)), + true = is_binary(ServerNode), + send(Config, N-1, dict:update_counter(ServerNode, 1, Dict)). + +%% =========================================================================== + +str(Cause) -> + #diameter_base_STR{'Destination-Realm' = ?REALM, + 'Auth-Application-Id' = ?DICT:id(), + 'Termination-Cause' = Cause}. + +%% send/2 + +send(Config, Req) -> + {Node, _} = lists:last(?util:read_priv(Config, nodes)), + rpc:call(Node, ?MODULE, call, [Req]). + +%% call/1 + +call(Req) -> + diameter:call(?CLIENT, ?DICT, Req, []). + +%% sname/0 + +sname() -> + ?A(hd(string:tokens(?L(node()), "@"))). + +%% =========================================================================== +%% diameter callbacks + +%% peer_up/3 + +peer_up(_SvcName, _Peer, State) -> + State. + +%% peer_down/3 + +peer_down(_SvcName, _Peer, State) -> + State. + +%% pick_peer/4 + +pick_peer([Peer], [], ?CLIENT, _State) -> + {ok, Peer}. + +%% prepare_request/3 + +prepare_request(Pkt, ?CLIENT, {_Ref, Caps}) -> + #diameter_packet{msg = Req} + = Pkt, + #diameter_caps{origin_host = {OH, _}, + origin_realm = {OR, _}} + = Caps, + {send, Req#diameter_base_STR{'Origin-Host' = OH, + 'Origin-Realm' = OR, + 'Session-Id' = diameter:session_id(OH)}}. + +%% prepare_retransmit/3 + +prepare_retransmit(_, ?CLIENT, _) -> + discard. + +%% handle_answer/5 + +handle_answer(Pkt, _Req, ?CLIENT, _Peer) -> + #diameter_packet{msg = Rec, errors = []} = Pkt, + Rec. + +%% handle_error/5 + +handle_error(Reason, _Req, ?CLIENT, _Peer) -> + {error, Reason}. + +%% handle_request/3 + +handle_request(Pkt, ?SERVER, {_, Caps}) -> + #diameter_packet{msg = #diameter_base_STR{'Session-Id' = SId}} + = Pkt, + #diameter_caps{origin_host = {OH, _}, + origin_realm = {OR, _}} + = Caps, + {reply, #diameter_base_STA{'Result-Code' = ?SUCCESS, + 'Session-Id' = SId, + 'Origin-Host' = OH, + 'Origin-Realm' = OR, + 'User-Name' = [atom_to_binary(node(), utf8)]}}. diff --git a/lib/diameter/test/diameter_distribution_SUITE.erl b/lib/diameter/test/diameter_distribution_SUITE.erl index 5146f68ff1..5fe02284ae 100644 --- a/lib/diameter/test/diameter_distribution_SUITE.erl +++ b/lib/diameter/test/diameter_distribution_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2016. All Rights Reserved. +%% Copyright Ericsson AB 2013-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -76,6 +76,7 @@ {share_peers, peers()}, {use_shared_peers, peers()}, {restrict_connections, false}, + {spawn_opt, {diameter_dist, spawn_local, []}}, {sequence, fun sequence/0}, {application, [{dictionary, ?DICT}, {module, ?MODULE}, @@ -125,7 +126,7 @@ all() -> %% enslave/1 %% %% Start four slave nodes, one to implement a Diameter server, -%% two three to implement a client. +%% three to implement a client. enslave() -> [{timetrap, {seconds, 30*length(?NODES)}}]. @@ -331,6 +332,8 @@ prepare_request(Pkt, ?CLIENT, {_Ref, Caps}, {_, client0}) -> 'Origin-Realm' = OR, 'Session-Id' = diameter:session_id(OH)}}. +%% prepare_retransmit/4 + prepare_retransmit(Pkt, ?CLIENT, _, {_, client0}) -> #diameter_packet{msg = #diameter_base_STR{'Termination-Cause' = ?MOVED}} = Pkt, %% assert diff --git a/lib/diameter/test/diameter_pool_SUITE.erl b/lib/diameter/test/diameter_pool_SUITE.erl index 97c16940ff..a36a4fa17a 100644 --- a/lib/diameter/test/diameter_pool_SUITE.erl +++ b/lib/diameter/test/diameter_pool_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2015-2017. All Rights Reserved. +%% Copyright Ericsson AB 2015-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -51,6 +51,7 @@ {'Auth-Application-Id', [0]}, %% common {'Acct-Application-Id', [3]}, %% accounting {restrict_connections, false}, + {spawn_opt, {diameter_dist, route_session, []}}, {application, [{alias, common}, {dictionary, diameter_gen_base_rfc6733}, {module, diameter_callback}]}, diff --git a/lib/diameter/test/diameter_traffic_SUITE.erl b/lib/diameter/test/diameter_traffic_SUITE.erl index 434aef01dd..47b00c25a2 100644 --- a/lib/diameter/test/diameter_traffic_SUITE.erl +++ b/lib/diameter/test/diameter_traffic_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2018. All Rights Reserved. +%% Copyright Ericsson AB 2010-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -539,8 +539,7 @@ add_transports(Config) -> ++ [{unordered, unordered()} || T == sctp], [{capabilities_cb, fun capx/2}, {pool_size, 8} - | server_apps()] - ++ [{spawn_opt, {erlang, spawn, []}} || CS]), + | server_apps()]), Cs = [?util:connect(CN, [T, {sender, CS} | client_opts(T)], LRef, diff --git a/lib/diameter/test/modules.mk b/lib/diameter/test/modules.mk index 0c73adca12..90b0a25d5f 100644 --- a/lib/diameter/test/modules.mk +++ b/lib/diameter/test/modules.mk @@ -1,7 +1,7 @@ # %CopyrightBegin% # -# Copyright Ericsson AB 2010-2017. All Rights Reserved. +# Copyright Ericsson AB 2010-2019. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ MODULES = \ diameter_codec_test \ diameter_config_SUITE \ diameter_compiler_SUITE \ + diameter_dist_SUITE \ diameter_distribution_SUITE \ diameter_dpr_SUITE \ diameter_event_SUITE \ diff --git a/lib/diameter/vsn.mk b/lib/diameter/vsn.mk index 8c75c9e55e..a900e8f28e 100644 --- a/lib/diameter/vsn.mk +++ b/lib/diameter/vsn.mk @@ -17,5 +17,5 @@ # %CopyrightEnd% APPLICATION = diameter -DIAMETER_VSN = 2.1.6 +DIAMETER_VSN = 2.2 APP_VSN = $(APPLICATION)-$(DIAMETER_VSN)$(PRE_VSN) diff --git a/lib/edoc/doc/src/notes.xml b/lib/edoc/doc/src/notes.xml index e818887eb8..145856bcaa 100644 --- a/lib/edoc/doc/src/notes.xml +++ b/lib/edoc/doc/src/notes.xml @@ -32,6 +32,22 @@ <p>This document describes the changes made to the EDoc application.</p> +<section><title>Edoc 0.10</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> Print a helpful message explaining that adding + <c>{preprocess, true}</c> can help if reading a source + file fails. </p> + <p> + Own Id: OTP-15605 Aux Id: ERL-841 </p> + </item> + </list> + </section> + +</section> + <section><title>Edoc 0.9.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/edoc/src/edoc.erl b/lib/edoc/src/edoc.erl index b641118c5d..e9d62d3283 100644 --- a/lib/edoc/src/edoc.erl +++ b/lib/edoc/src/edoc.erl @@ -578,7 +578,7 @@ read_source(Name, Opts0) -> Opts = expand_opts(Opts0), case read_source_1(Name, Opts) of {ok, Forms} -> - check_forms(Forms, Name), + check_forms(Forms, Name, Opts), Forms; {error, R} -> edoc_report:error({"error reading file '~ts'.", @@ -692,13 +692,19 @@ fll([T | L], LastLine, Ts) -> fll(L, _LastLine, Ts) -> lists:reverse(L, Ts). -check_forms(Fs, Name) -> +check_forms(Fs, Name, Opts) -> Fun = fun (F) -> case erl_syntax:type(F) of error_marker -> case erl_syntax:error_marker_info(F) of {L, M, D} -> - edoc_report:error(L, Name, {format_error, M, D}); + edoc_report:error(L, Name, {format_error, M, D}), + case proplists:get_bool(preprocess, Opts) of + true -> + ok; + false -> + helpful_message(Name) + end; Other -> edoc_report:report(Name, "unknown error in " "source code: ~w.", [Other]) @@ -710,6 +716,11 @@ check_forms(Fs, Name) -> end, lists:foreach(Fun, Fs). +helpful_message(Name) -> + Ms = ["If the error is caused by too exotic macro", + "definitions or uses of macros, adding option", + "{preprocess, true} can help. See also edoc(3)."], + lists:foreach(fun(M) -> edoc_report:report(Name, M, []) end, Ms). %% @spec get_doc(File::filename()) -> {ModuleName, edoc_module()} %% @equiv get_doc(File, []) diff --git a/lib/edoc/vsn.mk b/lib/edoc/vsn.mk index 0b3636f030..b6e1422623 100644 --- a/lib/edoc/vsn.mk +++ b/lib/edoc/vsn.mk @@ -1 +1 @@ -EDOC_VSN = 0.9.4 +EDOC_VSN = 0.10 diff --git a/lib/erl_docgen/doc/src/notes.xml b/lib/erl_docgen/doc/src/notes.xml index 97c842a324..54f0a36b27 100644 --- a/lib/erl_docgen/doc/src/notes.xml +++ b/lib/erl_docgen/doc/src/notes.xml @@ -31,7 +31,31 @@ </header> <p>This document describes the changes made to the <em>erl_docgen</em> application.</p> - <section><title>Erl_Docgen 0.8.1</title> + <section><title>Erl_Docgen 0.9</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The HTML reference documentation shows the OTP version + where modules and functions were first introduced. + Versions older than R13B04 are not shown.</p> + <p> + Own Id: OTP-15460</p> + </item> + <item> + <p> + Make html documentation of C functions with many + arguments more readable.</p> + <p> + Own Id: OTP-15637 Aux Id: PR-2160 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erl_Docgen 0.8.1</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/erl_docgen/priv/css/otp_doc.css b/lib/erl_docgen/priv/css/otp_doc.css index 89b278215c..17d9f8dd56 100644 --- a/lib/erl_docgen/priv/css/otp_doc.css +++ b/lib/erl_docgen/priv/css/otp_doc.css @@ -74,7 +74,7 @@ a:visited { color: #1b6ec2; text-decoration: none } /* Invisible table for function specs, * just to get since-version out in right margin */ -.func-table, .func-tr, .func-td, .func-since-td { +.func-table, .func-tr, .func-td, .cfunc-td, .func-since-td { width: 200%; border: 0; padding: 0; @@ -86,7 +86,13 @@ a:visited { color: #1b6ec2; text-decoration: none } } .func-td { - width: 50% + width: 50%; +} + +.cfunc-td { + width: 50%; + padding-left: 100px; + text-indent: -100px; } .func-since-td { diff --git a/lib/erl_docgen/priv/xsl/db_html.xsl b/lib/erl_docgen/priv/xsl/db_html.xsl index c5150d447c..c9be926e1e 100644 --- a/lib/erl_docgen/priv/xsl/db_html.xsl +++ b/lib/erl_docgen/priv/xsl/db_html.xsl @@ -2119,17 +2119,10 @@ <xsl:when test="ancestor::cref"> <table class="func-table"> <tr class="func-tr"> - <td class="func-td"> + <td class="cfunc-td"> <span class="bold_code bc-7"> <xsl:call-template name="title_link"> <xsl:with-param name="link" select="substring-before(nametext, '(')"/> - <xsl:with-param name="title"> - <xsl:value-of select="ret"/> - <xsl:call-template name="maybe-space-after-ret"> - <xsl:with-param name="s" select="ret"/> - </xsl:call-template> - <xsl:value-of select="nametext"/> - </xsl:with-param> </xsl:call-template> </span> </td> @@ -2199,18 +2192,6 @@ </xsl:template> - <xsl:template name="maybe-space-after-ret"> - <xsl:param name="s"/> - <xsl:variable name="last_char" - select="substring($s, string-length($s), 1)"/> - <xsl:choose> - <xsl:when test="$last_char != '*'"> - <xsl:text> </xsl:text> - </xsl:when> - </xsl:choose> - </xsl:template> - - <!-- Type --> <xsl:template match="type"> <xsl:param name="partnum"/> @@ -2264,7 +2245,7 @@ </xsl:template> <xsl:template name="title_link"> - <xsl:param name="title"/> + <xsl:param name="title" select="'APPLY'"/> <xsl:param name="link" select="erl:to-link(title)"/> <xsl:param name="ghlink" select="ancestor-or-self::*[@ghlink][position() = 1]/@ghlink"/> <xsl:variable name="id" select="concat(concat($link,'-'), generate-id(.))"/> @@ -2274,7 +2255,16 @@ <xsl:with-param name="id" select="$id"/> <xsl:with-param name="ghlink" select="$ghlink"/> </xsl:call-template> - <a class="title_link" name="{$link}" href="#{$link}"><xsl:value-of select="$title"/></a> + <a class="title_link" name="{$link}" href="#{$link}"> + <xsl:choose> + <xsl:when test="$title = 'APPLY'"> + <xsl:apply-templates/> <!-- like <ret> and <nametext> --> + </xsl:when> + <xsl:otherwise> + <xsl:value-of select="$title"/> + </xsl:otherwise> + </xsl:choose> + </a> </span> </xsl:template> @@ -2704,4 +2694,46 @@ <xsl:value-of select="normalize-space(.)"/> </xsl:template> + <xsl:template match="ret"> + <xsl:value-of select="."/> + <xsl:variable name="last_char" select="substring(., string-length(.), 1)"/> + <xsl:if test="$last_char != '*'"> + <xsl:text> </xsl:text> + </xsl:if> + </xsl:template> + + <xsl:template match="nametext"> + <xsl:value-of select="substring-before(.,'(')"/> + <xsl:text>(</xsl:text> + <xsl:variable name="arglist" select="substring-after(.,'(')"/> + <xsl:choose> + <xsl:when test="$arglist = ')' or $arglist = 'void)'"> + <xsl:value-of select="$arglist"/> + </xsl:when> + <xsl:otherwise> + <br/> + <xsl:call-template name="cfunc-arglist"> + <xsl:with-param name="text" select="$arglist"/> + </xsl:call-template> + </xsl:otherwise> + </xsl:choose> + </xsl:template> + + <!-- Format C function argument list with <br> after comma --> + <xsl:template name="cfunc-arglist"> + <xsl:param name="text"/> + <xsl:variable name="line" select="normalize-space($text)"/> + <xsl:choose> + <xsl:when test="contains($line,',')"> + <xsl:value-of select="substring-before($line,',')"/>,<br/> + <xsl:call-template name="cfunc-arglist"> + <xsl:with-param name="text" select="substring-after($line,',')"/> + </xsl:call-template> + </xsl:when> + <xsl:otherwise> + <xsl:value-of select="$line"/> + </xsl:otherwise> + </xsl:choose> + </xsl:template> + </xsl:stylesheet> diff --git a/lib/erl_docgen/vsn.mk b/lib/erl_docgen/vsn.mk index 3b2f6db6a1..6321f229dd 100644 --- a/lib/erl_docgen/vsn.mk +++ b/lib/erl_docgen/vsn.mk @@ -1 +1 @@ -ERL_DOCGEN_VSN = 0.8.1 +ERL_DOCGEN_VSN = 0.9 diff --git a/lib/erl_interface/doc/src/ei.xml b/lib/erl_interface/doc/src/ei.xml index ae322255ad..179dbbd966 100644 --- a/lib/erl_interface/doc/src/ei.xml +++ b/lib/erl_interface/doc/src/ei.xml @@ -733,7 +733,7 @@ ei_encode_tuple_header(buf, &i, 0);</pre> </func> <func> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_init(void)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_init(void)</nametext></name> <fsummary>Initialize the ei library.</fsummary> <desc> <p>Initialize the <c>ei</c> library. This function should be called once diff --git a/lib/erl_interface/doc/src/ei_connect.xml b/lib/erl_interface/doc/src/ei_connect.xml index e318dd6664..2dd0a285fe 100644 --- a/lib/erl_interface/doc/src/ei_connect.xml +++ b/lib/erl_interface/doc/src/ei_connect.xml @@ -409,7 +409,7 @@ typedef struct { </func> <func> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_close_connection(int fd)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_close_connection(int fd)</nametext></name> <fsummary>Close a connection.</fsummary> <desc> <p>Closes a previously opened connection or listen socket.</p> @@ -469,9 +469,9 @@ fd = ei_xconnect(&ec, &addr, ALIVE); <func> <name since=""><ret>int</ret><nametext>ei_connect_init(ei_cnode* ec, const char* this_node_name, const char *cookie, short creation)</nametext></name> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_connect_init_ussi(ei_cnode* ec, const char* this_node_name, const char *cookie, short creation, ei_socket_callbacks *cbs, int cbs_sz, void *setup_context)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_connect_init_ussi(ei_cnode* ec, const char* this_node_name, const char *cookie, short creation, ei_socket_callbacks *cbs, int cbs_sz, void *setup_context)</nametext></name> <name since=""><ret>int</ret><nametext>ei_connect_xinit(ei_cnode* ec, const char *thishostname, const char *thisalivename, const char *thisnodename, Erl_IpAddr thisipaddr, const char *cookie, short creation)</nametext></name> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_connect_xinit_ussi(ei_cnode* ec, const char *thishostname, const char *thisalivename, const char *thisnodename, Erl_IpAddr thisipaddr, const char *cookie, short creation, ei_socket_callbacks *cbs, int cbs_sz, void *setup_context)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_connect_xinit_ussi(ei_cnode* ec, const char *thishostname, const char *thisalivename, const char *thisnodename, Erl_IpAddr thisipaddr, const char *cookie, short creation, ei_socket_callbacks *cbs, int cbs_sz, void *setup_context)</nametext></name> <fsummary>Initialize for a connection.</fsummary> <desc> <p>Initializes the <c>ec</c> structure, to @@ -592,8 +592,8 @@ if (ei_connect_init(&ec, "madonna", "cookie...", n++) < 0) { </func> <func> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_listen(ei_cnode *ec, int *port, int backlog)</nametext></name> - <name since="OTP @OTP-15442@"><ret>int</ret><nametext>ei_xlisten(ei_cnode *ec, Erl_IpAddr adr, int *port, int backlog)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_listen(ei_cnode *ec, int *port, int backlog)</nametext></name> + <name since="OTP 21.3"><ret>int</ret><nametext>ei_xlisten(ei_cnode *ec, Erl_IpAddr adr, int *port, int backlog)</nametext></name> <fsummary>Create a listen socket.</fsummary> <desc> <p>Used by a server process to setup a listen socket which diff --git a/lib/erl_interface/doc/src/notes.xml b/lib/erl_interface/doc/src/notes.xml index 07ddd82718..32d28b853b 100644 --- a/lib/erl_interface/doc/src/notes.xml +++ b/lib/erl_interface/doc/src/notes.xml @@ -31,6 +31,43 @@ </header> <p>This document describes the changes made to the Erl_interface application.</p> +<section><title>Erl_Interface 3.11.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed two bugs in the <c>erl_call</c> program. A missing + initialization (introduced in <c>erl_interface-3.11</c>) + which either caused a crash or failure to connect to or + start a node, and an incorrectly calculated timeout which + could cause failure to start an erlang node. These bugs + only caused failures on some platforms.</p> + <p> + Own Id: OTP-15676 Aux Id: OTP-15442, ERL-881 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erl_Interface 3.11</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Support for plugin of a <seealso + marker="ei_connect#ussi">user supplied socket + implementation</seealso> has been added.</p> + <p> + Own Id: OTP-15442 Aux Id: ERIERL-258 </p> + </item> + </list> + </section> + +</section> + <section><title>Erl_Interface 3.10.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/erl_interface/src/prog/ei_fake_prog.c b/lib/erl_interface/src/prog/ei_fake_prog.c index c7a16dc7c4..158464b385 100644 --- a/lib/erl_interface/src/prog/ei_fake_prog.c +++ b/lib/erl_interface/src/prog/ei_fake_prog.c @@ -98,11 +98,18 @@ int main(void) EI_ULONGLONG ulonglongx = 0; #endif erlang_char_encoding enc; + ei_socket_callbacks cbs; intx = erl_errno; + ei_init(); + + ei_close_connection(intx); + ei_connect_init(&xec, charp, charp, creation); + ei_connect_init_ussi(&xec, charp, charp, creation, &cbs, sizeof(cbs), NULL); ei_connect_xinit (&xec, charp, charp, charp, thisipaddr, charp, creation); + ei_connect_xinit_ussi(&xec, charp, charp, charp, thisipaddr, charp, creation, &cbs, sizeof(cbs), NULL); ei_connect(&xec, charp); ei_xconnect (&xec, thisipaddr, charp); @@ -121,6 +128,8 @@ int main(void) ei_publish(&xec, intx); ei_accept(&xec, intx, &conp); ei_unpublish(&xec); + ei_listen(&xec, intp, intx); + ei_xlisten(&xec, thisipaddr, intp, intx); ei_thisnodename(&xec); ei_thishostname(&xec); @@ -187,7 +196,7 @@ int main(void) ei_decode_char(charp, intp, charp); ei_decode_string(charp, intp, charp); ei_decode_atom(charp, intp, charp); - ei_decode_atom_as(charp, intp, charp, MAXATOMLEN_UTF8, ERLANG_WHATEVER, &enc, &enc); + ei_decode_atom_as(charp, intp, charp, MAXATOMLEN_UTF8, ERLANG_UTF8, &enc, &enc); ei_decode_binary(charp, intp, (void *)0, longp); ei_decode_fun(charp, intp, &efun); free_fun(&efun); diff --git a/lib/erl_interface/src/prog/erl_call.c b/lib/erl_interface/src/prog/erl_call.c index 52ad6885e8..ab91157035 100644 --- a/lib/erl_interface/src/prog/erl_call.c +++ b/lib/erl_interface/src/prog/erl_call.c @@ -88,10 +88,6 @@ #include "ei_resolve.h" #include "erl_start.h" /* FIXME remove dependency */ -#ifdef __WIN32__ -static void initWinSock(void); -#endif - /* * Some nice global variables * (I don't think "nice" is the right word actually... -gordon) @@ -157,6 +153,8 @@ int erl_call(int argc, char **argv) char* progname = argv[0]; ei_cnode ec; + ei_init(); + /* Get the command line options */ while (i < argc) { if (argv[i][0] != '-') { @@ -317,14 +315,6 @@ int erl_call(int argc, char **argv) struct in_addr h_ipadr; char* ct; -#ifdef __WIN32__ - /* - * FIXME Extremly ugly, but needed to get ei_gethostbyname() below - * to work. - */ - initWinSock(); -#endif - /* gethostname requires len to be max(hostname) + 1 */ if (gethostname(h_hostname, EI_MAXHOSTNAMELEN+1) < 0) { fprintf(stderr,"erl_call: failed to get host name: %d\n", errno); @@ -857,46 +847,6 @@ static void usage(const char *progname) { exit(0); } - -/*************************************************************************** - * - * OS specific functions - * - ***************************************************************************/ - -#ifdef __WIN32__ -/* - * FIXME This should not be here. This is a quick fix to make erl_call - * work at all on Windows NT. - */ -static void initWinSock(void) -{ - WORD wVersionRequested; - WSADATA wsaData; - int err; - static int initialized; - - wVersionRequested = MAKEWORD(1, 1); - if (!initialized) { - initialized = 1; - err = WSAStartup(wVersionRequested, &wsaData); - - if (err != 0) { - fprintf(stderr,"erl_call: " - "Can't initialize windows sockets: %d\n", err); - } - - if ( LOBYTE( wsaData.wVersion ) != 1 || - HIBYTE( wsaData.wVersion ) != 1 ) { - fprintf(stderr,"erl_call: This version of " - "windows sockets not supported\n"); - WSACleanup(); - } - } -} -#endif - - /*************************************************************************** * * Utility functions diff --git a/lib/erl_interface/src/prog/erl_start.c b/lib/erl_interface/src/prog/erl_start.c index 670a5900c9..c766f4780e 100644 --- a/lib/erl_interface/src/prog/erl_start.c +++ b/lib/erl_interface/src/prog/erl_start.c @@ -657,7 +657,7 @@ static int wait_for_erlang(int sockd, int magic, struct timeval *timeout) gettimeofday(&now,NULL); to.tv_sec = stop_time.tv_sec - now.tv_sec; to.tv_usec = stop_time.tv_usec - now.tv_usec; - while ((to.tv_usec <= 0) && (to.tv_sec >= 0)) { + while ((to.tv_usec < 0) && (to.tv_sec > 0)) { to.tv_usec += 1000000; to.tv_sec--; } diff --git a/lib/erl_interface/test/Makefile b/lib/erl_interface/test/Makefile index 94f4b422d6..f8f2ef0156 100644 --- a/lib/erl_interface/test/Makefile +++ b/lib/erl_interface/test/Makefile @@ -33,6 +33,7 @@ MODULES= \ ei_format_SUITE \ ei_print_SUITE \ ei_tmo_SUITE \ + erl_call_SUITE \ erl_connect_SUITE \ erl_global_SUITE \ erl_eterm_SUITE \ diff --git a/lib/erl_interface/test/erl_call_SUITE.erl b/lib/erl_interface/test/erl_call_SUITE.erl new file mode 100644 index 0000000000..9e2b2e4251 --- /dev/null +++ b/lib/erl_interface/test/erl_call_SUITE.erl @@ -0,0 +1,96 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2019. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +-module(erl_call_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([all/0, smoke/1]). + +all() -> + [smoke]. + +smoke(Config) when is_list(Config) -> + ErlCall = find_erl_call(), + NameSwitch = case net_kernel:longnames() of + true -> + "-name"; + false -> + "-sname" + end, + Name = atom_to_list(?MODULE) + ++ "-" + ++ integer_to_list(erlang:system_time(microsecond)), + + ArgsList = ["-s", "-a", "erlang node", NameSwitch, Name], + io:format("erl_call: \"~ts\"\n~nargs list: ~p~n", [ErlCall, ArgsList]), + CmdRes = get_smoke_port_res(open_port({spawn_executable, ErlCall}, + [{args, ArgsList}, eof]), []), + io:format("CmdRes: ~p~n", [CmdRes]), + + [_, Hostname] = string:lexemes(atom_to_list(node()), "@"), + NodeName = list_to_atom(Name ++ "@" ++ Hostname), + io:format("NodeName: ~p~n~n", [NodeName]), + + pong = net_adm:ping(NodeName), + rpc:cast(NodeName, erlang, halt, []), + NodeName = list_to_atom(string:trim(CmdRes, both, "'")), + ok. + +% +% Utility functions... +% + +find_erl_call() -> + ErlCallName = case os:type() of + {win32, _} -> "erl_call.exe"; + _ -> "erl_call" + end, + LibDir = code:lib_dir(erl_interface), + InstalledErlCall = filename:join([LibDir, "bin", ErlCallName]), + TargetDir = erlang:system_info(system_architecture), + TargetErlCall = filename:join([LibDir, "bin", TargetDir, ErlCallName]), + + try + lists:foreach(fun (F) -> + io:format("Checking: \"~ts\"~n", [F]), + case file:read_file_info(F) of + {ok, _} -> + throw(F); + _ -> + ok + end + end, + [InstalledErlCall, TargetErlCall]), + exit({missing, erl_call}) + catch + throw:ErlCall -> + ErlCall + end. + +get_smoke_port_res(Port, Acc) when is_port(Port) -> + receive + {Port, {data, Data}} -> + get_smoke_port_res(Port, [Acc|Data]); + {Port, eof} -> + lists:flatten(Acc) + end. + diff --git a/lib/erl_interface/vsn.mk b/lib/erl_interface/vsn.mk index 06ef907d6c..dae6052d55 100644 --- a/lib/erl_interface/vsn.mk +++ b/lib/erl_interface/vsn.mk @@ -1,2 +1,2 @@ -EI_VSN = 3.10.4 +EI_VSN = 3.11.1 ERL_INTERFACE_VSN = $(EI_VSN) diff --git a/lib/ftp/doc/src/notes.xml b/lib/ftp/doc/src/notes.xml index 01c1f88cf1..61da079900 100644 --- a/lib/ftp/doc/src/notes.xml +++ b/lib/ftp/doc/src/notes.xml @@ -33,7 +33,23 @@ <file>notes.xml</file> </header> - <section><title>Ftp 1.0.1</title> + <section><title>Ftp 1.0.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed timing related bug that could make ftp functions + behave badly.</p> + <p> + Own Id: OTP-15659 Aux Id: ERIERL-316 </p> + </item> + </list> + </section> + +</section> + +<section><title>Ftp 1.0.1</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/ftp/src/ftp.erl b/lib/ftp/src/ftp.erl index 40f6b53fa3..18cd8c7524 100644 --- a/lib/ftp/src/ftp.erl +++ b/lib/ftp/src/ftp.erl @@ -1065,9 +1065,9 @@ handle_call({_, {open, ip_comm, Opts, {CtrlOpts, DataPassOpts, DataActOpts}}}, F end end; -handle_call({_, {open, tls_upgrade, TLSOptions}}, From, State) -> - _ = send_ctrl_message(State, mk_cmd("AUTH TLS", [])), - activate_ctrl_connection(State), +handle_call({_, {open, tls_upgrade, TLSOptions}}, From, State0) -> + _ = send_ctrl_message(State0, mk_cmd("AUTH TLS", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = open, tls_options = TLSOptions}}; handle_call({_, {user, User, Password}}, From, @@ -1081,17 +1081,17 @@ handle_call({_, {user, User, Password, Acc}}, From, handle_call({_, {account, Acc}}, From, State)-> handle_user_account(Acc, State#state{client = From}); -handle_call({_, pwd}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("PWD", [])), - activate_ctrl_connection(State), +handle_call({_, pwd}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("PWD", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = pwd}}; handle_call({_, lpwd}, From, #state{ldir = LDir} = State) -> {reply, {ok, LDir}, State#state{client = From}}; -handle_call({_, {cd, Dir}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("CWD ~s", [Dir])), - activate_ctrl_connection(State), +handle_call({_, {cd, Dir}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("CWD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = cd}}; handle_call({_,{lcd, Dir}}, _From, #state{ldir = LDir0} = State) -> @@ -1108,41 +1108,41 @@ handle_call({_, {dir, Len, Dir}}, {_Pid, _} = From, setup_data_connection(State#state{caller = {dir, Dir, Len}, client = From}); handle_call({_, {rename, CurrFile, NewFile}}, From, - #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("RNFR ~s", [CurrFile])), - activate_ctrl_connection(State), + #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RNFR ~s", [CurrFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {rename, NewFile}, client = From}}; handle_call({_, {delete, File}}, {_Pid, _} = From, - #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("DELE ~s", [File])), - activate_ctrl_connection(State), + #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("DELE ~s", [File])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From}}; -handle_call({_, {mkdir, Dir}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("MKD ~s", [Dir])), - activate_ctrl_connection(State), +handle_call({_, {mkdir, Dir}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("MKD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From}}; -handle_call({_,{rmdir, Dir}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("RMD ~s", [Dir])), - activate_ctrl_connection(State), +handle_call({_,{rmdir, Dir}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RMD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From}}; -handle_call({_,{type, Type}}, From, #state{chunk = false} = State) -> +handle_call({_,{type, Type}}, From, #state{chunk = false} = State0) -> case Type of ascii -> - _ = send_ctrl_message(State, mk_cmd("TYPE A", [])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("TYPE A", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = type, type = ascii, client = From}}; binary -> - _ = send_ctrl_message(State, mk_cmd("TYPE I", [])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("TYPE I", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = type, type = binary, client = From}}; _ -> - {reply, {error, etype}, State} + {reply, {error, etype}, State0} end; handle_call({_,{recv, RemoteFile, LocalFile}}, From, @@ -1181,8 +1181,8 @@ handle_call({_, recv_chunk}, _From, #state{chunk = true, } = State0) -> %% The ftp:recv_chunk call was the last event we waited for, finnish and clean up ?DBG("recv_chunk_closing ftp:recv_chunk, last event",[]), - activate_ctrl_connection(State0), - {reply, ok, State0#state{caller = undefined, + State = activate_ctrl_connection(State0), + {reply, ok, State#state{caller = undefined, chunk = false, client = undefined}}; @@ -1238,18 +1238,18 @@ handle_call({_, {transfer_chunk, Bin}}, _, #state{chunk = true} = State) -> handle_call({_, {transfer_chunk, _}}, _, #state{chunk = false} = State) -> {reply, {error, echunk}, State}; -handle_call({_, chunk_end}, From, #state{chunk = true} = State) -> - close_data_connection(State), - activate_ctrl_connection(State), +handle_call({_, chunk_end}, From, #state{chunk = true} = State0) -> + close_data_connection(State0), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, dsock = undefined, caller = end_chunk_transfer, chunk = false}}; handle_call({_, chunk_end}, _, #state{chunk = false} = State) -> {reply, {error, echunk}, State}; -handle_call({_, {quote, Cmd}}, From, #state{chunk = false} = State) -> - _ = send_ctrl_message(State, mk_cmd(Cmd, [])), - activate_ctrl_connection(State), +handle_call({_, {quote, Cmd}}, From, #state{chunk = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd(Cmd, [])), + State = activate_ctrl_connection(State0), {noreply, State#state{client = From, caller = quote}}; handle_call({_, _Req}, _From, #state{csock = CSock} = State) @@ -1325,38 +1325,38 @@ handle_info({Trpt, Socket, Data}, #state{dsock = {Trpt,Socket}} = State0) when T Data/binary>>}}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, - caller = {recv_file, Fd}} = State) + caller = {recv_file, Fd}} = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> file_close(Fd), - progress_report({transfer_size, 0}, State), - activate_ctrl_connection(State), + progress_report({transfer_size, 0}, State0), + State = activate_ctrl_connection(State0), ?DBG("Data channel close",[]), {noreply, State#state{dsock = undefined, data = <<>>}}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, client = Client, - caller = recv_chunk} = State) + caller = recv_chunk} = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> ?DBG("Data channel close recv_chunk",[]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{dsock = undefined, caller = #recv_chunk_closing{dconn_closed = true, client_called_us = Client =/= undefined} }}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, caller = recv_bin, - data = Data} = State) + data = Data} = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> ?DBG("Data channel close",[]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{dsock = undefined, data = <<>>, caller = {recv_bin, Data}}}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, data = Data, - caller = {handle_dir_result, Dir}} - = State) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> + caller = {handle_dir_result, Dir}} + = State0) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> ?DBG("Data channel close",[]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{dsock = undefined, caller = {handle_dir_result, Dir, Data}, % data = <<?CR,?LF>>}}; @@ -1377,7 +1377,7 @@ handle_info({Transport, Socket, Data}, #state{csock = {Transport, Socket}, client = From, ctrl_data = {CtrlData, AccLines, LineStatus}} - = State) -> + = State0) -> ?DBG('--ctrl ~p ----> ~s~p~n',[Socket,<<CtrlData/binary, Data/binary>>,State]), case ftp_response:parse_lines(<<CtrlData/binary, Data/binary>>, AccLines, LineStatus) of @@ -1387,21 +1387,21 @@ handle_info({Transport, Socket, Data}, #state{csock = {Transport, Socket}, case Caller of quote -> gen_server:reply(From, string:tokens(Lines, [?CR, ?LF])), - {noreply, State#state{client = undefined, - caller = undefined, - latest_ctrl_response = Lines, - ctrl_data = {NextMsgData, [], - start}}}; + {noreply, State0#state{client = undefined, + caller = undefined, + latest_ctrl_response = Lines, + ctrl_data = {NextMsgData, [], + start}}}; _ -> ?DBG(' ...handle_ctrl_result(~p,...) ctrl_data=~p~n',[CtrlResult,{NextMsgData, [], start}]), handle_ctrl_result(CtrlResult, - State#state{latest_ctrl_response = Lines, - ctrl_data = - {NextMsgData, [], start}}) + State0#state{latest_ctrl_response = Lines, + ctrl_data = + {NextMsgData, [], start}}) end; {continue, NewCtrlData} -> ?DBG(' ...Continue... ctrl_data=~p~n',[NewCtrlData]), - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{ctrl_data = NewCtrlData}} end; @@ -1567,19 +1567,19 @@ start_link(Opts, GenServerOptions) -> %%% Help functions to handle_call and/or handle_ctrl_result %%-------------------------------------------------------------------------- %% User handling -handle_user(User, Password, Acc, State) -> - _ = send_ctrl_message(State, mk_cmd("USER ~s", [User])), - activate_ctrl_connection(State), +handle_user(User, Password, Acc, State0) -> + _ = send_ctrl_message(State0, mk_cmd("USER ~s", [User])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_user, Password, Acc}}}. -handle_user_passwd(Password, Acc, State) -> - _ = send_ctrl_message(State, mk_cmd("PASS ~s", [Password])), - activate_ctrl_connection(State), +handle_user_passwd(Password, Acc, State0) -> + _ = send_ctrl_message(State0, mk_cmd("PASS ~s", [Password])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_user_passwd, Acc}}}. -handle_user_account(Acc, State) -> - _ = send_ctrl_message(State, mk_cmd("ACCT ~s", [Acc])), - activate_ctrl_connection(State), +handle_user_account(Acc, State0) -> + _ = send_ctrl_message(State0, mk_cmd("ACCT ~s", [Acc])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = handle_user_account}}. @@ -1594,9 +1594,9 @@ handle_ctrl_result({tls_upgrade, _}, #state{csock = {tcp, Socket}, ?DBG('<--ctrl ssl:connect(~p, ~p)~n~p~n',[Socket,TLSOptions,State0]), case ssl:connect(Socket, TLSOptions, Timeout) of {ok, TLSSocket} -> - State = State0#state{csock = {ssl,TLSSocket}}, - _ = send_ctrl_message(State, mk_cmd("PBSZ 0", [])), - activate_ctrl_connection(State), + State1 = State0#state{csock = {ssl,TLSSocket}}, + _ = send_ctrl_message(State1, mk_cmd("PBSZ 0", [])), + State = activate_ctrl_connection(State1), {noreply, State#state{tls_upgrading_data_connection = {true, pbsz}} }; {error, _} = Error -> gen_server:reply(From, {Error, self()}), @@ -1605,9 +1605,9 @@ handle_ctrl_result({tls_upgrade, _}, #state{csock = {tcp, Socket}, tls_upgrading_data_connection = false}} end; -handle_ctrl_result({pos_compl, _}, #state{tls_upgrading_data_connection = {true, pbsz}} = State) -> - _ = send_ctrl_message(State, mk_cmd("PROT P", [])), - activate_ctrl_connection(State), +handle_ctrl_result({pos_compl, _}, #state{tls_upgrading_data_connection = {true, pbsz}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("PROT P", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{tls_upgrading_data_connection = {true, prot}}}; handle_ctrl_result({pos_compl, _}, #state{tls_upgrading_data_connection = {true, prot}, @@ -1801,10 +1801,10 @@ handle_ctrl_result({pos_compl, _}, #state{caller = {handle_dir_result, Dir, handle_ctrl_result({pos_compl, Lines}, #state{caller = {handle_dir_data, Dir, DirData}} = - State) -> + State0) -> OldDir = pwd_result(Lines), - _ = send_ctrl_message(State, mk_cmd("CWD ~s", [Dir])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("CWD ~s", [Dir])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_dir_data_second_phase, OldDir, DirData}}}; handle_ctrl_result({Status, _}, @@ -1818,9 +1818,9 @@ handle_ctrl_result(S={_Status, _}, handle_ctrl_result({pos_compl, _}, #state{caller = {handle_dir_data_second_phase, OldDir, - DirData}} = State) -> - _ = send_ctrl_message(State, mk_cmd("CWD ~s", [OldDir])), - activate_ctrl_connection(State), + DirData}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("CWD ~s", [OldDir])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {handle_dir_data_third_phase, DirData}}}; handle_ctrl_result({Status, _}, #state{caller = {handle_dir_data_second_phase, _, _}} @@ -1840,9 +1840,9 @@ handle_ctrl_result(Status={epath, _}, #state{caller = {dir,_}} = State) -> %%-------------------------------------------------------------------------- %% File renaming handle_ctrl_result({pos_interm, _}, #state{caller = {rename, NewFile}} - = State) -> - _ = send_ctrl_message(State, mk_cmd("RNTO ~s", [NewFile])), - activate_ctrl_connection(State), + = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RNTO ~s", [NewFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = rename_second_phase}}; handle_ctrl_result({Status, _}, @@ -1916,10 +1916,10 @@ handle_ctrl_result({pos_compl, _}, #state{client = From, %% The pos_compl was the last event we waited for, finnish and clean up ?DBG("recv_chunk_closing pos_compl, last event",[]), gen_server:reply(From, ok), - activate_ctrl_connection(State0), - {noreply, State0#state{caller = undefined, - chunk = false, - client = undefined}}; + State = activate_ctrl_connection(State0), + {noreply, State#state{caller = undefined, + chunk = false, + client = undefined}}; handle_ctrl_result({pos_compl, _}, #state{caller = #recv_chunk_closing{}=R} = State0) -> @@ -2013,71 +2013,71 @@ ctrl_result_response(_, #state{client = From} = State, ErrorMsg) -> {noreply, State#state{client = undefined, caller = undefined}}. %%-------------------------------------------------------------------------- -handle_caller(#state{caller = {dir, Dir, Len}} = State) -> +handle_caller(#state{caller = {dir, Dir, Len}} = State0) -> Cmd = case Len of short -> "NLST"; long -> "LIST" end, _ = case Dir of "" -> - send_ctrl_message(State, mk_cmd(Cmd, "")); + send_ctrl_message(State0, mk_cmd(Cmd, "")); _ -> - send_ctrl_message(State, mk_cmd(Cmd ++ " ~s", [Dir])) + send_ctrl_message(State0, mk_cmd(Cmd ++ " ~s", [Dir])) end, - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {dir, Dir}}}; -handle_caller(#state{caller = {recv_bin, RemoteFile}} = State) -> - _ = send_ctrl_message(State, mk_cmd("RETR ~s", [RemoteFile])), - activate_ctrl_connection(State), +handle_caller(#state{caller = {recv_bin, RemoteFile}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RETR ~s", [RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = recv_bin}}; handle_caller(#state{caller = {start_chunk_transfer, Cmd, RemoteFile}} = - State) -> - _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), - activate_ctrl_connection(State), + State0) -> + _ = send_ctrl_message(State0, mk_cmd("~s ~s", [Cmd, RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = start_chunk_transfer}}; -handle_caller(#state{caller = {recv_file, RemoteFile, Fd}} = State) -> - _ = send_ctrl_message(State, mk_cmd("RETR ~s", [RemoteFile])), - activate_ctrl_connection(State), +handle_caller(#state{caller = {recv_file, RemoteFile, Fd}} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("RETR ~s", [RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {recv_file, Fd}}}; handle_caller(#state{caller = {transfer_file, {Cmd, LocalFile, RemoteFile}}, - ldir = LocalDir, client = From} = State) -> + ldir = LocalDir, client = From} = State0) -> case file_open(filename:absname(LocalFile, LocalDir), read) of {ok, Fd} -> - _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, mk_cmd("~s ~s", [Cmd, RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {transfer_file, Fd}}}; {error, _} -> gen_server:reply(From, {error, epath}), - {noreply, State#state{client = undefined, caller = undefined, - dsock = undefined}} + {noreply, State0#state{client = undefined, caller = undefined, + dsock = undefined}} end; handle_caller(#state{caller = {transfer_data, {Cmd, Bin, RemoteFile}}} = - State) -> - _ = send_ctrl_message(State, mk_cmd("~s ~s", [Cmd, RemoteFile])), - activate_ctrl_connection(State), + State0) -> + _ = send_ctrl_message(State0, mk_cmd("~s ~s", [Cmd, RemoteFile])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {transfer_data, Bin}}}. %% ----------- FTP SERVER COMMUNICATION ------------------------- %% Connect to FTP server at Host (default is TCP port 21) %% in order to establish a control connection. -setup_ctrl_connection(Host, Port, Timeout, #state{sockopts_ctrl = SockOpts} = State) -> +setup_ctrl_connection(Host, Port, Timeout, #state{sockopts_ctrl = SockOpts} = State0) -> MsTime = erlang:monotonic_time(), - case connect(Host, Port, SockOpts, Timeout, State) of + case connect(Host, Port, SockOpts, Timeout, State0) of {ok, IpFam, CSock} -> - NewState = State#state{csock = {tcp, CSock}, ipfamily = IpFam}, - activate_ctrl_connection(NewState), + State1 = State0#state{csock = {tcp, CSock}, ipfamily = IpFam}, + State = activate_ctrl_connection(State1), case Timeout - millisec_passed(MsTime) of Timeout2 when (Timeout2 >= 0) -> - {ok, NewState#state{caller = open}, Timeout2}; + {ok, State#state{caller = open}, Timeout2}; _ -> %% Oups: Simulate timeout - {ok, NewState#state{caller = open}, 0} + {ok, State#state{caller = open}, 0} end; Error -> Error @@ -2087,7 +2087,7 @@ setup_data_connection(#state{mode = active, caller = Caller, csock = CSock, sockopts_data_active = SockOpts, - ftp_extension = FtpExt} = State) -> + ftp_extension = FtpExt} = State0) -> case (catch sockname(CSock)) of {ok, {{_, _, _, _, _, _, _, _} = IP0, _}} -> IP = proplists:get_value(ip, SockOpts, IP0), @@ -2098,8 +2098,8 @@ setup_data_connection(#state{mode = active, {ok, {_, Port}} = sockname({tcp,LSock}), IpAddress = inet_parse:ntoa(IP), Cmd = mk_cmd("EPRT |2|~s|~p|", [IpAddress, Port]), - _ = send_ctrl_message(State, Cmd), - activate_ctrl_connection(State), + _ = send_ctrl_message(State0, Cmd), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, {LSock, Caller}}}}; {ok, {{_,_,_,_} = IP0, _}} -> @@ -2112,37 +2112,37 @@ setup_data_connection(#state{mode = active, false -> {IP1, IP2, IP3, IP4} = IP, {Port1, Port2} = {Port div 256, Port rem 256}, - send_ctrl_message(State, + send_ctrl_message(State0, mk_cmd("PORT ~w,~w,~w,~w,~w,~w", [IP1, IP2, IP3, IP4, Port1, Port2])); true -> IpAddress = inet_parse:ntoa(IP), Cmd = mk_cmd("EPRT |1|~s|~p|", [IpAddress, Port]), - send_ctrl_message(State, Cmd) + send_ctrl_message(State0, Cmd) end, - activate_ctrl_connection(State), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, {LSock, Caller}}}} end; setup_data_connection(#state{mode = passive, ipfamily = inet6, - caller = Caller} = State) -> - _ = send_ctrl_message(State, mk_cmd("EPSV", [])), - activate_ctrl_connection(State), + caller = Caller} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("EPSV", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, Caller}}}; setup_data_connection(#state{mode = passive, ipfamily = inet, caller = Caller, - ftp_extension = false} = State) -> - _ = send_ctrl_message(State, mk_cmd("PASV", [])), - activate_ctrl_connection(State), + ftp_extension = false} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("PASV", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, Caller}}}; setup_data_connection(#state{mode = passive, ipfamily = inet, caller = Caller, - ftp_extension = true} = State) -> - _ = send_ctrl_message(State, mk_cmd("EPSV", [])), - activate_ctrl_connection(State), + ftp_extension = true} = State0) -> + _ = send_ctrl_message(State0, mk_cmd("EPSV", [])), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = {setup_data_connection, Caller}}}. connect(Host, Port, SockOpts, Timeout, #state{ipfamily = inet = IpFam}) -> @@ -2248,14 +2248,15 @@ send_message({tcp, Socket}, Message) -> send_message({ssl, Socket}, Message) -> ssl:send(Socket, Message). -activate_ctrl_connection(#state{csock = CSock, ctrl_data = {<<>>, _, _}}) -> - activate_connection(CSock); -activate_ctrl_connection(#state{csock = CSock}) -> +activate_ctrl_connection(#state{csock = CSock, ctrl_data = {<<>>, _, _}} = State) -> + activate_connection(CSock), + State; +activate_ctrl_connection(#state{csock = CSock} = State0) -> activate_connection(CSock), %% We have already received at least part of the next control message, %% that has been saved in ctrl_data, process this first. - self() ! {socket_type(CSock), unwrap_socket(CSock), <<>>}, - ok. + {noreply, State} = handle_info({socket_type(CSock), unwrap_socket(CSock), <<>>}, State0), + State. activate_data_connection(#state{dsock = DSock} = State) -> activate_connection(DSock), @@ -2290,22 +2291,22 @@ close_connection({ssl, Socket}) -> ignore_return_value( ssl:close(Socket) ). %% ------------ FILE HANDLING ---------------------------------------- send_file(#state{tls_upgrading_data_connection = {true, CTRL, _}} = State, Fd) -> {noreply, State#state{tls_upgrading_data_connection = {true, CTRL, ?MODULE, send_file, Fd}}}; -send_file(State, Fd) -> +send_file(State0, Fd) -> case file_read(Fd) of {ok, N, Bin} when N > 0 -> - send_data_message(State, Bin), - progress_report({binary, Bin}, State), - send_file(State, Fd); + send_data_message(State0, Bin), + progress_report({binary, Bin}, State0), + send_file(State0, Fd); {ok, _, _} -> file_close(Fd), - close_data_connection(State), - progress_report({transfer_size, 0}, State), - activate_ctrl_connection(State), + close_data_connection(State0), + progress_report({transfer_size, 0}, State0), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = transfer_file_second_phase, dsock = undefined}}; {error, Reason} -> - gen_server:reply(State#state.client, {error, Reason}), - {stop, normal, State#state{client = undefined}} + gen_server:reply(State0#state.client, {error, Reason}), + {stop, normal, State0#state{client = undefined}} end. file_open(File, Option) -> @@ -2347,10 +2348,10 @@ cast(GenServer, Msg) -> send_bin(#state{tls_upgrading_data_connection = {true, CTRL, _}} = State, Bin) -> State#state{tls_upgrading_data_connection = {true, CTRL, ?MODULE, send_bin, Bin}}; -send_bin(State, Bin) -> - send_data_message(State, Bin), - close_data_connection(State), - activate_ctrl_connection(State), +send_bin(State0, Bin) -> + send_data_message(State0, Bin), + close_data_connection(State0), + State = activate_ctrl_connection(State0), {noreply, State#state{caller = transfer_data_second_phase, dsock = undefined}}. diff --git a/lib/ftp/test/ftp_SUITE.erl b/lib/ftp/test/ftp_SUITE.erl index 7c87d5cbdb..0b070ee8cb 100644 --- a/lib/ftp/test/ftp_SUITE.erl +++ b/lib/ftp/test/ftp_SUITE.erl @@ -96,6 +96,7 @@ ftp_tests()-> recv_chunk, recv_chunk_twice, recv_chunk_three_times, + recv_chunk_delay, type, quote, error_elogin, @@ -669,9 +670,9 @@ recv_chunk(Config0) -> Contents = list_to_binary( lists:duplicate(1000, lists:seq(0,255)) ), Config = set_state([reset, {mkfile,File,Contents}], Config0), Pid = proplists:get_value(ftp, Config), - {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + {error, "ftp:recv_chunk_start/2 not called"} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File,Config)), - {ok, ReceivedContents, _Ncunks} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents} = do_recv_chunk(Pid), find_diff(ReceivedContents, Contents). recv_chunk_twice() -> @@ -683,11 +684,11 @@ recv_chunk_twice(Config0) -> Contents2 = crypto:strong_rand_bytes(1200), Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}], Config0), Pid = proplists:get_value(ftp, Config), - {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + {error, "ftp:recv_chunk_start/2 not called"} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), - {ok, ReceivedContents1, _Ncunks1} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents1} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File2,Config)), - {ok, ReceivedContents2, _Ncunks2} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents2} = do_recv_chunk(Pid), find_diff(ReceivedContents1, Contents1), find_diff(ReceivedContents2, Contents2). @@ -704,46 +705,56 @@ recv_chunk_three_times(Config0) -> Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}, {mkfile,File3,Contents3}], Config0), Pid = proplists:get_value(ftp, Config), - {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + {error, "ftp:recv_chunk_start/2 not called"} = do_recv_chunk(Pid), + ok = ftp:recv_chunk_start(Pid, id2ftp(File3,Config)), + {ok, ReceivedContents3} = do_recv_chunk(Pid), + ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), - {ok, ReceivedContents1, Nchunks1} = recv_chunk(Pid, <<>>), + {ok, ReceivedContents1} = do_recv_chunk(Pid), ok = ftp:recv_chunk_start(Pid, id2ftp(File2,Config)), - {ok, ReceivedContents2, _Nchunks2} = recv_chunk(Pid, <<>>), - - ok = ftp:recv_chunk_start(Pid, id2ftp(File3,Config)), - {ok, ReceivedContents3, _Nchunks3} = recv_chunk(Pid, <<>>, 10000, 0, Nchunks1), + {ok, ReceivedContents2} = do_recv_chunk(Pid), find_diff(ReceivedContents1, Contents1), find_diff(ReceivedContents2, Contents2), find_diff(ReceivedContents3, Contents3). - +do_recv_chunk(Pid) -> + recv_chunk(Pid, <<>>). recv_chunk(Pid, Acc) -> - recv_chunk(Pid, Acc, 0, 0, undefined). - - - -%% ExpectNchunks :: integer() | undefined -recv_chunk(Pid, Acc, DelayMilliSec, N, ExpectNchunks) when N+1 < ExpectNchunks -> - %% for all I in integer(), I < undefined - recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks); - -recv_chunk(Pid, Acc, DelayMilliSec, N, ExpectNchunks) -> - %% N >= ExpectNchunks-1 - timer:sleep(DelayMilliSec), - recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks). + case ftp:recv_chunk(Pid) of + ok -> + {ok, Acc}; + {ok, Bin} -> + recv_chunk(Pid, <<Acc/binary, Bin/binary>>); + Error -> + Error + end. +recv_chunk_delay(Config0) when is_list(Config0) -> + File1 = "big_file1.txt", + Contents = list_to_binary(lists:duplicate(1000, lists:seq(0,255))), + Config = set_state([reset, {mkfile,File1,Contents}], Config0), + Pid = proplists:get_value(ftp, Config), + ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), + {ok, ReceivedContents} = delay_recv_chunk(Pid), + find_diff(ReceivedContents, Contents). -recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks) -> - ct:log("Call ftp:recv_chunk",[]), +delay_recv_chunk(Pid) -> + delay_recv_chunk(Pid, <<>>). +delay_recv_chunk(Pid, Acc) -> + ct:pal("Recived size ~p", [byte_size(Acc)]), case ftp:recv_chunk(Pid) of - ok -> {ok, Acc, N}; - {ok, Bin} -> recv_chunk(Pid, <<Acc/binary, Bin/binary>>, DelayMilliSec, N+1, ExpectNchunks); - Error -> {Error, N} - end. + ok -> + {ok, Acc}; + {ok, Bin} -> + ct:sleep(100), + delay_recv_chunk(Pid, <<Acc/binary, Bin/binary>>); + Error -> + Error + end. %%------------------------------------------------------------------------- type() -> diff --git a/lib/ftp/vsn.mk b/lib/ftp/vsn.mk index d5d6c45b28..9f14658099 100644 --- a/lib/ftp/vsn.mk +++ b/lib/ftp/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = ftp -FTP_VSN = 1.0.1 +FTP_VSN = 1.0.2 PRE_VSN = APP_VSN = "$(APPLICATION)-$(FTP_VSN)$(PRE_VSN)" diff --git a/lib/hipe/doc/src/notes.xml b/lib/hipe/doc/src/notes.xml index e9cdf42018..9a803cb9df 100644 --- a/lib/hipe/doc/src/notes.xml +++ b/lib/hipe/doc/src/notes.xml @@ -31,6 +31,21 @@ </header> <p>This document describes the changes made to HiPE.</p> +<section><title>Hipe 3.18.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix a bug in the handling of the <c>Key</c> argument + of <c>lists:{keysearch, keyfind, keymember}</c>. </p> + <p> + Own Id: OTP-15570</p> + </item> + </list> + </section> + +</section> + <section><title>Hipe 3.18.2</title> <section><title>Improvements and New Features</title> diff --git a/lib/hipe/vsn.mk b/lib/hipe/vsn.mk index 12d621bf01..39565d721f 100644 --- a/lib/hipe/vsn.mk +++ b/lib/hipe/vsn.mk @@ -1 +1 @@ -HIPE_VSN = 3.18.2 +HIPE_VSN = 3.18.3 diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 3624c6e3d7..7a6f7168ff 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -33,7 +33,43 @@ <file>notes.xml</file> </header> - <section><title>Inets 7.0.5</title> + <section><title>Inets 7.0.6</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix the internal handling of the option + erl_script_timeout in httpd. When httpd was started with + explicit erl_script_timeout, the value of the option was + converted to milliseconds before storage. Subsequent + calls to httpd:info/1 returned the input value multiplied + by 1000.</p> + <p> + This change fixes the handing of erl_script_timeout by + storing the timeout in seconds and converting to + milliseconds before usage.</p> + <p> + Own Id: OTP-15669 Aux Id: ERIERL-321 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Enhance documentation</p> + <p> + Own Id: OTP-15508 Aux Id: ERL-816 </p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 7.0.5</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/inets/src/http_server/mod_esi.erl b/lib/inets/src/http_server/mod_esi.erl index 443b7ee564..f495f12f03 100644 --- a/lib/inets/src/http_server/mod_esi.erl +++ b/lib/inets/src/http_server/mod_esi.erl @@ -33,7 +33,7 @@ -include("httpd_internal.hrl"). -define(VMODULE,"ESI"). --define(DEFAULT_ERL_TIMEOUT,15000). +-define(DEFAULT_ERL_TIMEOUT,15). %%%========================================================================= @@ -174,7 +174,7 @@ store({erl_script_alias, Value}, _) -> {error, {wrong_type, {erl_script_alias, Value}}}; store({erl_script_timeout, TimeoutSec}, _) when is_integer(TimeoutSec) andalso (TimeoutSec >= 0) -> - {ok, {erl_script_timeout, TimeoutSec * 1000}}; + {ok, {erl_script_timeout, TimeoutSec}}; store({erl_script_timeout, Value}, _) -> {error, {wrong_type, {erl_script_timeout, Value}}}; store({erl_script_nocache, Value} = Conf, _) @@ -500,7 +500,7 @@ kill_esi_delivery_process(Pid) -> erl_script_timeout(Db) -> - httpd_util:lookup(Db, erl_script_timeout, ?DEFAULT_ERL_TIMEOUT). + httpd_util:lookup(Db, erl_script_timeout, ?DEFAULT_ERL_TIMEOUT * 1000). script_elements(FuncAndInput, Input) -> case input_type(FuncAndInput) of diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index 5b6740fba3..fcb9ad7905 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -78,7 +78,8 @@ all() -> {group, http_rel_path_script_alias}, {group, http_not_sup}, {group, https_not_sup}, - mime_types_format + mime_types_format, + erl_script_timeout_option ]. groups() -> @@ -1777,6 +1778,18 @@ mime_types_format(Config) when is_list(Config) -> {"hqx","application/mac-binhex40"}]} = httpd_conf:load_mime_types(MimeTypes). +erl_script_timeout_option(Config) when is_list(Config) -> + inets:start(), + {ok, Pid} = inets:start(httpd, [{erl_script_timeout, 215}, + {server_name, "test"}, + {port,0}, + {server_root, "."}, + {document_root, "."}]), + Info = httpd:info(Pid), + 215 = proplists:get_value(erl_script_timeout, Info), + inets:stop(). + + %%-------------------------------------------------------------------- %% Internal functions ----------------------------------- %%-------------------------------------------------------------------- diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index 921161dce1..b7ddf39ebd 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 7.0.5 +INETS_VSN = 7.0.6 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" diff --git a/lib/jinterface/doc/src/Makefile b/lib/jinterface/doc/src/Makefile index d80bb38ec1..f5cba9d074 100644 --- a/lib/jinterface/doc/src/Makefile +++ b/lib/jinterface/doc/src/Makefile @@ -137,6 +137,7 @@ clean clean_docs: jdoc:$(JAVA_SRC_FILES) (cd ../../java_src;$(JAVADOC) -sourcepath . -d $(JAVADOC_DEST) \ -windowtitle $(JAVADOC_TITLE) $(JAVADOC_PKGS)) + touch jdoc man: @@ -152,15 +153,8 @@ include $(ERL_TOP)/make/otp_release_targets.mk release_docs_spec: docs $(INSTALL_DIR) "$(RELSYSDIR)/doc/pdf" $(INSTALL_DATA) $(TOP_PDF_FILE) "$(RELSYSDIR)/doc/pdf" - $(INSTALL_DIR) "$(RELSYSDIR)/doc/html" - $(INSTALL_DIR) "$(RELSYSDIR)/doc/html/java/$(JAVA_PKG_PATH)" $(INSTALL_DATA) $(INFO_FILE) "$(RELSYSDIR)" - ($(CP) -rf ../html "$(RELSYSDIR)/doc") - -# $(INSTALL_DATA) $(GIF_FILES) $(EXTRA_FILES) $(HTML_FILES) \ -# "$(RELSYSDIR)/doc/html" -# $(INSTALL_DATA) $(JAVA_EXTRA_FILES) "$(RELSYSDIR)/doc/html/java" -# $(INSTALL_DATA) $(TOP_HTML_FILES) "$(RELSYSDIR)/doc" + $(INSTALL_DIR_DATA) $(HTMLDIR) "$(RELSYSDIR)/doc/html" release_spec: diff --git a/lib/kernel/doc/src/application.xml b/lib/kernel/doc/src/application.xml index 4e32c1a3a5..83a83ebad2 100644 --- a/lib/kernel/doc/src/application.xml +++ b/lib/kernel/doc/src/application.xml @@ -238,6 +238,41 @@ Nodes = [cp1@cave, {cp2@cave, cp3@cave}]</code> </desc> </func> <func> + <name name="set_env" arity="1" since="OTP 21.3"/> + <name name="set_env" arity="2" since="OTP 21.3"/> + <fsummary>Sets the configuration parameters of multiple applications.</fsummary> + <desc> + <p>Sets the configuration <c><anno>Config</anno></c> for multiple + applications. It is equivalent to calling <c>set_env/4</c> on + each application individially, except it is more efficient. + The given <c><anno>Config</anno></c> is validated before the + configuration is set.</p> + <p><c>set_env/2</c> uses the standard <c>gen_server</c> time-out + value (5000 ms). Option <c>timeout</c> can be specified + if another time-out value is useful, for example, in situations + where the application controller is heavily loaded.</p> + <p>Option <c>persistent</c> can be set to <c>true</c> + to guarantee that parameters set with <c>set_env/2</c> + are not overridden by those defined in the application resource + file on load. This means that persistent values will stick after the application + is loaded and also on application reload.</p> + <p>If an application is given more than once or if an application + has the same key given more than once, the behaviour is undefined + and a warning message will be logged. In future releases, an error + will be raised.</p> + <p><c>set_env/1</c> is equivalent to <c>set_env(Config, [])</c>.</p> + <warning> + <p>Use this function only if you know what you are doing, + that is, on your own applications. It is very + application-dependent and + configuration parameter-dependent when and how often + the value is read by the application. Careless use + of this function can put the application in a + weird, inconsistent, and malfunctioning state.</p> + </warning> + </desc> + </func> + <func> <name name="permit" arity="2" since=""/> <fsummary>Change the permission for an application to run at a node.</fsummary> <desc> diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml index 15dbdb47dc..dbd83e1a6e 100644 --- a/lib/kernel/doc/src/kernel_app.xml +++ b/lib/kernel/doc/src/kernel_app.xml @@ -510,11 +510,13 @@ MaxT = TickTime + TickTime / 4</code> parameters for Logger are not set.</p> <taglist> <tag><c>error_logger</c></tag> - <item>Replaced by setting the type of the default - <seealso marker="logger_std_h#type"><c>logger_std_h</c></seealso> - to the same value. Example: + <item>Replaced by setting the <seealso + marker="logger_std_h#type"><c>type</c></seealso>, and possibly + <seealso marker="logger_std_h#file"><c>file</c></seealso> and + <seealso marker="logger_std_h#modes"><c>modes</c></seealso> + parameters of the default <c>logger_std_h</c> handler. Example: <code type="none"> -erl -kernel logger '[{handler,default,logger_std_h,#{config=>#{type=>{file,"/tmp/erlang.log"}}}}]' +erl -kernel logger '[{handler,default,logger_std_h,#{config=>#{file=>"/tmp/erlang.log"}}}]' </code> </item> <tag><c>error_logger_format_depth</c></tag> diff --git a/lib/kernel/doc/src/logger.xml b/lib/kernel/doc/src/logger.xml index df2d081d76..5bdfcf91db 100644 --- a/lib/kernel/doc/src/logger.xml +++ b/lib/kernel/doc/src/logger.xml @@ -66,7 +66,7 @@ logger:error("error happened because: ~p", [Reason]). % Without macro [{kernel, [{logger, [{handler, default, logger_std_h, - #{config => #{type => {file, "path/to/file.log"}}}}]}]}]. + #{config => #{file => "path/to/file.log"}}}]}]}]. </code> <p> For more information about: @@ -190,7 +190,7 @@ logger:error("error happened because: ~p", [Reason]). % Without macro <list> <item><c>pid => self()</c></item> <item><c>gl => group_leader()</c></item> - <item><c>time => erlang:system_time(microsecond)</c></item> + <item><c>time => logger:timestamp()</c></item> </list> <p>When a log macro is used, Logger also inserts location information:</p> @@ -288,8 +288,8 @@ logger:error("error happened because: ~p", [Reason]). % Without macro <name name="timestamp"/> <desc> <p>A timestamp produced - with <seealso marker="erts:erlang#system_time-1"> - <c>erlang:system_time(microsecond)</c></seealso>.</p> + with <seealso marker="#timestamp-0"> + <c>logger:timestamp()</c></seealso>.</p> </desc> </datatype> </datatypes> @@ -1117,6 +1117,24 @@ logger:set_proxy_config(maps:merge(Old, Config)). a key-value list before formatting as such.</p> </desc> </func> + + <func> + <name name="timestamp" arity="0" since="OTP 21.3"/> + <fsummary>Return a timestamp to insert in meta data for a log + event.</fsummary> + <desc> + <p>Return a timestamp that can be inserted as the <c>time</c> + field in the meta data for a log event. It is produced with + <seealso marker="kernel:os#system_time-1"> + <c>os:system_time(microsecond)</c></seealso>.</p> + <p>Notice that Logger automatically inserts a timestamp in the + meta data unless it already exists. This function is + exported for the rare case when the timestamp must be taken + at a different point in time than when the log event is + issued.</p> + </desc> + </func> + </funcs> <section> diff --git a/lib/kernel/doc/src/logger_chapter.xml b/lib/kernel/doc/src/logger_chapter.xml index 5ed7397135..bfd0acf634 100644 --- a/lib/kernel/doc/src/logger_chapter.xml +++ b/lib/kernel/doc/src/logger_chapter.xml @@ -801,7 +801,7 @@ logger:debug(#{got => connection_request, id => Id, state => State}, [{kernel, [{logger, [{handler, default, logger_std_h, % {handler, HandlerId, Module, - #{config => #{type => {file,"log/erlang.log"}}}} % Config} + #{config => #{file => "log/erlang.log"}}} % Config} ]}]}]. </code> <p>Modify the default handler to print each log event as a @@ -831,10 +831,10 @@ logger:debug(#{got => connection_request, id => Id, state => State}, [{logger, [{handler, default, logger_std_h, #{level => error, - config => #{type => {file, "log/erlang.log"}}}}, + config => #{file => "log/erlang.log"}}}, {handler, info, logger_std_h, #{level => debug, - config => #{type => {file, "log/debug.log"}}}} + config => #{file => "log/debug.log"}}} ]}]}]. </code> </section> @@ -1004,10 +1004,10 @@ ok</pre> <p>Then, add a new handler which prints to file. You can use the handler module <seealso marker="logger_std_h"><c>logger_std_h</c></seealso>, - and specify type <c>{file,File}</c>.:</p> + and configure it to log to file:</p> <pre> -4> <input>Config = #{config => #{type => {file,"./info.log"}}, level => info}.</input> -#{config => #{type => {file,"./info.log"}},level => info} +4> <input>Config = #{config => #{file => "./info.log"}, level => info}.</input> +#{config => #{file => "./info.log"},level => info} 5> <input>logger:add_handler(myhandler, logger_std_h, Config).</input> ok</pre> <p>Since <c>filter_default</c> defaults to <c>log</c>, this @@ -1246,7 +1246,7 @@ do_log(Fd, LogEvent, #{formatter := {FModule, FConfig}}) -> <p>A configuration example:</p> <code type="none"> logger:add_handler(my_standard_h, logger_std_h, - #{config => #{type => {file,"./system_info.log"}, + #{config => #{file => "./system_info.log", sync_mode_qlen => 100, drop_mode_qlen => 1000, flush_qlen => 2000}}). diff --git a/lib/kernel/doc/src/logger_std_h.xml b/lib/kernel/doc/src/logger_std_h.xml index fcd180abd6..5ed1a2f210 100644 --- a/lib/kernel/doc/src/logger_std_h.xml +++ b/lib/kernel/doc/src/logger_std_h.xml @@ -55,30 +55,105 @@ is stored in a sub map with the key <c>config</c>, and can contain the following parameters:</p> <taglist> - <tag><marker id="type"/><c>type</c></tag> + <tag><marker id="type"/><c>type = standard_io | standard_error | file</c></tag> <item> - <p>This has the value <c>standard_io</c>, <c>standard_error</c>, - <c>{file,LogFileName}</c>, or <c>{file,LogFileName,LogFileOpts}</c>.</p> - <p>If <c>LogFileOpts</c> is specified, it replaces the default - list of options used when opening the log file. The default - list is <c>[raw,append,delayed_write]</c>. One reason to do - so can be to change <c>append</c> to, for - example, <c>write</c>, ensuring that the old log is - truncated when a node is restarted. See the reference manual - for <seealso marker="file#open-2"><c>file:open/2</c></seealso> - for more information about file options.</p> + <p>Specifies the log destination.</p> + <p>The value is set when the handler is added, and it can not + be changed in runtime.</p> + <p>Defaults to <c>standard_io</c>, unless + parameter <seealso marker="#file"><c>file</c></seealso> is + given, in which case it defaults to <c>file</c>.</p> + </item> + <tag><marker id="file"/><c>file = </c><seealso marker="file#type-filename"><c>file:filename()</c></seealso></tag> + <item> + <p>This specifies the name of the log file when the handler is + of type <c>file</c>.</p> + <p>The value is set when the handler is added, and it can not + be changed in runtime.</p> + <p>Defaults to the same name as the handler identity, in the + current directory.</p> + </item> + <tag><marker id="modes"/><c>modes = [</c><seealso marker="file#type-mode"><c>file:mode()</c></seealso><c>]</c></tag> + <item> + <p>This specifies the file modes to use when opening the log + file, + see <seealso marker="file#open-2"><c>file:open/2</c></seealso>. + If <c>modes</c> are not specified, the default list used + is <c>[raw,append,delayed_write]</c>. If <c>modes</c> are + specified, the list replaces the default modes list with the + following adjustments:</p> + <list> + <item> + If <c>raw</c> is not found in the list, it is added. + </item> + <item> + If none of <c>write</c>, <c>append</c> or <c>exclusive</c> is + found in the list, <c>append</c> is added.</item> + <item>If none of <c>delayed_write</c> + or <c>{delayed_write,Size,Delay}</c> is found in the + list, <c>delayed_write</c> is added.</item> + </list> <p>Log files are always UTF-8 encoded. The encoding can not be - changed by setting the option <c>{encoding,Encoding}</c> - in <c>LogFileOpts</c>.</p> - <p>Notice that the standard handler does not have support for - circular logging. Use the disk_log handler, - <seealso marker="logger_disk_log_h"><c>logger_disk_log_h</c></seealso>, - for this.</p> + changed by setting the mode <c>{encoding,Encoding}</c>.</p> <p>The value is set when the handler is added, and it can not be changed in runtime.</p> - <p>Defaults to <c>standard_io</c>.</p> + <p>Defaults to <c>[raw,append,delayed_write]</c>.</p> + </item> + <tag><marker id="max_no_bytes"/><c>max_no_bytes = pos_integer() | infinity</c></tag> + <item> + <p>This parameter specifies if the log file should be rotated + or not. The value <c>infinity</c> means the log file will + grow indefinitely, while an integer value specifies at which + file size (bytes) the file is rotated.</p> + <p>Defaults to <c>infinity</c>.</p> + </item> + <tag><marker id="max_no_files"/><c>max_no_files = non_neg_integer()</c></tag> + <item> + <p>This parameter specifies the number of rotated log file + archives to keep. This has meaning only + if <seealso marker="#max_no_bytes"><c>max_no_bytes</c></seealso> + is set to an integer value.</p> + <p>The log archives are + named <c>FileName.0</c>, <c>FileName.1</c>, + ... <c>FileName.N</c>, where <c>FileName</c> is the name of + the current log file. <c>FileName.0</c> is the newest of the + archives. The maximum value for <c>N</c> is the value + of <c>max_no_files</c> minus 1.</p> + <p>Notice that setting this value to <c>0</c> does not turn of + rotation. It only specifies that no archives are kept.</p> + <p>Defaults to <c>0</c>.</p> + </item> + <tag><marker id="compress_on_rotate"/><c>compress_on_rotate = boolean()</c></tag> + <item> + <p>This parameter specifies if the rotated log file archives + shall be compressed or not. If set to <c>true</c>, all + archives are compressed with <c>gzip</c>, and renamed + to <c>FileName.N.gz</c></p> + <p><c>compress_on_rotate</c> has no meaning if <seealso + marker="#max_no_bytes"><c>max_no_bytes</c></seealso> has the + value <c>infinity</c>.</p> + <p>Defaults to <c>false</c>.</p> + </item> + <tag><marker id="file_check"/><c>file_check = non_neg_integer()</c></tag> + <item> + <p>When <c>logger_std_h</c> logs to a file, it reads the file + information of the log file prior to each write + operation. This is to make sure the file still exists and + has the same inode as when it was opened. This implies some + performance loss, but ensures that no log events are lost in + the case when the file has been removed or renamed by an + external actor.</p> + <p>In order to allow minimizing the performance loss, the + <c>file_check</c> parameter can be set to a positive integer + value, <c>N</c>. The handler will then skip reading the file + information prior to writing, as long as no more + than <c>N</c> milliseconds have passed since it was last + read.</p> + <p>Notice that the risk of loosing log events grows when + the <c>file_check</c> value grows.</p> + <p>Defaults to 0.</p> </item> - <tag><c>filesync_repeat_interval</c></tag> + <tag><c>filesync_repeat_interval = pos_integer() | no_repeat</c></tag> <item> <p>This value, in milliseconds, specifies how often the handler does a file sync operation to write buffered data to disk. The handler attempts @@ -97,12 +172,13 @@ standard handler and the disk_log handler, and are documented in the <seealso marker="logger_chapter#overload_protection"><c>User's Guide</c> </seealso>.</p> - <p>Notice that if changing the configuration of the handler in runtime, - the <c>type</c> parameter must not be modified.</p> + <p>Notice that if changing the configuration of the handler in + runtime, the <c>type</c>, <c>file</c>, or <c>modes</c> parameters + must not be modified.</p> <p>Example of adding a standard handler:</p> <code type="none"> logger:add_handler(my_standard_h, logger_std_h, - #{config => #{type => {file,"./system_info.log"}, + #{config => #{file => "./system_info.log", filesync_repeat_interval => 1000}}). </code> <p>To set the default handler, that starts initially with @@ -110,7 +186,7 @@ logger:add_handler(my_standard_h, logger_std_h, change the Kernel default logger configuration. Example:</p> <code type="none"> erl -kernel logger '[{handler,default,logger_std_h, - #{config => #{type => {file,"./log.log"}}}}]' + #{config => #{file => "./log.log"}}}]' </code> <p>An example of how to replace the standard handler with a disk_log handler at startup is found in the diff --git a/lib/kernel/doc/src/notes.xml b/lib/kernel/doc/src/notes.xml index 021ecfa40d..0c187eb19f 100644 --- a/lib/kernel/doc/src/notes.xml +++ b/lib/kernel/doc/src/notes.xml @@ -31,6 +31,204 @@ </header> <p>This document describes the changes made to the Kernel application.</p> +<section><title>Kernel 6.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + If for example the <c>/etc/hosts</c> did not come into + existence until after the kernel application had started, + its content was never read. This bug has now been + corrected.</p> + <p> + Own Id: OTP-14702 Aux Id: PR-2066 </p> + </item> + <item> + <p> + Fix bug where doing <c>seq_trace:reset_trace()</c> while + another process was doing a garbage collection could + cause the run-time system to segfault.</p> + <p> + Own Id: OTP-15490</p> + </item> + <item> + <p> + Fix <c>erl_epmd:port_please</c> spec to include + <c>atom()</c> and <c>string()</c>.</p> + <p> + Own Id: OTP-15557 Aux Id: PR-2117 </p> + </item> + <item> + <p> + The Logger handler logger_std_h now keeps track of the + inode of its log file in order to re-open the file if the + inode changes. This may happen, for instance, if the log + file is opened and saved by an editor.</p> + <p> + Own Id: OTP-15578 Aux Id: ERL-850 </p> + </item> + <item> + <p> + When user specific file modes are given to the logger + handler <c>logger_std_h</c>, they were earlier accepted + without any control. This is now changes, so Logger will + adjust the file modes as follows:</p> + <p> + - If <c>raw</c> is not found in the list, it is + added.<br/> - If none of <c>write</c>, <c>append</c> or + <c>exclusive</c> are found in the list, <c>append</c> is + added.<br/> - If none of <c>delayed_write</c> or + <c>{delayed_write,Size,Delay}</c> are found in the list, + <c>delayed_write</c> is added.</p> + <p> + Own Id: OTP-15602</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The standard logger handler, <c>logger_std_h</c>, now has + a new internal feature for log rotation. The rotation + scheme is as follows:</p> + <p> + The log file to which the handler currently writes always + has the same name, i.e. the name which is configured for + the handler. The archived files have the same name, but + with extension ".N", where N is an integer. The newest + archive has extension ".0", and the oldest archive has + the highest number. </p> + <p> + The size at which the log file is rotated, and the number + of archive files that are kept, is specified with the + handler specific configuration parameters + <c>max_no_bytes</c> and <c>max_no_files</c> respectively. </p> + <p> + Archives can be compressed, in which case they get a + ".gz" file extension after the integer. Compression is + specified with the handler specific configuration + parameter <c>compress_on_rotate</c>.</p> + <p> + Own Id: OTP-15479</p> + </item> + <item> + <p> + The new functions <c>logger:i/0</c> and <c>logger:i/1</c> + are added. These provide the same information as + <c>logger:get_config/0</c> and other + <c>logger:get_*_config</c> functions, but the information + is more logically sorted and more readable.</p> + <p> + Own Id: OTP-15600</p> + </item> + <item> + <p> + Logger is now protected against overload due to massive + amounts of log events from the emulator or from remote + nodes.</p> + <p> + Own Id: OTP-15601</p> + </item> + <item> + <p> + Logger now uses os:system_time/1 instead of + erlang:system_time/1 to generate log event timestamps.</p> + <p> + Own Id: OTP-15625</p> + </item> + <item> + <p> + Add functions <c>application:set_env/1,2</c> and + <c>application:set_env/2</c>. These take a list of + application configuration parameters, and the behaviour + is equivalent to calling <c>application:set_env/4</c> + individually for each application/key combination, except + it is more efficient.</p> + <p> + <c>set_env/1,2</c> warns about duplicated applications or + keys. The warning is also emitted during boot, if + applications or keys are duplicated within one + configuration file, e.g. sys.config.</p> + <p> + Own Id: OTP-15642 Aux Id: PR-2164 </p> + </item> + <item> + <p> + Handler specific configuration parameters for the + standard handler <c>logger_std_h</c> are changed to be + more intuitive and more similar to the disk_log handler.</p> + <p> + Earlier there was only one parameter, <c>type</c>, which + could have the values <c>standard_io</c>, + <c>standard_error</c>, <c>{file,FileName}</c> or + <c>{file,FileName,Modes}</c>.</p> + <p> + This is now changed, so the following parameters are + allowed:</p> + <p> + <c>type = standard_io | standard_error | file</c><br/> + <c>file = file:filename()</c><br/> <c>modes = + [file:mode()]</c></p> + <p> + All parameters are optional. <c>type</c> defaults to + <c>standard_io</c>, unless a file name is given, in which + case it defaults to <c>file</c>. If <c>type</c> is set to + <c>file</c>, the file name defaults to the same as the + handler id.</p> + <p> + The potential incompatibility is that + <c>logger:get_config/0</c> and + <c>logger:get_handler_config/1</c> now returns the new + parameters, even if the configuration was set with the + old variant, e.g. <c>#{type=>{file,FileName}}</c>.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-15662</p> + </item> + <item> + <p> + The new configuration parameter <c>file_check</c> is + added to the Logger handler <c>logger_std_h</c>. This + parameter specifies how long (in milliseconds) the + handler may wait before checking if the log file still + exists and the inode is the same as when it was opened.</p> + <p> + The default value is 0, which means that this check is + done prior to each write operation. Setting a higher + number may improve performance, but adds the risk of + loosing log events.</p> + <p> + Own Id: OTP-15663</p> + </item> + </list> + </section> + +</section> + +<section><title>Kernel 6.2.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Setting the <c>recbuf</c> size of an inet socket the + <c>buffer</c> is also automatically increased. Fix a bug + where the auto adjustment of inet buffer size would be + triggered even if an explicit inet buffer size had + already been set.</p> + <p> + Own Id: OTP-15651 Aux Id: ERIERL-304 </p> + </item> + </list> + </section> + +</section> + <section><title>Kernel 6.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/kernel/src/application.erl b/lib/kernel/src/application.erl index bc6be2f8f5..5c2e981e4b 100644 --- a/lib/kernel/src/application.erl +++ b/lib/kernel/src/application.erl @@ -25,7 +25,7 @@ which_applications/0, which_applications/1, loaded_applications/0, permit/2]). -export([ensure_started/1, ensure_started/2]). --export([set_env/3, set_env/4, unset_env/2, unset_env/3]). +-export([set_env/1, set_env/2, set_env/3, set_env/4, unset_env/2, unset_env/3]). -export([get_env/1, get_env/2, get_env/3, get_all_env/0, get_all_env/1]). -export([get_key/1, get_key/2, get_all_key/0, get_all_key/1]). -export([get_application/0, get_application/1, info/0]). @@ -279,6 +279,26 @@ loaded_applications() -> info() -> application_controller:info(). +-spec set_env(Config) -> 'ok' when + Config :: [{Application, Env}], + Application :: atom(), + Env :: [{Par :: atom(), Val :: term()}]. + +set_env(Config) when is_list(Config) -> + set_env(Config, []). + +-spec set_env(Config, Opts) -> 'ok' when + Config :: [{Application, Env}], + Application :: atom(), + Env :: [{Par :: atom(), Val :: term()}], + Opts :: [{timeout, timeout()} | {persistent, boolean()}]. + +set_env(Config, Opts) when is_list(Config), is_list(Opts) -> + case application_controller:set_env(Config, Opts) of + ok -> ok; + {error, Msg} -> erlang:error({badarg, Msg}, [Config, Opts]) + end. + -spec set_env(Application, Par, Val) -> 'ok' when Application :: atom(), Par :: atom(), diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl index a074d2e74b..9a8091fb2e 100644 --- a/lib/kernel/src/application_controller.erl +++ b/lib/kernel/src/application_controller.erl @@ -26,7 +26,7 @@ control_application/1, change_application_data/2, prep_config_change/0, config_change/1, which_applications/0, which_applications/1, - loaded_applications/0, info/0, + loaded_applications/0, info/0, set_env/2, get_pid_env/2, get_env/2, get_pid_all_env/1, get_all_env/1, get_pid_key/2, get_key/2, get_pid_all_key/1, get_all_key/1, get_master/1, get_application/1, get_application_module/1, @@ -345,9 +345,6 @@ get_all_env(AppName) -> map(fun([Key, Val]) -> {Key, Val} end, ets:match(ac_tab, {{env, AppName, '$1'}, '$2'})). - - - get_pid_key(Master, Key) -> case ets:match(ac_tab, {{application_master, '$1'}, Master}) of [[AppName]] -> get_key(AppName, Key); @@ -461,6 +458,15 @@ permit_application(ApplName, Flag) -> {permit_application, ApplName, Flag}, infinity). +set_env(Config, Opts) -> + case check_conf_data(Config) of + ok -> + Timeout = proplists:get_value(timeout, Opts, 5000), + gen_server:call(?AC, {set_env, Config, Opts}, Timeout); + + {error, _} = Error -> + Error + end. set_env(AppName, Key, Val) -> gen_server:call(?AC, {set_env, AppName, Key, Val, []}). @@ -528,19 +534,17 @@ check_conf_data([]) -> check_conf_data(ConfData) when is_list(ConfData) -> [Application | ConfDataRem] = ConfData, case Application of - {kernel, List} when is_list(List) -> - case check_para_kernel(List) of - ok -> - check_conf_data(ConfDataRem); - Error1 -> - Error1 - end; {AppName, List} when is_atom(AppName), is_list(List) -> - case check_para(List, atom_to_list(AppName)) of - ok -> - check_conf_data(ConfDataRem); - Error2 -> - Error2 + case lists:keymember(AppName, 1, ConfDataRem) of + true -> + ?LOG_WARNING("duplicate application config: " ++ atom_to_list(AppName)); + false -> + ok + end, + + case check_para(List, AppName) of + ok -> check_conf_data(ConfDataRem); + Error -> Error end; {AppName, List} when is_list(List) -> ErrMsg = "application: " @@ -553,36 +557,40 @@ check_conf_data(ConfData) when is_list(ConfData) -> ++ "; parameters must be a list", {error, ErrMsg}; Else -> - ErrMsg = "invalid application name: " ++ - lists:flatten(io_lib:format(" ~tp",[Else])), + ErrMsg = "invalid application config: " + ++ lists:flatten(io_lib:format("~tp",[Else])), {error, ErrMsg} end; check_conf_data(_ConfData) -> - {error, 'configuration must be a list ended by <dot><whitespace>'}. - + {error, "configuration must be a list ended by <dot><whitespace>"}. -%% Special check of distributed parameter for kernel -check_para_kernel([]) -> + +check_para([], _AppName) -> ok; -check_para_kernel([{distributed, Apps} | ParaList]) when is_list(Apps) -> - case check_distributed(Apps) of - {error, _ErrorMsg} = Error -> - Error; - _ -> - check_para_kernel(ParaList) +check_para([{Para, Val} | ParaList], AppName) when is_atom(Para) -> + case lists:keymember(Para, 1, ParaList) of + true -> + ?LOG_WARNING("application: " ++ atom_to_list(AppName) ++ + "; duplicate parameter: " ++ atom_to_list(Para)); + false -> + ok + end, + + case check_para_value(Para, Val, AppName) of + ok -> check_para(ParaList, AppName); + {error, _} = Error -> Error end; -check_para_kernel([{distributed, _Apps} | _ParaList]) -> - {error, "application: kernel; erroneous parameter: distributed"}; -check_para_kernel([{Para, _Val} | ParaList]) when is_atom(Para) -> - check_para_kernel(ParaList); -check_para_kernel([{Para, _Val} | _ParaList]) -> - {error, "application: kernel; invalid parameter: " ++ +check_para([{Para, _Val} | _ParaList], AppName) -> + {error, "application: " ++ atom_to_list(AppName) ++ "; invalid parameter name: " ++ lists:flatten(io_lib:format("~tp",[Para]))}; -check_para_kernel(Else) -> - {error, "application: kernel; invalid parameter list: " ++ +check_para([Else | _ParaList], AppName) -> + {error, "application: " ++ atom_to_list(AppName) ++ "; invalid parameter: " ++ lists:flatten(io_lib:format("~tp",[Else]))}. - +check_para_value(distributed, Apps, kernel) -> check_distributed(Apps); +check_para_value(_Para, _Val, _AppName) -> ok. + +%% Special check of distributed parameter for kernel check_distributed([]) -> ok; check_distributed([{App, List} | Apps]) when is_atom(App), is_list(List) -> @@ -595,18 +603,6 @@ check_distributed(_Else) -> {error, "application: kernel; erroneous parameter: distributed"}. -check_para([], _AppName) -> - ok; -check_para([{Para, _Val} | ParaList], AppName) when is_atom(Para) -> - check_para(ParaList, AppName); -check_para([{Para, _Val} | _ParaList], AppName) -> - {error, "application: " ++ AppName ++ "; invalid parameter: " ++ - lists:flatten(io_lib:format("~tp",[Para]))}; -check_para([Else | _ParaList], AppName) -> - {error, "application: " ++ AppName ++ "; invalid parameter: " ++ - lists:flatten(io_lib:format("~tp",[Else]))}. - - -type calls() :: 'info' | 'prep_config_change' | 'which_applications' | {'config_change' | 'control_application' | 'load_application' | 'start_type' | 'stop_application' | @@ -863,6 +859,16 @@ handle_call(which_applications, _From, S) -> end, S#state.running), {reply, Reply, S}; +handle_call({set_env, Config, Opts}, _From, S) -> + _ = [add_env(AppName, Env) || {AppName, Env} <- Config], + + case proplists:get_value(persistent, Opts, false) of + true -> + {reply, ok, S#state{conf_data = merge_env(S#state.conf_data, Config)}}; + false -> + {reply, ok, S} + end; + handle_call({set_env, AppName, Key, Val, Opts}, _From, S) -> ets:insert(ac_tab, {{env, AppName, Key}, Val}), case proplists:get_value(persistent, Opts, false) of diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl index 1b4a67ecb7..68e1205301 100644 --- a/lib/kernel/src/code_server.erl +++ b/lib/kernel/src/code_server.erl @@ -1434,19 +1434,25 @@ all_loaded(Db) -> -spec error_msg(io:format(), [term()]) -> 'ok'. error_msg(Format, Args) -> + %% This is equal to calling logger:error/3 which we don't want to + %% do from code_server. We don't want to call logger:timestamp() + %% either. logger ! {log,error,Format,Args, #{pid=>self(), gl=>group_leader(), - time=>erlang:system_time(microsecond), + time=>os:system_time(microsecond), error_logger=>#{tag=>error}}}, ok. -spec info_msg(io:format(), [term()]) -> 'ok'. info_msg(Format, Args) -> + %% This is equal to calling logger:info/3 which we don't want to + %% do from code_server. We don't want to call logger:timestamp() + %% either. logger ! {log,info,Format,Args, #{pid=>self(), gl=>group_leader(), - time=>erlang:system_time(microsecond), + time=>os:system_time(microsecond), error_logger=>#{tag=>info_msg}}}, ok. diff --git a/lib/kernel/src/inet_db.erl b/lib/kernel/src/inet_db.erl index 6cbb6ac2da..3f5a2ea5ee 100644 --- a/lib/kernel/src/inet_db.erl +++ b/lib/kernel/src/inet_db.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2016. All Rights Reserved. +%% Copyright Ericsson AB 1997-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1223,7 +1223,10 @@ handle_set_file(Option, Fname, TagTm, TagInfo, ParseFun, From, {ok, B, _} -> B; _ -> <<>> end; - _ -> <<>> + _ -> + ets:insert(Db, {TagInfo, undefined}), + TimeZero = - (?RES_FILE_UPDATE_TM + 1), % Early enough + ets:insert(Db, {TagTm, TimeZero}) end, handle_set_file(ParseFun, Bin, From, State); false -> {reply,error,State} diff --git a/lib/kernel/src/kernel.app.src b/lib/kernel/src/kernel.app.src index a1d9e8e215..6bd5518a37 100644 --- a/lib/kernel/src/kernel.app.src +++ b/lib/kernel/src/kernel.app.src @@ -148,6 +148,6 @@ {logger_sasl_compatible, false} ]}, {mod, {kernel, []}}, - {runtime_dependencies, ["erts-10.1", "stdlib-3.5", "sasl-3.0"]} + {runtime_dependencies, ["erts-10.2.5", "stdlib-3.5", "sasl-3.0"]} ] }. diff --git a/lib/kernel/src/kernel.appup.src b/lib/kernel/src/kernel.appup.src index ccf0a82ced..8fa3f5c588 100644 --- a/lib/kernel/src/kernel.appup.src +++ b/lib/kernel/src/kernel.appup.src @@ -40,7 +40,10 @@ {<<"^6\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.1$">>,[restart_new_emulator]}, {<<"^6\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], + {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.2$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], [{<<"^5\\.3$">>,[restart_new_emulator]}, {<<"^5\\.3\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^5\\.3\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -54,4 +57,7 @@ {<<"^6\\.0\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, {<<"^6\\.1$">>,[restart_new_emulator]}, {<<"^6\\.1\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, - {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. + {<<"^6\\.1\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, + {<<"^6\\.2$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^6\\.2\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. diff --git a/lib/kernel/src/logger.erl b/lib/kernel/src/logger.erl index 7d36640f52..38bd2f481c 100644 --- a/lib/kernel/src/logger.erl +++ b/lib/kernel/src/logger.erl @@ -61,6 +61,7 @@ -export([set_process_metadata/1, update_process_metadata/1, unset_process_metadata/0, get_process_metadata/0]). -export([i/0, i/1]). +-export([timestamp/0]). %% Basic report formatting -export([format_report/1, format_otp_report/1]). @@ -154,7 +155,8 @@ filter_return/0, config_handler/0, formatter_config/0, - olp_config/0]). + olp_config/0, + timestamp/0]). %%%----------------------------------------------------------------- %%% API @@ -354,6 +356,10 @@ internal_log(Level,Term) when is_atom(Level) -> erlang:display_string("Logger - "++ atom_to_list(Level) ++ ": "), erlang:display(Term). +-spec timestamp() -> timestamp(). +timestamp() -> + os:system_time(microsecond). + %%%----------------------------------------------------------------- %%% Configuration -spec add_primary_filter(FilterId,Filter) -> ok | {error,term()} when @@ -1129,7 +1135,7 @@ proc_meta() -> default(pid) -> self(); default(gl) -> group_leader(); -default(time) -> erlang:system_time(microsecond). +default(time) -> timestamp(). %% Remove everything upto and including this module from the stacktrace filter_stacktrace(Module,[{Module,_,_,_}|_]) -> diff --git a/lib/kernel/src/logger_formatter.erl b/lib/kernel/src/logger_formatter.erl index ded89bac9f..8696adbd72 100644 --- a/lib/kernel/src/logger_formatter.erl +++ b/lib/kernel/src/logger_formatter.erl @@ -64,7 +64,7 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) Config; Size0 -> Size = - case Size0 - string:length([B,A]) of + case Size0 - io_lib:chars_length([B,A]) of S when S>=0 -> S; _ -> 0 end, @@ -75,7 +75,11 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) true -> %% Trim leading and trailing whitespaces, and replace %% newlines with ", " - re:replace(string:trim(MsgStr0),",?\r?\n\s*",", ", + T = lists:reverse( + trim( + lists:reverse( + trim(MsgStr0,false)),true)), + re:replace(T,",?\r?\n\s*",", ", [{return,list},global,unicode]); _false -> MsgStr0 @@ -83,7 +87,26 @@ format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) true -> "" end, - truncate([B,MsgStr,A],maps:get(max_size,Config)). + truncate(B,MsgStr,A,maps:get(max_size,Config)). + +trim([H|T],Rev) when H==$\s; H==$\r; H==$\n -> + trim(T,Rev); +trim([H|T],false) when is_list(H) -> + case trim(H,false) of + [] -> + trim(T,false); + TrimmedH -> + [TrimmedH|T] + end; +trim([H|T],true) when is_list(H) -> + case trim(lists:reverse(H),true) of + [] -> + trim(T,true); + TrimmedH -> + [lists:reverse(TrimmedH)|T] + end; +trim(String,_) -> + String. do_format(Level,Data,[level|Format],Config) -> [to_string(level,Level,Config)|do_format(Level,Data,Format,Config)]; @@ -239,21 +262,47 @@ chardata_to_list(Chardata) -> throw(Error) end. -truncate(String,unlimited) -> - String; -truncate(String,Size) -> - Length = string:length(String), +truncate(B,Msg,A,unlimited) -> + [B,Msg,A]; +truncate(B,Msg,A,Size) -> + String = [B,Msg,A], + Length = io_lib:chars_length(String), if Length>Size -> - case lists:reverse(lists:flatten(String)) of - [$\n|_] -> - string:slice(String,0,Size-4)++"...\n"; + {Last,FlatString} = + case A of + [] -> + case Msg of + [] -> + {get_last(B),lists:flatten(B)}; + _ -> + {get_last(Msg),lists:flatten([B,Msg])} + end; + _ -> + {get_last(A),lists:flatten(String)} + end, + case Last of + $\n-> + lists:sublist(FlatString,1,Size-4)++"...\n"; _ -> - string:slice(String,0,Size-3)++"..." + lists:sublist(FlatString,1,Size-3)++"..." end; true -> String end. +get_last(L) -> + get_first(lists:reverse(L)). + +get_first([]) -> + error; +get_first([C|_]) when is_integer(C) -> + C; +get_first([L|Rest]) when is_list(L) -> + case get_last(L) of + error -> get_first(Rest); + First -> First + end. + %% SysTime is the system time in microseconds format_time(SysTime,#{time_offset:=Offset,time_designator:=Des}) when is_integer(SysTime) -> diff --git a/lib/kernel/src/logger_h_common.erl b/lib/kernel/src/logger_h_common.erl index e69f6de38d..16946ff97c 100644 --- a/lib/kernel/src/logger_h_common.erl +++ b/lib/kernel/src/logger_h_common.erl @@ -142,8 +142,9 @@ changing_config(SetOrUpdate, maps:with(?OLP_KEYS,NewHConfig0)), case logger_olp:set_opts(Olp,NewOlpOpts) of ok -> - maybe_set_repeated_filesync(Olp,OldCommonConfig, - NewCommonConfig), + logger_olp:cast(Olp, {config_changed, + NewCommonConfig, + NewHandlerConfig}), ReadOnly = maps:with(?READ_ONLY_KEYS,OldHConfig), NewHConfig = maps:merge( @@ -281,11 +282,24 @@ handle_cast(repeated_filesync, State#{handler_state => HS, last_op => sync} end, {noreply,set_repeated_filesync(State1)}; - -handle_cast({set_repeated_filesync,FSyncInt},State) -> - State1 = State#{filesync_repeat_interval=>FSyncInt}, - State2 = set_repeated_filesync(cancel_repeated_filesync(State1)), - {noreply, State2}. +handle_cast({config_changed, CommonConfig, HConfig}, + State = #{id := Name, + module := Module, + handler_state := HandlerState, + filesync_repeat_interval := OldFSyncInt}) -> + State1 = + case maps:get(filesync_repeat_interval,CommonConfig) of + OldFSyncInt -> + State; + FSyncInt -> + set_repeated_filesync( + cancel_repeated_filesync( + State#{filesync_repeat_interval=>FSyncInt})) + end, + HS = try Module:config_changed(Name, HConfig, HandlerState) + catch error:undef -> HandlerState + end, + {noreply, State1#{handler_state => HS}}. handle_info(Info, #{id := Name, module := Module, handler_state := HandlerState} = State) -> @@ -351,7 +365,7 @@ log_handler_info(Name, Format, Args, #{module:=Module, {ok,Conf} -> Conf; _ -> #{formatter=>{?DEFAULT_FORMATTER,?DEFAULT_FORMAT_CONFIG}} end, - Meta = #{time=>erlang:system_time(microsecond)}, + Meta = #{time=>logger:timestamp()}, Bin = log_to_binary(#{level => notice, msg => {Format,Args}, meta => Meta}, Config), @@ -447,10 +461,3 @@ cancel_repeated_filesync(State) -> end. error_notify(Term) -> ?internal_log(error, Term). - -maybe_set_repeated_filesync(_Olp, - #{filesync_repeat_interval:=FSyncInt}, - #{filesync_repeat_interval:=FSyncInt}) -> - ok; -maybe_set_repeated_filesync(Olp,_,#{filesync_repeat_interval:=FSyncInt}) -> - logger_olp:cast(Olp,{set_repeated_filesync,FSyncInt}). diff --git a/lib/kernel/src/logger_olp.erl b/lib/kernel/src/logger_olp.erl index 009280a9c9..8365383fe2 100644 --- a/lib/kernel/src/logger_olp.erl +++ b/lib/kernel/src/logger_olp.erl @@ -515,10 +515,11 @@ check_load(State = #{id:=_Name, mode_ref := ModeRef, mode := Mode, end, State1 = ?update_other(drops,DROPS,_NewDrops,State), State2 = ?update_max_qlen(QLen,State1), - State3 = maybe_notify_mode_change(Mode1,State2), + State3 = ?update_max_mem(Mem,State2), + State4 = maybe_notify_mode_change(Mode1,State3), {Mode1, QLen, Mem, ?update_other(flushes,FLUSHES,_NewFlushes, - State3#{last_qlen => QLen})}. + State4#{last_qlen => QLen})}. limit_burst(#{burst_limit_enable := false}=State) -> {true,State}; diff --git a/lib/kernel/src/logger_olp.hrl b/lib/kernel/src/logger_olp.hrl index 9b4f5ebf27..d68b5c048d 100644 --- a/lib/kernel/src/logger_olp.hrl +++ b/lib/kernel/src/logger_olp.hrl @@ -114,12 +114,16 @@ flushes => 0, flushed => 0, drops => 0, burst_drops => 0, casts => 0, calls => 0, writes => 0, max_qlen => 0, max_time => 0, - freq => {TIME,0,0}} end). + max_mem => 0, freq => {TIME,0,0}} end). -define(update_max_qlen(QLEN, STATE), begin #{max_qlen := QLEN0} = STATE, STATE#{max_qlen => ?max(QLEN0,QLEN)} end). + -define(update_max_mem(MEM, STATE), + begin #{max_mem := MEM0} = STATE, + STATE#{max_mem => ?max(MEM0,MEM)} end). + -define(update_calls_or_casts(CALL_OR_CAST, INC, STATE), case CALL_OR_CAST of cast -> @@ -154,6 +158,7 @@ -else. % DEFAULT! -define(merge_with_stats(STATE), STATE). -define(update_max_qlen(_QLEN, STATE), STATE). + -define(update_max_mem(_MEM, STATE), STATE). -define(update_calls_or_casts(_CALL_OR_CAST, _INC, STATE), STATE). -define(update_max_time(_TIME, STATE), STATE). -define(update_other(_OTHER, _VAR, _INCVAL, STATE), STATE). diff --git a/lib/kernel/src/logger_simple_h.erl b/lib/kernel/src/logger_simple_h.erl index fe181722f3..a0d51dba25 100644 --- a/lib/kernel/src/logger_simple_h.erl +++ b/lib/kernel/src/logger_simple_h.erl @@ -69,7 +69,7 @@ log(#{msg:=_,meta:=#{time:=_}}=Log,_Config) -> do_log( #{level=>error, msg=>{report,{error,simple_handler_process_dead}}, - meta=>#{time=>erlang:system_time(microsecond)}}), + meta=>#{time=>logger:timestamp()}}), do_log(Log); _ -> ?MODULE ! {log,Log} @@ -129,7 +129,7 @@ drop_msg(0) -> drop_msg(N) -> [#{level=>info, msg=>{"Simple handler buffer full, dropped ~w messages",[N]}, - meta=>#{time=>erlang:system_time(microsecond)}}]. + meta=>#{time=>logger:timestamp()}}]. %%%----------------------------------------------------------------- %%% Internal diff --git a/lib/kernel/src/logger_std_h.erl b/lib/kernel/src/logger_std_h.erl index 65f5b3876e..c8f1acfca4 100644 --- a/lib/kernel/src/logger_std_h.erl +++ b/lib/kernel/src/logger_std_h.erl @@ -29,7 +29,7 @@ -export([filesync/1]). %% logger_h_common callbacks --export([init/2, check_config/4, reset_state/2, +-export([init/2, check_config/4, config_changed/3, reset_state/2, filesync/3, write/4, handle_info/3, terminate/3]). %% logger callbacks @@ -105,85 +105,169 @@ filter_config(Config) -> %%%=================================================================== %%% logger_h_common callbacks %%%=================================================================== -init(Name, #{type := Type}) -> - case open_log_file(Name, Type) of +init(Name, Config) -> + MyConfig = maps:with([type,file,modes,file_check,max_no_bytes, + max_no_files,compress_on_rotate],Config), + case file_ctrl_start(Name, MyConfig) of {ok,FileCtrlPid} -> - {ok,#{type=>Type,file_ctrl_pid=>FileCtrlPid}}; + {ok,MyConfig#{file_ctrl_pid=>FileCtrlPid}}; Error -> Error end. -check_config(_Name,set,undefined,NewHConfig) -> - check_config(maps:merge(get_default_config(),NewHConfig)); -check_config(_Name,SetOrUpdate,OldHConfig,NewHConfig0) -> - WriteOnce = maps:with([type],OldHConfig), +check_config(Name,set,undefined,NewHConfig) -> + check_h_config(merge_default_config(Name,normalize_config(NewHConfig))); +check_config(Name,SetOrUpdate,OldHConfig,NewHConfig0) -> + WriteOnce = maps:with([type,file,modes],OldHConfig), Default = case SetOrUpdate of set -> %% Do not reset write-once fields to defaults - maps:merge(get_default_config(),WriteOnce); + merge_default_config(Name,WriteOnce); update -> OldHConfig end, - NewHConfig = maps:merge(Default, NewHConfig0), + NewHConfig = maps:merge(Default, normalize_config(NewHConfig0)), %% Fail if write-once fields are changed - case maps:with([type],NewHConfig) of + case maps:with([type,file,modes],NewHConfig) of WriteOnce -> - check_config(NewHConfig); + check_h_config(NewHConfig); Other -> {error,{illegal_config_change,?MODULE,WriteOnce,Other}} end. -check_config(#{type:=Type}=HConfig) -> - case check_h_config(maps:to_list(HConfig)) of - ok when is_atom(Type) -> - {ok,HConfig#{filesync_repeat_interval=>no_repeat}}; +check_h_config(HConfig) -> + case check_h_config(maps:get(type,HConfig),maps:to_list(HConfig)) of ok -> - {ok,HConfig}; + {ok,fix_file_opts(HConfig)}; {error,{Key,Value}} -> {error,{invalid_config,?MODULE,#{Key=>Value}}} end. -check_h_config([{type,Type} | Config]) when Type == standard_io; - Type == standard_error -> - check_h_config(Config); -check_h_config([{type,{file,File}} | Config]) when is_list(File) -> - check_h_config(Config); -check_h_config([{type,{file,File,Modes}} | Config]) when is_list(File), - is_list(Modes) -> - check_h_config(Config); -check_h_config([Other | _]) -> +check_h_config(Type,[{type,Type} | Config]) when Type =:= standard_io; + Type =:= standard_error; + Type =:= file -> + check_h_config(Type,Config); +check_h_config(file,[{file,File} | Config]) when is_list(File) -> + check_h_config(file,Config); +check_h_config(file,[{modes,Modes} | Config]) when is_list(Modes) -> + check_h_config(file,Config); +check_h_config(file,[{max_no_bytes,Size} | Config]) + when (is_integer(Size) andalso Size>0) orelse Size=:=infinity -> + check_h_config(file,Config); +check_h_config(file,[{max_no_files,Num} | Config]) when is_integer(Num), Num>=0 -> + check_h_config(file,Config); +check_h_config(file,[{compress_on_rotate,Bool} | Config]) when is_boolean(Bool) -> + check_h_config(file,Config); +check_h_config(file,[{file_check,FileCheck} | Config]) + when is_integer(FileCheck), FileCheck>=0 -> + check_h_config(file,Config); +check_h_config(_Type,[Other | _]) -> {error,Other}; -check_h_config([]) -> +check_h_config(_Type,[]) -> ok. -get_default_config() -> - #{type => standard_io}. +normalize_config(#{type:={file,File}}=HConfig) -> + HConfig#{type=>file,file=>File}; +normalize_config(#{type:={file,File,Modes}}=HConfig) -> + HConfig#{type=>file,file=>File,modes=>Modes}; +normalize_config(HConfig) -> + HConfig. + +merge_default_config(Name,#{type:=Type}=HConfig) -> + merge_default_config(Name,Type,HConfig); +merge_default_config(Name,#{file:=_}=HConfig) -> + merge_default_config(Name,file,HConfig); +merge_default_config(Name,HConfig) -> + merge_default_config(Name,standard_io,HConfig). + +merge_default_config(Name,Type,HConfig) -> + maps:merge(get_default_config(Name,Type),HConfig). + +get_default_config(Name,file) -> + #{type => file, + file => atom_to_list(Name), + modes => [raw,append], + file_check => 0, + max_no_bytes => infinity, + max_no_files => 0, + compress_on_rotate => false}; +get_default_config(_Name,Type) -> + #{type => Type}. + +fix_file_opts(#{modes:=Modes}=HConfig) -> + HConfig#{modes=>fix_modes(Modes)}; +fix_file_opts(HConfig) -> + HConfig#{filesync_repeat_interval=>no_repeat}. + +fix_modes(Modes) -> + %% Ensure write|append|exclusive + Modes1 = + case [M || M <- Modes, + lists:member(M,[write,append,exclusive])] of + [] -> [append|Modes]; + _ -> Modes + end, + %% Ensure raw + Modes2 = + case lists:member(raw,Modes) of + false -> [raw|Modes1]; + true -> Modes1 + end, + %% Ensure delayed_write + case lists:partition(fun(delayed_write) -> true; + ({delayed_write,_,_}) -> true; + (_) -> false + end, Modes2) of + {[],_} -> + [delayed_write|Modes2]; + _ -> + Modes2 + end. -filesync(_Name, _Mode, #{type := Type}=State) when is_atom(Type) -> - {ok,State}; -filesync(_Name, async, #{file_ctrl_pid := FileCtrlPid} = State) -> - ok = file_ctrl_filesync_async(FileCtrlPid), - {ok,State}; -filesync(_Name, sync, #{file_ctrl_pid := FileCtrlPid} = State) -> - Result = file_ctrl_filesync_sync(FileCtrlPid), +config_changed(_Name, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}=State) -> + State; +config_changed(_Name, + #{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}, + #{file_ctrl_pid := FileCtrlPid} = State) -> + FileCtrlPid ! {update_config,#{file_check=>FileCheck, + max_no_bytes=>Size, + max_no_files=>Count, + compress_on_rotate=>Compress}}, + State#{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}; +config_changed(_Name,_NewHConfig,State) -> + State. + +filesync(_Name, SyncAsync, #{file_ctrl_pid := FileCtrlPid} = State) -> + Result = file_ctrl_filesync(SyncAsync, FileCtrlPid), {Result,State}. -write(_Name, async, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> - ok = file_write_async(FileCtrlPid, Bin), - {ok,State}; -write(_Name, sync, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> - Result = file_write_sync(FileCtrlPid, Bin), +write(_Name, SyncAsync, Bin, #{file_ctrl_pid:=FileCtrlPid} = State) -> + Result = file_write(SyncAsync, FileCtrlPid, Bin), {Result,State}. reset_state(_Name, State) -> State. -handle_info(_Name, {'EXIT',Pid,Why}, #{type := FileInfo, file_ctrl_pid := Pid}) -> +handle_info(_Name, {'EXIT',Pid,Why}, #{file_ctrl_pid := Pid}=State) -> %% file_ctrl_pid died, file error, terminate handler - exit({error,{write_failed,FileInfo,Why}}); + exit({error,{write_failed,maps:with([type,file,modes],State),Why}}); handle_info(_, _, State) -> State. @@ -211,27 +295,33 @@ terminate(_Name, _Reason, #{file_ctrl_pid:=FWPid}) -> %%%----------------------------------------------------------------- %%% -open_log_file(HandlerName, FileInfo) -> - case file_ctrl_start(HandlerName, FileInfo) of - OK = {ok,_FileCtrlPid} -> OK; - Error -> Error - end. - -do_open_log_file({file,FileName}) -> - do_open_log_file({file,FileName,[raw,append,delayed_write]}); - -do_open_log_file({file,FileName,[]}) -> - do_open_log_file({file,FileName,[raw,append,delayed_write]}); - -do_open_log_file({file,FileName,Modes}) -> +open_log_file(HandlerName,#{type:=file, + file:=FileName, + modes:=Modes, + file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}) -> try case filelib:ensure_dir(FileName) of ok -> case file:open(FileName, Modes) of {ok, Fd} -> {ok,#file_info{inode=INode}} = - file:read_file_info(FileName), - {ok, {Fd, INode}}; + file:read_file_info(FileName,[raw]), + UpdateModes = [append | Modes--[write,append,exclusive]], + State0 = #{handler_name=>HandlerName, + file_name=>FileName, + modes=>UpdateModes, + file_check=>FileCheck, + fd=>Fd, + inode=>INode, + last_check=>timestamp(), + synced=>false, + write_res=>ok, + sync_res=>ok}, + State = update_rotation({Size,Count,Compress},State0), + {ok,State}; Error -> Error end; @@ -242,21 +332,23 @@ do_open_log_file({file,FileName,Modes}) -> _:Reason -> {error,Reason} end. -close_log_file(Std) when Std == standard_io; Std == standard_error -> - ok; -close_log_file({Fd,_}) -> +close_log_file(#{fd:=Fd}) -> _ = file:datasync(Fd), - _ = file:close(Fd). + _ = file:close(Fd), + ok; +close_log_file(_) -> + ok. + %%%----------------------------------------------------------------- %%% File control process -file_ctrl_start(HandlerName, FileInfo) -> +file_ctrl_start(HandlerName, HConfig) -> Starter = self(), FileCtrlPid = spawn_link(fun() -> - file_ctrl_init(HandlerName, FileInfo, Starter) + file_ctrl_init(HandlerName, HConfig, Starter) end), receive {FileCtrlPid,ok} -> @@ -271,18 +363,16 @@ file_ctrl_start(HandlerName, FileInfo) -> file_ctrl_stop(Pid) -> Pid ! stop. -file_write_async(Pid, Bin) -> +file_write(async, Pid, Bin) -> Pid ! {log,Bin}, - ok. - -file_write_sync(Pid, Bin) -> + ok; +file_write(sync, Pid, Bin) -> file_ctrl_call(Pid, {log,Bin}). -file_ctrl_filesync_async(Pid) -> +file_ctrl_filesync(async, Pid) -> Pid ! filesync, - ok. - -file_ctrl_filesync_sync(Pid) -> + ok; +file_ctrl_filesync(sync, Pid) -> file_ctrl_call(Pid, filesync). file_ctrl_call(Pid, Msg) -> @@ -299,98 +389,255 @@ file_ctrl_call(Pid, Msg) -> {error,{no_response,Pid}} end. -file_ctrl_init(HandlerName, FileInfo, Starter) when is_tuple(FileInfo) -> +file_ctrl_init(HandlerName, + #{type:=file, + file:=FileName} = HConfig, + Starter) -> process_flag(message_queue_data, off_heap), - FileName = element(2, FileInfo), - case do_open_log_file(FileInfo) of - {ok,File} -> + case open_log_file(HandlerName,HConfig) of + {ok,State} -> Starter ! {self(),ok}, - file_ctrl_loop(File, FileName, false, ok, ok, HandlerName); + file_ctrl_loop(State); {error,Reason} -> Starter ! {self(),{error,{open_failed,FileName,Reason}}} end; -file_ctrl_init(HandlerName, StdDev, Starter) -> +file_ctrl_init(HandlerName, #{type:=StdDev}, Starter) -> Starter ! {self(),ok}, - file_ctrl_loop(StdDev, StdDev, false, ok, ok, HandlerName). + file_ctrl_loop(#{handler_name=>HandlerName,dev=>StdDev}). -file_ctrl_loop(File, DevName, Synced, - PrevWriteResult, PrevSyncResult, HandlerName) -> +file_ctrl_loop(State) -> receive %% asynchronous event {log,Bin} -> - File1 = ensure(File, DevName), - Result = write_to_dev(File1, Bin, DevName, - PrevWriteResult, HandlerName), - file_ctrl_loop(File1, DevName, false, - Result, PrevSyncResult, HandlerName); + State1 = write_to_dev(Bin,State), + file_ctrl_loop(State1); %% synchronous event {{log,Bin},{From,MRef}} -> - File1 = ensure(File, DevName), - Result = write_to_dev(File1, Bin, DevName, - PrevWriteResult, HandlerName), + State1 = ensure_file(State), + State2 = write_to_dev(Bin,State1), From ! {MRef,ok}, - file_ctrl_loop(File1, DevName, false, - Result, PrevSyncResult, HandlerName); + file_ctrl_loop(State2); filesync -> - File1 = ensure(File, DevName), - Result = sync_dev(File1, DevName, Synced, - PrevSyncResult, HandlerName), - file_ctrl_loop(File1, DevName, true, - PrevWriteResult, Result, HandlerName); + State1 = sync_dev(State), + file_ctrl_loop(State1); {filesync,{From,MRef}} -> - File1 = ensure(File, DevName), - Result = sync_dev(File1, DevName, Synced, - PrevSyncResult, HandlerName), + State1 = ensure_file(State), + State2 = sync_dev(State1), From ! {MRef,ok}, - file_ctrl_loop(File1, DevName, true, - PrevWriteResult, Result, HandlerName); + file_ctrl_loop(State2); + + {update_config,#{file_check:=FileCheck, + max_no_bytes:=Size, + max_no_files:=Count, + compress_on_rotate:=Compress}} -> + State1 = update_rotation({Size,Count,Compress},State), + file_ctrl_loop(State1#{file_check=>FileCheck}); stop -> - _ = close_log_file(File), + close_log_file(State), stopped end. +maybe_ensure_file(#{file_check:=0}=State) -> + ensure_file(State); +maybe_ensure_file(#{last_check:=T0,file_check:=CheckInt}=State) + when is_integer(CheckInt) -> + T = timestamp(), + if T-T0 > CheckInt -> ensure_file(State); + true -> State + end; +maybe_ensure_file(State) -> + State. + %% In order to play well with tools like logrotate, we need to be able %% to re-create the file if it has disappeared (e.g. if rotated by %% logrotate) -ensure(Fd,DevName) when is_atom(DevName) -> - Fd; -ensure({Fd,INode},FileName) -> - case file:read_file_info(FileName) of - {ok,#file_info{inode=INode}} -> - {Fd,INode}; +ensure_file(#{fd:=Fd0,inode:=INode0,file_name:=FileName,modes:=Modes}=State) -> + case file:read_file_info(FileName,[raw]) of + {ok,#file_info{inode=INode0}} -> + State#{last_check=>timestamp()}; _ -> - _ = file:close(Fd), - _ = file:close(Fd), % delayed_write cause close not to close - case do_open_log_file({file,FileName}) of - {ok,File} -> - File; + close_log_file(Fd0), + case file:open(FileName,Modes) of + {ok,Fd} -> + {ok,#file_info{inode=INode}} = + file:read_file_info(FileName,[raw]), + State#{fd=>Fd,inode=>INode, + last_check=>timestamp(), + synced=>true,sync_res=>ok}; Error -> exit({could_not_reopen_file,Error}) end - end. + end; +ensure_file(State) -> + State. -write_to_dev(DevName, Bin, _DevName, _PrevWriteResult, _HandlerName) - when is_atom(DevName) -> - io:put_chars(DevName, Bin); -write_to_dev({Fd,_}, Bin, FileName, PrevWriteResult, HandlerName) -> +write_to_dev(Bin,#{dev:=DevName}=State) -> + io:put_chars(DevName, Bin), + State; +write_to_dev(Bin, State) -> + State1 = #{fd:=Fd} = maybe_ensure_file(State), Result = ?file_write(Fd, Bin), - maybe_notify_error(write,Result,PrevWriteResult,FileName,HandlerName). + State2 = maybe_rotate_file(Bin,State1), + maybe_notify_error(write,Result,State2), + State2#{synced=>false,write_res=>Result}. -sync_dev(_, _FileName, true, PrevSyncResult, _HandlerName) -> - PrevSyncResult; -sync_dev({Fd,_}, FileName, false, PrevSyncResult, HandlerName) -> +sync_dev(#{synced:=false}=State) -> + State1 = #{fd:=Fd} = maybe_ensure_file(State), Result = ?file_datasync(Fd), - maybe_notify_error(filesync,Result,PrevSyncResult,FileName,HandlerName). + maybe_notify_error(filesync,Result,State1), + State1#{synced=>true,sync_res=>Result}; +sync_dev(State) -> + State. -maybe_notify_error(_Op, ok, _PrevResult, _FileName, _HandlerName) -> +update_rotation({infinity,_,_},State) -> + maybe_remove_archives(0,State), + maps:remove(rotation,State); +update_rotation({Size,Count,Compress},#{file_name:=FileName} = State) -> + maybe_remove_archives(Count,State), + {ok,#file_info{size=CurrSize}} = file:read_file_info(FileName,[raw]), + State1 = State#{rotation=>#{size=>Size, + count=>Count, + compress=>Compress, + curr_size=>CurrSize}}, + maybe_update_compress(0,State1), + maybe_rotate_file(0,State1). + +maybe_remove_archives(Count,#{file_name:=FileName}=State) -> + Archive = rot_file_name(FileName,Count,false), + CompressedArchive = rot_file_name(FileName,Count,true), + case {file:read_file_info(Archive,[raw]), + file:read_file_info(CompressedArchive,[raw])} of + {{error,enoent},{error,enoent}} -> + ok; + _ -> + _ = file:delete(Archive), + _ = file:delete(CompressedArchive), + maybe_remove_archives(Count+1,State) + end. + +maybe_update_compress(Count,#{rotation:=#{count:=Count}}) -> + ok; +maybe_update_compress(N,#{file_name:=FileName, + rotation:=#{compress:=Compress}}=State) -> + Archive = rot_file_name(FileName,N,not Compress), + case file:read_file_info(Archive,[raw]) of + {ok,_} when Compress -> + compress_file(Archive); + {ok,_} -> + decompress_file(Archive); + _ -> + ok + end, + maybe_update_compress(N+1,State). + +maybe_rotate_file(Bin,#{rotation:=_}=State) when is_binary(Bin) -> + maybe_rotate_file(byte_size(Bin),State); +maybe_rotate_file(AddSize,#{rotation:=#{size:=RotSize, + curr_size:=CurrSize}=Rotation}=State) -> + NewSize = CurrSize + AddSize, + if NewSize>RotSize -> + rotate_file(State#{rotation=>Rotation#{curr_size=>NewSize}}); + true -> + State#{rotation=>Rotation#{curr_size=>NewSize}} + end; +maybe_rotate_file(_Bin,State) -> + State. + +rotate_file(#{fd:=Fd0,file_name:=FileName,modes:=Modes,rotation:=Rotation}=State) -> + State1 = sync_dev(State), + _ = file:close(Fd0), + _ = file:close(Fd0), + rotate_files(FileName,maps:get(count,Rotation),maps:get(compress,Rotation)), + case file:open(FileName,Modes) of + {ok,Fd} -> + {ok,#file_info{inode=INode}} = file:read_file_info(FileName,[raw]), + State1#{fd=>Fd,inode=>INode,rotation=>Rotation#{curr_size=>0}}; + Error -> + exit({could_not_reopen_file,Error}) + end. + +rotate_files(FileName,0,_Compress) -> + _ = file:delete(FileName), + ok; +rotate_files(FileName,1,Compress) -> + FileName0 = FileName++".0", + _ = file:rename(FileName,FileName0), + if Compress -> compress_file(FileName0); + true -> ok + end, ok; -maybe_notify_error(_Op, PrevResult, PrevResult, _FileName, _HandlerName) -> +rotate_files(FileName,Count,Compress) -> + _ = file:rename(rot_file_name(FileName,Count-2,Compress), + rot_file_name(FileName,Count-1,Compress)), + rotate_files(FileName,Count-1,Compress). + +rot_file_name(FileName,Count,false) -> + FileName ++ "." ++ integer_to_list(Count); +rot_file_name(FileName,Count,true) -> + rot_file_name(FileName,Count,false) ++ ".gz". + +compress_file(FileName) -> + {ok,In} = file:open(FileName,[read,binary]), + {ok,Out} = file:open(FileName++".gz",[write]), + Z = zlib:open(), + zlib:deflateInit(Z, default, deflated, 31, 8, default), + compress_data(Z,In,Out), + zlib:deflateEnd(Z), + zlib:close(Z), + _ = file:close(In), + _ = file:close(Out), + _ = file:delete(FileName), + ok. + +compress_data(Z,In,Out) -> + case file:read(In,100000) of + {ok,Data} -> + Compressed = zlib:deflate(Z, Data), + _ = file:write(Out,Compressed), + compress_data(Z,In,Out); + eof -> + Compressed = zlib:deflate(Z, <<>>, finish), + _ = file:write(Out,Compressed), + ok + end. + +decompress_file(FileName) -> + {ok,In} = file:open(FileName,[read,binary]), + {ok,Out} = file:open(filename:rootname(FileName,".gz"),[write]), + Z = zlib:open(), + zlib:inflateInit(Z, 31), + decompress_data(Z,In,Out), + zlib:inflateEnd(Z), + zlib:close(Z), + _ = file:close(In), + _ = file:close(Out), + _ = file:delete(FileName), + ok. + +decompress_data(Z,In,Out) -> + case file:read(In,1000) of + {ok,Data} -> + Decompressed = zlib:inflate(Z, Data), + _ = file:write(Out,Decompressed), + decompress_data(Z,In,Out); + eof -> + ok + end. + +maybe_notify_error(_Op, ok, _State) -> + ok; +maybe_notify_error(Op, Result, #{write_res:=WR,sync_res:=SR}) + when (Op==write andalso Result==WR) orelse + (Op==filesync andalso Result==SR) -> %% don't report same error twice - PrevResult; -maybe_notify_error(Op, Error, _PrevResult, FileName, HandlerName) -> + ok; +maybe_notify_error(Op, Error, #{handler_name:=HandlerName,file_name:=FileName}) -> logger_h_common:error_notify({HandlerName,Op,FileName,Error}), - Error. + ok. + +timestamp() -> + erlang:monotonic_time(millisecond). diff --git a/lib/kernel/test/application_SUITE.erl b/lib/kernel/test/application_SUITE.erl index 5c35b82207..94d7c17712 100644 --- a/lib/kernel/test/application_SUITE.erl +++ b/lib/kernel/test/application_SUITE.erl @@ -31,6 +31,7 @@ otp_3002/1, otp_3184/1, otp_4066/1, otp_4227/1, otp_5363/1, otp_5606/1, start_phases/1, get_key/1, get_env/1, + set_env/1, set_env_persistent/1, set_env_errors/1, permit_false_start_local/1, permit_false_start_dist/1, script_start/1, nodedown_start/1, init2973/0, loop2973/0, loop5606/1]). @@ -55,6 +56,7 @@ all() -> load_use_cache, ensure_started, {group, reported_bugs}, start_phases, script_start, nodedown_start, permit_false_start_local, permit_false_start_dist, get_key, get_env, ensure_all_started, + set_env, set_env_persistent, set_env_errors, {group, distr_changed}, config_change, shutdown_func, shutdown_timeout, shutdown_deadlock, config_relative_paths, persistent_env]. @@ -1944,6 +1946,101 @@ get_appls([_ | T], Res) -> get_appls([], Res) -> Res. +%% Test set_env/1. +set_env(Conf) when is_list(Conf) -> + ok = application:set_env([{appinc, [{own2, persist}, {not_in_app, persist}]}, + {unknown_app, [{key, persist}]}]), + + %% own_env1 and own2 are set in appinc + undefined = application:get_env(appinc, own_env1), + {ok, persist} = application:get_env(appinc, own2), + {ok, persist} = application:get_env(appinc, not_in_app), + {ok, persist} = application:get_env(unknown_app, key), + + ok = application:load(appinc()), + {ok, value1} = application:get_env(appinc, own_env1), + {ok, val2} = application:get_env(appinc, own2), + {ok, persist} = application:get_env(appinc, not_in_app), + {ok, persist} = application:get_env(unknown_app, key), + + %% On reload, values are lost + ok = application:unload(appinc), + ok = application:load(appinc()), + {ok, value1} = application:get_env(appinc, own_env1), + {ok, val2} = application:get_env(appinc, own2), + undefined = application:get_env(appinc, not_in_app), + + %% Clean up + ok = application:unload(appinc). + +%% Test set_env/2 with persistent true. +set_env_persistent(Conf) when is_list(Conf) -> + Opts = [{persistent, true}], + ok = application:set_env([{appinc, [{own2, persist}, {not_in_app, persist}]}, + {unknown_app, [{key, persist}]}], Opts), + + %% own_env1 and own2 are set in appinc + undefined = application:get_env(appinc, own_env1), + {ok, persist} = application:get_env(appinc, own2), + {ok, persist} = application:get_env(appinc, not_in_app), + {ok, persist} = application:get_env(unknown_app, key), + + ok = application:load(appinc()), + {ok, value1} = application:get_env(appinc, own_env1), + {ok, persist} = application:get_env(appinc, own2), + {ok, persist} = application:get_env(appinc, not_in_app), + {ok, persist} = application:get_env(unknown_app, key), + + %% On reload, values are not lost + ok = application:unload(appinc), + ok = application:load(appinc()), + {ok, value1} = application:get_env(appinc, own_env1), + {ok, persist} = application:get_env(appinc, own2), + {ok, persist} = application:get_env(appinc, not_in_app), + + %% Clean up + ok = application:unload(appinc). + +set_env_errors(Conf) when is_list(Conf) -> + "application: 1; application name must be an atom" = + badarg_msg(fun() -> application:set_env([{1, []}]) end), + + "application: foo; parameters must be a list" = + badarg_msg(fun() -> application:set_env([{foo, bar}]) end), + + "invalid application config: foo_bar" = + badarg_msg(fun() -> application:set_env([foo_bar]) end), + + "application: foo; invalid parameter name: 1" = + badarg_msg(fun() -> application:set_env([{foo, [{1, 2}]}]) end), + + "application: foo; invalid parameter: config" = + badarg_msg(fun() -> application:set_env([{foo, [config]}]) end), + + "application: kernel; erroneous parameter: distributed" = + badarg_msg(fun() -> application:set_env([{kernel, [{distributed, config}]}]) end), + + %% This will raise in the future + ct:capture_start(), + _ = application:set_env([{foo, []}, {foo, []}]), + timer:sleep(100), + ct:capture_stop(), + [_ | _] = string:find(ct:capture_get(), "duplicate application config: foo"), + + ct:capture_start(), + _ = application:set_env([{foo, [{bar, baz}, {bar, bat}]}]), + timer:sleep(100), + ct:capture_stop(), + [_ | _] = string:find(ct:capture_get(), "application: foo; duplicate parameter: bar"), + + ok. + +badarg_msg(Fun) -> + try Fun() of + _ -> ct:fail(try_succeeded) + catch + error:{badarg, Msg} -> Msg + end. %% Test set_env/4 and unset_env/3 with persistent true. persistent_env(Conf) when is_list(Conf) -> diff --git a/lib/kernel/test/inet_SUITE.erl b/lib/kernel/test/inet_SUITE.erl index f436eafad3..5a2d809aa4 100644 --- a/lib/kernel/test/inet_SUITE.erl +++ b/lib/kernel/test/inet_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("kernel/include/inet.hrl"). +-include_lib("kernel/src/inet_res.hrl"). -include_lib("kernel/src/inet_dns.hrl"). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, @@ -34,7 +35,7 @@ ipv4_to_ipv6/0, ipv4_to_ipv6/1, host_and_addr/0, host_and_addr/1, t_gethostnative/1, - gethostnative_parallell/1, cname_loop/1, + gethostnative_parallell/1, cname_loop/1, missing_hosts_reload/1, gethostnative_soft_restart/0, gethostnative_soft_restart/1, gethostnative_debug_level/0, gethostnative_debug_level/1, lookup_bad_search_option/1, @@ -56,7 +57,7 @@ all() -> [t_gethostbyaddr, t_gethostbyname, t_getaddr, t_gethostbyaddr_v6, t_gethostbyname_v6, t_getaddr_v6, ipv4_to_ipv6, host_and_addr, {group, parse}, - t_gethostnative, gethostnative_parallell, cname_loop, + t_gethostnative, gethostnative_parallell, cname_loop, missing_hosts_reload, gethostnative_debug_level, gethostnative_soft_restart, lookup_bad_search_option, getif, getif_ifr_name_overflow, getservbyname_overflow, @@ -840,6 +841,32 @@ cname_loop(Config) when is_list(Config) -> ok. +%% Test that hosts file gets reloaded correctly in case when it +% was missing during initial startup +missing_hosts_reload(Config) when is_list(Config) -> + RootDir = proplists:get_value(priv_dir,Config), + HostsFile = filename:join(RootDir, atom_to_list(?MODULE) ++ ".hosts"), + InetRc = filename:join(RootDir, "inetrc"), + ok = file:write_file(InetRc, "{hosts_file, \"" ++ HostsFile ++ "\"}.\n"), + {error, enoent} = file:read_file_info(HostsFile), + % start a node + Pa = filename:dirname(code:which(?MODULE)), + {ok, TestNode} = test_server:start_node(?MODULE, slave, + [{args, "-pa " ++ Pa ++ " -kernel inetrc '\"" ++ InetRc ++ "\"'"}]), + % ensure it has our RC + Rc = rpc:call(TestNode, inet_db, get_rc, []), + {hosts_file, HostsFile} = lists:keyfind(hosts_file, 1, Rc), + % ensure it does not resolve + {error, nxdomain} = rpc:call(TestNode, inet_hosts, gethostbyname, ["somehost"]), + % write hosts file + ok = file:write_file(HostsFile, "1.2.3.4 somehost"), + % wait for cached timestamp to expire + timer:sleep(?RES_FILE_UPDATE_TM * 1000 + 100), + % ensure it DOES resolve + {ok,{hostent,"somehost",[],inet,4,[{1,2,3,4}]}} = + rpc:call(TestNode, inet_hosts, gethostbyname, ["somehost"]), + % cleanup + true = test_server:stop_node(TestNode). %% These must be run in the whole suite since they need %% the host list and require inet_gethost_native to be started. diff --git a/lib/kernel/test/kernel.spec b/lib/kernel/test/kernel.spec index 62afc9f97b..eaa17f3a59 100644 --- a/lib/kernel/test/kernel.spec +++ b/lib/kernel/test/kernel.spec @@ -2,3 +2,4 @@ {config, "../test_server/ts.unix.config"}. {suites,"../kernel_test", all}. +{skip_suites,"../kernel_test",[logger_stress_SUITE],"Benchmarks only"}. diff --git a/lib/kernel/test/logger_SUITE.erl b/lib/kernel/test/logger_SUITE.erl index 2dad651f9c..035e5d8974 100644 --- a/lib/kernel/test/logger_SUITE.erl +++ b/lib/kernel/test/logger_SUITE.erl @@ -899,14 +899,14 @@ process_metadata(_Config) -> undefined = logger:get_process_metadata(), {error,badarg} = ?TRY(logger:set_process_metadata(bad)), ok = logger:add_handler(h1,?MODULE,#{level=>notice,filter_default=>log}), - Time = erlang:system_time(microsecond), + Time = logger:timestamp(), ProcMeta = #{time=>Time,line=>0,custom=>proc}, ok = logger:set_process_metadata(ProcMeta), S1 = ?str, ?LOG_NOTICE(S1,#{custom=>macro}), check_logged(notice,S1,#{time=>Time,line=>0,custom=>macro}), - Time2 = erlang:system_time(microsecond), + Time2 = logger:timestamp(), S2 = ?str, ?LOG_NOTICE(S2,#{time=>Time2,line=>1,custom=>macro}), check_logged(notice,S2,#{time=>Time2,line=>1,custom=>macro}), @@ -1048,8 +1048,11 @@ kernel_config(Config) -> ok = rpc:call(Node,logger,internal_init_logger,[]), ok = rpc:call(Node,logger,add_handlers,[kernel]), #{primary:=#{filter_default:=log,filters:=[]}, - handlers:=[#{id:=default,filters:=DF,config:=#{type:={file,F}}}], + handlers:=[#{id:=default,filters:=DF, + config:=#{type:=file,file:=F,modes:=Modes}}], module_levels:=[]} = rpc:call(Node,logger,get_config,[]), + [append,delayed_write,raw] = lists:sort(Modes), + %% Same, but using 'logger' parameter instead of 'error_logger' ok = rpc:call(Node,logger,remove_handler,[default]),% so it can be added again @@ -1060,26 +1063,27 @@ kernel_config(Config) -> ok = rpc:call(Node,logger,internal_init_logger,[]), ok = rpc:call(Node,logger,add_handlers,[kernel]), #{primary:=#{filter_default:=log,filters:=[]}, - handlers:=[#{id:=default,filters:=DF,config:=#{type:={file,F}}}], + handlers:=[#{id:=default,filters:=DF, + config:=#{type:=file,file:=F,modes:=Modes}}], module_levels:=[]} = rpc:call(Node,logger,get_config,[]), %% Same, but with type={file,File,Modes} ok = rpc:call(Node,logger,remove_handler,[default]),% so it can be added again ok = rpc:call(Node,application,unset_env,[kernel,error_logger]), - M = [raw,write,delayed_write], + M = [raw,write], ok = rpc:call(Node,application,set_env,[kernel,logger, [{handler,default,logger_std_h, #{config=>#{type=>{file,F,M}}}}]]), ok = rpc:call(Node,logger,internal_init_logger,[]), ok = rpc:call(Node,logger,add_handlers,[kernel]), #{primary:=#{filter_default:=log,filters:=[]}, - handlers:=[#{id:=default,filters:=DF,config:=#{type:={file,F,M}}}], + handlers:=[#{id:=default,filters:=DF, + config:=#{type:=file,file:=F,modes:=[delayed_write|M]}}], module_levels:=[]} = rpc:call(Node,logger,get_config,[]), %% Same, but with disk_log handler ok = rpc:call(Node,logger,remove_handler,[default]),% so it can be added again ok = rpc:call(Node,application,unset_env,[kernel,error_logger]), - M = [raw,write,delayed_write], ok = rpc:call(Node,application,set_env,[kernel,logger, [{handler,default,logger_disk_log_h, #{config=>#{file=>F}}}]]), diff --git a/lib/kernel/test/logger_formatter_SUITE.erl b/lib/kernel/test/logger_formatter_SUITE.erl index 8c13f0f908..83e3e6c40a 100644 --- a/lib/kernel/test/logger_formatter_SUITE.erl +++ b/lib/kernel/test/logger_formatter_SUITE.erl @@ -867,7 +867,7 @@ my_try(Fun) -> try Fun() catch C:R:S -> {C,R,hd(S)} end. timestamp() -> - erlang:system_time(microsecond). + logger:timestamp(). %% necessary? add_time(#{time:=_}=Meta) -> diff --git a/lib/kernel/test/logger_std_h_SUITE.erl b/lib/kernel/test/logger_std_h_SUITE.erl index b2c2c8ba67..0c5516f82b 100644 --- a/lib/kernel/test/logger_std_h_SUITE.erl +++ b/lib/kernel/test/logger_std_h_SUITE.erl @@ -112,6 +112,8 @@ all() -> add_remove_instance_standard_error, add_remove_instance_file1, add_remove_instance_file2, + add_remove_instance_file3, + add_remove_instance_file4, default_formatter, filter_config, errors, @@ -142,7 +144,12 @@ all() -> restart_after, handler_requests_under_load, recreate_deleted_log, - reopen_changed_log + reopen_changed_log, + rotate_size, + rotate_size_compressed, + rotate_size_reopen, + rotation_opts, + rotation_opts_restart_handler ]. add_remove_instance_tty(_Config) -> @@ -179,10 +186,27 @@ add_remove_instance_file2(Config) -> add_remove_instance_file2(cleanup,_Config) -> logger_std_h_remove(). -add_remove_instance_file(Log, Type) -> +add_remove_instance_file3(_Config) -> + Log = atom_to_list(?MODULE), + StdHConfig = #{type=>file}, + add_remove_instance_file(Log, StdHConfig). +add_remove_instance_file3(cleanup,_Config) -> + logger_std_h_remove(). + +add_remove_instance_file4(Config) -> + Dir = ?config(priv_dir,Config), + Log = filename:join(Dir,"stdlog4.txt"), + StdHConfig = #{file=>Log,modes=>[]}, + add_remove_instance_file(Log, StdHConfig). +add_remove_instance_file4(cleanup,_Config) -> + logger_std_h_remove(). + +add_remove_instance_file(Log, Type) when not is_map(Type) -> + add_remove_instance_file(Log,#{type=>Type}); +add_remove_instance_file(Log, StdHConfig) when is_map(StdHConfig) -> ok = logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => Type}, + #{config => StdHConfig, filter_default=>stop, filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), formatter=>{?MODULE,self()}}), @@ -257,9 +281,10 @@ errors(Config) -> end, {error, - {handler_not_added,{open_failed,Log,_}}} = + {handler_not_added, + {invalid_config,logger_std_h,#{modes:=bad_file_opt}}}} = logger:add_handler(myh3,logger_std_h, - #{config=>#{type=>{file,Log,[bad_file_opt]}}}), + #{config=>#{type=>{file,Log,bad_file_opt}}}), ok = logger:notice(?msg). @@ -607,24 +632,51 @@ reconfig(cleanup, _Config) -> file_opts(Config) -> Dir = ?config(priv_dir,Config), Log = filename:join(Dir, lists:concat([?FUNCTION_NAME,".log"])), - BadFileOpts = [raw], - BadType = {file,Log,BadFileOpts}, - {error,{handler_not_added,{open_failed,Log,enoent}}} = - logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => BadType}}), + MissingOpts = [raw], + Type1 = {file,Log,MissingOpts}, + ok = logger:add_handler(?MODULE, logger_std_h, + #{config => #{type => Type1}}), + {ok,#{config:=#{type:=file,file:=Log,modes:=Modes1}}} = + logger:get_handler_config(?MODULE), + [append,delayed_write,raw] = lists:sort(Modes1), + ok = logger:remove_handler(?MODULE), OkFileOpts = [raw,append], OkType = {file,Log,OkFileOpts}, ok = logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => OkType}, + #{config => #{type => OkType}, % old format filter_default=>log, filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), formatter=>{?MODULE,self()}}), - #{cb_state := #{handler_state := #{type := OkType}}} = + ModOpts = [delayed_write|OkFileOpts], + #{cb_state := #{handler_state := #{type:=file, + file:=Log, + modes:=ModOpts}}} = logger_olp:info(h_proc_name()), - {ok,#{config := #{type := OkType}}} = logger:get_handler_config(?MODULE), + {ok,#{config := #{type:=file, + file:=Log, + modes:=ModOpts}}} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + + ok = logger:add_handler(?MODULE, + logger_std_h, + #{config => #{type => file, + file => Log, + modes => OkFileOpts}, % new format + filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}), + + #{cb_state := #{handler_state := #{type:=file, + file:=Log, + modes:=ModOpts}}} = + logger_olp:info(h_proc_name()), + {ok,#{config := #{type:=file, + file:=Log, + modes:=ModOpts}}} = + logger:get_handler_config(?MODULE), logger:notice(M1=?msg,?domain), ?check(M1), B1 = ?bin(M1), @@ -640,13 +692,14 @@ sync(Config) -> Type = {file,Log}, ok = logger:add_handler(?MODULE, logger_std_h, - #{config => #{type => Type}, + #{config => #{type => Type, + file_check => 10000}, filter_default=>log, filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), formatter=>{?MODULE,nl}}), %% check repeated filesync happens - start_tracer([{logger_std_h, write_to_dev, 5}, + start_tracer([{logger_std_h, write_to_dev, 2}, {file, datasync, 1}], [{logger_std_h, write_to_dev, <<"first\n">>}, {file,datasync}]), @@ -656,7 +709,7 @@ sync(Config) -> check_tracer(filesync_rep_int()*2), %% check that explicit filesync is only done once - start_tracer([{logger_std_h, write_to_dev, 5}, + start_tracer([{logger_std_h, write_to_dev, 2}, {file, datasync, 1}], [{logger_std_h, write_to_dev, <<"second\n">>}, {file,datasync}, @@ -675,7 +728,7 @@ sync(Config) -> #{filesync_repeat_interval => no_repeat}), no_repeat = maps:get(filesync_repeat_interval, maps:get(cb_state, logger_olp:info(h_proc_name()))), - start_tracer([{logger_std_h, write_to_dev, 5}, + start_tracer([{logger_std_h, write_to_dev, 2}, {file, datasync, 1}], [{logger_std_h, write_to_dev, <<"third\n">>}, {file,datasync}, @@ -1285,6 +1338,331 @@ reopen_changed_log(Config) -> reopen_changed_log(cleanup, _Config) -> ok = stop_handler(?MODULE). +rotate_size(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE,#{config=>#{max_no_bytes=>1000, + max_no_files=>2}}), + + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + + logger:notice(Str,?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + {error,enoent} = file:read_file_info(Log++".1"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,51)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".2"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".2"), + + logger:notice("bbbb",?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=1005}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".2"), + + ok. +rotate_size(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotate_size_compressed(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE, + #{config=>#{max_no_bytes=>1000, + max_no_files=>2, + compress_on_rotate=>true}}), + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {error,enoent} = file:read_file_info(Log++".0.gz"), + + logger:notice(Str,?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {error,enoent} = file:read_file_info(Log++".1.gz"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,51)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".1.gz"), + {error,enoent} = file:read_file_info(Log++".2"), + {error,enoent} = file:read_file_info(Log++".2.gz"), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,50)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=1000}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".1.gz"), + {error,enoent} = file:read_file_info(Log++".2"), + {error,enoent} = file:read_file_info(Log++".2.gz"), + + logger:notice("bbbb",?domain), + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {error,enoent} = file:read_file_info(Log++".0"), + {ok,#file_info{size=38}} = file:read_file_info(Log++".0.gz"), + {error,enoent} = file:read_file_info(Log++".1"), + {ok,#file_info{size=35}} = file:read_file_info(Log++".1.gz"), + {error,enoent} = file:read_file_info(Log++".2"), + {error,enoent} = file:read_file_info(Log++".2.gz"), + + ok. +rotate_size_compressed(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotate_size_reopen(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE,#{config=>#{max_no_bytes=>1000, + max_no_files=>2}}), + + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,40)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=800}} = file:read_file_info(Log), + + {ok,HConfig} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler(?MODULE,maps:get(module,HConfig),HConfig), + {ok,#file_info{size=800}} = file:read_file_info(Log), + + [logger:notice(Str,?domain) || _ <- lists:seq(1,40)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=580}} = file:read_file_info(Log), + {ok,#file_info{size=1020}} = file:read_file_info(Log++".0"), + ok. +rotate_size_reopen(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotation_opts(Config) -> + {Log,_HConfig,StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + #{max_no_bytes:=infinity, + max_no_files:=0, + compress_on_rotate:=false} = StdHConfig, + + %% Test bad rotation config + {error,{invalid_config,_,_}} = + logger:update_handler_config(?MODULE,config,#{max_no_bytes=>0}), + {error,{invalid_config,_,_}} = + logger:update_handler_config(?MODULE,config,#{max_no_files=>infinity}), + {error,{invalid_config,_,_}} = + logger:update_handler_config(?MODULE,config, + #{compress_on_rotate=>undefined}), + + + %% Test good rotation config - start with no rotation + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=200}} = file:read_file_info(Log), + [] = filelib:wildcard(Log++".*"), + + %% Turn on rotation, check that existing file is rotated since its + %% size exceeds max_no_bytes + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>100, + max_no_files=>2}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=0}} = file:read_file_info(Log), + Log0 = Log++".0", + {ok,#file_info{size=200}} = file:read_file_info(Log0), + [Log0] = filelib:wildcard(Log++".*"), + + %% Fill all logs + [logger:notice(Str,?domain) || _ <- lists:seq(1,13)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log0), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Extend size and count and check that nothing changes with existing files + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>200, + max_no_files=>3}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log0), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Add more log events and see that extended size and count works + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=220}} = file:read_file_info(Log0), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + {ok,#file_info{size=120}} = file:read_file_info(Log++".2"), + [_,_,_] = filelib:wildcard(Log++".*"), + + %% Reduce count and check that archive files that exceed the new + %% count are moved + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_files=>1}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=220}} = file:read_file_info(Log0), + [Log0] = filelib:wildcard(Log++".*"), + + %% Extend size and count again, and turn on compression. Check + %% that archives are compressed + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>100, + max_no_files=>2, + compress_on_rotate=>true}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=0}} = file:read_file_info(Log), + Log0gz = Log0++".gz", + {ok,#file_info{size=29}} = file:read_file_info(Log0gz), + [Log0gz] = filelib:wildcard(Log++".*"), + + %% Fill all logs + [logger:notice(Str,?domain) || _ <- lists:seq(1,13)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=29}} = file:read_file_info(Log0gz), + {ok,#file_info{size=29}} = file:read_file_info(Log++".1.gz"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Reduce count and turn off compression. Check that archives that + %% exceeds the new count are removed, and the rest are + %% uncompressed. + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_files=>1, + compress_on_rotate=>false}), + timer:sleep(100), % give some time to execute config_changed + {ok,#file_info{size=20}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log0), + [Log0] = filelib:wildcard(Log++".*"), + + %% Check that config and handler state agree on the current rotation settings + {ok,#{config:=#{max_no_bytes:=100, + max_no_files:=1, + compress_on_rotate:=false}}} = + logger:get_handler_config(?MODULE), + #{cb_state:=#{handler_state:=#{max_no_bytes:=100, + max_no_files:=1, + compress_on_rotate:=false}}} = + logger_olp:info(h_proc_name()), + ok. +rotation_opts(cleanup,_Config) -> + ok = stop_handler(?MODULE). + +rotation_opts_restart_handler(Config) -> + {Log,_HConfig,_StdHConfig} = + start_handler(?MODULE, ?FUNCTION_NAME, Config), + ok = logger:update_handler_config(?MODULE, + config, + #{max_no_bytes=>100, + max_no_files=>2}), + + %% Fill all logs + Str = lists:duplicate(19,$a), + [logger:notice(Str,?domain) || _ <- lists:seq(1,15)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=60}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=120}} = file:read_file_info(Log++".1"), + [_,_] = filelib:wildcard(Log++".*"), + + %% Stop/start handler and turn off rotation. Check that archives are removed. + {ok,#{config:=StdHConfig1}=HConfig1} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig1#{config=>StdHConfig1#{max_no_bytes=>infinity}}), + timer:sleep(100), + {ok,#file_info{size=60}} = file:read_file_info(Log), + [] = filelib:wildcard(Log++".*"), + + %% Add some log events and check that file is no longer rotated. + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=260}} = file:read_file_info(Log), + [] = filelib:wildcard(Log++".*"), + + %% Stop/start handler and trun on rotation. Check that file is rotated. + {ok,#{config:=StdHConfig2}=HConfig2} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig2#{config=>StdHConfig2#{max_no_bytes=>100, + max_no_files=>2}}), + timer:sleep(100), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=260}} = file:read_file_info(Log++".0"), + [_] = filelib:wildcard(Log++".*"), + + %% Fill all logs + [logger:notice(Str,?domain) || _ <- lists:seq(1,10)], + logger_std_h:filesync(?MODULE), + {ok,#file_info{size=80}} = file:read_file_info(Log), + {ok,#file_info{size=120}} = file:read_file_info(Log++".0"), + {ok,#file_info{size=260}} = file:read_file_info(Log++".1"), + + %% Stop/start handler, reduce count and turn on compression. Check + %% that excess archives are removed, and the rest compressed. + {ok,#{config:=StdHConfig3}=HConfig3} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig3#{config=>StdHConfig3#{max_no_bytes=>75, + max_no_files=>1, + compress_on_rotate=>true}}), + timer:sleep(100), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=29}} = file:read_file_info(Log++".0.gz"), + [_] = filelib:wildcard(Log++".*"), + + %% Stop/start handler and turn off compression. Check that achives + %% are decompressed. + {ok,#{config:=StdHConfig4}=HConfig4} = logger:get_handler_config(?MODULE), + ok = logger:remove_handler(?MODULE), + ok = logger:add_handler( + ?MODULE,logger_std_h, + HConfig4#{config=>StdHConfig4#{compress_on_rotate=>false}}), + timer:sleep(100), + {ok,#file_info{size=0}} = file:read_file_info(Log), + {ok,#file_info{size=80}} = file:read_file_info(Log++".0"), + [_] = filelib:wildcard(Log++".*"), + + ok. +rotation_opts_restart_handler(cleanup,_Config) -> + ok = stop_handler(?MODULE). + %%%----------------------------------------------------------------- %%% send_requests(TO, Reqs = [{Mod,Func,Args,Res}|Rs]) -> @@ -1305,8 +1683,8 @@ start_handler(Name, TTY, _Config) when TTY == standard_io; ok = logger:add_handler(Name, logger_std_h, #{config => #{type => TTY}, - filter_default=>log, - filters=>?DEFAULT_HANDLER_FILTERS([Name]), + filter_default=>stop, + filters=>filter_only_this_domain(Name), formatter=>{?MODULE,op}}), {ok,HConfig = #{config := StdHConfig}} = logger:get_handler_config(Name), {HConfig,StdHConfig}; @@ -1320,12 +1698,17 @@ start_handler(Name, FuncName, Config) -> ok = logger:add_handler(Name, logger_std_h, #{config => #{type => Type}, - filter_default=>log, - filters=>?DEFAULT_HANDLER_FILTERS([Name]), + filter_default=>stop, + filters=>filter_only_this_domain(Name), formatter=>{?MODULE,op}}), {ok,HConfig = #{config := StdHConfig}} = logger:get_handler_config(Name), {Log,HConfig,StdHConfig}. + +filter_only_this_domain(Name) -> + [{remote_gl,{fun logger_filters:remote_gl/2,stop}}, + {domain,{fun logger_filters:domain/2,{log,super,[Name]}}}]. + stop_handler(Name) -> R = logger:remove_handler(Name), ct:pal("Handler ~p stopped! Result: ~p", [Name,R]), @@ -1658,7 +2041,7 @@ tpl([]) -> tracer({trace,_,call,{logger_h_common,handle_cast,[Op|_]}}, {Pid,[{Mod,Func,Op}|Expected]}) -> maybe_tracer_done(Pid,Expected,{Mod,Func,Op}); -tracer({trace,_,call,{Mod=logger_std_h,Func=write_to_dev,[_,Data,_,_,_]}}, +tracer({trace,_,call,{Mod=logger_std_h,Func=write_to_dev,[Data,_]}}, {Pid,[{Mod,Func,Data}|Expected]}) -> maybe_tracer_done(Pid,Expected,{Mod,Func,Data}); tracer({trace,_,call,{Mod,Func,_}}, {Pid,[{Mod,Func}|Expected]}) -> @@ -1742,4 +2125,3 @@ filesync_rep_int() -> file_delete(Log) -> file:delete(Log). - diff --git a/lib/kernel/test/logger_stress_SUITE.erl b/lib/kernel/test/logger_stress_SUITE.erl index 4072e8c86a..1a278fb1b2 100644 --- a/lib/kernel/test/logger_stress_SUITE.erl +++ b/lib/kernel/test/logger_stress_SUITE.erl @@ -67,6 +67,10 @@ all() -> reject_events, std_handler, disk_log_handler, + std_handler_time, + std_handler_time_big, + disk_log_handler_time, + disk_log_handler_time_big, emulator_events, remote_events, remote_to_disk_log, @@ -123,6 +127,20 @@ std_handler(cleanup,_Config) -> _ = file:delete("default.log"), ok. +%% Disable overload protection and just print a lot - measure time. +%% The IOPS reported is the number of log events written per millisecond. +std_handler_time(Config) -> + measure_handler_time(logger_std_h,#{type=>{file,"default.log"}},Config). +std_handler_time(cleanup,_Config) -> + _ = file:delete("default.log"), + ok. + +std_handler_time_big(Config) -> + measure_handler_time_big(logger_std_h,#{type=>{file,"default.log"}},Config). +std_handler_time_big(cleanup,_Config) -> + _ = file:delete("default.log"), + ok. + %% Cascading failure that produce gen_server and proc_lib reports - %% how many of the produced log events are actually written to a log %% with logger_disk_log_h wrap file handler. @@ -140,6 +158,20 @@ disk_log_handler(cleanup,_Config) -> [_ = file:delete(F) || F <- Files], ok. +%% Disable overload protection and just print a lot - measure time. +%% The IOPS reported is the number of log events written per millisecond. +disk_log_handler_time(Config) -> + measure_handler_time(logger_disk_log_h,#{type=>halt},Config). +disk_log_handler_time(cleanup,_Config) -> + _ = file:delete("default"), + ok. + +disk_log_handler_time_big(Config) -> + measure_handler_time_big(logger_disk_log_h,#{type=>halt},Config). +disk_log_handler_time_big(cleanup,_Config) -> + _ = file:delete("default"), + ok. + %% Cascading failure that produce log events from the emulator - how %% many of the produced log events pass through the proxy. emulator_events(Config) -> @@ -221,6 +253,68 @@ remote_emulator_to_disk_log(cleanup,_Config) -> %%%----------------------------------------------------------------- %%% Internal functions +measure_handler_time(Module,HCfg,Config) -> + measure_handler_time(Module,100000,small_fa(),millisecond,HCfg,#{},Config). + +measure_handler_time_big(Module,HCfg,Config) -> + FCfg = #{chars_limit=>4096, max_size=>1024}, + measure_handler_time(Module,100,big_fa(),second,HCfg,FCfg,Config). + +measure_handler_time(Module,N,FA,Unit,HCfg,FCfg,Config) -> + {ok,_,Node} = + logger_test_lib:setup( + Config, + [{logger, + [{handler,default,Module, + #{formatter => {logger_formatter, + maps:merge(#{legacy_header=>false, + single_line=>true},FCfg)}, + config=>maps:merge(#{sync_mode_qlen => N+1, + drop_mode_qlen => N+1, + flush_qlen => N+1, + burst_limit_enable => false, + filesync_repeat_interval => no_repeat}, + HCfg)}}]}]), + %% HPid = rpc:call(Node,erlang,whereis,[?name_to_reg_name(Module,default)]), + %% {links,[_,FCPid]} = rpc:call(Node,erlang,process_info,[HPid,links]), + T0 = erlang:monotonic_time(millisecond), + ok = rpc:call(Node,?MODULE,nlogs_wait,[N,FA,Module]), + %% ok = rpc:call(Node,fprof,apply,[fun ?MODULE:nlogs_wait/2,[N div 10,FA,Module],[{procs,[HPid,FCPid]}]]), + T1 = erlang:monotonic_time(millisecond), + T = T1-T0, + M = case Unit of + millisecond -> 1; + second -> 1000 + end, + IOPS = M*N/T, + ct:pal("N: ~p~nT: ~p~nIOPS: ~.2f events pr ~w",[N,T,IOPS,Unit]), + %% Stats = rpc:call(Node,logger_olp,info,[?name_to_reg_name(Module,default)]), + %% ct:pal("Stats: ~p",[Stats]), + ct_event:notify(#event{name = benchmark_data, + data = [{value,IOPS}]}), + {comment,io_lib:format("~.2f events written pr ~w",[IOPS,Unit])}. + +nlogs_wait(N,{F,A},Module) -> + group_leader(whereis(user),self()), + [?LOG_NOTICE(F,A) || _ <- lists:seq(1,N)], + wait(Module). + +wait(Module) -> + case Module:filesync(default) of + {error,handler_busy} -> + wait(Module); + ok -> + ok + end. + +small_fa() -> + Str = "\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "[\\]^_`abcdefghijklmnopqr", + {"~ts",[Str]}. + +big_fa() -> + {"~p",[lists:duplicate(1048576,"a")]}. + nlogs(N) -> group_leader(whereis(user),self()), Str = "\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ" diff --git a/lib/kernel/vsn.mk b/lib/kernel/vsn.mk index 4b43c6ae9d..7bebe1ba70 100644 --- a/lib/kernel/vsn.mk +++ b/lib/kernel/vsn.mk @@ -1 +1 @@ -KERNEL_VSN = 6.2 +KERNEL_VSN = 6.3 diff --git a/lib/mnesia/doc/src/notes.xml b/lib/mnesia/doc/src/notes.xml index 29c35d221c..22fb5e8c48 100644 --- a/lib/mnesia/doc/src/notes.xml +++ b/lib/mnesia/doc/src/notes.xml @@ -39,7 +39,22 @@ thus constitutes one section in this document. The title of each section is the version number of Mnesia.</p> - <section><title>Mnesia 4.15.5</title> + <section><title>Mnesia 4.15.6</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Avoid overload warnings caused by a race condition.</p> + <p> + Own Id: OTP-15619 Aux Id: ERIERL-310 </p> + </item> + </list> + </section> + +</section> + +<section><title>Mnesia 4.15.5</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/mnesia/vsn.mk b/lib/mnesia/vsn.mk index 1cfb35c774..781a4830a0 100644 --- a/lib/mnesia/vsn.mk +++ b/lib/mnesia/vsn.mk @@ -1 +1 @@ -MNESIA_VSN = 4.15.5 +MNESIA_VSN = 4.15.6 diff --git a/lib/observer/doc/src/notes.xml b/lib/observer/doc/src/notes.xml index 22035af982..2d914f8c61 100644 --- a/lib/observer/doc/src/notes.xml +++ b/lib/observer/doc/src/notes.xml @@ -32,6 +32,45 @@ <p>This document describes the changes made to the Observer application.</p> +<section><title>Observer 2.9</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Since Logger was introduced in OTP-21.0, menu choice + <em>Log > Toggle Log View</em> in observer would cause + a crash unless an <c>error_logger</c> event handler was + explicitly installed. This is now corrected.</p> + <p> + Own Id: OTP-15553 Aux Id: ERL-848 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Since persistent_term was introduced, observer would + sometimes crash when expanding a term from a process + state. This is now corrected.</p> + <p> + Own Id: OTP-15493 Aux Id: ERL-810 </p> + </item> + <item> + <p> + Add <c>OBSERVER_SCALE</c> environment variable for HiDPI + support.</p> + <p> + Own Id: OTP-15586 Aux Id: PR-2105 </p> + </item> + </list> + </section> + +</section> + <section><title>Observer 2.8.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/observer/vsn.mk b/lib/observer/vsn.mk index 5ce0aca589..0e9c8b302c 100644 --- a/lib/observer/vsn.mk +++ b/lib/observer/vsn.mk @@ -1 +1 @@ -OBSERVER_VSN = 2.8.2 +OBSERVER_VSN = 2.9 diff --git a/lib/odbc/doc/src/notes.xml b/lib/odbc/doc/src/notes.xml index dba7663bb9..696fcaa479 100644 --- a/lib/odbc/doc/src/notes.xml +++ b/lib/odbc/doc/src/notes.xml @@ -32,7 +32,22 @@ <p>This document describes the changes made to the odbc application. </p> - <section><title>ODBC 2.12.2</title> + <section><title>ODBC 2.12.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Enhance error handling to avoid stack corruption</p> + <p> + Own Id: OTP-15667 Aux Id: ERL-808, PR-2065 </p> + </item> + </list> + </section> + +</section> + +<section><title>ODBC 2.12.2</title> <section><title>Fixed Bugs and Malfunctions</title> <list> diff --git a/lib/odbc/vsn.mk b/lib/odbc/vsn.mk index bb21016fad..ff023e666b 100644 --- a/lib/odbc/vsn.mk +++ b/lib/odbc/vsn.mk @@ -1 +1 @@ -ODBC_VSN = 2.12.2 +ODBC_VSN = 2.12.3 diff --git a/lib/public_key/doc/src/notes.xml b/lib/public_key/doc/src/notes.xml index b3e6023c41..f6bc0dc797 100644 --- a/lib/public_key/doc/src/notes.xml +++ b/lib/public_key/doc/src/notes.xml @@ -35,6 +35,21 @@ <file>notes.xml</file> </header> +<section><title>Public_Key 1.6.5</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add export of dialyzer type</p> + <p> + Own Id: OTP-15624</p> + </item> + </list> + </section> + +</section> + <section><title>Public_Key 1.6.4</title> <section><title>Improvements and New Features</title> diff --git a/lib/public_key/vsn.mk b/lib/public_key/vsn.mk index 5e2643f0ea..11c06fb158 100644 --- a/lib/public_key/vsn.mk +++ b/lib/public_key/vsn.mk @@ -1 +1 @@ -PUBLIC_KEY_VSN = 1.6.4 +PUBLIC_KEY_VSN = 1.6.5 diff --git a/lib/runtime_tools/doc/src/notes.xml b/lib/runtime_tools/doc/src/notes.xml index 810bb8207c..58a2a66c4b 100644 --- a/lib/runtime_tools/doc/src/notes.xml +++ b/lib/runtime_tools/doc/src/notes.xml @@ -32,6 +32,21 @@ <p>This document describes the changes made to the Runtime_Tools application.</p> +<section><title>Runtime_Tools 1.13.2</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Update of systemtap trace example scripts.</p> + <p> + Own Id: OTP-15670</p> + </item> + </list> + </section> + +</section> + <section><title>Runtime_Tools 1.13.1</title> <section><title>Improvements and New Features</title> diff --git a/lib/runtime_tools/vsn.mk b/lib/runtime_tools/vsn.mk index aa3d702997..fa2f338ec2 100644 --- a/lib/runtime_tools/vsn.mk +++ b/lib/runtime_tools/vsn.mk @@ -1 +1 @@ -RUNTIME_TOOLS_VSN = 1.13.1 +RUNTIME_TOOLS_VSN = 1.13.2 diff --git a/lib/ssh/doc/src/notes.xml b/lib/ssh/doc/src/notes.xml index 2e1b946ebb..68a0afcbee 100644 --- a/lib/ssh/doc/src/notes.xml +++ b/lib/ssh/doc/src/notes.xml @@ -30,6 +30,23 @@ <file>notes.xml</file> </header> +<section><title>Ssh 4.7.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + SSH sftp daemon now accepts an SSH_FXP_STAT message + encoded according to the wrong sftp version. Some clients + sends such messages.</p> + <p> + Own Id: OTP-15498 Aux Id: ERL-822, PR-2077 </p> + </item> + </list> + </section> + +</section> + <section><title>Ssh 4.7.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssh/vsn.mk b/lib/ssh/vsn.mk index 2890d7fe5b..0f9eee887c 100644 --- a/lib/ssh/vsn.mk +++ b/lib/ssh/vsn.mk @@ -1,4 +1,4 @@ #-*-makefile-*- ; force emacs to enter makefile-mode -SSH_VSN = 4.7.3 +SSH_VSN = 4.7.4 APP_VSN = "ssh-$(SSH_VSN)" diff --git a/lib/ssl/doc/src/notes.xml b/lib/ssl/doc/src/notes.xml index 674e38b054..b036463e87 100644 --- a/lib/ssl/doc/src/notes.xml +++ b/lib/ssl/doc/src/notes.xml @@ -27,6 +27,117 @@ </header> <p>This document describes the changes made to the SSL application.</p> +<section><title>SSL 9.2.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The timeout for a passive receive was sometimes not + cancelled and later caused a server crash. This bug has + now been corrected.</p> + <p> + Own Id: OTP-14701 Aux Id: ERL-883, ERL-884 </p> + </item> + <item> + <p> + Add tag for passive message (active N) in cb_info to + retain transport transparency.</p> + <p> + Own Id: OTP-15679 Aux Id: ERL-861 </p> + </item> + </list> + </section> + +</section> + +<section><title>SSL 9.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fix bug that an incorrect return value for gen_statem + could be created when alert was a result of handling + renegotiation info extension</p> + <p> + Own Id: OTP-15502</p> + </item> + <item> + <p> + Correct check for 3des_ede_cbc, could cause ssl to claim + to support 3des_ede_cbc when cryptolib does not.</p> + <p> + Own Id: OTP-15539</p> + </item> + <item> + <p> + Improved DTLS error handling, avoids unexpected + connection failure in rare cases.</p> + <p> + Own Id: OTP-15561</p> + </item> + <item> + <p> + Corrected active once emulation bug that could cause the + ssl_closed meassage to not be sent. Bug introduced by + OTP-15449</p> + <p> + Own Id: OTP-15666 Aux Id: ERIERL-316, </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add client option {reuse_session, SessionID::binary()} + that can be used together with new option value + {reuse_sessions, save}. This makes it possible to reuse a + session from a specific connection establishment.</p> + <p> + Own Id: OTP-15369</p> + </item> + <item> + <p> + The Reason part of of the error return from the functions + connect and handshake has a better and documented format. + This will sometimes differ from previous returned + reasons, however those where only documented as term() + and should for that reason not be relied on.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-15423</p> + </item> + <item> + <p> + Refactor of state handling to improve TLS application + data throughput and reduce CPU overhead</p> + <p> + Own Id: OTP-15445</p> + </item> + <item> + <p> + The SSL code has been optimized in many small ways to + reduce CPU load for encryption/decryption, especially for + Erlang's distribution protocol over TLS.</p> + <p> + Own Id: OTP-15529</p> + </item> + <item> + <p> + Add support for active N</p> + <p> + Own Id: OTP-15665 Aux Id: ERL-811, PR-2072 </p> + </item> + </list> + </section> + +</section> + <section><title>SSL 9.1.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index 67ec93809f..24d063e1bd 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -89,25 +89,33 @@ <datatype> <name name="active_msgs"/> <desc> - <p>When an TLS/DTLS socket is in active mode (the default), data from the + <p>When a TLS/DTLS socket is in active mode (the default), data from the socket is delivered to the owner of the socket in the form of messages as described above.</p> + <p>The <c>ssl_passive</c> message is sent only when the socket is in + <c>{active, N}</c> mode and the counter dropped to 0. It indicates + that the socket has transitioned to passive (<c>{active, false}</c>) mode.</p> </desc> </datatype> <datatype> <name name="transport_option"/> <desc> - <p>Defaults to <c>{gen_tcp, tcp, tcp_closed, tcp_error}</c> - for TLS and <c>{gen_udp, udp, udp_closed, udp_error}</c> for - DTLS. Can be used to customize the transport layer. The tag - values should be the values used by the underlying transport - in its active mode messages. For TLS the callback module must implement a - reliable transport protocol, behave as <c>gen_tcp</c>, and have functions - corresponding to <c>inet:setopts/2</c>, <c>inet:getopts/2</c>, - <c>inet:peername/1</c>, <c>inet:sockname/1</c>, and <c>inet:port/1</c>. - The callback <c>gen_tcp</c> is treated specially and calls <c>inet</c> - directly. For DTLS this feature must be considered exprimental. + <p>Defaults to <c>{gen_tcp, tcp, tcp_closed, tcp_error, + tcp_passive}</c> for TLS (for backward compatibility a four + tuple will be converted to a five tuple with the last element + "second_element"_passive) and <c>{gen_udp, udp, udp_closed, + udp_error}</c> for DTLS (might also be changed to five tuple in + the future). Can be used to customize the transport layer. The + tag values should be the values used by the underlying + transport in its active mode messages. For TLS the callback + module must implement a reliable transport protocol, behave as + <c>gen_tcp</c>, and have functions corresponding to + <c>inet:setopts/2</c>, <c>inet:getopts/2</c>, + <c>inet:peername/1</c>, <c>inet:sockname/1</c>, and + <c>inet:port/1</c>. The callback <c>gen_tcp</c> is treated + specially and calls <c>inet</c> directly. For DTLS this + feature must be considered exprimental. </p> </desc> </datatype> @@ -1171,8 +1179,8 @@ fun(srp, Username :: string(), UserState :: term()) -> marker="#handshake_cancel-1"><c>handshake_cancel/1</c></seealso>. </p> - <p> If the option <c>active</c> is set to <c>once</c> or <c>true</c> the - process owning the sslsocket will receive messages of type + <p> If the option <c>active</c> is set to <c>once</c>, <c>true</c> or an integer value, + the process owning the sslsocket will receive messages of type <seealso marker="#type-active_msgs"> active_msgs() </seealso> </p> </desc> @@ -1219,8 +1227,8 @@ fun(srp, Username :: string(), UserState :: term()) -> marker="#handshake_cancel-1"><c>handshake_cancel/1</c></seealso>. </p> - <p> If the option <c>active</c> is set to <c>once</c> or <c>true</c> the - process owning the sslsocket will receive messages of type + <p> If the option <c>active</c> is set to <c>once</c>, <c>true</c> or an integer value, + the process owning the sslsocket will receive messages of type <seealso marker="#type-active_msgs"> active_msgs() </seealso> </p> </desc> @@ -1381,8 +1389,8 @@ fun(srp, Username :: string(), UserState :: term()) -> <p>Performs the SSL/TLS/DTLS server-side handshake.</p> <p>Returns a new TLS/DTLS socket if the handshake is successful.</p> - <p> If the option <c>active</c> is set to <c>once</c> or <c>true</c> the - process owning the sslsocket will receive messages of type + <p> If the option <c>active</c> is set to <c>once</c>, <c>true</c> or an integer value, + the process owning the sslsocket will receive messages of type <seealso marker="#type-active_msgs"> active_msgs() </seealso> </p> </desc> @@ -1426,8 +1434,8 @@ fun(srp, Username :: string(), UserState :: term()) -> marker="#handshake_cancel-1"><c>handshake_cancel/1</c></seealso>. </p> - <p> If the option <c>active</c> is set to <c>once</c> or <c>true</c> the - process owning the sslsocket will receive messages of type + <p> If the option <c>active</c> is set to <c>once</c>, <c>true</c> or an integer value, + the process owning the sslsocket will receive messages of type <seealso marker="#type-active_msgs"> active_msgs() </seealso> </p> diff --git a/lib/ssl/src/dtls_packet_demux.erl b/lib/ssl/src/dtls_packet_demux.erl index dccc22a448..092366b7c0 100644 --- a/lib/ssl/src/dtls_packet_demux.erl +++ b/lib/ssl/src/dtls_packet_demux.erl @@ -297,6 +297,9 @@ do_set_emulated_opts([], Opts) -> Opts; do_set_emulated_opts([{mode, Value} | Rest], Opts) -> do_set_emulated_opts(Rest, Opts#socket_options{mode = Value}); +do_set_emulated_opts([{active, N0} | Rest], Opts=#socket_options{active = Active}) when is_integer(N0) -> + N = tls_socket:update_active_n(N0, Active), + do_set_emulated_opts(Rest, Opts#socket_options{active = N}); do_set_emulated_opts([{active, Value} | Rest], Opts) -> do_set_emulated_opts(Rest, Opts#socket_options{active = Value}). diff --git a/lib/ssl/src/dtls_socket.erl b/lib/ssl/src/dtls_socket.erl index 2001afd02f..4d07372e31 100644 --- a/lib/ssl/src/dtls_socket.erl +++ b/lib/ssl/src/dtls_socket.erl @@ -38,7 +38,9 @@ listen(Port, #config{transport_info = TransportInfo, case dtls_listener_sup:start_child([Port, TransportInfo, emulated_socket_options(EmOpts, #socket_options{}), Options ++ internal_inet_values(), SslOpts]) of {ok, Pid} -> - {ok, #sslsocket{pid = {dtls, Config#config{dtls_handler = {Pid, Port}}}}}; + Socket = #sslsocket{pid = {dtls, Config#config{dtls_handler = {Pid, Port}}}}, + check_active_n(EmOpts, Socket), + {ok, Socket}; Err = {error, _} -> Err end. @@ -81,8 +83,9 @@ socket(Pids, Transport, Socket, ConnectionCb) -> #sslsocket{pid = Pids, %% "The name "fd" is keept for backwards compatibility fd = {Transport, Socket, ConnectionCb}}. -setopts(_, #sslsocket{pid = {dtls, #config{dtls_handler = {ListenPid, _}}}}, Options) -> - SplitOpts = tls_socket:split_options(Options), +setopts(_, Socket = #sslsocket{pid = {dtls, #config{dtls_handler = {ListenPid, _}}}}, Options) -> + SplitOpts = {_, EmOpts} = tls_socket:split_options(Options), + check_active_n(EmOpts, Socket), dtls_packet_demux:set_sock_opts(ListenPid, SplitOpts); %%% Following clauses will not be called for emulated options, they are handled in the connection process setopts(gen_udp, Socket, Options) -> @@ -90,6 +93,32 @@ setopts(gen_udp, Socket, Options) -> setopts(Transport, Socket, Options) -> Transport:setopts(Socket, Options). +check_active_n(EmulatedOpts, Socket = #sslsocket{pid = {dtls, #config{dtls_handler = {ListenPid, _}}}}) -> + %% We check the resulting options to send an ssl_passive message if necessary. + case proplists:lookup(active, EmulatedOpts) of + %% The provided value is out of bound. + {_, N} when is_integer(N), N < -32768 -> + throw(einval); + {_, N} when is_integer(N), N > 32767 -> + throw(einval); + {_, N} when is_integer(N) -> + {ok, #socket_options{active = Active}, _} = dtls_packet_demux:get_all_opts(ListenPid), + case Active of + Atom when is_atom(Atom), N =< 0 -> + self() ! {ssl_passive, Socket}; + %% The result of the addition is out of bound. + %% We do not need to check < -32768 because Active can't be below 1. + A when is_integer(A), A + N > 32767 -> + throw(einval); + A when is_integer(A), A + N =< 0 -> + self() ! {ssl_passive, Socket}; + _ -> + ok + end; + _ -> + ok + end. + getopts(_, #sslsocket{pid = {dtls, #config{dtls_handler = {ListenPid, _}}}}, Options) -> SplitOpts = tls_socket:split_options(Options), dtls_packet_demux:get_sock_opts(ListenPid, SplitOpts); @@ -161,9 +190,18 @@ emulated_socket_options(InetValues, #socket_options{ mode = proplists:get_value(mode, InetValues, Mode), packet = proplists:get_value(packet, InetValues, Packet), packet_size = proplists:get_value(packet_size, InetValues, PacketSize), - active = proplists:get_value(active, InetValues, Active) + active = emulated_active_option(InetValues, Active) }. +emulated_active_option([], Active) -> + Active; +emulated_active_option([{active, Active} | _], _) when Active =< 0 -> + false; +emulated_active_option([{active, Active} | _], _) -> + Active; +emulated_active_option([_|Tail], Active) -> + emulated_active_option(Tail, Active). + emulated_options([{mode, Value} = Opt |Opts], Inet, Emulated) -> validate_inet_option(mode, Value), emulated_options(Opts, Inet, [Opt | proplists:delete(mode, Emulated)]); @@ -185,6 +223,9 @@ validate_inet_option(mode, Value) when Value =/= list, Value =/= binary -> throw({error, {options, {mode,Value}}}); validate_inet_option(active, Value) + when Value >= -32768, Value =< 32767 -> + ok; +validate_inet_option(active, Value) when Value =/= true, Value =/= false, Value =/= once -> throw({error, {options, {active,Value}}}); validate_inet_option(_, _) -> diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 50e74d5eb7..7f8f1ec71c 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -92,9 +92,11 @@ -type tls_client_option() :: client_option() | common_option() | socket_option() | transport_option(). -type tls_server_option() :: server_option() | common_option() | socket_option() | transport_option(). -type active_msgs() :: {ssl, sslsocket(), Data::binary() | list()} | {ssl_closed, sslsocket()} | - {ssl_error, sslsocket(), Reason::term()}. + {ssl_error, sslsocket(), Reason::term()} | {ssl_passive, sslsocket()}. -type transport_option() :: {cb_info, {CallbackModule::atom(), DataTag::atom(), - ClosedTag::atom(), ErrTag::atom()}}. + ClosedTag::atom(), ErrTag::atom()}} | + {cb_info, {CallbackModule::atom(), DataTag::atom(), + ClosedTag::atom(), ErrTag::atom(), PassiveTag::atom()}}. -type host() :: hostname() | ip_address(). -type hostname() :: string(). -type ip_address() :: inet:ip_address(). @@ -400,9 +402,9 @@ connect(Socket, SslOptions) when is_port(Socket) -> timeout() | list()) -> {ok, #sslsocket{}} | {error, reason()}. connect(Socket, SslOptions0, Timeout) when is_port(Socket), - (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> - {Transport,_,_,_} = proplists:get_value(cb_info, SslOptions0, - {gen_tcp, tcp, tcp_closed, tcp_error}), + (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> + CbInfo = handle_option(cb_info, SslOptions0, default_cb_info(tls)), + Transport = element(1, CbInfo), EmulatedOptions = tls_socket:emulated_options(), {ok, SocketValues} = tls_socket:getopts(Transport, Socket, EmulatedOptions), try handle_options(SslOptions0 ++ SocketValues, client) of @@ -542,8 +544,8 @@ handshake(#sslsocket{pid = [Pid|_], fd = {_, _, _}} = Socket, SslOpts, Timeout) end; handshake(Socket, SslOptions, Timeout) when is_port(Socket), (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> - {Transport,_,_,_} = - proplists:get_value(cb_info, SslOptions, {gen_tcp, tcp, tcp_closed, tcp_error}), + CbInfo = handle_option(cb_info, SslOptions, default_cb_info(tls)), + Transport = element(1, CbInfo), EmulatedOptions = tls_socket:emulated_options(), {ok, SocketValues} = tls_socket:getopts(Transport, Socket, EmulatedOptions), ConnetionCb = connection_cb(SslOptions), @@ -595,7 +597,7 @@ close(#sslsocket{pid = [Pid|_]}) when is_pid(Pid) -> ssl_connection:close(Pid, {close, ?DEFAULT_TIMEOUT}); close(#sslsocket{pid = {dtls, #config{dtls_handler = {Pid, _}}}}) -> dtls_packet_demux:close(Pid); -close(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport,_, _, _}}}}) -> +close(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport,_,_,_,_}}}}) -> Transport:close(ListenSocket). %%-------------------------------------------------------------------- @@ -611,7 +613,7 @@ close(#sslsocket{pid = [TLSPid|_]}, close(#sslsocket{pid = [TLSPid|_]}, Timeout) when is_pid(TLSPid), (is_integer(Timeout) andalso Timeout >= 0) or (Timeout == infinity) -> ssl_connection:close(TLSPid, {close, Timeout}); -close(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport,_, _, _}}}}, _) -> +close(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport,_,_,_,_}}}}, _) -> Transport:close(ListenSocket). %%-------------------------------------------------------------------- @@ -627,7 +629,8 @@ send(#sslsocket{pid = {_, #config{transport_info={_, udp, _, _}}}}, _) -> {error,enotconn}; %% Emulate connection behaviour send(#sslsocket{pid = {dtls,_}}, _) -> {error,enotconn}; %% Emulate connection behaviour -send(#sslsocket{pid = {ListenSocket, #config{transport_info={Transport, _, _, _}}}}, Data) -> +send(#sslsocket{pid = {ListenSocket, #config{transport_info = Info}}}, Data) -> + Transport = element(1, Info), Transport:send(ListenSocket, Data). %% {error,enotconn} %%-------------------------------------------------------------------- @@ -645,7 +648,8 @@ recv(#sslsocket{pid = [Pid|_]}, Length, Timeout) when is_pid(Pid), recv(#sslsocket{pid = {dtls,_}}, _, _) -> {error,enotconn}; recv(#sslsocket{pid = {Listen, - #config{transport_info = {Transport, _, _, _}}}}, _,_) when is_port(Listen)-> + #config{transport_info = Info}}},_,_) when is_port(Listen)-> + Transport = element(1, Info), Transport:recv(Listen, 0). %% {error,enotconn} %%-------------------------------------------------------------------- @@ -660,7 +664,7 @@ controlling_process(#sslsocket{pid = {dtls, _}}, NewOwner) when is_pid(NewOwner) -> ok; %% Meaningless but let it be allowed to conform with TLS controlling_process(#sslsocket{pid = {Listen, - #config{transport_info = {Transport, _, _, _}}}}, + #config{transport_info = {Transport,_,_,_,_}}}}, NewOwner) when is_port(Listen), is_pid(NewOwner) -> %% Meaningless but let it be allowed to conform with normal sockets @@ -703,13 +707,13 @@ connection_information(#sslsocket{pid = [Pid|_]}, Items) when is_pid(Pid) -> %% %% Description: same as inet:peername/1. %%-------------------------------------------------------------------- -peername(#sslsocket{pid = [Pid|_], fd = {Transport, Socket, _}}) when is_pid(Pid)-> +peername(#sslsocket{pid = [Pid|_], fd = {Transport, Socket,_}}) when is_pid(Pid)-> dtls_socket:peername(Transport, Socket); -peername(#sslsocket{pid = [Pid|_], fd = {Transport, Socket, _, _}}) when is_pid(Pid)-> +peername(#sslsocket{pid = [Pid|_], fd = {Transport, Socket,_,_}}) when is_pid(Pid)-> tls_socket:peername(Transport, Socket); -peername(#sslsocket{pid = {dtls, #config{dtls_handler = {_Pid, _}}}}) -> +peername(#sslsocket{pid = {dtls, #config{dtls_handler = {_Pid,_}}}}) -> dtls_socket:peername(dtls, undefined); -peername(#sslsocket{pid = {ListenSocket, #config{transport_info = {Transport,_,_,_}}}}) -> +peername(#sslsocket{pid = {ListenSocket, #config{transport_info = {Transport,_,_,_,_}}}}) -> tls_socket:peername(Transport, ListenSocket); %% Will return {error, enotconn} peername(#sslsocket{pid = {dtls,_}}) -> {error,enotconn}. @@ -886,7 +890,7 @@ getopts(#sslsocket{pid = {dtls, #config{transport_info = {Transport,_,_,_}}}} = _:Error -> {error, {options, {socket_options, OptionTags, Error}}} end; -getopts(#sslsocket{pid = {_, #config{transport_info = {Transport,_,_,_}}}} = ListenSocket, +getopts(#sslsocket{pid = {_, #config{transport_info = {Transport,_,_,_,_}}}} = ListenSocket, OptionTags) when is_list(OptionTags) -> try tls_socket:getopts(Transport, ListenSocket, OptionTags) of {ok, _} = Result -> @@ -943,7 +947,7 @@ setopts(#sslsocket{pid = {dtls, #config{transport_info = {Transport,_,_,_}}}} = _:Error -> {error, {options, {socket_options, Options, Error}}} end; -setopts(#sslsocket{pid = {_, #config{transport_info = {Transport,_,_,_}}}} = ListenSocket, Options) when is_list(Options) -> +setopts(#sslsocket{pid = {_, #config{transport_info = {Transport,_,_,_,_}}}} = ListenSocket, Options) when is_list(Options) -> try tls_socket:setopts(Transport, ListenSocket, Options) of ok -> ok; @@ -987,8 +991,9 @@ getstat(#sslsocket{pid = [Pid|_], fd = {Transport, Socket, _, _}}, Options) when %% %% Description: Same as gen_tcp:shutdown/2 %%-------------------------------------------------------------------- -shutdown(#sslsocket{pid = {Listen, #config{transport_info = {Transport,_, _, _}}}}, +shutdown(#sslsocket{pid = {Listen, #config{transport_info = Info}}}, How) when is_port(Listen) -> + Transport = element(1, Info), Transport:shutdown(Listen, How); shutdown(#sslsocket{pid = {dtls,_}},_) -> {error, enotconn}; @@ -1000,13 +1005,13 @@ shutdown(#sslsocket{pid = [Pid|_]}, How) when is_pid(Pid) -> %% %% Description: Same as inet:sockname/1 %%-------------------------------------------------------------------- -sockname(#sslsocket{pid = {Listen, #config{transport_info = {Transport, _, _, _}}}}) when is_port(Listen) -> +sockname(#sslsocket{pid = {Listen, #config{transport_info = {Transport,_,_,_,_}}}}) when is_port(Listen) -> tls_socket:sockname(Transport, Listen); sockname(#sslsocket{pid = {dtls, #config{dtls_handler = {Pid, _}}}}) -> dtls_packet_demux:sockname(Pid); -sockname(#sslsocket{pid = [Pid|_], fd = {Transport, Socket, _}}) when is_pid(Pid) -> +sockname(#sslsocket{pid = [Pid|_], fd = {Transport, Socket,_}}) when is_pid(Pid) -> dtls_socket:sockname(Transport, Socket); -sockname(#sslsocket{pid = [Pid| _], fd = {Transport, Socket, _, _}}) when is_pid(Pid) -> +sockname(#sslsocket{pid = [Pid| _], fd = {Transport, Socket,_,_}}) when is_pid(Pid) -> tls_socket:sockname(Transport, Socket). %%--------------------------------------------------------------- @@ -1140,7 +1145,7 @@ supported_suites(all, Version) -> supported_suites(anonymous, Version) -> ssl_cipher:anonymous_suites(Version). -do_listen(Port, #config{transport_info = {Transport, _, _, _}} = Config, tls_connection) -> +do_listen(Port, #config{transport_info = {Transport, _, _, _,_}} = Config, tls_connection) -> tls_socket:listen(Transport, Port, Config); do_listen(Port, Config, dtls_connection) -> @@ -1288,7 +1293,7 @@ handle_options(Opts0, Role, Host) -> customize_hostname_check = handle_option(customize_hostname_check, Opts, []) }, - CbInfo = proplists:get_value(cb_info, Opts, default_cb_info(Protocol)), + CbInfo = handle_option(cb_info, Opts, default_cb_info(Protocol)), SslOptions = [protocol, versions, verify, verify_fun, partial_chain, fail_if_no_peer_cert, verify_client_once, depth, cert, certfile, key, keyfile, @@ -1330,6 +1335,10 @@ handle_option(sni_fun, Opts, Default) -> _ -> throw({error, {conflict_options, [sni_fun, sni_hosts]}}) end; +handle_option(cb_info, Opts, Default) -> + CbInfo = proplists:get_value(cb_info, Opts, Default), + true = validate_option(cb_info, CbInfo), + handle_cb_info(CbInfo, Default); handle_option(OptionName, Opts, Default) -> validate_option(OptionName, proplists:get_value(OptionName, Opts, Default)). @@ -1551,9 +1560,29 @@ validate_option(handshake, full = Value) -> Value; validate_option(customize_hostname_check, Value) when is_list(Value) -> Value; +validate_option(cb_info, {V1, V2, V3, V4}) when is_atom(V1), + is_atom(V2), + is_atom(V3), + is_atom(V4) + -> + true; +validate_option(cb_info, {V1, V2, V3, V4, V5}) when is_atom(V1), + is_atom(V2), + is_atom(V3), + is_atom(V4), + is_atom(V5) + -> + true; +validate_option(cb_info, _) -> + false; validate_option(Opt, Value) -> throw({error, {options, {Opt, Value}}}). +handle_cb_info({V1, V2, V3, V4}, {_,_,_,_,_}) -> + {V1,V2,V3,V4, list_to_atom(atom_to_list(V2) ++ "passive")}; +handle_cb_info(CbInfo, _) -> + CbInfo. + handle_hashsigns_option(Value, Version) when is_list(Value) andalso Version >= {3, 3} -> case tls_v1:signature_algs(Version, Value) of @@ -1964,7 +1993,7 @@ default_option_role(_,_,_) -> undefined. default_cb_info(tls) -> - {gen_tcp, tcp, tcp_closed, tcp_error}; + {gen_tcp, tcp, tcp_closed, tcp_error, tcp_passive}; default_cb_info(dtls) -> {gen_udp, udp, udp_closed, udp_error}. diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index 9e037313bb..ad81288f64 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -1381,10 +1381,22 @@ handle_call({get_opts, OptTags}, From, _, {keep_state_and_data, [{reply, From, OptsReply}]}; handle_call({set_opts, Opts0}, From, StateName, #state{static_env = #static_env{socket = Socket, - transport_cb = Transport}, + transport_cb = Transport, + tracker = Tracker}, + connection_env = + #connection_env{user_application = {_Mon, Pid}}, socket_options = Opts1 } = State0, Connection) -> {Reply, Opts} = set_socket_opts(Connection, Transport, Socket, Opts0, Opts1, []), + case {proplists:lookup(active, Opts0), Opts} of + {{_, N}, #socket_options{active=false}} when is_integer(N) -> + send_user( + Pid, + format_passive( + Connection:pids(State0), Transport, Socket, Tracker, Connection)); + _ -> + ok + end, State = State0#state{socket_options = Opts}, handle_active_option(Opts#socket_options.active, StateName, From, Reply, State); @@ -2455,6 +2467,30 @@ set_socket_opts(ConnectionCb, Transport, Socket, [{active, Active}| Opts], SockO Active == false -> set_socket_opts(ConnectionCb, Transport, Socket, Opts, SockOpts#socket_options{active = Active}, Other); +set_socket_opts(ConnectionCb, Transport, Socket, [{active, Active1} = Opt| Opts], + SockOpts=#socket_options{active = Active0}, Other) + when Active1 >= -32768, Active1 =< 32767 -> + Active = if + is_integer(Active0), Active0 + Active1 < -32768 -> + error; + is_integer(Active0), Active0 + Active1 =< 0 -> + false; + is_integer(Active0), Active0 + Active1 > 32767 -> + error; + Active1 =< 0 -> + false; + is_integer(Active0) -> + Active0 + Active1; + true -> + Active1 + end, + case Active of + error -> + {{error, {options, {socket_options, Opt}} }, SockOpts}; + _ -> + set_socket_opts(ConnectionCb, Transport, Socket, Opts, + SockOpts#socket_options{active = Active}, Other) + end; set_socket_opts(_,_, _, [{active, _} = Opt| _], SockOpts, _) -> {{error, {options, {socket_options, Opt}} }, SockOpts}; set_socket_opts(ConnectionCb, Transport, Socket, [Opt | Opts], SockOpts, Other) -> @@ -2655,6 +2691,11 @@ ssl_options_list([Key | Keys], [Value | Values], Acc) -> handle_active_option(false, connection = StateName, To, Reply, State) -> hibernate_after(StateName, State, [{reply, To, Reply}]); +handle_active_option(_, connection = StateName, To, _Reply, #state{connection_env = #connection_env{terminated = true}, + user_data_buffer = {_,0,_}} = State) -> + handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY, all_data_deliverd), StateName, + State#state{start_or_recv_from = To}), + {stop,{shutdown, peer_close}, State}; handle_active_option(_, connection = StateName0, To, Reply, #state{static_env = #static_env{protocol_cb = Connection}, user_data_buffer = {_,0,_}} = State0) -> case Connection:next_event(StateName0, no_record, State0) of @@ -2750,6 +2791,14 @@ deliver_app_data( case Active of once -> SO#socket_options{active=false}; + 1 -> + send_user( + Pid, + format_passive( + CPids, Transport, Socket, Tracker, Connection)), + SO#socket_options{active=false}; + N when is_integer(N) -> + SO#socket_options{active=N - 1}; _ -> SO end. @@ -2786,6 +2835,9 @@ do_format_reply(list, Packet, _, Data) do_format_reply(list, _,_, Data) -> binary_to_list(Data). +format_passive(CPids, Transport, Socket, Tracker, Connection) -> + {ssl_passive, Connection:socket(CPids, Transport, Socket, Tracker)}. + header(0, <<>>) -> <<>>; header(_, <<>>) -> diff --git a/lib/ssl/src/ssl_connection.hrl b/lib/ssl/src/ssl_connection.hrl index 9efd65b2d2..c90fe926b7 100644 --- a/lib/ssl/src/ssl_connection.hrl +++ b/lib/ssl/src/ssl_connection.hrl @@ -40,6 +40,7 @@ data_tag :: atom(), % ex tcp. close_tag :: atom(), % ex tcp_closed error_tag :: atom(), % ex tcp_error + passive_tag :: atom(), % ex tcp_passive host :: string() | inet:ip_address(), port :: integer(), socket :: port() | tuple(), %% TODO: dtls socket diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 3229004c9d..114710a92e 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -79,7 +79,7 @@ %% Setup %%==================================================================== start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = false},_, Tracker} = Opts, - User, {CbModule, _,_, _} = CbInfo, + User, {CbModule, _,_, _, _} = CbInfo, Timeout) -> try {ok, Sender} = tls_sender:start(), @@ -93,7 +93,7 @@ start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = false},_, Tracker} end; start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = true},_, Tracker} = Opts, - User, {CbModule, _,_, _} = CbInfo, + User, {CbModule, _,_, _, _} = CbInfo, Timeout) -> try {ok, Sender} = tls_sender:start([{spawn_opt, ?DIST_CNTRL_SPAWN_OPTS}]), @@ -223,7 +223,22 @@ next_event(StateName, Record, State, Actions) -> %%% TLS record protocol level application data messages - +handle_protocol_record(#ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, StateName, + #state{start_or_recv_from = From, + socket_options = #socket_options{active = false}} = State0) when From =/= undefined -> + case ssl_connection:read_application_data(Data, State0) of + {stop, _, _} = Stop-> + Stop; + {Record, #state{start_or_recv_from = Caller} = State1} -> + TimerAction = case Caller of + undefined -> %% Passive recv complete cancel timer + [{{timeout, recv}, infinity, timeout}]; + _ -> + [] + end, + {next_state, StateName, State, Actions} = next_event(StateName, Record, State1, TimerAction), + ssl_connection:hibernate_after(StateName, State, Actions) + end; handle_protocol_record(#ssl_tls{type = ?APPLICATION_DATA, fragment = Data}, StateName, State0) -> case ssl_connection:read_application_data(Data, State0) of {stop, _, _} = Stop-> @@ -765,7 +780,7 @@ code_change(_OldVsn, StateName, State, _) -> %%% Internal functions %%-------------------------------------------------------------------- initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Tracker}, User, - {CbModule, DataTag, CloseTag, ErrorTag}) -> + {CbModule, DataTag, CloseTag, ErrorTag, PassiveTag}) -> #ssl_options{beast_mitigation = BeastMitigation, erl_dist = IsErlDist} = SSLOptions, ConnectionStates = tls_record:init_connection_states(Role, BeastMitigation), @@ -790,6 +805,7 @@ initial_state(Role, Sender, Host, Port, Socket, {SSLOptions, SocketOptions, Trac data_tag = DataTag, close_tag = CloseTag, error_tag = ErrorTag, + passive_tag = PassiveTag, host = Host, port = Port, socket = Socket, @@ -879,8 +895,9 @@ handle_info({Protocol, _, Data}, StateName, ssl_connection:handle_normal_shutdown(Alert, StateName, State0), {stop, {shutdown, own_alert}, State0} end; -handle_info({tcp_passive, Socket}, StateName, - #state{static_env = #static_env{socket = Socket}, +handle_info({PassiveTag, Socket}, StateName, + #state{static_env = #static_env{socket = Socket, + passive_tag = PassiveTag}, protocol_specific = PS } = State) -> next_event(StateName, no_record, diff --git a/lib/ssl/src/tls_socket.erl b/lib/ssl/src/tls_socket.erl index a391bc53de..6c32e6fa04 100644 --- a/lib/ssl/src/tls_socket.erl +++ b/lib/ssl/src/tls_socket.erl @@ -32,6 +32,7 @@ emulated_socket_options/2, get_emulated_opts/1, set_emulated_opts/2, get_all_opts/1, handle_call/3, handle_cast/2, handle_info/2, code_change/3]). +-export([update_active_n/2]). -record(state, { emulated_opts, @@ -45,18 +46,20 @@ send(Transport, Socket, Data) -> Transport:send(Socket, Data). -listen(Transport, Port, #config{transport_info = {Transport, _, _, _}, +listen(Transport, Port, #config{transport_info = {Transport, _, _, _, _}, inet_user = Options, ssl = SslOpts, emulated = EmOpts} = Config) -> case Transport:listen(Port, Options ++ internal_inet_values()) of {ok, ListenSocket} -> {ok, Tracker} = inherit_tracker(ListenSocket, EmOpts, SslOpts), - {ok, #sslsocket{pid = {ListenSocket, Config#config{emulated = Tracker}}}}; + Socket = #sslsocket{pid = {ListenSocket, Config#config{emulated = Tracker}}}, + check_active_n(EmOpts, Socket), + {ok, Socket}; Err = {error, _} -> Err end. -accept(ListenSocket, #config{transport_info = {Transport,_,_,_} = CbInfo, +accept(ListenSocket, #config{transport_info = {Transport,_,_,_,_} = CbInfo, connection_cb = ConnectionCb, ssl = SslOpts, emulated = Tracker}, Timeout) -> @@ -77,7 +80,7 @@ accept(ListenSocket, #config{transport_info = {Transport,_,_,_} = CbInfo, {error, Reason} end. -upgrade(Socket, #config{transport_info = {Transport,_,_,_}= CbInfo, +upgrade(Socket, #config{transport_info = {Transport,_,_,_,_}= CbInfo, ssl = SslOptions, emulated = EmOpts, connection_cb = ConnectionCb}, Timeout) -> ok = setopts(Transport, Socket, tls_socket:internal_inet_values()), @@ -95,7 +98,7 @@ connect(Address, Port, #config{transport_info = CbInfo, inet_user = UserOpts, ssl = SslOpts, emulated = EmOpts, inet_ssl = SocketOpts, connection_cb = ConnetionCb}, Timeout) -> - {Transport, _, _, _} = CbInfo, + {Transport, _, _, _, _} = CbInfo, try Transport:connect(Address, Port, SocketOpts, Timeout) of {ok, Socket} -> ssl_connection:connect(ConnetionCb, Address, Port, Socket, @@ -117,14 +120,16 @@ socket(Pids, Transport, Socket, ConnectionCb, Tracker) -> #sslsocket{pid = Pids, %% "The name "fd" is keept for backwards compatibility fd = {Transport, Socket, ConnectionCb, Tracker}}. -setopts(gen_tcp, #sslsocket{pid = {ListenSocket, #config{emulated = Tracker}}}, Options) -> +setopts(gen_tcp, Socket = #sslsocket{pid = {ListenSocket, #config{emulated = Tracker}}}, Options) -> {SockOpts, EmulatedOpts} = split_options(Options), ok = set_emulated_opts(Tracker, EmulatedOpts), + check_active_n(EmulatedOpts, Socket), inet:setopts(ListenSocket, SockOpts); -setopts(_, #sslsocket{pid = {ListenSocket, #config{transport_info = {Transport,_,_,_}, +setopts(_, Socket = #sslsocket{pid = {ListenSocket, #config{transport_info = {Transport,_,_,_,_}, emulated = Tracker}}}, Options) -> {SockOpts, EmulatedOpts} = split_options(Options), ok = set_emulated_opts(Tracker, EmulatedOpts), + check_active_n(EmulatedOpts, Socket), Transport:setopts(ListenSocket, SockOpts); %%% Following clauses will not be called for emulated options, they are handled in the connection process setopts(gen_tcp, Socket, Options) -> @@ -132,6 +137,31 @@ setopts(gen_tcp, Socket, Options) -> setopts(Transport, Socket, Options) -> Transport:setopts(Socket, Options). +check_active_n(EmulatedOpts, Socket = #sslsocket{pid = {_, #config{emulated = Tracker}}}) -> + %% We check the resulting options to send an ssl_passive message if necessary. + case proplists:lookup(active, EmulatedOpts) of + %% The provided value is out of bound. + {_, N} when is_integer(N), N < -32768 -> + throw(einval); + {_, N} when is_integer(N), N > 32767 -> + throw(einval); + {_, N} when is_integer(N) -> + case get_emulated_opts(Tracker, [active]) of + [{_, false}] -> + self() ! {ssl_passive, Socket}, + ok; + %% The result of the addition is out of bound. + [{_, A}] when is_integer(A), A < -32768 -> + throw(einval); + [{_, A}] when is_integer(A), A > 32767 -> + throw(einval); + _ -> + ok + end; + _ -> + ok + end. + getopts(gen_tcp, #sslsocket{pid = {ListenSocket, #config{emulated = Tracker}}}, Options) -> {SockOptNames, EmulatedOptNames} = split_options(Options), EmulatedOpts = get_emulated_opts(Tracker, EmulatedOptNames), @@ -209,7 +239,7 @@ start_link(Port, SockOpts, SslOpts) -> init([Port, Opts, SslOpts]) -> process_flag(trap_exit, true), true = link(Port), - {ok, #state{emulated_opts = Opts, port = Port, ssl_opts = SslOpts}}. + {ok, #state{emulated_opts = do_set_emulated_opts(Opts, []), port = Port, ssl_opts = SslOpts}}. %%-------------------------------------------------------------------- -spec handle_call(msg(), from(), #state{}) -> {reply, reply(), #state{}}. @@ -304,9 +334,24 @@ split_options([Name | Opts], Emu, SocketOptNames, EmuOptNames) -> do_set_emulated_opts([], Opts) -> Opts; +do_set_emulated_opts([{active, N0} | Rest], Opts) when is_integer(N0) -> + N = update_active_n(N0, proplists:get_value(active, Opts, false)), + do_set_emulated_opts(Rest, [{active, N} | proplists:delete(active, Opts)]); do_set_emulated_opts([{Name,_} = Opt | Rest], Opts) -> do_set_emulated_opts(Rest, [Opt | proplists:delete(Name, Opts)]). +update_active_n(New, Current) -> + if + is_integer(Current), New + Current =< 0 -> + false; + is_integer(Current) -> + New + Current; + New =< 0 -> + false; + true -> + New + end. + get_socket_opts(_, [], _) -> []; get_socket_opts(ListenSocket, SockOptNames, Cb) -> @@ -366,6 +411,9 @@ validate_inet_option(header, Value) when not is_integer(Value) -> throw({error, {options, {header,Value}}}); validate_inet_option(active, Value) + when Value >= -32768, Value =< 32767 -> + ok; +validate_inet_option(active, Value) when Value =/= true, Value =/= false, Value =/= once -> throw({error, {options, {active,Value}}}); validate_inet_option(_, _) -> diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index 0a66dfa0b1..f55d1c8ea7 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -164,6 +164,7 @@ api_tests() -> accept_pool, prf, socket_options, + active_n, cipher_suites, handshake_continue, handshake_continue_timeout, @@ -243,6 +244,7 @@ error_handling_tests()-> [close_transport_accept, recv_active, recv_active_once, + recv_active_n, recv_error_handling, call_in_error_state, close_in_error_state, @@ -1978,7 +1980,7 @@ recv_active(Config) when is_list(Config) -> %%-------------------------------------------------------------------- recv_active_once() -> - [{doc,"Test recv on active socket"}]. + [{doc,"Test recv on active (once) socket"}]. recv_active_once(Config) when is_list(Config) -> ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), @@ -2003,6 +2005,178 @@ recv_active_once(Config) when is_list(Config) -> ssl_test_lib:close(Client). %%-------------------------------------------------------------------- +recv_active_n() -> + [{doc,"Test recv on active (n) socket"}]. + +recv_active_n(Config) when is_list(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config), + Server = + ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, + {from, self()}, + {mfa, {?MODULE, try_recv_active_once, []}}, + {options, [{active, 1} | ServerOpts]}]), + Port = ssl_test_lib:inet_port(Server), + Client = + ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, + {host, Hostname}, + {from, self()}, + {mfa, {?MODULE, try_recv_active_once, []}}, + {options, [{active, 1} | ClientOpts]}]), + + ssl_test_lib:check_result(Server, ok, Client, ok), + + ssl_test_lib:close(Server), + ssl_test_lib:close(Client). + +%%-------------------------------------------------------------------- +%% Test case adapted from gen_tcp_misc_SUITE. +active_n() -> + [{doc,"Test {active,N} option"}]. + +active_n(Config) when is_list(Config) -> + ClientOpts = ssl_test_lib:ssl_options(client_opts, Config), + ServerOpts = ssl_test_lib:ssl_options(server_opts, Config), + Port = ssl_test_lib:inet_port(node()), + N = 3, + LS = ok(ssl:listen(Port, [{active,N}|ServerOpts])), + [{active,N}] = ok(ssl:getopts(LS, [active])), + active_n_common(LS, N), + Self = self(), + spawn_link(fun() -> + S0 = ok(ssl:transport_accept(LS)), + {ok, S} = ssl:handshake(S0), + ok = ssl:setopts(S, [{active,N}]), + [{active,N}] = ok(ssl:getopts(S, [active])), + ssl:controlling_process(S, Self), + Self ! {server, S} + end), + C = ok(ssl:connect("localhost", Port, [{active,N}|ClientOpts])), + [{active,N}] = ok(ssl:getopts(C, [active])), + S = receive + {server, S0} -> S0 + after + 1000 -> + exit({error, connect}) + end, + active_n_common(C, N), + active_n_common(S, N), + ok = ssl:setopts(C, [{active,N}]), + ok = ssl:setopts(S, [{active,N}]), + ReceiveMsg = fun(Socket, Msg) -> + receive + {ssl,Socket,Msg} -> + ok; + {ssl,Socket,Begin} -> + receive + {ssl,Socket,End} -> + Msg = Begin ++ End, + ok + after 1000 -> + exit(timeout) + end + after 1000 -> + exit(timeout) + end + end, + repeat(3, fun(I) -> + Msg = "message "++integer_to_list(I), + ok = ssl:send(C, Msg), + ReceiveMsg(S, Msg), + ok = ssl:send(S, Msg), + ReceiveMsg(C, Msg) + end), + receive + {ssl_passive,S} -> + [{active,false}] = ok(ssl:getopts(S, [active])) + after + 1000 -> + exit({error,ssl_passive}) + end, + receive + {ssl_passive,C} -> + [{active,false}] = ok(ssl:getopts(C, [active])) + after + 1000 -> + exit({error,ssl_passive}) + end, + LS2 = ok(ssl:listen(0, [{active,0}])), + receive + {ssl_passive,LS2} -> + [{active,false}] = ok(ssl:getopts(LS2, [active])) + after + 1000 -> + exit({error,ssl_passive}) + end, + ok = ssl:close(LS2), + ok = ssl:close(C), + ok = ssl:close(S), + ok = ssl:close(LS), + ok. + +active_n_common(S, N) -> + ok = ssl:setopts(S, [{active,-N}]), + receive + {ssl_passive, S} -> ok + after + 1000 -> + error({error,ssl_passive_failure}) + end, + [{active,false}] = ok(ssl:getopts(S, [active])), + ok = ssl:setopts(S, [{active,0}]), + receive + {ssl_passive, S} -> ok + after + 1000 -> + error({error,ssl_passive_failure}) + end, + ok = ssl:setopts(S, [{active,32767}]), + {error,{options,_}} = ssl:setopts(S, [{active,1}]), + {error,{options,_}} = ssl:setopts(S, [{active,-32769}]), + ok = ssl:setopts(S, [{active,-32768}]), + receive + {ssl_passive, S} -> ok + after + 1000 -> + error({error,ssl_passive_failure}) + end, + [{active,false}] = ok(ssl:getopts(S, [active])), + ok = ssl:setopts(S, [{active,N}]), + ok = ssl:setopts(S, [{active,true}]), + [{active,true}] = ok(ssl:getopts(S, [active])), + receive + _ -> error({error,active_n}) + after + 0 -> + ok + end, + ok = ssl:setopts(S, [{active,N}]), + ok = ssl:setopts(S, [{active,once}]), + [{active,once}] = ok(ssl:getopts(S, [active])), + receive + _ -> error({error,active_n}) + after + 0 -> + ok + end, + {error,{options,_}} = ssl:setopts(S, [{active,32768}]), + ok = ssl:setopts(S, [{active,false}]), + [{active,false}] = ok(ssl:getopts(S, [active])), + ok. + +ok({ok,V}) -> V. + +repeat(N, Fun) -> + repeat(N, N, Fun). + +repeat(N, T, Fun) when is_integer(N), N > 0 -> + Fun(T-N), + repeat(N-1, T, Fun); +repeat(_, _, _) -> + ok. + +%%-------------------------------------------------------------------- dh_params() -> [{doc,"Test to specify DH-params file in server."}]. @@ -3889,7 +4063,7 @@ tls_tcp_error_propagation_in_active_mode(Config) when is_list(Config) -> [_, _,_, _, Prop] = StatusInfo, State = ssl_test_lib:state(Prop), StaticEnv = element(2, State), - Socket = element(10, StaticEnv), + Socket = element(11, StaticEnv), %% Fake tcp error Pid ! {tcp_error, Socket, etimedout}, diff --git a/lib/ssl/vsn.mk b/lib/ssl/vsn.mk index 3527062a8a..c4bcc1560c 100644 --- a/lib/ssl/vsn.mk +++ b/lib/ssl/vsn.mk @@ -1 +1 @@ -SSL_VSN = 9.1.2 +SSL_VSN = 9.2.1 diff --git a/lib/stdlib/doc/src/notes.xml b/lib/stdlib/doc/src/notes.xml index 993945b9c7..278560af1d 100644 --- a/lib/stdlib/doc/src/notes.xml +++ b/lib/stdlib/doc/src/notes.xml @@ -31,6 +31,57 @@ </header> <p>This document describes the changes made to the STDLIB application.</p> +<section><title>STDLIB 3.8</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix a bug in the Erlang Pretty Printer: long atom + names in combination with <c><<>></c> could + cause a crash. </p> + <p> + Own Id: OTP-15592 Aux Id: ERL-818 </p> + </item> + <item> + <p> Fix bugs that could cause wrong results or bad + performance when formatting lists of characters using the + control sequences <c>p</c> or <c>P</c> and limiting the + output with the option <c>chars_limit</c>. </p> + <p> + Own Id: OTP-15639</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Improved ETS documentation about safe table traversal and + the partially bound key optimization for + <c>ordered_set</c>.</p> + <p> + Own Id: OTP-15545 Aux Id: PR-2103, PR-2139 </p> + </item> + <item> + <p> Optimize <c>calendar:gregorian_days_to_date/1</c>. + </p> + <p> + Own Id: OTP-15572 Aux Id: PR-2121 </p> + </item> + <item> + <p> Optimize functions + <c>calendar:rfc3339_to_system_time()</c> and + <c>calendar:system_time_to_rfc3339()</c>. </p> + <p> + Own Id: OTP-15630</p> + </item> + </list> + </section> + +</section> + <section><title>STDLIB 3.7.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/stdlib/src/calendar.erl b/lib/stdlib/src/calendar.erl index 3a083d9fda..3a8fe2211b 100644 --- a/lib/stdlib/src/calendar.erl +++ b/lib/stdlib/src/calendar.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2018. All Rights Reserved. +%% Copyright Ericsson AB 1996-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -357,13 +357,17 @@ rfc3339_to_system_time(DateTimeString) -> rfc3339_to_system_time(DateTimeString, Options) -> Unit = proplists:get_value(unit, Options, second), %% _T is the character separating the date and the time: - {DateStr, [_T|TimeStr]} = lists:split(10, DateTimeString), - {TimeStr2, TimeStr3} = lists:split(8, TimeStr), - {ok, [Hour, Min, Sec], []} = io_lib:fread("~d:~d:~d", TimeStr2), - {ok, [Year, Month, Day], []} = io_lib:fread("~d-~d-~d", DateStr), + [Y1, Y2, Y3, Y4, $-, Mon1, Mon2, $-, D1, D2, _T, + H1, H2, $:, Min1, Min2, $:, S1, S2 | TimeStr] = DateTimeString, + Hour = list_to_integer([H1, H2]), + Min = list_to_integer([Min1, Min2]), + Sec = list_to_integer([S1, S2]), + Year = list_to_integer([Y1, Y2, Y3, Y4]), + Month = list_to_integer([Mon1, Mon2]), + Day = list_to_integer([D1, D2]), DateTime = {{Year, Month, Day}, {Hour, Min, Sec}}, IsFractionChar = fun(C) -> C >= $0 andalso C =< $9 orelse C =:= $. end, - {FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr3), + {FractionStr, UtcOffset} = lists:splitwith(IsFractionChar, TimeStr), Time = datetime_to_system_time(DateTime), Secs = Time - offset_adjustment(Time, second, UtcOffset), check(DateTimeString, Options, Secs), @@ -451,8 +455,9 @@ system_time_to_rfc3339(Time, Options) -> DateTime = system_time_to_datetime(Secs), {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime, FractionStr = fraction_str(Factor, AdjustedTime), - flat_fwrite("~4.10.0B-~2.10.0B-~2.10.0B~c~2.10.0B:~2.10.0B:~2.10.0B~s~s", - [Year, Month, Day, T, Hour, Min, Sec, FractionStr, Offset]). + L = [pad4(Year), "-", pad2(Month), "-", pad2(Day), [T], + pad2(Hour), ":", pad2(Min), ":", pad2(Sec), FractionStr, Offset], + lists:append(L). %% time_difference(T1, T2) = Tdiff %% @@ -680,7 +685,7 @@ offset(OffsetOption, Secs0) when OffsetOption =:= ""; Secs = abs(Secs0), Hour = Secs div 3600, Min = (Secs rem 3600) div 60, - io_lib:fwrite("~c~2.10.0B:~2.10.0B", [Sign, Hour, Min]); + [Sign | lists:append([pad2(Hour), ":", pad2(Min)])]; offset(OffsetOption, _Secs) -> OffsetOption. @@ -695,8 +700,10 @@ offset_string_adjustment(_Time, _Unit, "Z") -> 0; offset_string_adjustment(_Time, _Unit, "z") -> 0; -offset_string_adjustment(_Time, _Unit, [Sign|Tz]) -> - {ok, [Hour, Min], []} = io_lib:fread("~d:~d", Tz), +offset_string_adjustment(_Time, _Unit, Tz) -> + [Sign, H1, H2, $:, M1, M2] = Tz, + Hour = list_to_integer([H1, H2]), + Min = list_to_integer([M1, M2]), Adjustment = 3600 * Hour + 60 * Min, case Sign of $- -> -Adjustment; @@ -704,8 +711,9 @@ offset_string_adjustment(_Time, _Unit, [Sign|Tz]) -> end. local_offset(SystemTime, Unit) -> - LocalTime = system_time_to_local_time(SystemTime, Unit), + %% Not optimized for special cases. UniversalTime = system_time_to_universal_time(SystemTime, Unit), + LocalTime = erlang:universaltime_to_localtime(UniversalTime), LocalSecs = datetime_to_gregorian_seconds(LocalTime), UniversalSecs = datetime_to_gregorian_seconds(UniversalTime), LocalSecs - UniversalSecs. @@ -714,7 +722,8 @@ fraction_str(1, _Time) -> ""; fraction_str(Factor, Time) -> Fraction = Time rem Factor, - io_lib:fwrite(".~*..0B", [log10(Factor), abs(Fraction)]). + S = integer_to_list(abs(Fraction)), + [$. | pad(log10(Factor) - length(S), S)]. fraction(second, _) -> 0; @@ -735,5 +744,21 @@ log10(1000) -> 3; log10(1000000) -> 6; log10(1000000000) -> 9. -flat_fwrite(F, S) -> - lists:flatten(io_lib:fwrite(F, S)). +pad(0, S) -> + S; +pad(I, S) -> + [$0 | pad(I - 1, S)]. + +pad2(N) when N < 10 -> + [$0 | integer_to_list(N)]; +pad2(N) -> + integer_to_list(N). + +pad4(N) when N < 10 -> + [$0, $0, $0 | integer_to_list(N)]; +pad4(N) when N < 100 -> + [$0, $0 | integer_to_list(N)]; +pad4(N) when N < 1000 -> + [$0 | integer_to_list(N)]; +pad4(N) -> + integer_to_list(N). diff --git a/lib/stdlib/src/io_lib_pretty.erl b/lib/stdlib/src/io_lib_pretty.erl index 5483ea87b5..8f2fd7ea8f 100644 --- a/lib/stdlib/src/io_lib_pretty.erl +++ b/lib/stdlib/src/io_lib_pretty.erl @@ -721,7 +721,7 @@ printable_list(_L, 1, _T, _Enc) -> printable_list(L, _D, T, latin1) when T < 0 -> io_lib:printable_latin1_list(L); printable_list(L, _D, T, Enc) when T >= 0 -> - case slice(L, tsub(T, 2)) of + case slice(L, tsub(T, 2), Enc) of false -> false; {prefix, Prefix} when Enc =:= latin1 -> @@ -737,20 +737,46 @@ printable_list(L, _D, T, Enc) when T >= 0 -> printable_list(L, _D, T, _Uni) when T < 0-> io_lib:printable_list(L). -slice(L, N) -> - try io_lib:chars_length(L) =< N of - true -> +slice(L, N, latin1) -> + try lists:split(N, L) of + {_, []} -> all; - false -> - case string:slice(L, 0, N) of - "" -> - false; - Prefix -> - {prefix, Prefix} + {[], _} -> + false; + {L1, _} -> + {prefix, L1} + catch + _:_ -> + all + end; +slice(L, N, _Uni) -> + %% Be careful not to traverse more of L than necessary. + try string:slice(L, 0, N) of + "" -> + false; + Prefix -> + %% Assume no binaries are introduced by string:slice(). + case is_flat(L, lists:flatlength(Prefix)) of + true -> + case string:equal(Prefix, L) of + true -> + all; + false -> + {prefix, Prefix} + end; + false -> + false end catch _:_ -> false end. +is_flat(_L, 0) -> + true; +is_flat([C|Cs], N) when is_integer(C) -> + is_flat(Cs, N - 1); +is_flat(_, _N) -> + false. + printable_bin0(Bin, D, T, Enc) -> Len = case D >= 0 of true -> diff --git a/lib/stdlib/src/stdlib.appup.src b/lib/stdlib/src/stdlib.appup.src index 9e5d6a3bd8..37ea97c353 100644 --- a/lib/stdlib/src/stdlib.appup.src +++ b/lib/stdlib/src/stdlib.appup.src @@ -40,7 +40,8 @@ {<<"^3\\.6$">>,[restart_new_emulator]}, {<<"^3\\.6\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.7$">>,[restart_new_emulator]}, - {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}], + {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}], [{<<"^3\\.4$">>,[restart_new_emulator]}, {<<"^3\\.4\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.4\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}, @@ -54,4 +55,5 @@ {<<"^3\\.6$">>,[restart_new_emulator]}, {<<"^3\\.6\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, {<<"^3\\.7$">>,[restart_new_emulator]}, - {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}]}. + {<<"^3\\.7\\.0(?:\\.[0-9]+)+$">>,[restart_new_emulator]}, + {<<"^3\\.7\\.1(?:\\.[0-9]+)*$">>,[restart_new_emulator]}]}. diff --git a/lib/stdlib/test/calendar_SUITE.erl b/lib/stdlib/test/calendar_SUITE.erl index c6d9dbca4a..224c0d5625 100644 --- a/lib/stdlib/test/calendar_SUITE.erl +++ b/lib/stdlib/test/calendar_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2018. All Rights Reserved. +%% Copyright Ericsson AB 1997-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/stdlib/test/ets_SUITE.erl b/lib/stdlib/test/ets_SUITE.erl index aa2e352c29..433b812fd5 100644 --- a/lib/stdlib/test/ets_SUITE.erl +++ b/lib/stdlib/test/ets_SUITE.erl @@ -59,6 +59,7 @@ -export([otp_5340/1]). -export([otp_6338/1]). -export([otp_6842_select_1000/1]). +-export([select_mbuf_trapping/1]). -export([otp_7665/1]). -export([meta_wb/1]). -export([grow_shrink/1, grow_pseudo_deleted/1, shrink_pseudo_deleted/1]). @@ -130,6 +131,7 @@ all() -> t_named_select, select_fail, t_insert_new, t_repair_continuation, otp_5340, otp_6338, otp_6842_select_1000, otp_7665, + select_mbuf_trapping, otp_8732, meta_wb, grow_shrink, grow_pseudo_deleted, shrink_pseudo_deleted, {group, meta_smp}, smp_insert, smp_fixed_delete, smp_unfix_fix, smp_select_replace, @@ -5131,6 +5133,61 @@ otp_6338(Config) when is_list(Config) -> [[4839,recv]] = ets:match(T,{[{sbm,ppb2_bs12@blade_0_8},'$1'],'$2'}), ets:delete(T). +%% OTP-15660: Verify select not doing excessive trapping +%% when process have mbuf heap fragments. +select_mbuf_trapping(Config) when is_list(Config) -> + select_mbuf_trapping_do(set), + select_mbuf_trapping_do(ordered_set). + +select_mbuf_trapping_do(Type) -> + T = ets:new(xxx, [Type]), + NKeys = 50, + [ets:insert(T, {K, value}) || K <- lists:seq(1,NKeys)], + + {priority, Prio} = process_info(self(), priority), + Tracee = self(), + [SchedTracer] + = start_loopers(1, Prio, + fun (SC) -> + receive + {trace, Tracee, out, _} -> + SC+1; + done -> + Tracee ! {schedule_count, SC}, + exit(normal) + end + end, + 0), + + erlang:garbage_collect(), + 1 = erlang:trace(self(), true, [running,{tracer,SchedTracer}]), + + %% Artificially create an mbuf heap fragment + MbufTerm = "Frag me up", + MbufTerm = erts_debug:set_internal_state(mbuf, MbufTerm), + + Keys = ets:select(T, [{{'$1', value}, [], ['$1']}]), + NKeys = length(Keys), + + 1 = erlang:trace(self(), false, [running]), + Ref = erlang:trace_delivered(Tracee), + receive + {trace_delivered, Tracee, Ref} -> + SchedTracer ! done + end, + receive + {schedule_count, N} -> + io:format("~p context switches: ~p", [Type,N]), + if + N < 3 -> ok; + true -> ct:fail(failed) + end + end, + true = ets:delete(T), + ok. + + + %% Elements could come in the wrong order in a bag if a rehash occurred. otp_5340(Config) when is_list(Config) -> repeat_for_opts(fun otp_5340_do/1). diff --git a/lib/stdlib/test/io_SUITE.erl b/lib/stdlib/test/io_SUITE.erl index f097552e8c..824f5d19f2 100644 --- a/lib/stdlib/test/io_SUITE.erl +++ b/lib/stdlib/test/io_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2018. All Rights Reserved. +%% Copyright Ericsson AB 1999-2019. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -32,7 +32,7 @@ io_with_huge_message_queue/1, format_string/1, maps/1, coverage/1, otp_14178_unicode_atoms/1, otp_14175/1, otp_14285/1, limit_term/1, otp_14983/1, otp_15103/1, otp_15076/1, - otp_15159/1]). + otp_15159/1, otp_15639/1]). -export([pretty/2, trf/3]). @@ -64,7 +64,8 @@ all() -> io_lib_print_binary_depth_one, otp_10302, otp_10755, otp_10836, io_lib_width_too_small, io_with_huge_message_queue, format_string, maps, coverage, otp_14178_unicode_atoms, otp_14175, - otp_14285, limit_term, otp_14983, otp_15103, otp_15076, otp_15159]. + otp_14285, limit_term, otp_14983, otp_15103, otp_15076, otp_15159, + otp_15639]. %% Error cases for output. error_1(Config) when is_list(Config) -> @@ -2647,3 +2648,35 @@ otp_15076(_Config) -> {'EXIT', {badarg, _}} = (catch io_lib:build_text(L)), {'EXIT', {badarg, _}} = (catch io_lib:build_text(L, [])), ok. + +otp_15639(_Config) -> + L = lists:duplicate(10, "a"), + LOpts = [{encoding, latin1}, {chars_limit, 10}], + UOpts = [{encoding, unicode}, {chars_limit, 10}], + "[[...]|...]" = pretty(L, LOpts), + "[[...]|...]" = pretty(L, UOpts), + "[\"a\",[...]|...]" = + pretty(L, [{chars_limit, 12}, {encoding, latin1}]), + "[\"a\",[...]|...]" = + pretty(L, [{chars_limit, 12}, {encoding, unicode}]), + + %% Latin-1 + "\"12345678\"" = pretty("12345678", LOpts), + "\"12345678\"..." = pretty("123456789", LOpts), + "\"\\r\\n123456\"..." = pretty("\r\n1234567", LOpts), + "\"\\r1234567\"..." = pretty("\r12345678", LOpts), + "\"\\r\\n123456\"..." = pretty("\r\n12345678", LOpts), + "\"12345678\"..." = pretty("12345678"++[x], LOpts), + "[49,50|...]" = pretty("1234567"++[x], LOpts), + "[49,x]" = pretty("1"++[x], LOpts), + "[[...]|...]" = pretty(["1","2","3","4","5","6","7","8"], LOpts), + %% Unicode + "\"12345678\"" = pretty("12345678", UOpts), + "\"12345678\"..." = pretty("123456789", UOpts), + "\"\\r\\n1234567\"" = pretty("\r\n1234567", UOpts), + "\"\\r1234567\"..." = pretty("\r12345678", UOpts), + "\"\\r\\n1234567\"..." = pretty("\r\n12345678", UOpts), + "[49,50|...]" = pretty("12345678"++[x], UOpts), + "\"12345678\"..." = pretty("123456789"++[x], UOpts), + "[[...]|...]" = pretty(["1","2","3","4","5","6","7","8"], UOpts), + ok. diff --git a/lib/stdlib/vsn.mk b/lib/stdlib/vsn.mk index d46173497b..cbefd6590a 100644 --- a/lib/stdlib/vsn.mk +++ b/lib/stdlib/vsn.mk @@ -1 +1 @@ -STDLIB_VSN = 3.7.1 +STDLIB_VSN = 3.8 diff --git a/lib/syntax_tools/doc/src/notes.xml b/lib/syntax_tools/doc/src/notes.xml index dc13fe474b..772f5e6e04 100644 --- a/lib/syntax_tools/doc/src/notes.xml +++ b/lib/syntax_tools/doc/src/notes.xml @@ -32,6 +32,20 @@ <p>This document describes the changes made to the Syntax_Tools application.</p> +<section><title>Syntax_Tools 2.1.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix pretty-printing of type funs. </p> + <p> + Own Id: OTP-15519 Aux Id: ERL-815 </p> + </item> + </list> + </section> + +</section> + <section><title>Syntax_Tools 2.1.6</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/syntax_tools/vsn.mk b/lib/syntax_tools/vsn.mk index 8959ebbd04..538c71dc24 100644 --- a/lib/syntax_tools/vsn.mk +++ b/lib/syntax_tools/vsn.mk @@ -1 +1 @@ -SYNTAX_TOOLS_VSN = 2.1.6 +SYNTAX_TOOLS_VSN = 2.1.7 diff --git a/lib/tools/c_src/Makefile.in b/lib/tools/c_src/Makefile.in index cfe91917f8..289322b6fa 100644 --- a/lib/tools/c_src/Makefile.in +++ b/lib/tools/c_src/Makefile.in @@ -29,7 +29,6 @@ CC=@CC@ LD=@LD@ AR=@AR@ RANLIB=@RANLIB@ -RM=@RM@ MKDIR=@MKDIR@ INSTALL=@INSTALL@ INSTALL_DIR=@INSTALL_DIR@ @@ -158,9 +157,9 @@ $(ERTS_LIB): docs: clean: - $(RM) -rf ../obj/* - $(RM) -rf ../bin/* - $(RM) -f ./*~ + $(RM) -r ../obj/* + $(RM) -r ../bin/* + $(RM) ./*~ .PHONY: all erts_lib docs clean diff --git a/lib/tools/doc/src/notes.xml b/lib/tools/doc/src/notes.xml index a6781dfdb3..28f8346a19 100644 --- a/lib/tools/doc/src/notes.xml +++ b/lib/tools/doc/src/notes.xml @@ -31,6 +31,45 @@ </header> <p>This document describes the changes made to the Tools application.</p> +<section><title>Tools 3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Minor fixes for <c>make clean</c>.</p> + <p> + Own Id: OTP-15657</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + In the HTML file generated by + <c>cover:analyse_to_file/1,2</c>, a link is now added to + the line number. This makes it easier to share pointers + to specific lines.</p> + <p> + Own Id: OTP-15541</p> + </item> + <item> + <p> + Uncovered lines are now marked with a sad face, + <c>:-(</c>, in the HTML output from + <c>cover:analyse_to_file/1,2</c>. This is to make these + lines easier to find by search.</p> + <p> + Own Id: OTP-15542</p> + </item> + </list> + </section> + +</section> + <section><title>Tools 3.0.2</title> <section><title>Improvements and New Features</title> diff --git a/lib/tools/vsn.mk b/lib/tools/vsn.mk index bb8305e9f1..5700885549 100644 --- a/lib/tools/vsn.mk +++ b/lib/tools/vsn.mk @@ -1 +1 @@ -TOOLS_VSN = 3.0.2 +TOOLS_VSN = 3.1 diff --git a/lib/wx/doc/src/notes.xml b/lib/wx/doc/src/notes.xml index 1061e73138..33d02e22ba 100644 --- a/lib/wx/doc/src/notes.xml +++ b/lib/wx/doc/src/notes.xml @@ -32,6 +32,25 @@ <p>This document describes the changes made to the wxErlang application.</p> +<section><title>Wx 1.8.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Improved support for wxWidgets 3.1.3 which have changed + <c>wxFONTWEIGTH</c>, also added <c>wxGCDC</c> and + <c>wxDisplay</c> modules.</p> + <p> + Fixed a crash on Mojave and check for events more often.</p> + <p> + Own Id: OTP-15587</p> + </item> + </list> + </section> + +</section> + <section><title>Wx 1.8.6</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/wx/vsn.mk b/lib/wx/vsn.mk index d241a7a1b4..dac219fa98 100644 --- a/lib/wx/vsn.mk +++ b/lib/wx/vsn.mk @@ -1 +1 @@ -WX_VSN = 1.8.6 +WX_VSN = 1.8.7 diff --git a/make/install_dir_data.sh.in b/make/install_dir_data.sh.in new file mode 100644 index 0000000000..8c1dc3d889 --- /dev/null +++ b/make/install_dir_data.sh.in @@ -0,0 +1,70 @@ +#!/bin/sh +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2019. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# + +# +# install_dir_data.sh <SourceDir> <DestDir> +# +# Install all content in <SourceDir> including subdirectories +# into <DestDir>. +# + +INSTALL="@INSTALL@" +INSTALL_DIR="@INSTALL_DIR@" +INSTALL_DATA="@INSTALL_DATA@" + +debug=yes + +error () { + echo "ERROR: $1" 1>&2 + exit 1 +} + +usage () { + error "$1\n Usage $progname <SourceDir> <DestDir>" +} + +cmd () { + [ $debug = no ] || echo "$@" + "$@" || exit 1 +} + +progname="$0" + +[ $# -eq 2 ] || usage "Invalid amount of arguments" + +src="$1" +dest="$2" + +cmd cd "$src" + +for dir in `find . -type d`; do + destdir="$dest" + [ "$dir" = "." ] || destdir="$dest/$dir" + cmd $INSTALL_DIR "$destdir" +done + +for file in `find . -type f`; do + subdir=`dirname "$file"` + destdir="$dest" + [ "$subdir" = "." ] || destdir="$dest/$subdir" + cmd $INSTALL_DATA "$file" "$destdir" +done + +exit 0 diff --git a/make/otp.mk.in b/make/otp.mk.in index df29d26833..ceff8f7c31 100644 --- a/make/otp.mk.in +++ b/make/otp.mk.in @@ -73,6 +73,7 @@ INSTALL_DIR = @INSTALL_DIR@ INSTALL_PROGRAM = @INSTALL_PROGRAM@ INSTALL_SCRIPT = @INSTALL_SCRIPT@ INSTALL_DATA = @INSTALL_DATA@ +INSTALL_DIR_DATA = $(ERL_TOP)/make/install_dir_data.sh CC = @CC@ GCC = @GCC@ @@ -274,6 +275,7 @@ XSLTPROC = @XSLTPROC@ FOP = @FOP@ XMLLINT = @XMLLINT@ CP = @CP@ +MKDIR = @MKDIR@ DOCGEN=$(ERL_TOP)/lib/erl_docgen FOP_CONFIG = $(DOCGEN)/priv/fop.xconf diff --git a/make/otp_version_tickets_in_merge b/make/otp_version_tickets_in_merge index 32e6923ce9..cfd555fd49 100644 --- a/make/otp_version_tickets_in_merge +++ b/make/otp_version_tickets_in_merge @@ -1,2 +1,3 @@ -OTP-14728 -OTP-15573 +OTP-14701 +OTP-15676 +OTP-15679 diff --git a/otp_patch_apply b/otp_patch_apply index 0127b96a73..3ff929ccbb 100755 --- a/otp_patch_apply +++ b/otp_patch_apply @@ -19,7 +19,7 @@ # %CopyrightEnd% # -version="1.0.1" +version="1.0.2" force= lib_path= @@ -388,9 +388,7 @@ if [ $install_docs = yes ]; then TESTROOT="$idir" release_docs) || exit 1 done - (cd "$sdir/system/doc/top" && $MAKE clean) - - (cd "$sdir/system/doc/top" && \ + (cd "$sdir/system/doc" && \ $MAKE MAKE="$MAKE" RELEASE_ROOT="$idir" RELEASE_PATH="$idir" \ TESTROOT="$idir" release_docs) || exit 1 diff --git a/otp_versions.table b/otp_versions.table index aea1e8d5cd..a31759a8ef 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -1,3 +1,6 @@ +OTP-21.3.1 : erl_interface-3.11.1 ssl-9.2.1 # asn1-5.0.8 common_test-1.17 compiler-7.3.2 crypto-4.4.1 debugger-4.2.6 dialyzer-3.3.2 diameter-2.2 edoc-0.10 eldap-1.2.6 erl_docgen-0.9 erts-10.3 et-1.6.4 eunit-2.3.7 ftp-1.0.2 hipe-3.18.3 inets-7.0.6 jinterface-1.9.1 kernel-6.3 megaco-3.18.4 mnesia-4.15.6 observer-2.9 odbc-2.12.3 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.5 reltool-0.7.8 runtime_tools-1.13.2 sasl-3.3 snmp-5.2.12 ssh-4.7.4 stdlib-3.8 syntax_tools-2.1.7 tftp-1.0.1 tools-3.1 wx-1.8.7 xmerl-1.3.19 : +OTP-21.3 : common_test-1.17 compiler-7.3.2 crypto-4.4.1 dialyzer-3.3.2 diameter-2.2 edoc-0.10 erl_docgen-0.9 erl_interface-3.11 erts-10.3 ftp-1.0.2 hipe-3.18.3 inets-7.0.6 kernel-6.3 mnesia-4.15.6 observer-2.9 odbc-2.12.3 public_key-1.6.5 runtime_tools-1.13.2 ssh-4.7.4 ssl-9.2 stdlib-3.8 syntax_tools-2.1.7 tools-3.1 wx-1.8.7 # asn1-5.0.8 debugger-4.2.6 eldap-1.2.6 et-1.6.4 eunit-2.3.7 jinterface-1.9.1 megaco-3.18.4 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 reltool-0.7.8 sasl-3.3 snmp-5.2.12 tftp-1.0.1 xmerl-1.3.19 : +OTP-21.2.7 : erts-10.2.5 kernel-6.2.1 # asn1-5.0.8 common_test-1.16.1 compiler-7.3.1 crypto-4.4 debugger-4.2.6 dialyzer-3.3.1 diameter-2.1.6 edoc-0.9.4 eldap-1.2.6 erl_docgen-0.8.1 erl_interface-3.10.4 et-1.6.4 eunit-2.3.7 ftp-1.0.1 hipe-3.18.2 inets-7.0.5 jinterface-1.9.1 megaco-3.18.4 mnesia-4.15.5 observer-2.8.2 odbc-2.12.2 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.4 reltool-0.7.8 runtime_tools-1.13.1 sasl-3.3 snmp-5.2.12 ssh-4.7.3 ssl-9.1.2 stdlib-3.7.1 syntax_tools-2.1.6 tftp-1.0.1 tools-3.0.2 wx-1.8.6 xmerl-1.3.19 : OTP-21.2.6 : erts-10.2.4 stdlib-3.7.1 # asn1-5.0.8 common_test-1.16.1 compiler-7.3.1 crypto-4.4 debugger-4.2.6 dialyzer-3.3.1 diameter-2.1.6 edoc-0.9.4 eldap-1.2.6 erl_docgen-0.8.1 erl_interface-3.10.4 et-1.6.4 eunit-2.3.7 ftp-1.0.1 hipe-3.18.2 inets-7.0.5 jinterface-1.9.1 kernel-6.2 megaco-3.18.4 mnesia-4.15.5 observer-2.8.2 odbc-2.12.2 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.4 reltool-0.7.8 runtime_tools-1.13.1 sasl-3.3 snmp-5.2.12 ssh-4.7.3 ssl-9.1.2 syntax_tools-2.1.6 tftp-1.0.1 tools-3.0.2 wx-1.8.6 xmerl-1.3.19 : OTP-21.2.5 : inets-7.0.5 # asn1-5.0.8 common_test-1.16.1 compiler-7.3.1 crypto-4.4 debugger-4.2.6 dialyzer-3.3.1 diameter-2.1.6 edoc-0.9.4 eldap-1.2.6 erl_docgen-0.8.1 erl_interface-3.10.4 erts-10.2.3 et-1.6.4 eunit-2.3.7 ftp-1.0.1 hipe-3.18.2 jinterface-1.9.1 kernel-6.2 megaco-3.18.4 mnesia-4.15.5 observer-2.8.2 odbc-2.12.2 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.4 reltool-0.7.8 runtime_tools-1.13.1 sasl-3.3 snmp-5.2.12 ssh-4.7.3 ssl-9.1.2 stdlib-3.7 syntax_tools-2.1.6 tftp-1.0.1 tools-3.0.2 wx-1.8.6 xmerl-1.3.19 : OTP-21.2.4 : erts-10.2.3 inets-7.0.4 # asn1-5.0.8 common_test-1.16.1 compiler-7.3.1 crypto-4.4 debugger-4.2.6 dialyzer-3.3.1 diameter-2.1.6 edoc-0.9.4 eldap-1.2.6 erl_docgen-0.8.1 erl_interface-3.10.4 et-1.6.4 eunit-2.3.7 ftp-1.0.1 hipe-3.18.2 jinterface-1.9.1 kernel-6.2 megaco-3.18.4 mnesia-4.15.5 observer-2.8.2 odbc-2.12.2 os_mon-2.4.7 otp_mibs-1.2.1 parsetools-2.1.8 public_key-1.6.4 reltool-0.7.8 runtime_tools-1.13.1 sasl-3.3 snmp-5.2.12 ssh-4.7.3 ssl-9.1.2 stdlib-3.7 syntax_tools-2.1.6 tftp-1.0.1 tools-3.0.2 wx-1.8.6 xmerl-1.3.19 : diff --git a/system/doc/design_principles/Makefile b/system/doc/design_principles/Makefile index 242bf1c9a4..2fbd7d087f 100644 --- a/system/doc/design_principles/Makefile +++ b/system/doc/design_principles/Makefile @@ -88,7 +88,6 @@ DVIPS_FLAGS += # ---------------------------------------------------- # Targets # ---------------------------------------------------- -_create_dirs := $(shell mkdir -p $(HTMLDIR)) $(HTMLDIR)/%.gif: %.gif $(INSTALL_DATA) $< $@ @@ -107,8 +106,8 @@ images: $(IMAGE_FILES:%=$(HTMLDIR)/%) debug opt: clean clean_docs: - rm -rf $(HTMLDIR) - rm -rf $(XMLDIR) + rm -f $(XMLDIR)/*.xml + rm -f $(HTMLDIR)/*.gif $(HTMLDIR)/*.html rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) rm -f errs core *~ diff --git a/system/doc/efficiency_guide/Makefile b/system/doc/efficiency_guide/Makefile index 72bcd2ee73..a2742a1354 100644 --- a/system/doc/efficiency_guide/Makefile +++ b/system/doc/efficiency_guide/Makefile @@ -87,7 +87,6 @@ DVIPS_FLAGS += # ---------------------------------------------------- # Targets # ---------------------------------------------------- -_create_dirs := $(shell mkdir -p $(HTMLDIR)) docs: html @@ -98,8 +97,8 @@ html: $(GIF_FILES) $(HTML_UG_FILE) debug opt: clean clean_docs: - rm -rf $(HTMLDIR) - rm -rf $(XMLDIR) + rm -f $(XMLDIR)/*.xml + rm -f $(HTMLDIR)/*.gif $(HTMLDIR)/*.html rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) rm -f errs core *~ diff --git a/system/doc/embedded/Makefile b/system/doc/embedded/Makefile index 396aef276b..1604075312 100644 --- a/system/doc/embedded/Makefile +++ b/system/doc/embedded/Makefile @@ -75,7 +75,6 @@ DVIPS_FLAGS += # ---------------------------------------------------- # Targets # ---------------------------------------------------- -_create_dirs := $(shell mkdir -p $(HTMLDIR)) docs: html @@ -86,8 +85,8 @@ html: $(GIF_FILES) $(HTML_UG_FILE) debug opt: clean clean_docs: - rm -rf $(HTMLDIR) - rm -rf $(XMLDIR) + rm -f $(XMLDIR)/*.xml + rm -f $(HTMLDIR)/*.gif $(HTMLDIR)/*.html rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) rm -f errs core *~ diff --git a/system/doc/getting_started/Makefile b/system/doc/getting_started/Makefile index cdf1e121c2..1c917895d5 100644 --- a/system/doc/getting_started/Makefile +++ b/system/doc/getting_started/Makefile @@ -74,7 +74,6 @@ DVIPS_FLAGS += # ---------------------------------------------------- # Targets # ---------------------------------------------------- -_create_dirs := $(shell mkdir -p $(HTMLDIR)) docs: html @@ -85,8 +84,8 @@ html: $(GIF_FILES) $(HTML_UG_FILE) debug opt: clean clean_docs: - rm -rf $(HTMLDIR) - rm -rf $(XMLDIR) + rm -f $(XMLDIR)/*.xml + rm -f $(HTMLDIR)/*.gif $(HTMLDIR)/*.html rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) rm -f errs core *~ diff --git a/system/doc/html/design_principles/.gitignore b/system/doc/html/design_principles/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/html/design_principles/.gitignore diff --git a/system/doc/html/efficiency_guide/.gitignore b/system/doc/html/efficiency_guide/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/html/efficiency_guide/.gitignore diff --git a/system/doc/html/embedded/.gitignore b/system/doc/html/embedded/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/html/embedded/.gitignore diff --git a/system/doc/html/getting_started/.gitignore b/system/doc/html/getting_started/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/html/getting_started/.gitignore diff --git a/system/doc/html/installation_guide/.gitignore b/system/doc/html/installation_guide/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/html/installation_guide/.gitignore diff --git a/system/doc/html/installation_guide/source/.gitignore b/system/doc/html/installation_guide/source/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/html/installation_guide/source/.gitignore diff --git a/system/doc/html/js/.gitignore b/system/doc/html/js/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/html/js/.gitignore diff --git a/system/doc/html/oam/.gitignore b/system/doc/html/oam/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/html/oam/.gitignore diff --git a/system/doc/html/programming_examples/.gitignore b/system/doc/html/programming_examples/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/html/programming_examples/.gitignore diff --git a/system/doc/html/reference_manual/.gitignore b/system/doc/html/reference_manual/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/html/reference_manual/.gitignore diff --git a/system/doc/html/system_architecture_intro/.gitignore b/system/doc/html/system_architecture_intro/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/html/system_architecture_intro/.gitignore diff --git a/system/doc/html/system_principles/.gitignore b/system/doc/html/system_principles/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/html/system_principles/.gitignore diff --git a/system/doc/html/tutorial/.gitignore b/system/doc/html/tutorial/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/html/tutorial/.gitignore diff --git a/system/doc/installation_guide/Makefile b/system/doc/installation_guide/Makefile index 4a1335cf31..38252757d6 100644 --- a/system/doc/installation_guide/Makefile +++ b/system/doc/installation_guide/Makefile @@ -91,7 +91,6 @@ $(XMLDIR)/%.xml: $(ERL_TOP)/HOWTO/%.md $(ERL_TOP)/make/emd2exml $(ERL_TOP)/make/emd2exml $< $@ $(REDIRECT_HTML_DIR)/%.html: Makefile - test -d $(REDIRECT_HTML_DIR) || $(INSTALL_DIR) $(REDIRECT_HTML_DIR) echo "<html><head><meta HTTP-EQUIV=\"REFRESH\"" > $@ echo " content=\"5; url=../"$(notdir $@)"\">" >> $@ echo "<title>This page has moved</title></head><body>" >> $@ @@ -112,8 +111,8 @@ debug opt: clean clean_docs: rm -f $(GENERATED_XML_FILES) - rm -rf $(HTMLDIR) - rm -rf $(XMLDIR) + rm -f $(XMLDIR)/*.xml + rm -f $(HTMLDIR)/*.gif $(HTMLDIR)/*.html rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) rm -f errs core *~ diff --git a/system/doc/oam/Makefile b/system/doc/oam/Makefile index 147f56f885..2eb429e04d 100644 --- a/system/doc/oam/Makefile +++ b/system/doc/oam/Makefile @@ -71,10 +71,9 @@ DVIPS_FLAGS += # ---------------------------------------------------- # Targets # ---------------------------------------------------- -_create_dirs := $(shell mkdir -p $(HTMLDIR)) $(HTMLDIR)/%.gif: %.gif - $(INSTALL_DATA) $< $@ + $(CP) $< $@ docs: html @@ -87,8 +86,8 @@ gifs: $(GIF_FILES:%=$(HTMLDIR)/%) debug opt: clean clean_docs: - rm -rf $(HTMLDIR) - rm -rf $(XMLDIR) + rm -f $(XMLDIR)/*.xml + rm -f $(HTMLDIR)/*.gif $(HTMLDIR)/*.html rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) rm -f errs core *~ diff --git a/system/doc/programming_examples/Makefile b/system/doc/programming_examples/Makefile index e4737ba069..9c67c24b64 100644 --- a/system/doc/programming_examples/Makefile +++ b/system/doc/programming_examples/Makefile @@ -74,7 +74,6 @@ DVIPS_FLAGS += # ---------------------------------------------------- # Targets # ---------------------------------------------------- -_create_dirs := $(shell mkdir -p $(HTMLDIR)) docs: html local_docs: PDFDIR=../../pdf @@ -84,8 +83,8 @@ html: $(GIF_FILES) $(HTML_UG_FILE) debug opt: clean clean_docs: - rm -rf $(HTMLDIR) - rm -rf $(XMLDIR) + rm -f $(XMLDIR)/*.xml + rm -f $(HTMLDIR)/*.gif $(HTMLDIR)/*.html rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) rm -f errs core *~ diff --git a/system/doc/reference_manual/Makefile b/system/doc/reference_manual/Makefile index d034ad2ff8..809eb2c979 100644 --- a/system/doc/reference_manual/Makefile +++ b/system/doc/reference_manual/Makefile @@ -84,7 +84,6 @@ DVIPS_FLAGS += # ---------------------------------------------------- # Targets # ---------------------------------------------------- -_create_dirs := $(shell mkdir -p $(HTMLDIR)) docs: html @@ -95,8 +94,8 @@ html: $(GIF_FILES) $(HTML_UG_FILE) debug opt: clean clean_docs: - rm -rf $(HTMLDIR) - rm -rf $(XMLDIR) + rm -f $(XMLDIR)/*.xml + rm -f $(HTMLDIR)/*.gif $(HTMLDIR)/*.html rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) rm -f errs core *~ diff --git a/system/doc/system_architecture_intro/Makefile b/system/doc/system_architecture_intro/Makefile index eb885a744d..ea9ee85105 100644 --- a/system/doc/system_architecture_intro/Makefile +++ b/system/doc/system_architecture_intro/Makefile @@ -69,7 +69,6 @@ DVIPS_FLAGS += # ---------------------------------------------------- # Targets # ---------------------------------------------------- -_create_dirs := $(shell mkdir -p $(HTMLDIR)) docs: html @@ -80,8 +79,8 @@ html: $(GIF_FILES) $(HTML_UG_FILE) debug opt: clean clean_docs: - rm -rf $(HTMLDIR) - rm -rf $(XMLDIR) + rm -f $(XMLDIR)/*.xml + rm -f $(HTMLDIR)/*.gif $(HTMLDIR)/*.html rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) rm -f errs core *~ diff --git a/system/doc/system_principles/Makefile b/system/doc/system_principles/Makefile index 1979deda4c..5110b73373 100644 --- a/system/doc/system_principles/Makefile +++ b/system/doc/system_principles/Makefile @@ -70,7 +70,6 @@ DVIPS_FLAGS += # ---------------------------------------------------- # Targets # ---------------------------------------------------- -_create_dirs := $(shell mkdir -p $(HTMLDIR)) docs: html @@ -81,8 +80,8 @@ html: $(GIF_FILES) $(HTML_UG_FILE) debug opt: clean clean_docs: - rm -rf $(HTMLDIR) - rm -rf $(XMLDIR) + rm -f $(XMLDIR)/*.xml + rm -f $(HTMLDIR)/*.gif $(HTMLDIR)/*.html rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) rm -f errs core *~ diff --git a/system/doc/top/Makefile b/system/doc/top/Makefile index 0703b821f1..3267bf96df 100644 --- a/system/doc/top/Makefile +++ b/system/doc/top/Makefile @@ -98,6 +98,25 @@ PDFREFDIR= pdf TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf TOPDOC=true +ifdef RELEASE_PATH +INST_TYPE=rel +INST_TYPE_SRC_DIR=$(RELEASE_PATH) +# We build to the 'temporary' dir in order to be able to install +# results using INSTALL_DATA (in order to get correct access +# rights on installed files) +INST_TYPE_DEST_DIR=$(RELSYSDIR)/temporary +INST_TYPE_DEST_DIR_DEP=$(INST_TYPE_DEST_DIR) +INST_TYPE_JS_DEST_DIR=$(INST_TYPE_DEST_DIR) +INST_TYPE_VSN_FILE=$(INST_TYPE_DEST_DIR)/OTP_VERSION +else +INST_TYPE=src +INST_TYPE_SRC_DIR=$(ERL_TOP) +INST_TYPE_DEST_DIR=$(HTMLDIR) +INST_TYPE_DEST_DIR_DEP= +INST_TYPE_JS_DEST_DIR=$(INST_TYPE_DEST_DIR)/js +INST_TYPE_VSN_FILE=$(ERL_TOP)/OTP_VERSION +endif + #-------------------------------------------------------------------------- # We generate the index page from the installed system. This make # it important that this is done last. The file index.html.src @@ -107,17 +126,18 @@ EBIN = ebin INDEX_SCRIPT = $(EBIN)/erl_html_tools.$(EMULATOR) INDEX_SRC = src/erl_html_tools.erl -INDEX_FILES = \ - $(HTMLDIR)/index.html \ - $(HTMLDIR)/applications.html -JAVASCRIPT = $(HTMLDIR)/js/erlresolvelinks.js +INDEX_HTML=$(INST_TYPE_DEST_DIR)/index.html +APPLICATIONS_HTML=$(INST_TYPE_DEST_DIR)/applications.html +INDEX_FILES = $(INDEX_HTML) $(APPLICATIONS_HTML) + +JAVASCRIPT = $(INST_TYPE_JS_DEST_DIR)/erlresolvelinks.js JAVASCRIPT_BUILD_SCRIPT = $(EBIN)/erlresolvelinks.$(EMULATOR) JAVASCRIPT_BUILD_SCRIPT_SRC = src/erlresolvelinks.erl MAN_INDEX_SCRIPT = $(EBIN)/otp_man_index.$(EMULATOR) MAN_INDEX_SRC = src/otp_man_index.erl -MAN_INDEX = $(HTMLDIR)/man_index.html +MAN_INDEX = $(INST_TYPE_DEST_DIR)/man_index.html GLOSSARY = $(HTMLDIR)/glossary.html GLOSSARY_SRC = $(ERL_TOP)/system/internal_tools/doctools/src/glossary.erl @@ -132,45 +152,38 @@ TEMPLATES = \ $(INDEX_SCRIPT): $(INDEX_SRC) $(ERLC) -o$(EBIN) +warn_unused_vars $< -# We don't list toc_*.html as targets because we don't know -$(HTMLDIR)/index.html + $(HTMLDIR)/applications.html: $(INDEX_SCRIPT) $(TEMPLATES) - echo "Generating index $@" -# Check if we are building the index from source or an installed release - if test "$$RELEASE_ROOT" = "" ; then \ - $(ERL) -noshell -pa $(EBIN) -s erl_html_tools top_index src $(ERL_TOP) \ - $(HTMLDIR) `cat "$(ERL_TOP)/OTP_VERSION"` -s erlang halt ;\ +$(INST_TYPE_DEST_DIR)/OTP_VERSION: $(INST_TYPE_DEST_DIR_DEP) + if test -f "$(RELEASE_PATH)/releases/$(SYSTEM_VSN)/OTP_VERSION"; then \ + $(CP) "$(RELEASE_PATH)/releases/$(SYSTEM_VSN)/OTP_VERSION" $@; \ else \ - $(ERL) -noshell -pa $(EBIN) -s erl_html_tools top_index rel $(RELEASE_ROOT) \ - $(HTMLDIR) `cat "$(RELEASE_ROOT)/releases/$(SYSTEM_VSN)/OTP_VERSION"` \ - -s erlang halt ;\ + $(CP) $(ERL_TOP)/OTP_VERSION $@; \ fi +# We don't list toc_*.html as targets because we don't know +$(INDEX_HTML) + $(APPLICATIONS_HTML): $(INST_TYPE_DEST_DIR_DEP) $(INDEX_SCRIPT) $(TEMPLATES) $(INST_TYPE_VSN_FILE) + echo "Generating index $@" + $(ERL) -noshell -pa $(EBIN) -s erl_html_tools top_index $(INST_TYPE) \ + $(INST_TYPE_SRC_DIR) $(INST_TYPE_DEST_DIR) \ + `cat "$(INST_TYPE_VSN_FILE)"` -s erlang halt + #-------------------------------------------------------------------------- $(JAVASCRIPT_BUILD_SCRIPT): $(JAVASCRIPT_BUILD_SCRIPT_SRC) $(ERLC) -o$(EBIN) +warn_unused_vars $< -$(JAVASCRIPT): $(JAVASCRIPT_BUILD_SCRIPT) - erl -noshell -pa $(EBIN) -s erlresolvelinks make -s erlang halt - $(INSTALL_DIR) $(HTMLDIR)/js - $(INSTALL_DATA) erlresolvelinks.js $(JAVASCRIPT) +$(JAVASCRIPT): $(INST_TYPE_DEST_DIR_DEP) $(JAVASCRIPT_BUILD_SCRIPT) + erl -noshell -pa $(EBIN) -run erlresolvelinks make $(ERL_TOP) \ + $(INST_TYPE_SRC_DIR) $(INST_TYPE_JS_DEST_DIR) -s erlang halt #-------------------------------------------------------------------------- $(MAN_INDEX_SCRIPT): $(MAN_INDEX_SRC) $(ERLC) -o$(EBIN) +warn_unused_vars $< -$(MAN_INDEX): $(MAN_INDEX_SCRIPT) -# Check if we are building the index from source or an installed release - if test "$$RELEASE_ROOT" = "" ; then \ - $(ERL) -noshell -pa $(EBIN) -s otp_man_index gen src $(ERL_TOP) $@ \ - -s erlang halt ;\ - else \ - $(ERL) -noshell -pa $(EBIN) -s otp_man_index gen rel $(RELEASE_ROOT) $@ \ - -s erlang halt ;\ - fi - +$(MAN_INDEX): $(INST_TYPE_DEST_DIR_DEP) $(MAN_INDEX_SCRIPT) + $(ERL) -noshell -pa $(EBIN) -s otp_man_index gen $(INST_TYPE) \ + $(INST_TYPE_SRC_DIR) $@ -s erlang halt #-------------------------------------------------------------------------- @@ -248,18 +261,22 @@ html: $(INDEX_FILES) \ debug opt: clean: - rm -rf ../html/js - rm -f PR.template - rm -f $(INDEX_FILES) $(MAN_INDEX) - rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) - rm -f $(INDEX_SCRIPT) $(GLOSSARY_SCRIPT) \ - $(JAVASCRIPT_BUILD_SCRIPT) - rm -f erl_crash.dump errs core *~ + $(RM) ../html/js/*.js + $(RM) PR.template + $(RM) $(XMLDIR)/*.xml + $(RM) $(INDEX_FILES) $(MAN_INDEX) + $(RM) $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) + $(RM) $(INDEX_SCRIPT) $(GLOSSARY_SCRIPT) $(JAVASCRIPT_BUILD_SCRIPT) + $(RM) erl_crash.dump errs core *~ + # ---------------------------------------------------- # Release Target # ---------------------------------------------------- include $(ERL_TOP)/make/otp_release_targets.mk +$(RELSYSDIR)/temporary: + $(INSTALL_DIR) $(RELSYSDIR)/temporary + release_docs_spec: docs $(INSTALL_DIR) "$(RELEASE_PATH)" $(INSTALL_DATA) $(INFO_FILES) "$(RELEASE_PATH)" @@ -268,13 +285,13 @@ release_docs_spec: docs $(INSTALL_DATA) \ $(TOP_PDF_FILE) $(RELSYSDIR)/pdf $(INSTALL_DIR) $(RELSYSDIR)/js - $(INSTALL_DATA) \ - $(JAVASCRIPT) $(RELSYSDIR)/js + $(INSTALL_DATA) $(JAVASCRIPT) $(RELSYSDIR)/js $(INSTALL_DATA) $(INDEX_FILES) $(MAN_INDEX) $(RELSYSDIR) $(INSTALL_DIR) $(RELSYSDIR)/docbuild $(INSTALL_DATA) $(INDEX_SCRIPT) $(MAN_INDEX_SCRIPT) $(JAVASCRIPT_BUILD_SCRIPT) \ $(INDEX_SRC) $(MAN_INDEX_SRC) $(JAVASCRIPT_BUILD_SCRIPT_SRC) \ $(TEMPLATES) $(RELSYSDIR)/docbuild + $(RM) -r $(RELSYSDIR)/temporary release_spec: diff --git a/system/doc/top/src/erl_html_tools.erl b/system/doc/top/src/erl_html_tools.erl index 28a0649658..dee1871342 100644 --- a/system/doc/top/src/erl_html_tools.erl +++ b/system/doc/top/src/erl_html_tools.erl @@ -508,7 +508,7 @@ subst_app(App, [{VSN,_Path,Link,Text} | VerInfos]) -> " <a href=\"",Link,"\" target=\"_top\">",uc(App), "</a>\n", " <a href=\"",Link,"\" target=\"_top\">",VSN,"</a>\n", - " <td class=appnums>\n", + " <br/>\n", subst_vsn(VerInfos), " </td>\n", " <td>\n", @@ -522,7 +522,7 @@ subst_vsn([{VSN,_Path,Link,_Text} | VSNs]) -> [ " <font size=\"2\"><a class=anum href=\"",Link,"\" target=\"_top\">", VSN, - "</a></font><br>\n", + "</a></font><br/>\n", subst_vsn(VSNs) ]; subst_vsn([]) -> diff --git a/system/doc/top/src/erlresolvelinks.erl b/system/doc/top/src/erlresolvelinks.erl index cfe8d0fa0b..c1285fa1c3 100644 --- a/system/doc/top/src/erlresolvelinks.erl +++ b/system/doc/top/src/erlresolvelinks.erl @@ -27,40 +27,28 @@ %%----------------------------------------------------------------- -module(erlresolvelinks). --export([make/0, make/1]). +-export([make/1]). -include_lib("kernel/include/file.hrl"). -define(JAVASCRIPT_NAME, "erlresolvelinks.js"). -make() -> - case os:getenv("ERL_TOP") of - false -> - io:format("Variable ERL_TOP is required\n",[]); - Value -> - make_from_src(Value, ".") - end. - -make([RootDir, DestDir]) -> - do_make(RootDir, DestDir); -make(RootDir) when is_atom(RootDir) -> - DestDir = filename:join(RootDir, "doc"), - do_make(RootDir, DestDir). - -do_make(_RootDir, _DestDir) -> - ok. +make([ErlTop, RootDir, DestDir]) -> + make(ErlTop, RootDir, DestDir). -make_from_src(RootDir, DestDir) -> +make(ErlTop, RootDir, DestDir) -> %% doc/Dir %% erts-Vsn %% lib/App-Vsn Name = ?JAVASCRIPT_NAME, - DocDirs0 = get_dirs(filename:join([RootDir, "system/doc"])), + DocDirs0 = get_dirs(filename:join([ErlTop, "system/doc"])), DocDirs = lists:map(fun({Dir, _DirPath}) -> D = filename:join(["doc", Dir]), {D, D} end, DocDirs0), - ErtsDirs = latest_app_dirs(RootDir, ""), - AppDirs = latest_app_dirs(RootDir, "lib"), + Released = ErlTop /= RootDir, + + ErtsDirs = latest_app_dirs(Released, RootDir, ""), + AppDirs = latest_app_dirs(Released, RootDir, "lib"), AllAppDirs = lists:map( @@ -106,30 +94,46 @@ is_dir({File, AFile}) -> false end. -latest_app_dirs(RootDir, Dir) -> +released_app_vsns([]) -> + []; +released_app_vsns([{AppVsn, Dir} | AVDirs]) -> + try + {ok, _} = file:read_file_info(filename:join([Dir, "doc", "html"])), + [App, Vsn] = string:tokens(AppVsn, "-"), + VsnNumList = vsnstr_to_numlist(Vsn), + [_Maj, _Min | _] = VsnNumList, + [{{App, VsnNumList}, AppVsn} | released_app_vsns(AVDirs)] + catch + _:_ -> released_app_vsns(AVDirs) + end. + +latest_app_dirs(Release, RootDir, Dir) -> ADir = filename:join(RootDir, Dir), RDirs0 = get_dirs(ADir), - RDirs1 = lists:filter(fun is_app_dir/1, RDirs0), - - SDirs0 = - lists:map(fun({App, Dir1}) -> - File = filename:join(Dir1, "vsn.mk"), - case file:read_file(File) of - {ok, Bin} -> - case re:run(Bin, ".*VSN\s*=\s*([0-9\.]+).*",[{capture,[1],list}]) of - {match, [VsnStr]} -> - VsnNumList = vsnstr_to_numlist(VsnStr), - {{App, VsnNumList}, App++"-"++VsnStr}; - nomatch -> - io:format("No VSN variable found in ~s\n", [File]), - error - end; - {error, Reason} -> - io:format("~p : ~s\n", [Reason, File]), - error - end - end, - RDirs1), + SDirs0 = case Release of + true -> + released_app_vsns(RDirs0); + false -> + lists:map(fun({App, Dir1}) -> + File = filename:join(Dir1, "vsn.mk"), + case file:read_file(File) of + {ok, Bin} -> + case re:run(Bin, ".*VSN\s*=\s*([0-9\.]+).*",[{capture,[1],list}]) of + {match, [VsnStr]} -> + VsnNumList = vsnstr_to_numlist(VsnStr), + {{App, VsnNumList}, App++"-"++VsnStr}; + nomatch -> + io:format("No VSN variable found in ~s\n", [File]), + error + end; + {error, Reason} -> + io:format("~p : ~s\n", [Reason, File]), + error + end + end, + lists:filter(fun is_app_dir/1, RDirs0)) + end, + SDirs1 = lists:keysort(1, SDirs0), App2Dirs = lists:foldr(fun({{App, _VsnNumList}, AppVsn}, Acc) -> case lists:keymember(App, 1, Acc) of diff --git a/system/doc/tutorial/Makefile b/system/doc/tutorial/Makefile index 5867096fc8..4c62deeffd 100644 --- a/system/doc/tutorial/Makefile +++ b/system/doc/tutorial/Makefile @@ -93,10 +93,9 @@ DVIPS_FLAGS += # ---------------------------------------------------- # Targets # ---------------------------------------------------- -_create_dirs := $(shell mkdir -p $(HTMLDIR)) $(HTMLDIR)/%.gif: %.gif - $(INSTALL_DATA) $< $@ + $(CP) $< $@ docs: html @@ -109,8 +108,8 @@ gifs: $(GIF_FILES:%=$(HTMLDIR)/%) debug opt: clean clean_docs: - rm -rf $(HTMLDIR) - rm -rf $(XMLDIR) + rm -f $(XMLDIR)/*.xml + rm -f $(HTMLDIR)/*.gif $(HTMLDIR)/*.html rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) rm -f errs core *~ diff --git a/system/doc/xml/design_principles/.gitignore b/system/doc/xml/design_principles/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/xml/design_principles/.gitignore diff --git a/system/doc/xml/efficiency_guide/.gitignore b/system/doc/xml/efficiency_guide/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/xml/efficiency_guide/.gitignore diff --git a/system/doc/xml/embedded/.gitignore b/system/doc/xml/embedded/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/xml/embedded/.gitignore diff --git a/system/doc/xml/getting_started/.gitignore b/system/doc/xml/getting_started/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/xml/getting_started/.gitignore diff --git a/system/doc/xml/installation_guide/.gitignore b/system/doc/xml/installation_guide/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/xml/installation_guide/.gitignore diff --git a/system/doc/xml/oam/.gitignore b/system/doc/xml/oam/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/xml/oam/.gitignore diff --git a/system/doc/xml/programming_examples/.gitignore b/system/doc/xml/programming_examples/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/xml/programming_examples/.gitignore diff --git a/system/doc/xml/reference_manual/.gitignore b/system/doc/xml/reference_manual/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/xml/reference_manual/.gitignore diff --git a/system/doc/xml/system_architecture_intro/.gitignore b/system/doc/xml/system_architecture_intro/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/xml/system_architecture_intro/.gitignore diff --git a/system/doc/xml/system_principles/.gitignore b/system/doc/xml/system_principles/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/xml/system_principles/.gitignore diff --git a/system/doc/xml/tutorial/.gitignore b/system/doc/xml/tutorial/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/system/doc/xml/tutorial/.gitignore |