diff options
110 files changed, 4011 insertions, 1374 deletions
diff --git a/Makefile.in b/Makefile.in index cc9a9375c9..a07a779f10 100644 --- a/Makefile.in +++ b/Makefile.in @@ -161,6 +161,8 @@ ERLANG_LIBDIR = $(DESTDIR)$(ERLANG_INST_LIBDIR) # Must be GNU make! MAKE = @MAKE_PROG@ +PERL = @PERL@ + NATIVE_LIBS_ENABLED = @NATIVE_LIBS_ENABLED@ ifeq ($(NATIVE_LIBS_ENABLED),yes) @@ -327,16 +329,16 @@ ifneq ($(CROSS_COMPILING),yes) # Not cross compiling ifeq ($(BOOTSTRAP_ONLY),yes) -all: bootstrap +all: bootstrap check_dev_rt_dep else # The normal case; not cross compiling, and not bootstrap only build. -all: bootstrap libs local_setup +all: bootstrap libs local_setup check_dev_rt_dep endif else # Cross compiling -all: cross_check_erl depend emulator libs start_scripts +all: cross_check_erl depend emulator libs start_scripts check_dev_rt_dep endif @@ -356,7 +358,10 @@ erlang_inst_libdir_configured: bootstrap: depend all_bootstraps - +check_dev_rt_dep: + @if `grep DEVELOPMENT "$(ERL_TOP)/make/otp_version_tickets" 1>/dev/null 2>&1`; then \ + LANG=C "$(PERL)" "$(ERL_TOP)/make/fixup_development_runtime_dependencies" "$(ERL_TOP)"; \ + fi ifeq ($(OTP_STRICT_INSTALL),yes) diff --git a/bootstrap/lib/compiler/ebin/beam_asm.beam b/bootstrap/lib/compiler/ebin/beam_asm.beam Binary files differindex 4c7d37270e..6753eda31d 100644 --- a/bootstrap/lib/compiler/ebin/beam_asm.beam +++ b/bootstrap/lib/compiler/ebin/beam_asm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_jump.beam b/bootstrap/lib/compiler/ebin/beam_jump.beam Binary files differindex bc6e006047..cfdb0c7c70 100644 --- a/bootstrap/lib/compiler/ebin/beam_jump.beam +++ b/bootstrap/lib/compiler/ebin/beam_jump.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa.beam b/bootstrap/lib/compiler/ebin/beam_ssa.beam Binary files differindex 8b8ab2b69e..956ece3826 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam b/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam Binary files differindex d04b8767d2..b0c07c6965 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_bsm.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_funs.beam b/bootstrap/lib/compiler/ebin/beam_ssa_funs.beam Binary files differnew file mode 100644 index 0000000000..dc26c2bf66 --- /dev/null +++ b/bootstrap/lib/compiler/ebin/beam_ssa_funs.beam diff --git a/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam b/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam Binary files differindex 34c29161e6..3fc568be02 100644 --- a/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam +++ b/bootstrap/lib/compiler/ebin/beam_ssa_pre_codegen.beam diff --git a/bootstrap/lib/compiler/ebin/cerl_inline.beam b/bootstrap/lib/compiler/ebin/cerl_inline.beam Binary files differindex 7d6864335f..8fae7dc8f7 100644 --- a/bootstrap/lib/compiler/ebin/cerl_inline.beam +++ b/bootstrap/lib/compiler/ebin/cerl_inline.beam diff --git a/bootstrap/lib/compiler/ebin/compile.beam b/bootstrap/lib/compiler/ebin/compile.beam Binary files differindex ea3db59fea..c5e1c4701e 100644 --- a/bootstrap/lib/compiler/ebin/compile.beam +++ b/bootstrap/lib/compiler/ebin/compile.beam diff --git a/bootstrap/lib/compiler/ebin/compiler.app b/bootstrap/lib/compiler/ebin/compiler.app index 028ee62b42..539383e02e 100644 --- a/bootstrap/lib/compiler/ebin/compiler.app +++ b/bootstrap/lib/compiler/ebin/compiler.app @@ -19,7 +19,7 @@ {application, compiler, [{description, "ERTS CXC 138 10"}, - {vsn, "7.2.5"}, + {vsn, "7.2.6"}, {modules, [ beam_a, beam_asm, @@ -39,6 +39,7 @@ beam_ssa_bsm, beam_ssa_codegen, beam_ssa_dead, + beam_ssa_funs, beam_ssa_lint, beam_ssa_opt, beam_ssa_pp, diff --git a/bootstrap/lib/compiler/ebin/erl_bifs.beam b/bootstrap/lib/compiler/ebin/erl_bifs.beam Binary files differindex bbf38b0153..dc3ede0fce 100644 --- a/bootstrap/lib/compiler/ebin/erl_bifs.beam +++ b/bootstrap/lib/compiler/ebin/erl_bifs.beam diff --git a/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam b/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam Binary files differindex 02e2d081ed..db9b195aec 100644 --- a/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam +++ b/bootstrap/lib/compiler/ebin/sys_core_fold_lists.beam diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index f1465ae9ae..273cf7ff0b 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -1355,6 +1355,24 @@ </section> +<section><title>Erts 9.3.3.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed bug in <c>ets:select_replace</c> when called with a + fully bound key could cause a following call to + <c>ets:next</c> or <c>ets:prev</c> to crash the emulator + or return invalid result.</p> + <p> + Own Id: OTP-15346</p> + </item> + </list> + </section> + +</section> + <section><title>Erts 9.3.3.3</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -19172,4 +19190,3 @@ </section> </section> </chapter> - diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 3b45f968cf..4741d7451d 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -3624,6 +3624,10 @@ erts_internal_garbage_collect_1(BIF_ALIST_1) default: BIF_ERROR(BIF_P, BADARG); } erts_garbage_collect(BIF_P, 0, NULL, 0); + if (ERTS_PROC_IS_EXITING(BIF_P)) { + /* The max heap size limit was reached. */ + return THE_NON_VALUE; + } return am_true; } diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 36c46fd7aa..1d4b33c598 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -4072,6 +4072,9 @@ debug_free(ErtsAlcType_t type, void *extra, void *ptr) ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); + if (!ptr) + return; + dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE); #ifdef ERTS_ALC_A_EXEC diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index a65dbbf42b..47dd115c82 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1133,9 +1133,28 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, 0, p->arg_reg, p->arity, fcalls, ygen_usage); + if (ERTS_PROC_IS_EXITING(p)) { + return 0; + } ASSERT(!(p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC))); + if (MAX_HEAP_SIZE_GET(p)) { + Uint new_heap_size; + Uint old_heap_size; + Uint total_heap_size; + + new_heap_size = HEAP_END(p) - HEAP_START(p); + old_heap_size = erts_next_heap_size(lit_size, 0); + total_heap_size = new_heap_size + old_heap_size; + if (MAX_HEAP_SIZE_GET(p) < total_heap_size && + reached_max_heap_size(p, total_heap_size, + new_heap_size, old_heap_size)) { + erts_set_self_exiting(p, am_killed); + return 0; + } + } + /* * Set GC state. */ diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 0dabe705b3..dbe0201caf 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -812,6 +812,7 @@ static size_t my_strnlen(const char *s, size_t maxlen) #define INET_OPT_PKTOPTIONS 45 /* IP(V6)_PKTOPTIONS get ancillary data */ #define INET_OPT_TTL 46 /* IP_TTL */ #define INET_OPT_RECVTTL 47 /* IP_RECVTTL ancillary data */ +#define TCP_OPT_NOPUSH 48 /* super-Nagle, aka TCP_CORK */ /* SCTP options: a separate range, from 100: */ #define SCTP_OPT_RTOINFO 100 #define SCTP_OPT_ASSOCINFO 101 @@ -955,6 +956,12 @@ static size_t my_strnlen(const char *s, size_t maxlen) #endif +#if defined(TCP_CORK) +#define INET_TCP_NOPUSH TCP_CORK +#elif defined(TCP_NOPUSH) && !defined(__DARWIN__) +#define INET_TCP_NOPUSH TCP_NOPUSH +#endif + #define BIN_REALLOC_MARGIN(x) ((x)/4) /* 25% */ /* The general purpose sockaddr */ @@ -6598,6 +6605,19 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) (long)desc->port, desc->s, ival)); break; + case TCP_OPT_NOPUSH: +#if defined(INET_TCP_NOPUSH) + proto = IPPROTO_TCP; + type = INET_TCP_NOPUSH; + DEBUGF(("inet_set_opts(%ld): s=%d, t=%d TCP_NOPUSH=%d\r\n", + (long)desc->port, desc->s, type, ival)); + break; +#else + /* inet_fill_opts always returns a value for this option, + * so we need to ignore it if not implemented, just in case */ + continue; +#endif + #if defined(HAVE_MULTICAST_SUPPORT) && defined(IPPROTO_IP) case UDP_OPT_MULTICAST_TTL: @@ -7759,6 +7779,16 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc, proto = IPPROTO_TCP; type = TCP_NODELAY; break; + case TCP_OPT_NOPUSH: +#if defined(INET_TCP_NOPUSH) + proto = IPPROTO_TCP; + type = INET_TCP_NOPUSH; + break; +#else + *ptr++ = opt; + put_int32(0, ptr); + continue; +#endif #if defined(HAVE_MULTICAST_SUPPORT) && defined(IPPROTO_IP) case UDP_OPT_MULTICAST_TTL: diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 7e690fd870..493c6ebe99 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -28,7 +28,7 @@ fake_literals/1, false_dependency/1,coverage/1,fun_confusion/1, t_copy_literals/1, t_copy_literals_frags/1, - erl_544/1]). + erl_544/1, max_heap_size/1]). -define(line_trace, 1). -include_lib("common_test/include/ct.hrl"). @@ -43,7 +43,7 @@ all() -> constant_pools, constant_refc_binaries, fake_literals, false_dependency, coverage, fun_confusion, t_copy_literals, t_copy_literals_frags, - erl_544]. + erl_544, max_heap_size]. init_per_suite(Config) -> erts_debug:set_internal_state(available_internal_state, true), @@ -978,6 +978,39 @@ erl_544(Config) when is_list(Config) -> {skipped, "Only run when native file name encoding is utf8"} end. +%% Test that the copying of literals to a process during purging of +%% literals will cause the process to be killed if the max heap size +%% is exceeded. +max_heap_size(_Config) -> + Mod = ?FUNCTION_NAME, + Value = [I || I <- lists:seq(1, 5000)], + Code = gen_lit(Mod, [{term,Value}]), + {module,Mod} = erlang:load_module(Mod, Code), + SpawnOpts = [monitor, + {max_heap_size, + #{size=>1024, + kill=>true, + error_logger=>true}}], + {Pid,Ref} = spawn_opt(fun() -> + max_heap_size_proc(Mod) + end, SpawnOpts), + receive + {'DOWN',Ref,process,Pid,Reason} -> + killed = Reason; + Other -> + ct:fail({unexpected_message,Other}) + after 10000 -> + ct:fail({process_did_not_die, Pid, erlang:process_info(Pid)}) + end. + +max_heap_size_proc(Mod) -> + Value = Mod:term(), + code:delete(Mod), + code:purge(Mod), + receive + _ -> Value + end. + %% Utilities. make_sub_binary(Bin) when is_binary(Bin) -> diff --git a/erts/emulator/zlib/adler32.c b/erts/emulator/zlib/adler32.c index c693a42b7c..d0be4380a3 100644 --- a/erts/emulator/zlib/adler32.c +++ b/erts/emulator/zlib/adler32.c @@ -1,20 +1,15 @@ /* adler32.c -- compute the Adler-32 checksum of a data stream - * Copyright (C) 1995-2011 Mark Adler + * Copyright (C) 1995-2011, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif #include "zutil.h" -#define local static - local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); -#define BASE 65521 /* largest prime smaller than 65536 */ +#define BASE 65521U /* largest prime smaller than 65536 */ #define NMAX 5552 /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ @@ -65,10 +60,10 @@ local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); #endif /* ========================================================================= */ -uLong ZEXPORT adler32(adler, buf, len) +uLong ZEXPORT adler32_z(adler, buf, len) uLong adler; const Bytef *buf; - uInt len; + z_size_t len; { unsigned long sum2; unsigned n; @@ -136,6 +131,15 @@ uLong ZEXPORT adler32(adler, buf, len) } /* ========================================================================= */ +uLong ZEXPORT adler32(adler, buf, len) + uLong adler; + const Bytef *buf; + uInt len; +{ + return adler32_z(adler, buf, len); +} + +/* ========================================================================= */ local uLong adler32_combine_(adler1, adler2, len2) uLong adler1; uLong adler2; @@ -159,7 +163,7 @@ local uLong adler32_combine_(adler1, adler2, len2) sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem; if (sum1 >= BASE) sum1 -= BASE; if (sum1 >= BASE) sum1 -= BASE; - if (sum2 >= (BASE << 1)) sum2 -= (BASE << 1); + if (sum2 >= ((unsigned long)BASE << 1)) sum2 -= ((unsigned long)BASE << 1); if (sum2 >= BASE) sum2 -= BASE; return sum1 | (sum2 << 16); } diff --git a/erts/emulator/zlib/compress.c b/erts/emulator/zlib/compress.c index 8ecef0f790..e2db404abf 100644 --- a/erts/emulator/zlib/compress.c +++ b/erts/emulator/zlib/compress.c @@ -1,13 +1,10 @@ /* compress.c -- compress a memory buffer - * Copyright (C) 1995-2005 Jean-loup Gailly. + * Copyright (C) 1995-2005, 2014, 2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif #define ZLIB_INTERNAL #include "zlib.h" @@ -31,16 +28,11 @@ int ZEXPORT compress2 (dest, destLen, source, sourceLen, level) { z_stream stream; int err; + const uInt max = (uInt)-1; + uLong left; - stream.next_in = (z_const Bytef *)source; - stream.avail_in = (uInt)sourceLen; -#ifdef MAXSEG_64K - /* Check for source > 64K on 16-bit machine: */ - if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR; -#endif - stream.next_out = dest; - stream.avail_out = (uInt)*destLen; - if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR; + left = *destLen; + *destLen = 0; stream.zalloc = (alloc_func)0; stream.zfree = (free_func)0; @@ -49,15 +41,26 @@ int ZEXPORT compress2 (dest, destLen, source, sourceLen, level) err = deflateInit(&stream, level); if (err != Z_OK) return err; - err = deflate(&stream, Z_FINISH); - if (err != Z_STREAM_END) { - deflateEnd(&stream); - return err == Z_OK ? Z_BUF_ERROR : err; - } - *destLen = stream.total_out; + stream.next_out = dest; + stream.avail_out = 0; + stream.next_in = (z_const Bytef *)source; + stream.avail_in = 0; + + do { + if (stream.avail_out == 0) { + stream.avail_out = left > (uLong)max ? max : (uInt)left; + left -= stream.avail_out; + } + if (stream.avail_in == 0) { + stream.avail_in = sourceLen > (uLong)max ? max : (uInt)sourceLen; + sourceLen -= stream.avail_in; + } + err = deflate(&stream, sourceLen ? Z_NO_FLUSH : Z_FINISH); + } while (err == Z_OK); - err = deflateEnd(&stream); - return err; + *destLen = stream.total_out; + deflateEnd(&stream); + return err == Z_STREAM_END ? Z_OK : err; } /* =========================================================================== diff --git a/erts/emulator/zlib/crc32.c b/erts/emulator/zlib/crc32.c index ba506d8dd3..9580440c0e 100644 --- a/erts/emulator/zlib/crc32.c +++ b/erts/emulator/zlib/crc32.c @@ -1,5 +1,5 @@ /* crc32.c -- compute the CRC-32 of a data stream - * Copyright (C) 1995-2006, 2010, 2011, 2012 Mark Adler + * Copyright (C) 1995-2006, 2010, 2011, 2012, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h * * Thanks to Rodney Brown <[email protected]> for his contribution of faster @@ -28,23 +28,17 @@ # endif /* !DYNAMIC_CRC_TABLE */ #endif /* MAKECRCH */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - #include "zutil.h" /* for STDC and FAR definitions */ -#define local static - /* Definitions for doing the crc four data bytes at a time. */ #if !defined(NOBYFOUR) && defined(Z_U4) # define BYFOUR #endif #ifdef BYFOUR local unsigned long crc32_little OF((unsigned long, - const unsigned char FAR *, unsigned)); + const unsigned char FAR *, z_size_t)); local unsigned long crc32_big OF((unsigned long, - const unsigned char FAR *, unsigned)); + const unsigned char FAR *, z_size_t)); # define TBLS 8 #else # define TBLS 1 @@ -205,10 +199,10 @@ const z_crc_t FAR * ZEXPORT get_crc_table() #define DO8 DO1; DO1; DO1; DO1; DO1; DO1; DO1; DO1 /* ========================================================================= */ -unsigned long ZEXPORT crc32(crc, buf, len) +unsigned long ZEXPORT crc32_z(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - uInt len; + z_size_t len; { if (buf == Z_NULL) return 0UL; @@ -239,8 +233,29 @@ unsigned long ZEXPORT crc32(crc, buf, len) return crc ^ 0xffffffffUL; } +/* ========================================================================= */ +unsigned long ZEXPORT crc32(crc, buf, len) + unsigned long crc; + const unsigned char FAR *buf; + uInt len; +{ + return crc32_z(crc, buf, len); +} + #ifdef BYFOUR +/* + This BYFOUR code accesses the passed unsigned char * buffer with a 32-bit + integer pointer type. This violates the strict aliasing rule, where a + compiler can assume, for optimization purposes, that two pointers to + fundamentally different types won't ever point to the same memory. This can + manifest as a problem only if one of the pointers is written to. This code + only reads from those pointers. So long as this code remains isolated in + this compilation unit, there won't be a problem. For this reason, this code + should not be copied and pasted into a compilation unit in which other code + writes to the buffer that is passed to these routines. + */ + /* ========================================================================= */ #define DOLIT4 c ^= *buf4++; \ c = crc_table[3][c & 0xff] ^ crc_table[2][(c >> 8) & 0xff] ^ \ @@ -251,7 +266,7 @@ unsigned long ZEXPORT crc32(crc, buf, len) local unsigned long crc32_little(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - unsigned len; + z_size_t len; { register z_crc_t c; register const z_crc_t FAR *buf4; @@ -282,7 +297,7 @@ local unsigned long crc32_little(crc, buf, len) } /* ========================================================================= */ -#define DOBIG4 c ^= *++buf4; \ +#define DOBIG4 c ^= *buf4++; \ c = crc_table[4][c & 0xff] ^ crc_table[5][(c >> 8) & 0xff] ^ \ crc_table[6][(c >> 16) & 0xff] ^ crc_table[7][c >> 24] #define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4 @@ -291,7 +306,7 @@ local unsigned long crc32_little(crc, buf, len) local unsigned long crc32_big(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; - unsigned len; + z_size_t len; { register z_crc_t c; register const z_crc_t FAR *buf4; @@ -304,7 +319,6 @@ local unsigned long crc32_big(crc, buf, len) } buf4 = (const z_crc_t FAR *)(const void FAR *)buf; - buf4--; while (len >= 32) { DOBIG32; len -= 32; @@ -313,7 +327,6 @@ local unsigned long crc32_big(crc, buf, len) DOBIG4; len -= 4; } - buf4++; buf = (const unsigned char FAR *)buf4; if (len) do { diff --git a/erts/emulator/zlib/deflate.c b/erts/emulator/zlib/deflate.c index 943c26dfb2..1ec761448d 100644 --- a/erts/emulator/zlib/deflate.c +++ b/erts/emulator/zlib/deflate.c @@ -1,5 +1,5 @@ /* deflate.c -- compress data using the deflation algorithm - * Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler + * Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -49,13 +49,10 @@ /* @(#) $Id$ */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif #include "deflate.h" const char deflate_copyright[] = - " deflate 1.2.8 Copyright 1995-2013 Jean-loup Gailly and Mark Adler "; + " deflate 1.2.11 Copyright 1995-2017 Jean-loup Gailly and Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -76,6 +73,8 @@ typedef enum { typedef block_state (*compress_func) OF((deflate_state *s, int flush)); /* Compression function. Returns the block state after the call. */ +local int deflateStateCheck OF((z_streamp strm)); +local void slide_hash OF((deflate_state *s)); local void fill_window OF((deflate_state *s)); local block_state deflate_stored OF((deflate_state *s, int flush)); local block_state deflate_fast OF((deflate_state *s, int flush)); @@ -87,15 +86,16 @@ local block_state deflate_huff OF((deflate_state *s, int flush)); local void lm_init OF((deflate_state *s)); local void putShortMSB OF((deflate_state *s, uInt b)); local void flush_pending OF((z_streamp strm)); -local int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); +local unsigned read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); #ifdef ASMV +# pragma message("Assembler code may have bugs -- use at your own risk") void match_init OF((void)); /* asm code initialization */ uInt longest_match OF((deflate_state *s, IPos cur_match)); #else local uInt longest_match OF((deflate_state *s, IPos cur_match)); #endif -#ifdef DEBUG +#ifdef ZLIB_DEBUG local void check_match OF((deflate_state *s, IPos start, IPos match, int length)); #endif @@ -151,21 +151,14 @@ local const config configuration_table[10] = { * meaning. */ -#define EQUAL 0 -/* result of memcmp for equal strings */ - -#ifndef NO_DUMMY_DECL -struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ -#endif - /* rank Z_BLOCK between Z_NO_FLUSH and Z_PARTIAL_FLUSH */ -#define RANK(f) (((f) << 1) - ((f) > 4 ? 9 : 0)) +#define RANK(f) (((f) * 2) - ((f) > 4 ? 9 : 0)) /* =========================================================================== * Update a hash value with the given input byte - * IN assertion: all calls to to UPDATE_HASH are made with consecutive - * input characters, so that a running hash key can be computed from the - * previous key instead of complete recalculation each time. + * IN assertion: all calls to UPDATE_HASH are made with consecutive input + * characters, so that a running hash key can be computed from the previous + * key instead of complete recalculation each time. */ #define UPDATE_HASH(s,h,c) (h = (((h)<<s->hash_shift) ^ (c)) & s->hash_mask) @@ -176,9 +169,9 @@ struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ * the previous length of the hash chain. * If this file is compiled with -DFASTEST, the compression level is forced * to 1, and no hash chains are maintained. - * IN assertion: all calls to to INSERT_STRING are made with consecutive - * input characters and the first MIN_MATCH bytes of str are valid - * (except for the last MIN_MATCH-1 bytes of the input file). + * IN assertion: all calls to INSERT_STRING are made with consecutive input + * characters and the first MIN_MATCH bytes of str are valid (except for + * the last MIN_MATCH-1 bytes of the input file). */ #ifdef FASTEST #define INSERT_STRING(s, str, match_head) \ @@ -200,6 +193,37 @@ struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ s->head[s->hash_size-1] = NIL; \ zmemzero((Bytef *)s->head, (unsigned)(s->hash_size-1)*sizeof(*s->head)); +/* =========================================================================== + * Slide the hash table when sliding the window down (could be avoided with 32 + * bit values at the expense of memory usage). We slide even when level == 0 to + * keep the hash table consistent if we switch back to level > 0 later. + */ +local void slide_hash(s) + deflate_state *s; +{ + unsigned n, m; + Posf *p; + uInt wsize = s->w_size; + + n = s->hash_size; + p = &s->head[n]; + do { + m = *--p; + *p = (Pos)(m >= wsize ? m - wsize : NIL); + } while (--n); + n = wsize; +#ifndef FASTEST + p = &s->prev[n]; + do { + m = *--p; + *p = (Pos)(m >= wsize ? m - wsize : NIL); + /* If n is not on any hash chain, prev[n] is garbage but + * its value will never be used. + */ + } while (--n); +#endif +} + /* ========================================================================= */ int ZEXPORT deflateInit_(strm, level, version, stream_size) z_streamp strm; @@ -273,7 +297,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, #endif if (memLevel < 1 || memLevel > MAX_MEM_LEVEL || method != Z_DEFLATED || windowBits < 8 || windowBits > 15 || level < 0 || level > 9 || - strategy < 0 || strategy > Z_FIXED) { + strategy < 0 || strategy > Z_FIXED || (windowBits == 8 && wrap != 1)) { return Z_STREAM_ERROR; } if (windowBits == 8) windowBits = 9; /* until 256-byte window bug fixed */ @@ -281,14 +305,15 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, if (s == Z_NULL) return Z_MEM_ERROR; strm->state = (struct internal_state FAR *)s; s->strm = strm; + s->status = INIT_STATE; /* to pass state test in deflateReset() */ s->wrap = wrap; s->gzhead = Z_NULL; - s->w_bits = windowBits; + s->w_bits = (uInt)windowBits; s->w_size = 1 << s->w_bits; s->w_mask = s->w_size - 1; - s->hash_bits = memLevel + 7; + s->hash_bits = (uInt)memLevel + 7; s->hash_size = 1 << s->hash_bits; s->hash_mask = s->hash_size - 1; s->hash_shift = ((s->hash_bits+MIN_MATCH-1)/MIN_MATCH); @@ -322,6 +347,31 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, return deflateReset(strm); } +/* ========================================================================= + * Check for a valid deflate stream state. Return 0 if ok, 1 if not. + */ +local int deflateStateCheck (strm) + z_streamp strm; +{ + deflate_state *s; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + s = strm->state; + if (s == Z_NULL || s->strm != strm || (s->status != INIT_STATE && +#ifdef GZIP + s->status != GZIP_STATE && +#endif + s->status != EXTRA_STATE && + s->status != NAME_STATE && + s->status != COMMENT_STATE && + s->status != HCRC_STATE && + s->status != BUSY_STATE && + s->status != FINISH_STATE)) + return 1; + return 0; +} + /* ========================================================================= */ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) z_streamp strm; @@ -334,7 +384,7 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) unsigned avail; z_const unsigned char *next; - if (strm == Z_NULL || strm->state == Z_NULL || dictionary == Z_NULL) + if (deflateStateCheck(strm) || dictionary == Z_NULL) return Z_STREAM_ERROR; s = strm->state; wrap = s->wrap; @@ -392,13 +442,34 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) } /* ========================================================================= */ +int ZEXPORT deflateGetDictionary (strm, dictionary, dictLength) + z_streamp strm; + Bytef *dictionary; + uInt *dictLength; +{ + deflate_state *s; + uInt len; + + if (deflateStateCheck(strm)) + return Z_STREAM_ERROR; + s = strm->state; + len = s->strstart + s->lookahead; + if (len > s->w_size) + len = s->w_size; + if (dictionary != Z_NULL && len) + zmemcpy(dictionary, s->window + s->strstart + s->lookahead - len, len); + if (dictLength != Z_NULL) + *dictLength = len; + return Z_OK; +} + +/* ========================================================================= */ int ZEXPORT deflateResetKeep (strm) z_streamp strm; { deflate_state *s; - if (strm == Z_NULL || strm->state == Z_NULL || - strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) { + if (deflateStateCheck(strm)) { return Z_STREAM_ERROR; } @@ -413,7 +484,11 @@ int ZEXPORT deflateResetKeep (strm) if (s->wrap < 0) { s->wrap = -s->wrap; /* was made negative by deflate(..., Z_FINISH); */ } - s->status = s->wrap ? INIT_STATE : BUSY_STATE; + s->status = +#ifdef GZIP + s->wrap == 2 ? GZIP_STATE : +#endif + s->wrap ? INIT_STATE : BUSY_STATE; strm->adler = #ifdef GZIP s->wrap == 2 ? crc32(0L, Z_NULL, 0) : @@ -443,8 +518,8 @@ int ZEXPORT deflateSetHeader (strm, head) z_streamp strm; gz_headerp head; { - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; - if (strm->state->wrap != 2) return Z_STREAM_ERROR; + if (deflateStateCheck(strm) || strm->state->wrap != 2) + return Z_STREAM_ERROR; strm->state->gzhead = head; return Z_OK; } @@ -455,7 +530,7 @@ int ZEXPORT deflatePending (strm, pending, bits) int *bits; z_streamp strm; { - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; if (pending != Z_NULL) *pending = strm->state->pending; if (bits != Z_NULL) @@ -472,7 +547,7 @@ int ZEXPORT deflatePrime (strm, bits, value) deflate_state *s; int put; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; if ((Bytef *)(s->d_buf) < s->pending_out + ((Buf_size + 7) >> 3)) return Z_BUF_ERROR; @@ -497,9 +572,8 @@ int ZEXPORT deflateParams(strm, level, strategy) { deflate_state *s; compress_func func; - int err = Z_OK; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; #ifdef FASTEST @@ -513,13 +587,22 @@ int ZEXPORT deflateParams(strm, level, strategy) func = configuration_table[s->level].func; if ((strategy != s->strategy || func != configuration_table[level].func) && - strm->total_in != 0) { + s->high_water) { /* Flush the last buffer: */ - err = deflate(strm, Z_BLOCK); - if (err == Z_BUF_ERROR && s->pending == 0) - err = Z_OK; + int err = deflate(strm, Z_BLOCK); + if (err == Z_STREAM_ERROR) + return err; + if (strm->avail_out == 0) + return Z_BUF_ERROR; } if (s->level != level) { + if (s->level == 0 && s->matches != 0) { + if (s->matches == 1) + slide_hash(s); + else + CLEAR_HASH(s); + s->matches = 0; + } s->level = level; s->max_lazy_match = configuration_table[level].max_lazy; s->good_match = configuration_table[level].good_length; @@ -527,7 +610,7 @@ int ZEXPORT deflateParams(strm, level, strategy) s->max_chain_length = configuration_table[level].max_chain; } s->strategy = strategy; - return err; + return Z_OK; } /* ========================================================================= */ @@ -540,12 +623,12 @@ int ZEXPORT deflateTune(strm, good_length, max_lazy, nice_length, max_chain) { deflate_state *s; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; - s->good_match = good_length; - s->max_lazy_match = max_lazy; + s->good_match = (uInt)good_length; + s->max_lazy_match = (uInt)max_lazy; s->nice_match = nice_length; - s->max_chain_length = max_chain; + s->max_chain_length = (uInt)max_chain; return Z_OK; } @@ -572,14 +655,13 @@ uLong ZEXPORT deflateBound(strm, sourceLen) { deflate_state *s; uLong complen, wraplen; - Bytef *str; /* conservative upper bound for compressed data */ complen = sourceLen + ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 5; /* if can't get parameters, return conservative bound plus zlib wrapper */ - if (strm == Z_NULL || strm->state == Z_NULL) + if (deflateStateCheck(strm)) return complen + 6; /* compute wrapper length */ @@ -591,9 +673,11 @@ uLong ZEXPORT deflateBound(strm, sourceLen) case 1: /* zlib wrapper */ wraplen = 6 + (s->strstart ? 4 : 0); break; +#ifdef GZIP case 2: /* gzip wrapper */ wraplen = 18; if (s->gzhead != Z_NULL) { /* user-supplied gzip header */ + Bytef *str; if (s->gzhead->extra != Z_NULL) wraplen += 2 + s->gzhead->extra_len; str = s->gzhead->name; @@ -610,6 +694,7 @@ uLong ZEXPORT deflateBound(strm, sourceLen) wraplen += 2; } break; +#endif default: /* for compiler happiness */ wraplen = 6; } @@ -637,10 +722,10 @@ local void putShortMSB (s, b) } /* ========================================================================= - * Flush as much pending output as possible. All deflate() output goes - * through this function so some applications may wish to modify it - * to avoid allocating a large strm->next_out buffer and copying into it. - * (See also read_buf()). + * Flush as much pending output as possible. All deflate() output, except for + * some deflate_stored() output, goes through this function so some + * applications may wish to modify it to avoid allocating a large + * strm->next_out buffer and copying into it. (See also read_buf()). */ local void flush_pending(strm) z_streamp strm; @@ -657,13 +742,23 @@ local void flush_pending(strm) strm->next_out += len; s->pending_out += len; strm->total_out += len; - strm->avail_out -= len; - s->pending -= len; + strm->avail_out -= len; + s->pending -= len; if (s->pending == 0) { s->pending_out = s->pending_buf; } } +/* =========================================================================== + * Update the header CRC with the bytes s->pending_buf[beg..s->pending - 1]. + */ +#define HCRC_UPDATE(beg) \ + do { \ + if (s->gzhead->hcrc && s->pending > (beg)) \ + strm->adler = crc32(strm->adler, s->pending_buf + (beg), \ + s->pending - (beg)); \ + } while (0) + /* ========================================================================= */ int ZEXPORT deflate (strm, flush) z_streamp strm; @@ -672,230 +767,229 @@ int ZEXPORT deflate (strm, flush) int old_flush; /* value of flush param for previous deflate call */ deflate_state *s; - if (strm == Z_NULL || strm->state == Z_NULL || - flush > Z_BLOCK || flush < 0) { + if (deflateStateCheck(strm) || flush > Z_BLOCK || flush < 0) { return Z_STREAM_ERROR; } s = strm->state; if (strm->next_out == Z_NULL || - (strm->next_in == Z_NULL && strm->avail_in != 0) || + (strm->avail_in != 0 && strm->next_in == Z_NULL) || (s->status == FINISH_STATE && flush != Z_FINISH)) { ERR_RETURN(strm, Z_STREAM_ERROR); } if (strm->avail_out == 0) ERR_RETURN(strm, Z_BUF_ERROR); - s->strm = strm; /* just in case */ old_flush = s->last_flush; s->last_flush = flush; + /* Flush as much pending output as possible */ + if (s->pending != 0) { + flush_pending(strm); + if (strm->avail_out == 0) { + /* Since avail_out is 0, deflate will be called again with + * more output space, but possibly with both pending and + * avail_in equal to zero. There won't be anything to do, + * but this is not an error situation so make sure we + * return OK instead of BUF_ERROR at next call of deflate: + */ + s->last_flush = -1; + return Z_OK; + } + + /* Make sure there is something to do and avoid duplicate consecutive + * flushes. For repeated and useless calls with Z_FINISH, we keep + * returning Z_STREAM_END instead of Z_BUF_ERROR. + */ + } else if (strm->avail_in == 0 && RANK(flush) <= RANK(old_flush) && + flush != Z_FINISH) { + ERR_RETURN(strm, Z_BUF_ERROR); + } + + /* User must not provide more input after the first FINISH: */ + if (s->status == FINISH_STATE && strm->avail_in != 0) { + ERR_RETURN(strm, Z_BUF_ERROR); + } + /* Write the header */ if (s->status == INIT_STATE) { -#ifdef GZIP - if (s->wrap == 2) { - strm->adler = crc32(0L, Z_NULL, 0); - put_byte(s, 31); - put_byte(s, 139); - put_byte(s, 8); - if (s->gzhead == Z_NULL) { - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, 0); - put_byte(s, s->level == 9 ? 2 : - (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? - 4 : 0)); - put_byte(s, OS_CODE); - s->status = BUSY_STATE; - } - else { - put_byte(s, (s->gzhead->text ? 1 : 0) + - (s->gzhead->hcrc ? 2 : 0) + - (s->gzhead->extra == Z_NULL ? 0 : 4) + - (s->gzhead->name == Z_NULL ? 0 : 8) + - (s->gzhead->comment == Z_NULL ? 0 : 16) - ); - put_byte(s, (Byte)(s->gzhead->time & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 8) & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 16) & 0xff)); - put_byte(s, (Byte)((s->gzhead->time >> 24) & 0xff)); - put_byte(s, s->level == 9 ? 2 : - (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? - 4 : 0)); - put_byte(s, s->gzhead->os & 0xff); - if (s->gzhead->extra != Z_NULL) { - put_byte(s, s->gzhead->extra_len & 0xff); - put_byte(s, (s->gzhead->extra_len >> 8) & 0xff); - } - if (s->gzhead->hcrc) - strm->adler = crc32(strm->adler, s->pending_buf, - s->pending); - s->gzindex = 0; - s->status = EXTRA_STATE; - } - } + /* zlib header */ + uInt header = (Z_DEFLATED + ((s->w_bits-8)<<4)) << 8; + uInt level_flags; + + if (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2) + level_flags = 0; + else if (s->level < 6) + level_flags = 1; + else if (s->level == 6) + level_flags = 2; else -#endif - { - uInt header = (Z_DEFLATED + ((s->w_bits-8)<<4)) << 8; - uInt level_flags; - - if (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2) - level_flags = 0; - else if (s->level < 6) - level_flags = 1; - else if (s->level == 6) - level_flags = 2; - else - level_flags = 3; - header |= (level_flags << 6); - if (s->strstart != 0) header |= PRESET_DICT; - header += 31 - (header % 31); + level_flags = 3; + header |= (level_flags << 6); + if (s->strstart != 0) header |= PRESET_DICT; + header += 31 - (header % 31); + + putShortMSB(s, header); + + /* Save the adler32 of the preset dictionary: */ + if (s->strstart != 0) { + putShortMSB(s, (uInt)(strm->adler >> 16)); + putShortMSB(s, (uInt)(strm->adler & 0xffff)); + } + strm->adler = adler32(0L, Z_NULL, 0); + s->status = BUSY_STATE; + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } +#ifdef GZIP + if (s->status == GZIP_STATE) { + /* gzip header */ + strm->adler = crc32(0L, Z_NULL, 0); + put_byte(s, 31); + put_byte(s, 139); + put_byte(s, 8); + if (s->gzhead == Z_NULL) { + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, s->level == 9 ? 2 : + (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? + 4 : 0)); + put_byte(s, OS_CODE); s->status = BUSY_STATE; - putShortMSB(s, header); - /* Save the adler32 of the preset dictionary: */ - if (s->strstart != 0) { - putShortMSB(s, (uInt)(strm->adler >> 16)); - putShortMSB(s, (uInt)(strm->adler & 0xffff)); + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } + else { + put_byte(s, (s->gzhead->text ? 1 : 0) + + (s->gzhead->hcrc ? 2 : 0) + + (s->gzhead->extra == Z_NULL ? 0 : 4) + + (s->gzhead->name == Z_NULL ? 0 : 8) + + (s->gzhead->comment == Z_NULL ? 0 : 16) + ); + put_byte(s, (Byte)(s->gzhead->time & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 8) & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 16) & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 24) & 0xff)); + put_byte(s, s->level == 9 ? 2 : + (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? + 4 : 0)); + put_byte(s, s->gzhead->os & 0xff); + if (s->gzhead->extra != Z_NULL) { + put_byte(s, s->gzhead->extra_len & 0xff); + put_byte(s, (s->gzhead->extra_len >> 8) & 0xff); } - strm->adler = adler32(0L, Z_NULL, 0); + if (s->gzhead->hcrc) + strm->adler = crc32(strm->adler, s->pending_buf, + s->pending); + s->gzindex = 0; + s->status = EXTRA_STATE; } } -#ifdef GZIP if (s->status == EXTRA_STATE) { if (s->gzhead->extra != Z_NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ - - while (s->gzindex < (s->gzhead->extra_len & 0xffff)) { - if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) - break; + ulg beg = s->pending; /* start of bytes to update crc */ + uInt left = (s->gzhead->extra_len & 0xffff) - s->gzindex; + while (s->pending + left > s->pending_buf_size) { + uInt copy = s->pending_buf_size - s->pending; + zmemcpy(s->pending_buf + s->pending, + s->gzhead->extra + s->gzindex, copy); + s->pending = s->pending_buf_size; + HCRC_UPDATE(beg); + s->gzindex += copy; + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; } - put_byte(s, s->gzhead->extra[s->gzindex]); - s->gzindex++; - } - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (s->gzindex == s->gzhead->extra_len) { - s->gzindex = 0; - s->status = NAME_STATE; + beg = 0; + left -= copy; } + zmemcpy(s->pending_buf + s->pending, + s->gzhead->extra + s->gzindex, left); + s->pending += left; + HCRC_UPDATE(beg); + s->gzindex = 0; } - else - s->status = NAME_STATE; + s->status = NAME_STATE; } if (s->status == NAME_STATE) { if (s->gzhead->name != Z_NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ + ulg beg = s->pending; /* start of bytes to update crc */ int val; - do { if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); + HCRC_UPDATE(beg); flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) { - val = 1; - break; + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; } + beg = 0; } val = s->gzhead->name[s->gzindex++]; put_byte(s, val); } while (val != 0); - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (val == 0) { - s->gzindex = 0; - s->status = COMMENT_STATE; - } + HCRC_UPDATE(beg); + s->gzindex = 0; } - else - s->status = COMMENT_STATE; + s->status = COMMENT_STATE; } if (s->status == COMMENT_STATE) { if (s->gzhead->comment != Z_NULL) { - uInt beg = s->pending; /* start of bytes to update crc */ + ulg beg = s->pending; /* start of bytes to update crc */ int val; - do { if (s->pending == s->pending_buf_size) { - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); + HCRC_UPDATE(beg); flush_pending(strm); - beg = s->pending; - if (s->pending == s->pending_buf_size) { - val = 1; - break; + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; } + beg = 0; } val = s->gzhead->comment[s->gzindex++]; put_byte(s, val); } while (val != 0); - if (s->gzhead->hcrc && s->pending > beg) - strm->adler = crc32(strm->adler, s->pending_buf + beg, - s->pending - beg); - if (val == 0) - s->status = HCRC_STATE; + HCRC_UPDATE(beg); } - else - s->status = HCRC_STATE; + s->status = HCRC_STATE; } if (s->status == HCRC_STATE) { if (s->gzhead->hcrc) { - if (s->pending + 2 > s->pending_buf_size) + if (s->pending + 2 > s->pending_buf_size) { flush_pending(strm); - if (s->pending + 2 <= s->pending_buf_size) { - put_byte(s, (Byte)(strm->adler & 0xff)); - put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); - strm->adler = crc32(0L, Z_NULL, 0); - s->status = BUSY_STATE; + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } } + put_byte(s, (Byte)(strm->adler & 0xff)); + put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); + strm->adler = crc32(0L, Z_NULL, 0); } - else - s->status = BUSY_STATE; - } -#endif + s->status = BUSY_STATE; - /* Flush as much pending output as possible */ - if (s->pending != 0) { + /* Compression must start with an empty pending buffer */ flush_pending(strm); - if (strm->avail_out == 0) { - /* Since avail_out is 0, deflate will be called again with - * more output space, but possibly with both pending and - * avail_in equal to zero. There won't be anything to do, - * but this is not an error situation so make sure we - * return OK instead of BUF_ERROR at next call of deflate: - */ + if (s->pending != 0) { s->last_flush = -1; return Z_OK; } - - /* Make sure there is something to do and avoid duplicate consecutive - * flushes. For repeated and useless calls with Z_FINISH, we keep - * returning Z_STREAM_END instead of Z_BUF_ERROR. - */ - } else if (strm->avail_in == 0 && RANK(flush) <= RANK(old_flush) && - flush != Z_FINISH) { - ERR_RETURN(strm, Z_BUF_ERROR); - } - - /* User must not provide more input after the first FINISH: */ - if (s->status == FINISH_STATE && strm->avail_in != 0) { - ERR_RETURN(strm, Z_BUF_ERROR); } +#endif /* Start a new block or continue the current one. */ @@ -903,9 +997,10 @@ int ZEXPORT deflate (strm, flush) (flush != Z_NO_FLUSH && s->status != FINISH_STATE)) { block_state bstate; - bstate = s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : - (s->strategy == Z_RLE ? deflate_rle(s, flush) : - (*(configuration_table[s->level].func))(s, flush)); + bstate = s->level == 0 ? deflate_stored(s, flush) : + s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : + s->strategy == Z_RLE ? deflate_rle(s, flush) : + (*(configuration_table[s->level].func))(s, flush); if (bstate == finish_started || bstate == finish_done) { s->status = FINISH_STATE; @@ -947,7 +1042,6 @@ int ZEXPORT deflate (strm, flush) } } } - Assert(strm->avail_out > 0, "bug2"); if (flush != Z_FINISH) return Z_OK; if (s->wrap <= 0) return Z_STREAM_END; @@ -984,18 +1078,9 @@ int ZEXPORT deflateEnd (strm) { int status; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; status = strm->state->status; - if (status != INIT_STATE && - status != EXTRA_STATE && - status != NAME_STATE && - status != COMMENT_STATE && - status != HCRC_STATE && - status != BUSY_STATE && - status != FINISH_STATE) { - return Z_STREAM_ERROR; - } /* Deallocate in reverse order of allocations: */ TRY_FREE(strm, strm->state->pending_buf); @@ -1026,7 +1111,7 @@ int ZEXPORT deflateCopy (dest, source) ushf *overlay; - if (source == Z_NULL || dest == Z_NULL || source->state == Z_NULL) { + if (deflateStateCheck(source) || dest == Z_NULL) { return Z_STREAM_ERROR; } @@ -1076,7 +1161,7 @@ int ZEXPORT deflateCopy (dest, source) * allocating a large strm->next_in buffer and copying from it. * (See also flush_pending()). */ -local int read_buf(strm, buf, size) +local unsigned read_buf(strm, buf, size) z_streamp strm; Bytef *buf; unsigned size; @@ -1100,7 +1185,7 @@ local int read_buf(strm, buf, size) strm->next_in += len; strm->total_in += len; - return (int)len; + return len; } /* =========================================================================== @@ -1154,9 +1239,9 @@ local uInt longest_match(s, cur_match) { unsigned chain_length = s->max_chain_length;/* max hash chain length */ register Bytef *scan = s->window + s->strstart; /* current string */ - register Bytef *match; /* matched string */ + register Bytef *match; /* matched string */ register int len; /* length of current match */ - int best_len = s->prev_length; /* best match length so far */ + int best_len = (int)s->prev_length; /* best match length so far */ int nice_match = s->nice_match; /* stop if match long enough */ IPos limit = s->strstart > (IPos)MAX_DIST(s) ? s->strstart - (IPos)MAX_DIST(s) : NIL; @@ -1191,7 +1276,7 @@ local uInt longest_match(s, cur_match) /* Do not look for matches beyond the end of the input. This is necessary * to make deflate deterministic. */ - if ((uInt)nice_match > s->lookahead) nice_match = s->lookahead; + if ((uInt)nice_match > s->lookahead) nice_match = (int)s->lookahead; Assert((ulg)s->strstart <= s->window_size-MIN_LOOKAHEAD, "need lookahead"); @@ -1352,7 +1437,11 @@ local uInt longest_match(s, cur_match) #endif /* FASTEST */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG + +#define EQUAL 0 +/* result of memcmp for equal strings */ + /* =========================================================================== * Check that the match at match_start is indeed a match. */ @@ -1378,7 +1467,7 @@ local void check_match(s, start, match, length) } #else # define check_match(s, start, match, length) -#endif /* DEBUG */ +#endif /* ZLIB_DEBUG */ /* =========================================================================== * Fill the window when the lookahead becomes insufficient. @@ -1393,8 +1482,7 @@ local void check_match(s, start, match, length) local void fill_window(s) deflate_state *s; { - register unsigned n, m; - register Posf *p; + unsigned n; unsigned more; /* Amount of free space at the end of the window. */ uInt wsize = s->w_size; @@ -1421,35 +1509,11 @@ local void fill_window(s) */ if (s->strstart >= wsize+MAX_DIST(s)) { - zmemcpy(s->window, s->window+wsize, (unsigned)wsize); + zmemcpy(s->window, s->window+wsize, (unsigned)wsize - more); s->match_start -= wsize; s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ s->block_start -= (long) wsize; - - /* Slide the hash table (could be avoided with 32 bit values - at the expense of memory usage). We slide even when level == 0 - to keep the hash table consistent if we switch back to level > 0 - later. (Using level 0 permanently is not an optimal usage of - zlib, so we don't care about this pathological case.) - */ - n = s->hash_size; - p = &s->head[n]; - do { - m = *--p; - *p = (Pos)(m >= wsize ? m-wsize : NIL); - } while (--n); - - n = wsize; -#ifndef FASTEST - p = &s->prev[n]; - do { - m = *--p; - *p = (Pos)(m >= wsize ? m-wsize : NIL); - /* If n is not on any hash chain, prev[n] is garbage but - * its value will never be used. - */ - } while (--n); -#endif + slide_hash(s); more += wsize; } if (s->strm->avail_in == 0) break; @@ -1555,70 +1619,199 @@ local void fill_window(s) if (s->strm->avail_out == 0) return (last) ? finish_started : need_more; \ } +/* Maximum stored block length in deflate format (not including header). */ +#define MAX_STORED 65535 + +/* Minimum of a and b. */ +#define MIN(a, b) ((a) > (b) ? (b) : (a)) + /* =========================================================================== * Copy without compression as much as possible from the input stream, return * the current block state. - * This function does not insert new strings in the dictionary since - * uncompressible data is probably not useful. This function is used - * only for the level=0 compression option. - * NOTE: this function should be optimized to avoid extra copying from - * window to pending_buf. + * + * In case deflateParams() is used to later switch to a non-zero compression + * level, s->matches (otherwise unused when storing) keeps track of the number + * of hash table slides to perform. If s->matches is 1, then one hash table + * slide will be done when switching. If s->matches is 2, the maximum value + * allowed here, then the hash table will be cleared, since two or more slides + * is the same as a clear. + * + * deflate_stored() is written to minimize the number of times an input byte is + * copied. It is most efficient with large input and output buffers, which + * maximizes the opportunites to have a single copy from next_in to next_out. */ local block_state deflate_stored(s, flush) deflate_state *s; int flush; { - /* Stored blocks are limited to 0xffff bytes, pending_buf is limited - * to pending_buf_size, and each stored block has a 5 byte header: + /* Smallest worthy block size when not flushing or finishing. By default + * this is 32K. This can be as small as 507 bytes for memLevel == 1. For + * large input and output buffers, the stored block size will be larger. */ - ulg max_block_size = 0xffff; - ulg max_start; - - if (max_block_size > s->pending_buf_size - 5) { - max_block_size = s->pending_buf_size - 5; - } - - /* Copy as much as possible from input to output: */ - for (;;) { - /* Fill the window as much as possible: */ - if (s->lookahead <= 1) { + unsigned min_block = MIN(s->pending_buf_size - 5, s->w_size); - Assert(s->strstart < s->w_size+MAX_DIST(s) || - s->block_start >= (long)s->w_size, "slide too late"); + /* Copy as many min_block or larger stored blocks directly to next_out as + * possible. If flushing, copy the remaining available input to next_out as + * stored blocks, if there is enough space. + */ + unsigned len, left, have, last = 0; + unsigned used = s->strm->avail_in; + do { + /* Set len to the maximum size block that we can copy directly with the + * available input data and output space. Set left to how much of that + * would be copied from what's left in the window. + */ + len = MAX_STORED; /* maximum deflate stored block length */ + have = (s->bi_valid + 42) >> 3; /* number of header bytes */ + if (s->strm->avail_out < have) /* need room for header */ + break; + /* maximum stored block length that will fit in avail_out: */ + have = s->strm->avail_out - have; + left = s->strstart - s->block_start; /* bytes left in window */ + if (len > (ulg)left + s->strm->avail_in) + len = left + s->strm->avail_in; /* limit len to the input */ + if (len > have) + len = have; /* limit len to the output */ + + /* If the stored block would be less than min_block in length, or if + * unable to copy all of the available input when flushing, then try + * copying to the window and the pending buffer instead. Also don't + * write an empty block when flushing -- deflate() does that. + */ + if (len < min_block && ((len == 0 && flush != Z_FINISH) || + flush == Z_NO_FLUSH || + len != left + s->strm->avail_in)) + break; - fill_window(s); - if (s->lookahead == 0 && flush == Z_NO_FLUSH) return need_more; + /* Make a dummy stored block in pending to get the header bytes, + * including any pending bits. This also updates the debugging counts. + */ + last = flush == Z_FINISH && len == left + s->strm->avail_in ? 1 : 0; + _tr_stored_block(s, (char *)0, 0L, last); + + /* Replace the lengths in the dummy stored block with len. */ + s->pending_buf[s->pending - 4] = len; + s->pending_buf[s->pending - 3] = len >> 8; + s->pending_buf[s->pending - 2] = ~len; + s->pending_buf[s->pending - 1] = ~len >> 8; + + /* Write the stored block header bytes. */ + flush_pending(s->strm); + +#ifdef ZLIB_DEBUG + /* Update debugging counts for the data about to be copied. */ + s->compressed_len += len << 3; + s->bits_sent += len << 3; +#endif - if (s->lookahead == 0) break; /* flush the current block */ + /* Copy uncompressed bytes from the window to next_out. */ + if (left) { + if (left > len) + left = len; + zmemcpy(s->strm->next_out, s->window + s->block_start, left); + s->strm->next_out += left; + s->strm->avail_out -= left; + s->strm->total_out += left; + s->block_start += left; + len -= left; } - Assert(s->block_start >= 0L, "block gone"); - - s->strstart += s->lookahead; - s->lookahead = 0; - - /* Emit a stored block if pending_buf will be full: */ - max_start = s->block_start + max_block_size; - if (s->strstart == 0 || (ulg)s->strstart >= max_start) { - /* strstart == 0 is possible when wraparound on 16-bit machine */ - s->lookahead = (uInt)(s->strstart - max_start); - s->strstart = (uInt)max_start; - FLUSH_BLOCK(s, 0); + + /* Copy uncompressed bytes directly from next_in to next_out, updating + * the check value. + */ + if (len) { + read_buf(s->strm, s->strm->next_out, len); + s->strm->next_out += len; + s->strm->avail_out -= len; + s->strm->total_out += len; } - /* Flush if we may have to slide, otherwise block_start may become - * negative and the data will be gone: + } while (last == 0); + + /* Update the sliding window with the last s->w_size bytes of the copied + * data, or append all of the copied data to the existing window if less + * than s->w_size bytes were copied. Also update the number of bytes to + * insert in the hash tables, in the event that deflateParams() switches to + * a non-zero compression level. + */ + used -= s->strm->avail_in; /* number of input bytes directly copied */ + if (used) { + /* If any input was used, then no unused input remains in the window, + * therefore s->block_start == s->strstart. */ - if (s->strstart - (uInt)s->block_start >= MAX_DIST(s)) { - FLUSH_BLOCK(s, 0); + if (used >= s->w_size) { /* supplant the previous history */ + s->matches = 2; /* clear hash */ + zmemcpy(s->window, s->strm->next_in - s->w_size, s->w_size); + s->strstart = s->w_size; } + else { + if (s->window_size - s->strstart <= used) { + /* Slide the window down. */ + s->strstart -= s->w_size; + zmemcpy(s->window, s->window + s->w_size, s->strstart); + if (s->matches < 2) + s->matches++; /* add a pending slide_hash() */ + } + zmemcpy(s->window + s->strstart, s->strm->next_in - used, used); + s->strstart += used; + } + s->block_start = s->strstart; + s->insert += MIN(used, s->w_size - s->insert); } - s->insert = 0; - if (flush == Z_FINISH) { - FLUSH_BLOCK(s, 1); + if (s->high_water < s->strstart) + s->high_water = s->strstart; + + /* If the last block was written to next_out, then done. */ + if (last) return finish_done; + + /* If flushing and all input has been consumed, then done. */ + if (flush != Z_NO_FLUSH && flush != Z_FINISH && + s->strm->avail_in == 0 && (long)s->strstart == s->block_start) + return block_done; + + /* Fill the window with any remaining input. */ + have = s->window_size - s->strstart - 1; + if (s->strm->avail_in > have && s->block_start >= (long)s->w_size) { + /* Slide the window down. */ + s->block_start -= s->w_size; + s->strstart -= s->w_size; + zmemcpy(s->window, s->window + s->w_size, s->strstart); + if (s->matches < 2) + s->matches++; /* add a pending slide_hash() */ + have += s->w_size; /* more space now */ } - if ((long)s->strstart > s->block_start) - FLUSH_BLOCK(s, 0); - return block_done; + if (have > s->strm->avail_in) + have = s->strm->avail_in; + if (have) { + read_buf(s->strm, s->window + s->strstart, have); + s->strstart += have; + } + if (s->high_water < s->strstart) + s->high_water = s->strstart; + + /* There was not enough avail_out to write a complete worthy or flushed + * stored block to next_out. Write a stored block to pending instead, if we + * have enough input for a worthy block, or if flushing and there is enough + * room for the remaining input as a stored block in the pending buffer. + */ + have = (s->bi_valid + 42) >> 3; /* number of header bytes */ + /* maximum stored block length that will fit in pending: */ + have = MIN(s->pending_buf_size - have, MAX_STORED); + min_block = MIN(have, s->w_size); + left = s->strstart - s->block_start; + if (left >= min_block || + ((left || flush == Z_FINISH) && flush != Z_NO_FLUSH && + s->strm->avail_in == 0 && left <= have)) { + len = MIN(left, have); + last = flush == Z_FINISH && s->strm->avail_in == 0 && + len == left ? 1 : 0; + _tr_stored_block(s, (charf *)s->window + s->block_start, len, last); + s->block_start += len; + flush_pending(s->strm); + } + + /* We've done all we can with the available input and output. */ + return last ? finish_started : need_more; } /* =========================================================================== @@ -1895,7 +2088,7 @@ local block_state deflate_rle(s, flush) prev == *++scan && prev == *++scan && prev == *++scan && prev == *++scan && scan < strend); - s->match_length = MAX_MATCH - (int)(strend - scan); + s->match_length = MAX_MATCH - (uInt)(strend - scan); if (s->match_length > s->lookahead) s->match_length = s->lookahead; } diff --git a/erts/emulator/zlib/deflate.h b/erts/emulator/zlib/deflate.h index ce0299edd1..23ecdd312b 100644 --- a/erts/emulator/zlib/deflate.h +++ b/erts/emulator/zlib/deflate.h @@ -1,5 +1,5 @@ /* deflate.h -- internal compression state - * Copyright (C) 1995-2012 Jean-loup Gailly + * Copyright (C) 1995-2016 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -51,13 +51,16 @@ #define Buf_size 16 /* size of bit buffer in bi_buf */ -#define INIT_STATE 42 -#define EXTRA_STATE 69 -#define NAME_STATE 73 -#define COMMENT_STATE 91 -#define HCRC_STATE 103 -#define BUSY_STATE 113 -#define FINISH_STATE 666 +#define INIT_STATE 42 /* zlib header -> BUSY_STATE */ +#ifdef GZIP +# define GZIP_STATE 57 /* gzip header -> BUSY_STATE | EXTRA_STATE */ +#endif +#define EXTRA_STATE 69 /* gzip extra block -> NAME_STATE */ +#define NAME_STATE 73 /* gzip file name -> COMMENT_STATE */ +#define COMMENT_STATE 91 /* gzip comment -> HCRC_STATE */ +#define HCRC_STATE 103 /* gzip header CRC -> BUSY_STATE */ +#define BUSY_STATE 113 /* deflate -> FINISH_STATE */ +#define FINISH_STATE 666 /* stream complete */ /* Stream status */ @@ -83,7 +86,7 @@ typedef struct static_tree_desc_s static_tree_desc; typedef struct tree_desc_s { ct_data *dyn_tree; /* the dynamic tree */ int max_code; /* largest code with non zero frequency */ - static_tree_desc *stat_desc; /* the corresponding static tree */ + const static_tree_desc *stat_desc; /* the corresponding static tree */ } FAR tree_desc; typedef ush Pos; @@ -100,10 +103,10 @@ typedef struct internal_state { Bytef *pending_buf; /* output still pending */ ulg pending_buf_size; /* size of pending_buf */ Bytef *pending_out; /* next pending byte to output to the stream */ - uInt pending; /* nb of bytes in the pending buffer */ + ulg pending; /* nb of bytes in the pending buffer */ int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ gz_headerp gzhead; /* gzip header information to write */ - uInt gzindex; /* where in extra, name, or comment */ + ulg gzindex; /* where in extra, name, or comment */ Byte method; /* can only be DEFLATED */ int last_flush; /* value of flush param for previous deflate call */ @@ -249,7 +252,7 @@ typedef struct internal_state { uInt matches; /* number of string matches in current block */ uInt insert; /* bytes at end of window left to insert */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG ulg compressed_len; /* total bit length of compressed file mod 2^32 */ ulg bits_sent; /* bit length of compressed data sent mod 2^32 */ #endif @@ -275,7 +278,7 @@ typedef struct internal_state { /* Output a byte on the stream. * IN assertion: there is enough room in pending_buf. */ -#define put_byte(s, c) {s->pending_buf[s->pending++] = (c);} +#define put_byte(s, c) {s->pending_buf[s->pending++] = (Bytef)(c);} #define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1) @@ -309,7 +312,7 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, * used. */ -#ifndef DEBUG +#ifndef ZLIB_DEBUG /* Inline versions of _tr_tally for speed: */ #if defined(GEN_TREES_H) || !defined(STDC) @@ -328,8 +331,8 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, flush = (s->last_lit == s->lit_bufsize-1); \ } # define _tr_tally_dist(s, distance, length, flush) \ - { uch len = (length); \ - ush dist = (distance); \ + { uch len = (uch)(length); \ + ush dist = (ush)(distance); \ s->d_buf[s->last_lit] = dist; \ s->l_buf[s->last_lit++] = len; \ dist--; \ diff --git a/erts/emulator/zlib/gzguts.h b/erts/emulator/zlib/gzguts.h index d87659d031..990a4d2514 100644 --- a/erts/emulator/zlib/gzguts.h +++ b/erts/emulator/zlib/gzguts.h @@ -1,5 +1,5 @@ /* gzguts.h -- zlib internal header definitions for gz* operations - * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -25,6 +25,10 @@ # include <stdlib.h> # include <limits.h> #endif + +#ifndef _POSIX_SOURCE +# define _POSIX_SOURCE +#endif #include <fcntl.h> #ifdef _WIN32 @@ -35,6 +39,10 @@ # include <io.h> #endif +#if defined(_WIN32) || defined(__CYGWIN__) +# define WIDECHAR +#endif + #ifdef WINAPI_FAMILY # define open _open # define read _read @@ -95,18 +103,19 @@ # endif #endif -/* unlike snprintf (which is required in C99, yet still not supported by - Microsoft more than a decade later!), _snprintf does not guarantee null - termination of the result -- however this is only used in gzlib.c where +/* unlike snprintf (which is required in C99), _snprintf does not guarantee + null termination of the result -- however this is only used in gzlib.c where the result is assured to fit in the space provided */ -#ifdef _MSC_VER +#if defined(_MSC_VER) && _MSC_VER < 1900 # define snprintf _snprintf #endif #ifndef local # define local static #endif -/* compile with -Dlocal if your debugger can't find static symbols */ +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ /* gz* functions always use library allocation functions */ #ifndef STDC @@ -170,7 +179,7 @@ typedef struct { char *path; /* path or fd for error messages */ unsigned size; /* buffer size, zero if not allocated yet */ unsigned want; /* requested buffer size, default is GZBUFSIZE */ - unsigned char *in; /* input buffer */ + unsigned char *in; /* input buffer (double-sized when writing) */ unsigned char *out; /* output buffer (double-sized when reading) */ int direct; /* 0 if processing gzip, 1 if transparent */ /* just for reading */ diff --git a/erts/emulator/zlib/inffast.c b/erts/emulator/zlib/inffast.c index 5187743fde..0dbd1dbc09 100644 --- a/erts/emulator/zlib/inffast.c +++ b/erts/emulator/zlib/inffast.c @@ -1,36 +1,16 @@ /* inffast.c -- fast decoding - * Copyright (C) 1995-2008, 2010, 2013 Mark Adler + * Copyright (C) 1995-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif #include "zutil.h" #include "inftrees.h" #include "inflate.h" #include "inffast.h" -#ifndef ASMINF - -/* Allow machine dependent optimization for post-increment or pre-increment. - Based on testing to date, - Pre-increment preferred for: - - PowerPC G3 (Adler) - - MIPS R5000 (Randers-Pehrson) - Post-increment preferred for: - - none - No measurable difference: - - Pentium III (Anderson) - - M68060 (Nikl) - */ -#ifdef POSTINC -# define OFF 0 -# define PUP(a) *(a)++ +#ifdef ASMINF +# pragma message("Assembler code may have bugs -- use at your own risk") #else -# define OFF 1 -# define PUP(a) *++(a) -#endif /* Decode literal, length, and distance codes and write out the resulting @@ -99,9 +79,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ /* copy state to local variables */ state = (struct inflate_state FAR *)strm->state; - in = strm->next_in - OFF; + in = strm->next_in; last = in + (strm->avail_in - 5); - out = strm->next_out - OFF; + out = strm->next_out; beg = out - (start - strm->avail_out); end = out + (strm->avail_out - 257); #ifdef INFLATE_STRICT @@ -122,9 +102,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ input data or output space */ do { if (bits < 15) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } here = lcode[hold & lmask]; @@ -137,14 +117,14 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? "inflate: literal '%c'\n" : "inflate: literal 0x%02x\n", here.val)); - PUP(out) = (unsigned char)(here.val); + *out++ = (unsigned char)(here.val); } else if (op & 16) { /* length base */ len = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (op) { if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } len += (unsigned)hold & ((1U << op) - 1); @@ -153,9 +133,9 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ } Tracevv((stderr, "inflate: length %u\n", len)); if (bits < 15) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } here = dcode[hold & dmask]; @@ -168,10 +148,10 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ dist = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; if (bits < op) { - hold += (unsigned long)(PUP(in)) << bits; + hold += (unsigned long)(*in++) << bits; bits += 8; } } @@ -199,30 +179,30 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ #ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR if (len <= op - whave) { do { - PUP(out) = 0; + *out++ = 0; } while (--len); continue; } len -= op - whave; do { - PUP(out) = 0; + *out++ = 0; } while (--op > whave); if (op == 0) { from = out - dist; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--len); continue; } #endif } - from = window - OFF; + from = window; if (wnext == 0) { /* very common case */ from += wsize - op; if (op < len) { /* some from window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } @@ -233,14 +213,14 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ if (op < len) { /* some from end of window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); - from = window - OFF; + from = window; if (wnext < len) { /* some from start of window */ op = wnext; len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } @@ -251,35 +231,35 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ if (op < len) { /* some from window */ len -= op; do { - PUP(out) = PUP(from); + *out++ = *from++; } while (--op); from = out - dist; /* rest from output */ } } while (len > 2) { - PUP(out) = PUP(from); - PUP(out) = PUP(from); - PUP(out) = PUP(from); + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; len -= 3; } if (len) { - PUP(out) = PUP(from); + *out++ = *from++; if (len > 1) - PUP(out) = PUP(from); + *out++ = *from++; } } else { from = out - dist; /* copy direct from output */ do { /* minimum length is three */ - PUP(out) = PUP(from); - PUP(out) = PUP(from); - PUP(out) = PUP(from); + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; len -= 3; } while (len > 2); if (len) { - PUP(out) = PUP(from); + *out++ = *from++; if (len > 1) - PUP(out) = PUP(from); + *out++ = *from++; } } } @@ -316,8 +296,8 @@ unsigned start; /* inflate()'s starting value for strm->avail_out */ hold &= (1U << bits) - 1; /* update state and return */ - strm->next_in = in + OFF; - strm->next_out = out + OFF; + strm->next_in = in; + strm->next_out = out; strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last)); strm->avail_out = (unsigned)(out < end ? 257 + (end - out) : 257 - (out - end)); diff --git a/erts/emulator/zlib/inflate.c b/erts/emulator/zlib/inflate.c index 532330b06b..ac333e8c2e 100644 --- a/erts/emulator/zlib/inflate.c +++ b/erts/emulator/zlib/inflate.c @@ -1,5 +1,5 @@ /* inflate.c -- zlib decompression - * Copyright (C) 1995-2012 Mark Adler + * Copyright (C) 1995-2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -80,9 +80,6 @@ * The history for versions after 1.2.0 are in ChangeLog in zlib distribution. */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif #include "zutil.h" #include "inftrees.h" #include "inflate.h" @@ -95,6 +92,7 @@ #endif /* function prototypes */ +local int inflateStateCheck OF((z_streamp strm)); local void fixedtables OF((struct inflate_state FAR *state)); local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, unsigned copy)); @@ -104,12 +102,26 @@ local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, unsigned len)); +local int inflateStateCheck(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + state = (struct inflate_state FAR *)strm->state; + if (state == Z_NULL || state->strm != strm || + state->mode < HEAD || state->mode > SYNC) + return 1; + return 0; +} + int ZEXPORT inflateResetKeep(strm) z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; strm->total_in = strm->total_out = state->total = 0; strm->msg = Z_NULL; @@ -134,7 +146,7 @@ z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; state->wsize = 0; state->whave = 0; @@ -150,7 +162,7 @@ int windowBits; struct inflate_state FAR *state; /* get the state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; /* extract wrap request from windowBits parameter */ @@ -159,7 +171,7 @@ int windowBits; windowBits = -windowBits; } else { - wrap = (windowBits >> 4) + 1; + wrap = (windowBits >> 4) + 5; #ifdef GUNZIP if (windowBits < 48) windowBits &= 15; @@ -213,7 +225,9 @@ int stream_size; if (state == Z_NULL) return Z_MEM_ERROR; Tracev((stderr, "inflate: allocated\n")); strm->state = (struct internal_state FAR *)state; + state->strm = strm; state->window = Z_NULL; + state->mode = HEAD; /* to pass state test in inflateReset2() */ ret = inflateReset2(strm, windowBits); if (ret != Z_OK) { ZFREE(strm, state); @@ -237,17 +251,17 @@ int value; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (bits < 0) { state->hold = 0; state->bits = 0; return Z_OK; } - if (bits > 16 || state->bits + bits > 32) return Z_STREAM_ERROR; + if (bits > 16 || state->bits + (uInt)bits > 32) return Z_STREAM_ERROR; value &= (1L << bits) - 1; - state->hold += value << state->bits; - state->bits += bits; + state->hold += (unsigned)value << state->bits; + state->bits += (uInt)bits; return Z_OK; } @@ -628,7 +642,7 @@ int flush; static const unsigned short order[19] = /* permutation of code lengths */ {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; - if (strm == Z_NULL || strm->state == Z_NULL || strm->next_out == Z_NULL || + if (inflateStateCheck(strm) || strm->next_out == Z_NULL || (strm->next_in == Z_NULL && strm->avail_in != 0)) return Z_STREAM_ERROR; @@ -648,6 +662,8 @@ int flush; NEEDBITS(16); #ifdef GUNZIP if ((state->wrap & 2) && hold == 0x8b1f) { /* gzip header */ + if (state->wbits == 0) + state->wbits = 15; state->check = crc32(0L, Z_NULL, 0); CRC2(state->check, hold); INITBITS(); @@ -675,7 +691,7 @@ int flush; len = BITS(4) + 8; if (state->wbits == 0) state->wbits = len; - else if (len > state->wbits) { + if (len > 15 || len > state->wbits) { strm->msg = (char *)"invalid window size"; state->mode = BAD; break; @@ -702,14 +718,16 @@ int flush; } if (state->head != Z_NULL) state->head->text = (int)((hold >> 8) & 1); - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); state->mode = TIME; case TIME: NEEDBITS(32); if (state->head != Z_NULL) state->head->time = hold; - if (state->flags & 0x0200) CRC4(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC4(state->check, hold); INITBITS(); state->mode = OS; case OS: @@ -718,7 +736,8 @@ int flush; state->head->xflags = (int)(hold & 0xff); state->head->os = (int)(hold >> 8); } - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); state->mode = EXLEN; case EXLEN: @@ -727,7 +746,8 @@ int flush; state->length = (unsigned)(hold); if (state->head != Z_NULL) state->head->extra_len = (unsigned)hold; - if (state->flags & 0x0200) CRC2(state->check, hold); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); INITBITS(); } else if (state->head != Z_NULL) @@ -745,7 +765,7 @@ int flush; len + copy > state->head->extra_max ? state->head->extra_max - len : copy); } - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -764,9 +784,9 @@ int flush; if (state->head != Z_NULL && state->head->name != Z_NULL && state->length < state->head->name_max) - state->head->name[state->length++] = len; + state->head->name[state->length++] = (Bytef)len; } while (len && copy < have); - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -785,9 +805,9 @@ int flush; if (state->head != Z_NULL && state->head->comment != Z_NULL && state->length < state->head->comm_max) - state->head->comment[state->length++] = len; + state->head->comment[state->length++] = (Bytef)len; } while (len && copy < have); - if (state->flags & 0x0200) + if ((state->flags & 0x0200) && (state->wrap & 4)) state->check = crc32(state->check, next, copy); have -= copy; next += copy; @@ -799,7 +819,7 @@ int flush; case HCRC: if (state->flags & 0x0200) { NEEDBITS(16); - if (hold != (state->check & 0xffff)) { + if ((state->wrap & 4) && hold != (state->check & 0xffff)) { strm->msg = (char *)"header crc mismatch"; state->mode = BAD; break; @@ -1180,11 +1200,11 @@ int flush; out -= left; strm->total_out += out; state->total += out; - if (out) + if ((state->wrap & 4) && out) strm->adler = state->check = UPDATE(state->check, put - out, out); out = left; - if (( + if ((state->wrap & 4) && ( #ifdef GUNZIP state->flags ? hold : #endif @@ -1243,10 +1263,10 @@ int flush; strm->total_in += in; strm->total_out += out; state->total += out; - if (state->wrap && out) + if ((state->wrap & 4) && out) strm->adler = state->check = UPDATE(state->check, strm->next_out - out, out); - strm->data_type = state->bits + (state->last ? 64 : 0) + + strm->data_type = (int)state->bits + (state->last ? 64 : 0) + (state->mode == TYPE ? 128 : 0) + (state->mode == LEN_ || state->mode == COPY_ ? 256 : 0); if (((in == 0 && out == 0) || flush == Z_FINISH) && ret == Z_OK) @@ -1258,7 +1278,7 @@ int ZEXPORT inflateEnd(strm) z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL || strm->zfree == (free_func)0) + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (state->window != Z_NULL) ZFREE(strm, state->window); @@ -1276,7 +1296,7 @@ uInt *dictLength; struct inflate_state FAR *state; /* check state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; /* copy dictionary */ @@ -1301,7 +1321,7 @@ uInt dictLength; int ret; /* check state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (state->wrap != 0 && state->mode != DICT) return Z_STREAM_ERROR; @@ -1333,7 +1353,7 @@ gz_headerp head; struct inflate_state FAR *state; /* check state */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if ((state->wrap & 2) == 0) return Z_STREAM_ERROR; @@ -1386,7 +1406,7 @@ z_streamp strm; struct inflate_state FAR *state; /* check parameters */ - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (strm->avail_in == 0 && state->bits < 8) return Z_BUF_ERROR; @@ -1433,7 +1453,7 @@ z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; return state->mode == STORED && state->bits == 0; } @@ -1448,8 +1468,7 @@ z_streamp source; unsigned wsize; /* check input */ - if (dest == Z_NULL || source == Z_NULL || source->state == Z_NULL || - source->zalloc == (alloc_func)0 || source->zfree == (free_func)0) + if (inflateStateCheck(source) || dest == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)source->state; @@ -1470,6 +1489,7 @@ z_streamp source; /* copy state */ zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); zmemcpy((voidpf)copy, (voidpf)state, sizeof(struct inflate_state)); + copy->strm = dest; if (state->lencode >= state->codes && state->lencode <= state->codes + ENOUGH - 1) { copy->lencode = copy->codes + (state->lencode - state->codes); @@ -1491,25 +1511,51 @@ int subvert; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; - state->sane = !subvert; #ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + state->sane = !subvert; return Z_OK; #else + (void)subvert; state->sane = 1; return Z_DATA_ERROR; #endif } +int ZEXPORT inflateValidate(strm, check) +z_streamp strm; +int check; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (check) + state->wrap |= 4; + else + state->wrap &= ~4; + return Z_OK; +} + long ZEXPORT inflateMark(strm) z_streamp strm; { struct inflate_state FAR *state; - if (strm == Z_NULL || strm->state == Z_NULL) return -1L << 16; + if (inflateStateCheck(strm)) + return -(1L << 16); state = (struct inflate_state FAR *)strm->state; - return ((long)(state->back) << 16) + + return (long)(((unsigned long)((long)state->back)) << 16) + (state->mode == COPY ? state->length : (state->mode == MATCH ? state->was - state->length : 0)); } + +unsigned long ZEXPORT inflateCodesUsed(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) return (unsigned long)-1; + state = (struct inflate_state FAR *)strm->state; + return (unsigned long)(state->next - state->codes); +} diff --git a/erts/emulator/zlib/inflate.h b/erts/emulator/zlib/inflate.h index 95f4986d40..a46cce6b6d 100644 --- a/erts/emulator/zlib/inflate.h +++ b/erts/emulator/zlib/inflate.h @@ -1,5 +1,5 @@ /* inflate.h -- internal inflate state definition - * Copyright (C) 1995-2009 Mark Adler + * Copyright (C) 1995-2016 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -18,7 +18,7 @@ /* Possible inflate modes between inflate() calls */ typedef enum { - HEAD, /* i: waiting for magic header */ + HEAD = 16180, /* i: waiting for magic header */ FLAGS, /* i: waiting for method and flags (gzip) */ TIME, /* i: waiting for modification time (gzip) */ OS, /* i: waiting for extra flags and operating system (gzip) */ @@ -77,11 +77,14 @@ typedef enum { CHECK -> LENGTH -> DONE */ -/* state maintained between inflate() calls. Approximately 10K bytes. */ +/* State maintained between inflate() calls -- approximately 7K bytes, not + including the allocated sliding window, which is up to 32K bytes. */ struct inflate_state { + z_streamp strm; /* pointer back to this zlib stream */ inflate_mode mode; /* current inflate mode */ int last; /* true if processing last block */ - int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ + int wrap; /* bit 0 true for zlib, bit 1 true for gzip, + bit 2 true to validate check value */ int havedict; /* true if dictionary provided */ int flags; /* gzip header method and flags (0 if zlib) */ unsigned dmax; /* zlib header max distance (INFLATE_STRICT) */ diff --git a/erts/emulator/zlib/inftrees.c b/erts/emulator/zlib/inftrees.c index 3766fa2646..2ea08fc13e 100644 --- a/erts/emulator/zlib/inftrees.c +++ b/erts/emulator/zlib/inftrees.c @@ -1,18 +1,15 @@ /* inftrees.c -- generate Huffman trees for efficient decoding - * Copyright (C) 1995-2013 Mark Adler + * Copyright (C) 1995-2017 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif #include "zutil.h" #include "inftrees.h" #define MAXBITS 15 const char inflate_copyright[] = - " inflate 1.2.8 Copyright 1995-2013 Mark Adler "; + " inflate 1.2.11 Copyright 1995-2017 Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -57,7 +54,7 @@ unsigned short FAR *work; code FAR *next; /* next available space in table */ const unsigned short FAR *base; /* base value table to use */ const unsigned short FAR *extra; /* extra bits table to use */ - int end; /* use base and extra for symbol > end */ + unsigned match; /* use base and extra for symbol >= match */ unsigned short count[MAXBITS+1]; /* number of codes of each length */ unsigned short offs[MAXBITS+1]; /* offsets in table for each length */ static const unsigned short lbase[31] = { /* Length codes 257..285 base */ @@ -65,7 +62,7 @@ unsigned short FAR *work; 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0}; static const unsigned short lext[31] = { /* Length codes 257..285 extra */ 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, - 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 72, 78}; + 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 77, 202}; static const unsigned short dbase[32] = { /* Distance codes 0..29 base */ 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, @@ -184,19 +181,17 @@ unsigned short FAR *work; switch (type) { case CODES: base = extra = work; /* dummy value--not used */ - end = 19; + match = 20; break; case LENS: base = lbase; - base -= 257; extra = lext; - extra -= 257; - end = 256; + match = 257; break; - default: /* DISTS */ + default: /* DISTS */ base = dbase; extra = dext; - end = -1; + match = 0; } /* initialize state for loop */ @@ -219,13 +214,13 @@ unsigned short FAR *work; for (;;) { /* create table entry */ here.bits = (unsigned char)(len - drop); - if ((int)(work[sym]) < end) { + if (work[sym] + 1U < match) { here.op = (unsigned char)0; here.val = work[sym]; } - else if ((int)(work[sym]) > end) { - here.op = (unsigned char)(extra[work[sym]]); - here.val = base[work[sym]]; + else if (work[sym] >= match) { + here.op = (unsigned char)(extra[work[sym] - match]); + here.val = base[work[sym] - match]; } else { here.op = (unsigned char)(32 + 64); /* end of block */ diff --git a/erts/emulator/zlib/trees.c b/erts/emulator/zlib/trees.c index 465e944e5b..50cf4b4571 100644 --- a/erts/emulator/zlib/trees.c +++ b/erts/emulator/zlib/trees.c @@ -1,5 +1,5 @@ /* trees.c -- output deflated data using Huffman coding - * Copyright (C) 1995-2012 Jean-loup Gailly + * Copyright (C) 1995-2017 Jean-loup Gailly * detect_data_type() function provided freely by Cosmin Truta, 2006 * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -34,12 +34,9 @@ /* #define GEN_TREES_H */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif #include "deflate.h" -#ifdef DEBUG +#ifdef ZLIB_DEBUG # include <ctype.h> #endif @@ -125,13 +122,13 @@ struct static_tree_desc_s { int max_length; /* max bit length for the codes */ }; -local static_tree_desc static_l_desc = +local const static_tree_desc static_l_desc = {static_ltree, extra_lbits, LITERALS+1, L_CODES, MAX_BITS}; -local static_tree_desc static_d_desc = +local const static_tree_desc static_d_desc = {static_dtree, extra_dbits, 0, D_CODES, MAX_BITS}; -local static_tree_desc static_bl_desc = +local const static_tree_desc static_bl_desc = {(const ct_data *)0, extra_blbits, 0, BL_CODES, MAX_BL_BITS}; /* =========================================================================== @@ -155,18 +152,16 @@ local int detect_data_type OF((deflate_state *s)); local unsigned bi_reverse OF((unsigned value, int length)); local void bi_windup OF((deflate_state *s)); local void bi_flush OF((deflate_state *s)); -local void copy_block OF((deflate_state *s, charf *buf, unsigned len, - int header)); #ifdef GEN_TREES_H local void gen_trees_header OF((void)); #endif -#ifndef DEBUG +#ifndef ZLIB_DEBUG # define send_code(s, c, tree) send_bits(s, tree[c].Code, tree[c].Len) /* Send a code of the given tree. c and tree must not have side effects */ -#else /* DEBUG */ +#else /* !ZLIB_DEBUG */ # define send_code(s, c, tree) \ { if (z_verbose>2) fprintf(stderr,"\ncd %3d ",(c)); \ send_bits(s, tree[c].Code, tree[c].Len); } @@ -185,7 +180,7 @@ local void gen_trees_header OF((void)); * Send a value on a given number of bits. * IN assertion: length <= 16 and value fits in length bits. */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG local void send_bits OF((deflate_state *s, int value, int length)); local void send_bits(s, value, length) @@ -211,12 +206,12 @@ local void send_bits(s, value, length) s->bi_valid += length; } } -#else /* !DEBUG */ +#else /* !ZLIB_DEBUG */ #define send_bits(s, value, length) \ { int len = length;\ if (s->bi_valid > (int)Buf_size - len) {\ - int val = value;\ + int val = (int)value;\ s->bi_buf |= (ush)val << s->bi_valid;\ put_short(s, s->bi_buf);\ s->bi_buf = (ush)val >> (Buf_size - s->bi_valid);\ @@ -226,7 +221,7 @@ local void send_bits(s, value, length) s->bi_valid += len;\ }\ } -#endif /* DEBUG */ +#endif /* ZLIB_DEBUG */ /* the arguments must not have side effects */ @@ -320,7 +315,7 @@ local void tr_static_init() * Genererate the file trees.h describing the static trees. */ #ifdef GEN_TREES_H -# ifndef DEBUG +# ifndef ZLIB_DEBUG # include <stdio.h> # endif @@ -397,7 +392,7 @@ void ZLIB_INTERNAL _tr_init(s) s->bi_buf = 0; s->bi_valid = 0; -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len = 0L; s->bits_sent = 0L; #endif @@ -525,12 +520,12 @@ local void gen_bitlen(s, desc) xbits = 0; if (n >= base) xbits = extra[n-base]; f = tree[n].Freq; - s->opt_len += (ulg)f * (bits + xbits); - if (stree) s->static_len += (ulg)f * (stree[n].Len + xbits); + s->opt_len += (ulg)f * (unsigned)(bits + xbits); + if (stree) s->static_len += (ulg)f * (unsigned)(stree[n].Len + xbits); } if (overflow == 0) return; - Trace((stderr,"\nbit length overflow\n")); + Tracev((stderr,"\nbit length overflow\n")); /* This happens for example on obj2 and pic of the Calgary corpus */ /* Find the first bit length which could increase: */ @@ -557,9 +552,8 @@ local void gen_bitlen(s, desc) m = s->heap[--h]; if (m > max_code) continue; if ((unsigned) tree[m].Len != (unsigned) bits) { - Trace((stderr,"code %d bits %d->%d\n", m, tree[m].Len, bits)); - s->opt_len += ((long)bits - (long)tree[m].Len) - *(long)tree[m].Freq; + Tracev((stderr,"code %d bits %d->%d\n", m, tree[m].Len, bits)); + s->opt_len += ((ulg)bits - tree[m].Len) * tree[m].Freq; tree[m].Len = (ush)bits; } n--; @@ -581,7 +575,7 @@ local void gen_codes (tree, max_code, bl_count) ushf *bl_count; /* number of codes at each bit length */ { ush next_code[MAX_BITS+1]; /* next code value for each bit length */ - ush code = 0; /* running code value */ + unsigned code = 0; /* running code value */ int bits; /* bit index */ int n; /* code index */ @@ -589,7 +583,8 @@ local void gen_codes (tree, max_code, bl_count) * without bit reversal. */ for (bits = 1; bits <= MAX_BITS; bits++) { - next_code[bits] = code = (code + bl_count[bits-1]) << 1; + code = (code + bl_count[bits-1]) << 1; + next_code[bits] = (ush)code; } /* Check that the bit counts in bl_count are consistent. The last code * must be all ones. @@ -602,7 +597,7 @@ local void gen_codes (tree, max_code, bl_count) int len = tree[n].Len; if (len == 0) continue; /* Now reverse the bits */ - tree[n].Code = bi_reverse(next_code[len]++, len); + tree[n].Code = (ush)bi_reverse(next_code[len]++, len); Tracecv(tree != static_ltree, (stderr,"\nn %3d %c l %2d c %4x (%x) ", n, (isgraph(n) ? n : ' '), len, tree[n].Code, next_code[len]-1)); @@ -824,7 +819,7 @@ local int build_bl_tree(s) if (s->bl_tree[bl_order[max_blindex]].Len != 0) break; } /* Update opt_len to include the bit length tree and counts */ - s->opt_len += 3*(max_blindex+1) + 5+5+4; + s->opt_len += 3*((ulg)max_blindex+1) + 5+5+4; Tracev((stderr, "\ndyn trees: dyn %ld, stat %ld", s->opt_len, s->static_len)); @@ -872,11 +867,17 @@ void ZLIB_INTERNAL _tr_stored_block(s, buf, stored_len, last) int last; /* one if this is the last block for a file */ { send_bits(s, (STORED_BLOCK<<1)+last, 3); /* send block type */ -#ifdef DEBUG + bi_windup(s); /* align on byte boundary */ + put_short(s, (ush)stored_len); + put_short(s, (ush)~stored_len); + zmemcpy(s->pending_buf + s->pending, (Bytef *)buf, stored_len); + s->pending += stored_len; +#ifdef ZLIB_DEBUG s->compressed_len = (s->compressed_len + 3 + 7) & (ulg)~7L; s->compressed_len += (stored_len + 4) << 3; + s->bits_sent += 2*16; + s->bits_sent += stored_len<<3; #endif - copy_block(s, buf, (unsigned)stored_len, 1); /* with header */ } /* =========================================================================== @@ -897,7 +898,7 @@ void ZLIB_INTERNAL _tr_align(s) { send_bits(s, STATIC_TREES<<1, 3); send_code(s, END_BLOCK, static_ltree); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 10L; /* 3 for block type, 7 for EOB */ #endif bi_flush(s); @@ -905,7 +906,7 @@ void ZLIB_INTERNAL _tr_align(s) /* =========================================================================== * Determine the best encoding for the current block: dynamic trees, static - * trees or store, and output the encoded block to the zip file. + * trees or store, and write out the encoded block. */ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) deflate_state *s; @@ -977,7 +978,7 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) send_bits(s, (STATIC_TREES<<1)+last, 3); compress_block(s, (const ct_data *)static_ltree, (const ct_data *)static_dtree); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 3 + s->static_len; #endif } else { @@ -986,7 +987,7 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) max_blindex+1); compress_block(s, (const ct_data *)s->dyn_ltree, (const ct_data *)s->dyn_dtree); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 3 + s->opt_len; #endif } @@ -998,7 +999,7 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) if (last) { bi_windup(s); -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->compressed_len += 7; /* align on byte boundary */ #endif } @@ -1093,7 +1094,7 @@ local void compress_block(s, ltree, dtree) send_code(s, code, dtree); /* send the distance code */ extra = extra_dbits[code]; if (extra != 0) { - dist -= base_dist[code]; + dist -= (unsigned)base_dist[code]; send_bits(s, dist, extra); /* send the extra distance bits */ } } /* literal or match pair ? */ @@ -1196,34 +1197,7 @@ local void bi_windup(s) } s->bi_buf = 0; s->bi_valid = 0; -#ifdef DEBUG +#ifdef ZLIB_DEBUG s->bits_sent = (s->bits_sent+7) & ~7; #endif } - -/* =========================================================================== - * Copy a stored block, storing first the length and its - * one's complement if requested. - */ -local void copy_block(s, buf, len, header) - deflate_state *s; - charf *buf; /* the input data */ - unsigned len; /* its length */ - int header; /* true if block header must be written */ -{ - bi_windup(s); /* align on byte boundary */ - - if (header) { - put_short(s, (ush)len); - put_short(s, (ush)~len); -#ifdef DEBUG - s->bits_sent += 2*16; -#endif - } -#ifdef DEBUG - s->bits_sent += (ulg)len<<3; -#endif - while (len--) { - put_byte(s, *buf++); - } -} diff --git a/erts/emulator/zlib/uncompr.c b/erts/emulator/zlib/uncompr.c index 864d571719..f03a1a865e 100644 --- a/erts/emulator/zlib/uncompr.c +++ b/erts/emulator/zlib/uncompr.c @@ -1,62 +1,93 @@ /* uncompr.c -- decompress a memory buffer - * Copyright (C) 1995-2003, 2010 Jean-loup Gailly. + * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif #define ZLIB_INTERNAL #include "zlib.h" /* =========================================================================== - Decompresses the source buffer into the destination buffer. sourceLen is - the byte length of the source buffer. Upon entry, destLen is the total - size of the destination buffer, which must be large enough to hold the - entire uncompressed data. (The size of the uncompressed data must have - been saved previously by the compressor and transmitted to the decompressor - by some mechanism outside the scope of this compression library.) - Upon exit, destLen is the actual size of the compressed buffer. - - uncompress returns Z_OK if success, Z_MEM_ERROR if there was not - enough memory, Z_BUF_ERROR if there was not enough room in the output - buffer, or Z_DATA_ERROR if the input data was corrupted. + Decompresses the source buffer into the destination buffer. *sourceLen is + the byte length of the source buffer. Upon entry, *destLen is the total size + of the destination buffer, which must be large enough to hold the entire + uncompressed data. (The size of the uncompressed data must have been saved + previously by the compressor and transmitted to the decompressor by some + mechanism outside the scope of this compression library.) Upon exit, + *destLen is the size of the decompressed data and *sourceLen is the number + of source bytes consumed. Upon return, source + *sourceLen points to the + first unused input byte. + + uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_BUF_ERROR if there was not enough room in the output buffer, or + Z_DATA_ERROR if the input data was corrupted, including if the input data is + an incomplete zlib stream. */ -int ZEXPORT uncompress (dest, destLen, source, sourceLen) +int ZEXPORT uncompress2 (dest, destLen, source, sourceLen) Bytef *dest; uLongf *destLen; const Bytef *source; - uLong sourceLen; + uLong *sourceLen; { z_stream stream; int err; + const uInt max = (uInt)-1; + uLong len, left; + Byte buf[1]; /* for detection of incomplete stream when *destLen == 0 */ - stream.next_in = (z_const Bytef *)source; - stream.avail_in = (uInt)sourceLen; - /* Check for source > 64K on 16-bit machine: */ - if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR; - - stream.next_out = dest; - stream.avail_out = (uInt)*destLen; - if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR; + len = *sourceLen; + if (*destLen) { + left = *destLen; + *destLen = 0; + } + else { + left = 1; + dest = buf; + } + stream.next_in = (z_const Bytef *)source; + stream.avail_in = 0; stream.zalloc = (alloc_func)0; stream.zfree = (free_func)0; + stream.opaque = (voidpf)0; err = inflateInit(&stream); if (err != Z_OK) return err; - err = inflate(&stream, Z_FINISH); - if (err != Z_STREAM_END) { - inflateEnd(&stream); - if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0)) - return Z_DATA_ERROR; - return err; - } - *destLen = stream.total_out; + stream.next_out = dest; + stream.avail_out = 0; - err = inflateEnd(&stream); - return err; + do { + if (stream.avail_out == 0) { + stream.avail_out = left > (uLong)max ? max : (uInt)left; + left -= stream.avail_out; + } + if (stream.avail_in == 0) { + stream.avail_in = len > (uLong)max ? max : (uInt)len; + len -= stream.avail_in; + } + err = inflate(&stream, Z_NO_FLUSH); + } while (err == Z_OK); + + *sourceLen -= len + stream.avail_in; + if (dest != buf) + *destLen = stream.total_out; + else if (stream.total_out && err == Z_BUF_ERROR) + left = 1; + + inflateEnd(&stream); + return err == Z_STREAM_END ? Z_OK : + err == Z_NEED_DICT ? Z_DATA_ERROR : + err == Z_BUF_ERROR && left + stream.avail_out ? Z_DATA_ERROR : + err; +} + +int ZEXPORT uncompress (dest, destLen, source, sourceLen) + Bytef *dest; + uLongf *destLen; + const Bytef *source; + uLong sourceLen; +{ + return uncompress2(dest, destLen, source, &sourceLen); } diff --git a/erts/emulator/zlib/zconf.h b/erts/emulator/zlib/zconf.h index 9987a77553..5e1d68a004 100644 --- a/erts/emulator/zlib/zconf.h +++ b/erts/emulator/zlib/zconf.h @@ -1,5 +1,5 @@ /* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2013 Jean-loup Gailly. + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -17,7 +17,7 @@ #ifdef Z_PREFIX /* may be set to #if 1 by ./configure */ # define Z_PREFIX_SET -/* all linked symbols */ +/* all linked symbols and init macros */ # define _dist_code z__dist_code # define _length_code z__length_code # define _tr_align z__tr_align @@ -29,6 +29,7 @@ # define adler32 z_adler32 # define adler32_combine z_adler32_combine # define adler32_combine64 z_adler32_combine64 +# define adler32_z z_adler32_z # ifndef Z_SOLO # define compress z_compress # define compress2 z_compress2 @@ -37,10 +38,14 @@ # define crc32 z_crc32 # define crc32_combine z_crc32_combine # define crc32_combine64 z_crc32_combine64 +# define crc32_z z_crc32_z # define deflate z_deflate # define deflateBound z_deflateBound # define deflateCopy z_deflateCopy # define deflateEnd z_deflateEnd +# define deflateGetDictionary z_deflateGetDictionary +# define deflateInit z_deflateInit +# define deflateInit2 z_deflateInit2 # define deflateInit2_ z_deflateInit2_ # define deflateInit_ z_deflateInit_ # define deflateParams z_deflateParams @@ -67,6 +72,8 @@ # define gzeof z_gzeof # define gzerror z_gzerror # define gzflush z_gzflush +# define gzfread z_gzfread +# define gzfwrite z_gzfwrite # define gzgetc z_gzgetc # define gzgetc_ z_gzgetc_ # define gzgets z_gzgets @@ -78,7 +85,6 @@ # define gzopen_w z_gzopen_w # endif # define gzprintf z_gzprintf -# define gzvprintf z_gzvprintf # define gzputc z_gzputc # define gzputs z_gzputs # define gzread z_gzread @@ -89,32 +95,39 @@ # define gztell z_gztell # define gztell64 z_gztell64 # define gzungetc z_gzungetc +# define gzvprintf z_gzvprintf # define gzwrite z_gzwrite # endif # define inflate z_inflate # define inflateBack z_inflateBack # define inflateBackEnd z_inflateBackEnd +# define inflateBackInit z_inflateBackInit # define inflateBackInit_ z_inflateBackInit_ +# define inflateCodesUsed z_inflateCodesUsed # define inflateCopy z_inflateCopy # define inflateEnd z_inflateEnd +# define inflateGetDictionary z_inflateGetDictionary # define inflateGetHeader z_inflateGetHeader +# define inflateInit z_inflateInit +# define inflateInit2 z_inflateInit2 # define inflateInit2_ z_inflateInit2_ # define inflateInit_ z_inflateInit_ # define inflateMark z_inflateMark # define inflatePrime z_inflatePrime # define inflateReset z_inflateReset # define inflateReset2 z_inflateReset2 +# define inflateResetKeep z_inflateResetKeep # define inflateSetDictionary z_inflateSetDictionary -# define inflateGetDictionary z_inflateGetDictionary # define inflateSync z_inflateSync # define inflateSyncPoint z_inflateSyncPoint # define inflateUndermine z_inflateUndermine -# define inflateResetKeep z_inflateResetKeep +# define inflateValidate z_inflateValidate # define inflate_copyright z_inflate_copyright # define inflate_fast z_inflate_fast # define inflate_table z_inflate_table # ifndef Z_SOLO # define uncompress z_uncompress +# define uncompress2 z_uncompress2 # endif # define zError z_zError # ifndef Z_SOLO @@ -224,9 +237,19 @@ # define z_const #endif -/* Some Mac compilers merge all .h files incorrectly: */ -#if defined(__MWERKS__)||defined(applec)||defined(THINK_C)||defined(__SC__) -# define NO_DUMMY_DECL +#ifdef Z_SOLO + typedef unsigned long z_size_t; +#else +# define z_longlong long long +# if defined(NO_SIZE_T) + typedef unsigned NO_SIZE_T z_size_t; +# elif defined(STDC) +# include <stddef.h> + typedef size_t z_size_t; +# else + typedef unsigned long z_size_t; +# endif +# undef z_longlong #endif /* Maximum value for memLevel in deflateInit2 */ @@ -256,7 +279,7 @@ Of course this will generally degrade compression (there's no free lunch). The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes + that is, 32K for windowBits=15 (default value) plus about 7 kilobytes for small objects. */ diff --git a/erts/emulator/zlib/zlib.h b/erts/emulator/zlib/zlib.h index 3e0c7672ac..f09cdaf1e0 100644 --- a/erts/emulator/zlib/zlib.h +++ b/erts/emulator/zlib/zlib.h @@ -1,7 +1,7 @@ /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.8, April 28th, 2013 + version 1.2.11, January 15th, 2017 - Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -37,11 +37,11 @@ extern "C" { #endif -#define ZLIB_VERSION "1.2.8" -#define ZLIB_VERNUM 0x1280 +#define ZLIB_VERSION "1.2.11" +#define ZLIB_VERNUM 0x12b0 #define ZLIB_VER_MAJOR 1 #define ZLIB_VER_MINOR 2 -#define ZLIB_VER_REVISION 8 +#define ZLIB_VER_REVISION 11 #define ZLIB_VER_SUBREVISION 0 /* @@ -65,7 +65,8 @@ extern "C" { with "gz". The gzip format is different from the zlib format. gzip is a gzip wrapper, documented in RFC 1952, wrapped around a deflate stream. - This library can optionally read and write gzip streams in memory as well. + This library can optionally read and write gzip and raw deflate streams in + memory as well. The zlib format was designed to be compact and fast for use in memory and on communications channels. The gzip format was designed for single- @@ -74,7 +75,7 @@ extern "C" { The library does not install any signal handler. The decoder checks the consistency of the compressed data, so the library should never crash - even in case of corrupted input. + even in the case of corrupted input. */ typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size)); @@ -87,7 +88,7 @@ typedef struct z_stream_s { uInt avail_in; /* number of bytes available at next_in */ uLong total_in; /* total number of input bytes read so far */ - Bytef *next_out; /* next output byte should be put there */ + Bytef *next_out; /* next output byte will go here */ uInt avail_out; /* remaining free space at next_out */ uLong total_out; /* total number of bytes output so far */ @@ -98,8 +99,9 @@ typedef struct z_stream_s { free_func zfree; /* used to free the internal state */ voidpf opaque; /* private data object passed to zalloc and zfree */ - int data_type; /* best guess about the data type: binary or text */ - uLong adler; /* adler32 value of the uncompressed data */ + int data_type; /* best guess about the data type: binary or text + for deflate, or the decoding state for inflate */ + uLong adler; /* Adler-32 or CRC-32 value of the uncompressed data */ uLong reserved; /* reserved for future use */ } z_stream; @@ -142,7 +144,9 @@ typedef gz_header FAR *gz_headerp; zalloc must return Z_NULL if there is not enough memory for the object. If zlib is used in a multi-threaded application, zalloc and zfree must be - thread safe. + thread safe. In that case, zlib is thread-safe. When zalloc and zfree are + Z_NULL on entry to the initialization function, they are set to internal + routines that use the standard library functions malloc() and free(). On 16-bit systems, the functions zalloc and zfree must be able to allocate exactly 65536 bytes, but will not be required to allocate more than this if @@ -155,7 +159,7 @@ typedef gz_header FAR *gz_headerp; The fields total_in and total_out can be used for statistics or progress reports. After compression, total_in holds the total size of the - uncompressed data and may be saved for use in the decompressor (particularly + uncompressed data and may be saved for use by the decompressor (particularly if the decompressor wants to decompress everything in a single step). */ @@ -200,7 +204,7 @@ typedef gz_header FAR *gz_headerp; #define Z_TEXT 1 #define Z_ASCII Z_TEXT /* for compatibility with 1.2.2 and earlier */ #define Z_UNKNOWN 2 -/* Possible values of the data_type field (though see inflate()) */ +/* Possible values of the data_type field for deflate() */ #define Z_DEFLATED 8 /* The deflate compression method (the only one supported in this version) */ @@ -258,11 +262,11 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); enough room in the output buffer), next_in and avail_in are updated and processing will resume at this point for the next call of deflate(). - - Provide more output starting at next_out and update next_out and avail_out + - Generate more output starting at next_out and update next_out and avail_out accordingly. This action is forced if the parameter flush is non zero. Forcing flush frequently degrades the compression ratio, so this parameter - should be set only when necessary (in interactive applications). Some - output may be provided even if flush is not set. + should be set only when necessary. Some output may be provided even if + flush is zero. Before the call of deflate(), the application should ensure that at least one of the actions is possible, by providing more input and/or consuming more @@ -271,7 +275,9 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); output when it wants, for example when the output buffer is full (avail_out == 0), or after each call of deflate(). If deflate returns Z_OK and with zero avail_out, it must be called again after making room in the output - buffer because there might be more output pending. + buffer because there might be more output pending. See deflatePending(), + which can be used if desired to determine whether or not there is more ouput + in that case. Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to decide how much data to accumulate before producing output, in order to @@ -292,8 +298,8 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); input data so far will be available to the decompressor, as for Z_SYNC_FLUSH. This completes the current deflate block and follows it with an empty fixed codes block that is 10 bits long. This assures that enough bytes are output - in order for the decompressor to finish the block before the empty fixed code - block. + in order for the decompressor to finish the block before the empty fixed + codes block. If flush is set to Z_BLOCK, a deflate block is completed and emitted, as for Z_SYNC_FLUSH, but the output is not aligned on a byte boundary, and up to @@ -319,34 +325,38 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); If the parameter flush is set to Z_FINISH, pending input is processed, pending output is flushed and deflate returns with Z_STREAM_END if there was - enough output space; if deflate returns with Z_OK, this function must be - called again with Z_FINISH and more output space (updated avail_out) but no - more input data, until it returns with Z_STREAM_END or an error. After - deflate has returned Z_STREAM_END, the only possible operations on the stream - are deflateReset or deflateEnd. - - Z_FINISH can be used immediately after deflateInit if all the compression - is to be done in a single step. In this case, avail_out must be at least the - value returned by deflateBound (see below). Then deflate is guaranteed to - return Z_STREAM_END. If not enough output space is provided, deflate will - not return Z_STREAM_END, and it must be called again as described above. - - deflate() sets strm->adler to the adler32 checksum of all input read - so far (that is, total_in bytes). + enough output space. If deflate returns with Z_OK or Z_BUF_ERROR, this + function must be called again with Z_FINISH and more output space (updated + avail_out) but no more input data, until it returns with Z_STREAM_END or an + error. After deflate has returned Z_STREAM_END, the only possible operations + on the stream are deflateReset or deflateEnd. + + Z_FINISH can be used in the first deflate call after deflateInit if all the + compression is to be done in a single step. In order to complete in one + call, avail_out must be at least the value returned by deflateBound (see + below). Then deflate is guaranteed to return Z_STREAM_END. If not enough + output space is provided, deflate will not return Z_STREAM_END, and it must + be called again as described above. + + deflate() sets strm->adler to the Adler-32 checksum of all input read + so far (that is, total_in bytes). If a gzip stream is being generated, then + strm->adler will be the CRC-32 checksum of the input read so far. (See + deflateInit2 below.) deflate() may update strm->data_type if it can make a good guess about - the input data type (Z_BINARY or Z_TEXT). In doubt, the data is considered - binary. This field is only for information purposes and does not affect the - compression algorithm in any manner. + the input data type (Z_BINARY or Z_TEXT). If in doubt, the data is + considered binary. This field is only for information purposes and does not + affect the compression algorithm in any manner. deflate() returns Z_OK if some progress has been made (more input processed or more output produced), Z_STREAM_END if all input has been consumed and all output has been produced (only when flush is set to Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example - if next_in or next_out was Z_NULL), Z_BUF_ERROR if no progress is possible - (for example avail_in or avail_out was zero). Note that Z_BUF_ERROR is not - fatal, and deflate() can be called again with more input and more output - space to continue compressing. + if next_in or next_out was Z_NULL or the state was inadvertently written over + by the application), or Z_BUF_ERROR if no progress is possible (for example + avail_in or avail_out was zero). Note that Z_BUF_ERROR is not fatal, and + deflate() can be called again with more input and more output space to + continue compressing. */ @@ -369,23 +379,21 @@ ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm)); Initializes the internal stream state for decompression. The fields next_in, avail_in, zalloc, zfree and opaque must be initialized before by - the caller. If next_in is not Z_NULL and avail_in is large enough (the - exact value depends on the compression method), inflateInit determines the - compression method from the zlib header and allocates all data structures - accordingly; otherwise the allocation will be deferred to the first call of - inflate. If zalloc and zfree are set to Z_NULL, inflateInit updates them to - use default allocation functions. + the caller. In the current version of inflate, the provided input is not + read or consumed. The allocation of a sliding window will be deferred to + the first call of inflate (if the decompression does not complete on the + first call). If zalloc and zfree are set to Z_NULL, inflateInit updates + them to use default allocation functions. inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_VERSION_ERROR if the zlib library version is incompatible with the version assumed by the caller, or Z_STREAM_ERROR if the parameters are invalid, such as a null pointer to the structure. msg is set to null if - there is no error message. inflateInit does not perform any decompression - apart from possibly reading the zlib header if present: actual decompression - will be done by inflate(). (So next_in and avail_in may be modified, but - next_out and avail_out are unused and unchanged.) The current implementation - of inflateInit() does not process any header information -- that is deferred - until inflate() is called. + there is no error message. inflateInit does not perform any decompression. + Actual decompression will be done by inflate(). So next_in, and avail_in, + next_out, and avail_out are unused and unchanged. The current + implementation of inflateInit() does not process any header information -- + that is deferred until inflate() is called. */ @@ -401,17 +409,20 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); - Decompress more input starting at next_in and update next_in and avail_in accordingly. If not all input can be processed (because there is not - enough room in the output buffer), next_in is updated and processing will - resume at this point for the next call of inflate(). + enough room in the output buffer), then next_in and avail_in are updated + accordingly, and processing will resume at this point for the next call of + inflate(). - - Provide more output starting at next_out and update next_out and avail_out + - Generate more output starting at next_out and update next_out and avail_out accordingly. inflate() provides as much output as possible, until there is no more input data or no more space in the output buffer (see below about the flush parameter). Before the call of inflate(), the application should ensure that at least one of the actions is possible, by providing more input and/or consuming more - output, and updating the next_* and avail_* values accordingly. The + output, and updating the next_* and avail_* values accordingly. If the + caller of inflate() does not provide both available input and available + output space, it is possible that there will be no progress made. The application can consume the uncompressed output when it wants, for example when the output buffer is full (avail_out == 0), or after each call of inflate(). If inflate returns Z_OK and with zero avail_out, it must be @@ -428,7 +439,7 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); gets to the end of that block, or when it runs out of data. The Z_BLOCK option assists in appending to or combining deflate streams. - Also to assist in this, on return inflate() will set strm->data_type to the + To assist in this, on return inflate() always sets strm->data_type to the number of unused bits in the last byte taken from strm->next_in, plus 64 if inflate() is currently decoding the last block in the deflate stream, plus 128 if inflate() returned immediately after decoding an end-of-block code or @@ -454,7 +465,7 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); this case all pending input is processed and all pending output is flushed; avail_out must be large enough to hold all of the uncompressed data for the operation to complete. (The size of the uncompressed data may have been - saved by the compressor for this purpose.) The use of Z_FINISH is not + saved by the compressor for this purpose.) The use of Z_FINISH is not required to perform an inflation in one step. However it may be used to inform inflate that a faster approach can be used for the single inflate() call. Z_FINISH also informs inflate to not maintain a sliding window if the @@ -476,32 +487,33 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); chosen by the compressor and returns Z_NEED_DICT; otherwise it sets strm->adler to the Adler-32 checksum of all output produced so far (that is, total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described - below. At the end of the stream, inflate() checks that its computed adler32 + below. At the end of the stream, inflate() checks that its computed Adler-32 checksum is equal to that saved by the compressor and returns Z_STREAM_END only if the checksum is correct. inflate() can decompress and check either zlib-wrapped or gzip-wrapped deflate data. The header type is detected automatically, if requested when initializing with inflateInit2(). Any information contained in the gzip - header is not retained, so applications that need that information should - instead use raw inflate, see inflateInit2() below, or inflateBack() and - perform their own processing of the gzip header and trailer. When processing + header is not retained unless inflateGetHeader() is used. When processing gzip-wrapped deflate data, strm->adler32 is set to the CRC-32 of the output - producted so far. The CRC-32 is checked against the gzip trailer. + produced so far. The CRC-32 is checked against the gzip trailer, as is the + uncompressed length, modulo 2^32. inflate() returns Z_OK if some progress has been made (more input processed or more output produced), Z_STREAM_END if the end of the compressed data has been reached and all uncompressed output has been produced, Z_NEED_DICT if a preset dictionary is needed at this point, Z_DATA_ERROR if the input data was corrupted (input stream not conforming to the zlib format or incorrect check - value), Z_STREAM_ERROR if the stream structure was inconsistent (for example - next_in or next_out was Z_NULL), Z_MEM_ERROR if there was not enough memory, - Z_BUF_ERROR if no progress is possible or if there was not enough room in the - output buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and + value, in which case strm->msg points to a string with a more specific + error), Z_STREAM_ERROR if the stream structure was inconsistent (for example + next_in or next_out was Z_NULL, or the state was inadvertently written over + by the application), Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR + if no progress was possible or if there was not enough room in the output + buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and inflate() can be called again with more input and more output space to continue decompressing. If Z_DATA_ERROR is returned, the application may then call inflateSync() to look for a good compression block if a partial - recovery of the data is desired. + recovery of the data is to be attempted. */ @@ -511,9 +523,8 @@ ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm)); This function discards any unprocessed input and does not flush any pending output. - inflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state - was inconsistent. In the error case, msg may be set but then points to a - static string (which must not be deallocated). + inflateEnd returns Z_OK if success, or Z_STREAM_ERROR if the stream state + was inconsistent. */ @@ -544,16 +555,29 @@ ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm, compression at the expense of memory usage. The default value is 15 if deflateInit is used instead. + For the current implementation of deflate(), a windowBits value of 8 (a + window size of 256 bytes) is not supported. As a result, a request for 8 + will result in 9 (a 512-byte window). In that case, providing 8 to + inflateInit2() will result in an error when the zlib header with 9 is + checked against the initialization of inflate(). The remedy is to not use 8 + with deflateInit2() with this initialization, or at least in that case use 9 + with inflateInit2(). + windowBits can also be -8..-15 for raw deflate. In this case, -windowBits determines the window size. deflate() will then generate raw deflate data - with no zlib header or trailer, and will not compute an adler32 check value. + with no zlib header or trailer, and will not compute a check value. windowBits can also be greater than 15 for optional gzip encoding. Add 16 to windowBits to write a simple gzip header and trailer around the compressed data instead of a zlib wrapper. The gzip header will have no file name, no extra data, no comment, no modification time (set to zero), no - header crc, and the operating system will be set to 255 (unknown). If a - gzip stream is being written, strm->adler is a crc32 instead of an adler32. + header crc, and the operating system will be set to the appropriate value, + if the operating system was determined at compile time. If a gzip stream is + being written, strm->adler is a CRC-32 instead of an Adler-32. + + For raw deflate or gzip encoding, a request for a 256-byte window is + rejected as invalid, since only the zlib header provides a means of + transmitting the window size to the decompressor. The memLevel parameter specifies how much memory should be allocated for the internal compression state. memLevel=1 uses minimum memory but is @@ -614,12 +638,12 @@ ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, addition, the current implementation of deflate will use at most the window size minus 262 bytes of the provided dictionary. - Upon return of this function, strm->adler is set to the adler32 value + Upon return of this function, strm->adler is set to the Adler-32 value of the dictionary; the decompressor may later use this value to determine - which dictionary has been used by the compressor. (The adler32 value + which dictionary has been used by the compressor. (The Adler-32 value applies to the whole dictionary even if only a subset of the dictionary is actually used by the compressor.) If a raw deflate was requested, then the - adler32 value is not computed and strm->adler is not set. + Adler-32 value is not computed and strm->adler is not set. deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is @@ -628,6 +652,28 @@ ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, not perform any compression: this will be done by deflate(). */ +ZEXTERN int ZEXPORT deflateGetDictionary OF((z_streamp strm, + Bytef *dictionary, + uInt *dictLength)); +/* + Returns the sliding dictionary being maintained by deflate. dictLength is + set to the number of bytes in the dictionary, and that many bytes are copied + to dictionary. dictionary must have enough space, where 32768 bytes is + always enough. If deflateGetDictionary() is called with dictionary equal to + Z_NULL, then only the dictionary length is returned, and nothing is copied. + Similary, if dictLength is Z_NULL, then it is not set. + + deflateGetDictionary() may return a length less than the window size, even + when more than the window size in input has been provided. It may return up + to 258 bytes less in that case, due to how zlib's implementation of deflate + manages the sliding window and lookahead for matches, where matches can be + up to 258 bytes long. If the application needs the last window-size bytes of + input, then that would need to be saved by the application outside of zlib. + + deflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the + stream state is inconsistent. +*/ + ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, z_streamp source)); /* @@ -648,10 +694,10 @@ ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm)); /* - This function is equivalent to deflateEnd followed by deflateInit, - but does not free and reallocate all the internal compression state. The - stream will keep the same compression level and any other attributes that - may have been set by deflateInit2. + This function is equivalent to deflateEnd followed by deflateInit, but + does not free and reallocate the internal compression state. The stream + will leave the compression level and any other attributes that may have been + set unchanged. deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL). @@ -662,20 +708,36 @@ ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm, int strategy)); /* Dynamically update the compression level and compression strategy. The - interpretation of level and strategy is as in deflateInit2. This can be + interpretation of level and strategy is as in deflateInit2(). This can be used to switch between compression and straight copy of the input data, or to switch to a different kind of input data requiring a different strategy. - If the compression level is changed, the input available so far is - compressed with the old level (and may be flushed); the new level will take - effect only at the next call of deflate(). - - Before the call of deflateParams, the stream state must be set as for - a call of deflate(), since the currently available input may have to be - compressed and flushed. In particular, strm->avail_out must be non-zero. - - deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source - stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR if - strm->avail_out was zero. + If the compression approach (which is a function of the level) or the + strategy is changed, and if any input has been consumed in a previous + deflate() call, then the input available so far is compressed with the old + level and strategy using deflate(strm, Z_BLOCK). There are three approaches + for the compression levels 0, 1..3, and 4..9 respectively. The new level + and strategy will take effect at the next call of deflate(). + + If a deflate(strm, Z_BLOCK) is performed by deflateParams(), and it does + not have enough output space to complete, then the parameter change will not + take effect. In this case, deflateParams() can be called again with the + same parameters and more output space to try again. + + In order to assure a change in the parameters on the first try, the + deflate stream should be flushed using deflate() with Z_BLOCK or other flush + request until strm.avail_out is not zero, before calling deflateParams(). + Then no more input data should be provided before the deflateParams() call. + If this is done, the old level and strategy will be applied to the data + compressed before deflateParams(), and the new level and strategy will be + applied to the the data compressed after deflateParams(). + + deflateParams returns Z_OK on success, Z_STREAM_ERROR if the source stream + state was inconsistent or if a parameter was invalid, or Z_BUF_ERROR if + there was not enough output space to complete the compression of the + available input data before a change in the strategy or approach. Note that + in the case of a Z_BUF_ERROR, the parameters are not changed. A return + value of Z_BUF_ERROR is not fatal, in which case deflateParams() can be + retried with more output space. */ ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm, @@ -793,7 +855,7 @@ ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, is for use with other formats that use the deflate compressed data format such as zip. Those formats provide their own check values. If a custom format is developed using the raw deflate format for compressed data, it is - recommended that a check value such as an adler32 or a crc32 be applied to + recommended that a check value such as an Adler-32 or a CRC-32 be applied to the uncompressed data as is done in the zlib, gzip, and zip formats. For most applications, the zlib format should be used as is. Note that comments above on the use in deflateInit2() applies to the magnitude of windowBits. @@ -802,7 +864,10 @@ ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, 32 to windowBits to enable zlib and gzip decoding with automatic header detection, or add 16 to decode only the gzip format (the zlib format will return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a - crc32 instead of an adler32. + CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see + below), inflate() will not automatically decode concatenated gzip streams. + inflate() will return Z_STREAM_END at the end of the gzip stream. The state + would need to be reset to continue decoding a subsequent gzip stream. inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_VERSION_ERROR if the zlib library version is incompatible with the @@ -823,7 +888,7 @@ ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, Initializes the decompression dictionary from the given uncompressed byte sequence. This function must be called immediately after a call of inflate, if that call returned Z_NEED_DICT. The dictionary chosen by the compressor - can be determined from the adler32 value returned by that call of inflate. + can be determined from the Adler-32 value returned by that call of inflate. The compressor and decompressor must use exactly the same dictionary (see deflateSetDictionary). For raw inflate, this function can be called at any time to set the dictionary. If the provided dictionary is smaller than the @@ -834,7 +899,7 @@ ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the - expected one (incorrect adler32 value). inflateSetDictionary does not + expected one (incorrect Adler-32 value). inflateSetDictionary does not perform any decompression: this will be done by subsequent calls of inflate(). */ @@ -892,7 +957,7 @@ ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest, ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm)); /* This function is equivalent to inflateEnd followed by inflateInit, - but does not free and reallocate all the internal decompression state. The + but does not free and reallocate the internal decompression state. The stream will keep attributes that may have been set by inflateInit2. inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source @@ -904,7 +969,9 @@ ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm, /* This function is the same as inflateReset, but it also permits changing the wrap and window size requests. The windowBits parameter is interpreted - the same as it is for inflateInit2. + the same as it is for inflateInit2. If the window size is changed, then the + memory allocated for the window is freed, and the window will be reallocated + by inflate() if needed. inflateReset2 returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL), or if @@ -956,7 +1023,7 @@ ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm)); location in the input stream can be determined from avail_in and data_type as noted in the description for the Z_BLOCK flush parameter for inflate. - inflateMark returns the value noted above or -1 << 16 if the provided + inflateMark returns the value noted above, or -65536 if the provided source stream state was inconsistent. */ @@ -1048,9 +1115,9 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, This routine would normally be used in a utility that reads zip or gzip files and writes out uncompressed files. The utility would decode the header and process the trailer on its own, hence this routine expects only - the raw deflate stream to decompress. This is different from the normal - behavior of inflate(), which expects either a zlib or gzip header and - trailer around the deflate stream. + the raw deflate stream to decompress. This is different from the default + behavior of inflate(), which expects a zlib header and trailer around the + deflate stream. inflateBack() uses two subroutines supplied by the caller that are then called by inflateBack() for input and output. inflateBack() calls those @@ -1059,12 +1126,12 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, parameters and return types are defined above in the in_func and out_func typedefs. inflateBack() will call in(in_desc, &buf) which should return the number of bytes of provided input, and a pointer to that input in buf. If - there is no input available, in() must return zero--buf is ignored in that - case--and inflateBack() will return a buffer error. inflateBack() will call - out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. out() - should return zero on success, or non-zero on failure. If out() returns - non-zero, inflateBack() will return with an error. Neither in() nor out() - are permitted to change the contents of the window provided to + there is no input available, in() must return zero -- buf is ignored in that + case -- and inflateBack() will return a buffer error. inflateBack() will + call out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. + out() should return zero on success, or non-zero on failure. If out() + returns non-zero, inflateBack() will return with an error. Neither in() nor + out() are permitted to change the contents of the window provided to inflateBackInit(), which is also the buffer that out() uses to write from. The length written by out() will be at most the window size. Any non-zero amount of input may be provided by in(). @@ -1092,7 +1159,7 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, using strm->next_in which will be Z_NULL only if in() returned an error. If strm->next_in is not Z_NULL, then the Z_BUF_ERROR was due to out() returning non-zero. (in() will always be called before out(), so strm->next_in is - assured to be defined if out() returns non-zero.) Note that inflateBack() + assured to be defined if out() returns non-zero.) Note that inflateBack() cannot return Z_OK. */ @@ -1114,7 +1181,7 @@ ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void)); 7.6: size of z_off_t Compiler, assembler, and debug options: - 8: DEBUG + 8: ZLIB_DEBUG 9: ASMV or ASMINF -- use ASM code 10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention 11: 0 (reserved) @@ -1164,7 +1231,8 @@ ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, the byte length of the source buffer. Upon entry, destLen is the total size of the destination buffer, which must be at least the value returned by compressBound(sourceLen). Upon exit, destLen is the actual size of the - compressed buffer. + compressed data. compress() is equivalent to compress2() with a level + parameter of Z_DEFAULT_COMPRESSION. compress returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output @@ -1180,7 +1248,7 @@ ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen, length of the source buffer. Upon entry, destLen is the total size of the destination buffer, which must be at least the value returned by compressBound(sourceLen). Upon exit, destLen is the actual size of the - compressed buffer. + compressed data. compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output buffer, @@ -1203,7 +1271,7 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, uncompressed data. (The size of the uncompressed data must have been saved previously by the compressor and transmitted to the decompressor by some mechanism outside the scope of this compression library.) Upon exit, destLen - is the actual size of the uncompressed buffer. + is the actual size of the uncompressed data. uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output @@ -1212,6 +1280,14 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, buffer with the uncompressed data up to that point. */ +ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen)); +/* + Same as uncompress, except that sourceLen is a pointer, where the + length of the source is *sourceLen. On return, *sourceLen is the number of + source bytes consumed. +*/ + /* gzip file access functions */ /* @@ -1290,10 +1366,9 @@ ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); default buffer size is 8192 bytes. This function must be called after gzopen() or gzdopen(), and before any other calls that read or write the file. The buffer memory allocation is always deferred to the first read or - write. Two buffers are allocated, either both of the specified size when - writing, or one of the specified size and the other twice that size when - reading. A larger buffer size of, for example, 64K or 128K bytes will - noticeably increase the speed of decompression (reading). + write. Three times that size in buffer space is allocated. A larger buffer + size of, for example, 64K or 128K bytes will noticeably increase the speed + of decompression (reading). The new buffer size also affects the maximum length for gzprintf(). @@ -1304,10 +1379,12 @@ ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy)); /* Dynamically update the compression level or strategy. See the description - of deflateInit2 for the meaning of these parameters. + of deflateInit2 for the meaning of these parameters. Previously provided + data is flushed before the parameter change. - gzsetparams returns Z_OK if success, or Z_STREAM_ERROR if the file was not - opened for writing. + gzsetparams returns Z_OK if success, Z_STREAM_ERROR if the file was not + opened for writing, Z_ERRNO if there is an error writing the flushed data, + or Z_MEM_ERROR if there is a memory allocation error. */ ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); @@ -1335,7 +1412,35 @@ ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); case. gzread returns the number of uncompressed bytes actually read, less than - len for end of file, or -1 for error. + len for end of file, or -1 for error. If len is too large to fit in an int, + then nothing is read, -1 is returned, and the error state is set to + Z_STREAM_ERROR. +*/ + +ZEXTERN z_size_t ZEXPORT gzfread OF((voidp buf, z_size_t size, z_size_t nitems, + gzFile file)); +/* + Read up to nitems items of size size from file to buf, otherwise operating + as gzread() does. This duplicates the interface of stdio's fread(), with + size_t request and return types. If the library defines size_t, then + z_size_t is identical to size_t. If not, then z_size_t is an unsigned + integer type that can contain a pointer. + + gzfread() returns the number of full items read of size size, or zero if + the end of the file was reached and a full item could not be read, or if + there was an error. gzerror() must be consulted if zero is returned in + order to determine if there was an error. If the multiplication of size and + nitems overflows, i.e. the product does not fit in a z_size_t, then nothing + is read, zero is returned, and the error state is set to Z_STREAM_ERROR. + + In the event that the end of file is reached and only a partial item is + available at the end, i.e. the remaining uncompressed data length is not a + multiple of size, then the final partial item is nevetheless read into buf + and the end-of-file flag is set. The length of the partial item read is not + provided, but could be inferred from the result of gztell(). This behavior + is the same as the behavior of fread() implementations in common libraries, + but it prevents the direct use of gzfread() to read a concurrently written + file, reseting and retrying on end-of-file, when size is not 1. */ ZEXTERN int ZEXPORT gzwrite OF((gzFile file, @@ -1346,19 +1451,33 @@ ZEXTERN int ZEXPORT gzwrite OF((gzFile file, error. */ +ZEXTERN z_size_t ZEXPORT gzfwrite OF((voidpc buf, z_size_t size, + z_size_t nitems, gzFile file)); +/* + gzfwrite() writes nitems items of size size from buf to file, duplicating + the interface of stdio's fwrite(), with size_t request and return types. If + the library defines size_t, then z_size_t is identical to size_t. If not, + then z_size_t is an unsigned integer type that can contain a pointer. + + gzfwrite() returns the number of full items written of size size, or zero + if there was an error. If the multiplication of size and nitems overflows, + i.e. the product does not fit in a z_size_t, then nothing is written, zero + is returned, and the error state is set to Z_STREAM_ERROR. +*/ + ZEXTERN int ZEXPORTVA gzprintf Z_ARG((gzFile file, const char *format, ...)); /* Converts, formats, and writes the arguments to the compressed file under control of the format string, as in fprintf. gzprintf returns the number of - uncompressed bytes actually written, or 0 in case of error. The number of - uncompressed bytes written is limited to 8191, or one less than the buffer - size given to gzbuffer(). The caller should assure that this limit is not - exceeded. If it is exceeded, then gzprintf() will return an error (0) with - nothing written. In this case, there may also be a buffer overflow with - unpredictable consequences, which is possible only if zlib was compiled with - the insecure functions sprintf() or vsprintf() because the secure snprintf() - or vsnprintf() functions were not available. This can be determined using - zlibCompileFlags(). + uncompressed bytes actually written, or a negative zlib error code in case + of error. The number of uncompressed bytes written is limited to 8191, or + one less than the buffer size given to gzbuffer(). The caller should assure + that this limit is not exceeded. If it is exceeded, then gzprintf() will + return an error (0) with nothing written. In this case, there may also be a + buffer overflow with unpredictable consequences, which is possible only if + zlib was compiled with the insecure functions sprintf() or vsprintf() + because the secure snprintf() or vsnprintf() functions were not available. + This can be determined using zlibCompileFlags(). */ ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s)); @@ -1418,7 +1537,7 @@ ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush)); If the flush parameter is Z_FINISH, the remaining data is written and the gzip stream is completed in the output. If gzwrite() is called again, a new gzip stream will be started in the output. gzread() is able to read such - concatented gzip streams. + concatenated gzip streams. gzflush should be called only when strictly necessary because it will degrade compression if called too often. @@ -1572,7 +1691,7 @@ ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); return the updated checksum. If buf is Z_NULL, this function returns the required initial value for the checksum. - An Adler-32 checksum is almost as reliable as a CRC32 but can be computed + An Adler-32 checksum is almost as reliable as a CRC-32 but can be computed much faster. Usage example: @@ -1585,6 +1704,12 @@ ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); if (adler != original_adler) error(); */ +ZEXTERN uLong ZEXPORT adler32_z OF((uLong adler, const Bytef *buf, + z_size_t len)); +/* + Same as adler32(), but with a size_t length. +*/ + /* ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2, z_off_t len2)); @@ -1614,6 +1739,12 @@ ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)); if (crc != original_crc) error(); */ +ZEXTERN uLong ZEXPORT crc32_z OF((uLong adler, const Bytef *buf, + z_size_t len)); +/* + Same as crc32(), but with a size_t length. +*/ + /* ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2)); @@ -1644,19 +1775,35 @@ ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, unsigned char FAR *window, const char *version, int stream_size)); -#define deflateInit(strm, level) \ - deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) -#define inflateInit(strm) \ - inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) -#define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ - deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ - (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) -#define inflateInit2(strm, windowBits) \ - inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ - (int)sizeof(z_stream)) -#define inflateBackInit(strm, windowBits, window) \ - inflateBackInit_((strm), (windowBits), (window), \ - ZLIB_VERSION, (int)sizeof(z_stream)) +#ifdef Z_PREFIX_SET +# define z_deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define z_inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#else +# define deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#endif #ifndef Z_SOLO @@ -1676,10 +1823,10 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ #ifdef Z_PREFIX_SET # undef z_gzgetc # define z_gzgetc(g) \ - ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : gzgetc(g)) + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) #else # define gzgetc(g) \ - ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : gzgetc(g)) + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) #endif /* provide 64-bit offset functions if _LARGEFILE64_SOURCE defined, and/or @@ -1737,19 +1884,16 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ #endif /* !Z_SOLO */ -/* hack for buggy compilers */ -#if !defined(ZUTIL_H) && !defined(NO_DUMMY_DECL) - struct internal_state {int dummy;}; -#endif - /* undocumented functions */ ZEXTERN const char * ZEXPORT zError OF((int)); ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp)); ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table OF((void)); ZEXTERN int ZEXPORT inflateUndermine OF((z_streamp, int)); +ZEXTERN int ZEXPORT inflateValidate OF((z_streamp, int)); +ZEXTERN unsigned long ZEXPORT inflateCodesUsed OF ((z_streamp)); ZEXTERN int ZEXPORT inflateResetKeep OF((z_streamp)); ZEXTERN int ZEXPORT deflateResetKeep OF((z_streamp)); -#if defined(_WIN32) && !defined(Z_SOLO) +#if (defined(_WIN32) || defined(__CYGWIN__)) && !defined(Z_SOLO) ZEXTERN gzFile ZEXPORT gzopen_w OF((const wchar_t *path, const char *mode)); #endif diff --git a/erts/emulator/zlib/zlib.mk b/erts/emulator/zlib/zlib.mk index 3f0d64d250..b51b4ec8d6 100644 --- a/erts/emulator/zlib/zlib.mk +++ b/erts/emulator/zlib/zlib.mk @@ -52,7 +52,7 @@ ifeq ($(TYPE),gcov) ZLIB_CFLAGS = -O0 -fprofile-arcs -ftest-coverage $(DEBUG_CFLAGS) $(DEFS) $(THR_DEFS) else # gcov ifeq ($(TYPE),debug) -ZLIB_CFLAGS = $(DEBUG_CFLAGS) $(DEFS) $(THR_DEFS) +ZLIB_CFLAGS = -DZLIB_DEBUG=1 $(DEBUG_CFLAGS) $(DEFS) $(THR_DEFS) else # debug ZLIB_CFLAGS = $(subst -O2, -O3, $(CONFIGURE_CFLAGS) $(DEFS) $(THR_DEFS)) #ZLIB_CFLAGS=-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7 @@ -62,6 +62,9 @@ ZLIB_CFLAGS = $(subst -O2, -O3, $(CONFIGURE_CFLAGS) $(DEFS) $(THR_DEFS)) endif # debug endif # gcov +# Don't fail if _LFS64_LARGEFILE is undefined +ZLIB_CFLAGS := $(filter-out -Werror=undef,$(ZLIB_CFLAGS)) + ifeq ($(TARGET), win32) $(ZLIB_LIBRARY): $(ZLIB_OBJS) $(V_AR) -out:$@ $(ZLIB_OBJS) diff --git a/erts/emulator/zlib/zutil.c b/erts/emulator/zlib/zutil.c index 27a8af4a2b..a76c6b0c7e 100644 --- a/erts/emulator/zlib/zutil.c +++ b/erts/emulator/zlib/zutil.c @@ -1,33 +1,27 @@ /* zutil.c -- target dependent utility functions for the compression library - * Copyright (C) 1995-2005, 2010, 2011, 2012 Jean-loup Gailly. + * Copyright (C) 1995-2017 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif #include "zutil.h" #ifndef Z_SOLO # include "gzguts.h" #endif -#ifndef NO_DUMMY_DECL -struct internal_state {int dummy;}; /* for buggy compilers */ -#endif - z_const char * const z_errmsg[10] = { -"need dictionary", /* Z_NEED_DICT 2 */ -"stream end", /* Z_STREAM_END 1 */ -"", /* Z_OK 0 */ -"file error", /* Z_ERRNO (-1) */ -"stream error", /* Z_STREAM_ERROR (-2) */ -"data error", /* Z_DATA_ERROR (-3) */ -"insufficient memory", /* Z_MEM_ERROR (-4) */ -"buffer error", /* Z_BUF_ERROR (-5) */ -"incompatible version",/* Z_VERSION_ERROR (-6) */ -""}; + (z_const char *)"need dictionary", /* Z_NEED_DICT 2 */ + (z_const char *)"stream end", /* Z_STREAM_END 1 */ + (z_const char *)"", /* Z_OK 0 */ + (z_const char *)"file error", /* Z_ERRNO (-1) */ + (z_const char *)"stream error", /* Z_STREAM_ERROR (-2) */ + (z_const char *)"data error", /* Z_DATA_ERROR (-3) */ + (z_const char *)"insufficient memory", /* Z_MEM_ERROR (-4) */ + (z_const char *)"buffer error", /* Z_BUF_ERROR (-5) */ + (z_const char *)"incompatible version",/* Z_VERSION_ERROR (-6) */ + (z_const char *)"" +}; const char * ZEXPORT zlibVersion() @@ -64,7 +58,7 @@ uLong ZEXPORT zlibCompileFlags() case 8: flags += 2 << 6; break; default: flags += 3 << 6; } -#ifdef DEBUG +#ifdef ZLIB_DEBUG flags += 1 << 8; #endif #if defined(ASMV) || defined(ASMINF) @@ -118,8 +112,8 @@ uLong ZEXPORT zlibCompileFlags() return flags; } -#ifdef DEBUG - +#ifdef ZLIB_DEBUG +#include <stdlib.h> # ifndef verbose # define verbose 0 # endif @@ -222,9 +216,11 @@ local ptr_table table[MAX_PTR]; voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size) { - voidpf buf = opaque; /* just to make some compilers happy */ + voidpf buf; ulg bsize = (ulg)items*size; + (void)opaque; + /* If we allocate less than 65520 bytes, we assume that farmalloc * will return a usable pointer which doesn't have to be normalized. */ @@ -247,6 +243,9 @@ voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size) void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) { int n; + + (void)opaque; + if (*(ush*)&ptr != 0) { /* object < 64K */ farfree(ptr); return; @@ -262,7 +261,6 @@ void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) next_ptr--; return; } - ptr = opaque; /* just to make some compilers happy */ Assert(0, "zcfree: ptr not found"); } @@ -281,13 +279,13 @@ void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, uInt items, uInt size) { - if (opaque) opaque = 0; /* to make compiler happy */ + (void)opaque; return _halloc((long)items, size); } void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) { - if (opaque) opaque = 0; /* to make compiler happy */ + (void)opaque; _hfree(ptr); } @@ -309,7 +307,7 @@ voidpf ZLIB_INTERNAL zcalloc (opaque, items, size) unsigned items; unsigned size; { - if (opaque) items += size - size; /* make compiler happy */ + (void)opaque; return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) : (voidpf)calloc(items, size); } @@ -318,8 +316,8 @@ void ZLIB_INTERNAL zcfree (opaque, ptr) voidpf opaque; voidpf ptr; { + (void)opaque; free(ptr); - if (opaque) return; /* make compiler happy */ } #endif /* MY_ZCALLOC */ diff --git a/erts/emulator/zlib/zutil.h b/erts/emulator/zlib/zutil.h index 24ab06b1cf..b079ea6a80 100644 --- a/erts/emulator/zlib/zutil.h +++ b/erts/emulator/zlib/zutil.h @@ -1,5 +1,5 @@ /* zutil.h -- internal interface and configuration of the compression library - * Copyright (C) 1995-2013 Jean-loup Gailly. + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -36,7 +36,9 @@ #ifndef local # define local static #endif -/* compile with -Dlocal if your debugger can't find static symbols */ +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ typedef unsigned char uch; typedef uch FAR uchf; @@ -98,28 +100,38 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #endif #ifdef AMIGA -# define OS_CODE 0x01 +# define OS_CODE 1 #endif #if defined(VAXC) || defined(VMS) -# define OS_CODE 0x02 +# define OS_CODE 2 # define F_OPEN(name, mode) \ fopen((name), (mode), "mbc=60", "ctx=stm", "rfm=fix", "mrs=512") #endif +#ifdef __370__ +# if __TARGET_LIB__ < 0x20000000 +# define OS_CODE 4 +# elif __TARGET_LIB__ < 0x40000000 +# define OS_CODE 11 +# else +# define OS_CODE 8 +# endif +#endif + #if defined(ATARI) || defined(atarist) -# define OS_CODE 0x05 +# define OS_CODE 5 #endif #ifdef OS2 -# define OS_CODE 0x06 +# define OS_CODE 6 # if defined(M_I86) && !defined(Z_SOLO) # include <malloc.h> # endif #endif #if defined(MACOS) || defined(TARGET_OS_MAC) -# define OS_CODE 0x07 +# define OS_CODE 7 # ifndef Z_SOLO # if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os # include <unix.h> /* for fdopen */ @@ -131,18 +143,24 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # endif #endif -#ifdef TOPS20 -# define OS_CODE 0x0a +#ifdef __acorn +# define OS_CODE 13 #endif -#ifdef WIN32 -# ifndef __CYGWIN__ /* Cygwin is Unix, not Win32 */ -# define OS_CODE 0x0b -# endif +#if defined(WIN32) && !defined(__CYGWIN__) +# define OS_CODE 10 +#endif + +#ifdef _BEOS_ +# define OS_CODE 16 +#endif + +#ifdef __TOS_OS400__ +# define OS_CODE 18 #endif -#ifdef __50SERIES /* Prime/PRIMOS */ -# define OS_CODE 0x0f +#ifdef __APPLE__ +# define OS_CODE 19 #endif #if defined(_BEOS_) || defined(RISCOS) @@ -177,7 +195,7 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* common defaults */ #ifndef OS_CODE -# define OS_CODE 0x03 /* assume Unix */ +# define OS_CODE 3 /* assume Unix */ #endif #ifndef F_OPEN @@ -216,7 +234,7 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #endif /* Diagnostic functions */ -#ifdef DEBUG +#ifdef ZLIB_DEBUG # include <stdio.h> extern int ZLIB_INTERNAL z_verbose; extern void ZLIB_INTERNAL z_error OF((char *m)); diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam Binary files differindex 40d9f89378..23a8e8f210 100644 --- a/erts/preloaded/ebin/prim_inet.beam +++ b/erts/preloaded/ebin/prim_inet.beam diff --git a/erts/preloaded/src/prim_inet.erl b/erts/preloaded/src/prim_inet.erl index fa781d4a33..cc2711b540 100644 --- a/erts/preloaded/src/prim_inet.erl +++ b/erts/preloaded/src/prim_inet.erl @@ -49,9 +49,15 @@ -include("inet_sctp.hrl"). -include("inet_int.hrl"). -%-define(DEBUG, 1). +%%%-define(DEBUG, 1). -ifdef(DEBUG). --define(DBG_FORMAT(Format, Args), (io:format((Format), (Args)))). +-define( + DBG_FORMAT(Format, Args), + begin + %% io:format((Format), (Args)), + erlang:display(lists:flatten(io_lib:format((Format), (Args)))), + ok + end). -else. -define(DBG_FORMAT(Format, Args), ok). -endif. @@ -150,39 +156,96 @@ shutdown_1(S, How) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% close(S) when is_port(S) -> + ?DBG_FORMAT("prim_inet:close(~p)~n", [S]), case getopt(S, linger) of {ok,{true,0}} -> close_port(S); - _ -> - case subscribe(S, [subs_empty_out_q]) of - {ok, [{subs_empty_out_q,N}]} when N > 0 -> - close_pend_loop(S, N); %% wait for pending output to be sent - _ -> - close_port(S) - end + {ok,{true,T}} -> + %% Wait for T seconds for pending output to be sent + %% + %% Note that this handling of Linger may look ok, + %% but sweeps some problems under the rug since + %% there are OS buffers that may have remaining data + %% after the inet driver has emptied its buffers. + %% But Linger for nonblocking sockets is broken + %% anyway on all OS:es, according to hearsay, + %% and is a contradiction in itself. + %% We have hereby done our best... + %% + Tref = erlang:start_timer(T * 1000, self(), close_port), + close_pend_loop(S, Tref, undefined); + _ -> % Regard this as {ok,{false,_}} + case subscribe(S, [subs_empty_out_q]) of + {ok, [{subs_empty_out_q,N}]} when N > 0 -> + %% Wait for pending output to be sent + DefaultT = 180000, % Arbitrary system timeout 3 min + Tref = erlang:start_timer(DefaultT, self(), close_port), + close_pend_loop(S, Tref, N); + _ -> + %% Subscribe failed or empty out q - give up or done + close_port(S) + end end. -close_pend_loop(S, N) -> +close_pend_loop(S, Tref, N) -> + ?DBG_FORMAT("prim_inet:close_pend_loop(~p, _, ~p)~n", [S,N]), receive - {empty_out_q,S} -> - close_port(S) + {timeout,Tref,_} -> % Linger timeout + ?DBG_FORMAT("prim_inet:close_pend_loop(~p, _, _) timeout~n", [S]), + close_port(S); + {empty_out_q,S} when N =/= undefined -> + ?DBG_FORMAT( + "prim_inet:close_pend_loop(~p, _, _) empty_out_q~n", [S]), + close_port(S, Tref) after ?INET_CLOSE_TIMEOUT -> case getstat(S, [send_pend]) of {ok, [{send_pend,N1}]} -> + ?DBG_FORMAT( + "prim_inet:close_pend_loop(~p, _, _) send_pend ~p~n", + [S,N1]), if - N1 =:= N -> - close_port(S); - true -> - close_pend_loop(S, N1) + N1 =:= 0 -> + %% Empty outq - done + close_port(S, Tref); + N =:= undefined -> + %% Within linger time - wait some more + close_pend_loop(S, Tref, N); + N1 =:= N -> + %% Inactivity - give up + close_port(S, Tref); + true -> + %% Still moving - wait some more + close_pend_loop(S, Tref, N) end; - _ -> - close_port(S) - end + _Stat -> + %% Failed getstat - give up + ?DBG_FORMAT( + "prim_inet:close_pend_loop(~p, _, _) getstat ~p~n", + [S,_Stat]), + close_port(S, Tref) + end end. + +close_port(S, Tref) -> + ?DBG_FORMAT("prim_inet:close_port(~p, _)~n", [S]), + case erlang:cancel_timer(Tref) of + false -> + receive + {timeout,Tref,_} -> + ok + end; + _N -> + ok + end, + close_port(S). +%% close_port(S) -> - catch erlang:port_close(S), - receive {'EXIT',S,_} -> ok after 0 -> ok end. + ?DBG_FORMAT("prim_inet:close_port(~p)~n", [S]), + _Closed = (catch erlang:port_close(S)), + receive {'EXIT',S,_} -> ok after 0 -> ok end, + ?DBG_FORMAT("prim_inet:close_port(~p) ~p~n", [S,_Closed]), + ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% @@ -424,23 +487,49 @@ peeloff(S, AssocId) -> %% be called directly -- use "sendmsg" instead: %% send(S, Data, OptList) when is_port(S), is_list(OptList) -> - ?DBG_FORMAT("prim_inet:send(~p, ~p)~n", [S,Data]), + ?DBG_FORMAT("prim_inet:send(~p, _, ~p)~n", [S,OptList]), try erlang:port_command(S, Data, OptList) of false -> % Port busy and nosuspend option passed ?DBG_FORMAT("prim_inet:send() -> {error,busy}~n", []), {error,busy}; true -> - receive - {inet_reply,S,Status} -> - ?DBG_FORMAT("prim_inet:send() -> ~p~n", [Status]), - Status - end + send_recv_reply(S, undefined) catch error:_Error -> ?DBG_FORMAT("prim_inet:send() -> {error,einval}~n", []), {error,einval} end. +send_recv_reply(S, Mref) -> + ReplyTimeout = + case Mref of + undefined -> + ?INET_CLOSE_TIMEOUT; + _ -> + infinity + end, + receive + {inet_reply,S,Status} -> + ?DBG_FORMAT( + "prim_inet:send_recv_reply(~p, _): inet_reply ~p~n", + [S,Status]), + case Mref of + undefined -> ok; + _ -> + demonitor(Mref, [flush]), + ok + end, + Status; + {'DOWN',Mref,_,_,_Reason} when Mref =/= undefined -> + ?DBG_FORMAT( + "prim_inet:send_recv_reply(~p, _) 'DOWN' ~p~n", + [S,_Reason]), + {error,closed} + after ReplyTimeout -> + send_recv_reply(S, monitor(port, S)) + end. + + send(S, Data) -> send(S, Data, []). @@ -520,13 +609,35 @@ sendfile(S, FileHandle, Offset, Length) sendfile(S, FileHandle, Offset, Length) -> case erlang:port_info(S, connected) of {connected, Pid} when Pid =:= self() -> - sendfile_1(S, FileHandle, Offset, Length); + Uncork = sendfile_maybe_cork(S), + Result = sendfile_1(S, FileHandle, Offset, Length), + sendfile_maybe_uncork(S, Uncork), + Result; {connected, Pid} when Pid =/= self() -> {error, not_owner}; _Other -> {error, einval} end. +sendfile_maybe_cork(S) -> + case getprotocol(S) of + tcp -> + case getopts(S, [nopush]) of + {ok, [{nopush,false}]} -> + _ = setopts(S, [{nopush,true}]), + true; + _ -> + false + end; + _ -> false + end. + +sendfile_maybe_uncork(S, true) -> + _ = setopts(S, [{nopush,false}]), + ok; +sendfile_maybe_uncork(_, false) -> + ok. + sendfile_1(S, FileHandle, Offset, 0) -> sendfile_1(S, FileHandle, Offset, (1 bsl 63) - 1); sendfile_1(_S, _FileHandle, Offset, Length) when @@ -1318,6 +1429,7 @@ enc_opt(pktoptions) -> ?INET_OPT_PKTOPTIONS; enc_opt(ttl) -> ?INET_OPT_TTL; enc_opt(recvttl) -> ?INET_OPT_RECVTTL; enc_opt(nodelay) -> ?TCP_OPT_NODELAY; +enc_opt(nopush) -> ?TCP_OPT_NOPUSH; enc_opt(multicast_if) -> ?UDP_OPT_MULTICAST_IF; enc_opt(multicast_ttl) -> ?UDP_OPT_MULTICAST_TTL; enc_opt(multicast_loop) -> ?UDP_OPT_MULTICAST_LOOP; @@ -1379,6 +1491,7 @@ dec_opt(?INET_OPT_PRIORITY) -> priority; dec_opt(?INET_OPT_TOS) -> tos; dec_opt(?INET_OPT_TCLASS) -> tclass; dec_opt(?TCP_OPT_NODELAY) -> nodelay; +dec_opt(?TCP_OPT_NOPUSH) -> nopush; dec_opt(?INET_OPT_RECVTOS) -> recvtos; dec_opt(?INET_OPT_RECVTCLASS) -> recvtclass; dec_opt(?INET_OPT_PKTOPTIONS) -> pktoptions; @@ -1465,6 +1578,7 @@ type_opt_1(pktoptions) -> opts; type_opt_1(ttl) -> int; type_opt_1(recvttl) -> bool; type_opt_1(nodelay) -> bool; +type_opt_1(nopush) -> bool; type_opt_1(ipv6_v6only) -> bool; %% multicast type_opt_1(multicast_ttl) -> int; diff --git a/lib/compiler/src/beam_jump.erl b/lib/compiler/src/beam_jump.erl index fbff4cfd79..d694bd2a5b 100644 --- a/lib/compiler/src/beam_jump.erl +++ b/lib/compiler/src/beam_jump.erl @@ -196,22 +196,19 @@ no_fallthrough([I|_]) -> is_unreachable_after(I). already_has_value(Lit, Lbl, Reg, D) -> - Key = {Lbl,Reg}, case D of - #{Lbl:=unsafe} -> - false; - #{Key:=Lit} -> + #{Lbl:={Reg,Lit}} -> true; #{} -> false end. update_value_dict([Lit,{f,Lbl}|T], Reg, D0) -> - Key = {Lbl,Reg}, D = case D0 of - #{Key := inconsistent} -> D0; - #{Key := _} -> D0#{Key := inconsistent}; - _ -> D0#{Key => Lit} + #{Lbl:=unsafe} -> D0; + #{Lbl:={Reg,Lit}} -> D0; + #{Lbl:=_} -> D0#{Lbl:=unsafe}; + #{} -> D0#{Lbl=>{Reg,Lit}} end, update_value_dict(T, Reg, D); update_value_dict([], _, D) -> D. diff --git a/lib/compiler/src/erl_bifs.erl b/lib/compiler/src/erl_bifs.erl index 71ab0e872a..ce9762899e 100644 --- a/lib/compiler/src/erl_bifs.erl +++ b/lib/compiler/src/erl_bifs.erl @@ -108,6 +108,7 @@ is_pure(erlang, list_to_atom, 1) -> true; is_pure(erlang, list_to_binary, 1) -> true; is_pure(erlang, list_to_float, 1) -> true; is_pure(erlang, list_to_integer, 1) -> true; +is_pure(erlang, list_to_integer, 2) -> true; is_pure(erlang, list_to_pid, 1) -> true; is_pure(erlang, list_to_tuple, 1) -> true; is_pure(erlang, max, 2) -> true; diff --git a/lib/compiler/test/beam_jump_SUITE.erl b/lib/compiler/test/beam_jump_SUITE.erl index c61e4ab65c..f6cddbf1b1 100644 --- a/lib/compiler/test/beam_jump_SUITE.erl +++ b/lib/compiler/test/beam_jump_SUITE.erl @@ -21,7 +21,8 @@ -export([all/0,suite/0,groups/0,init_per_suite/1,end_per_suite/1, init_per_group/2,end_per_group/2, - undefined_label/1,ambiguous_catch_try_state/1]). + undefined_label/1,ambiguous_catch_try_state/1, + unsafe_move_elimination/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -32,7 +33,8 @@ all() -> groups() -> [{p,[parallel], [undefined_label, - ambiguous_catch_try_state + ambiguous_catch_try_state, + unsafe_move_elimination ]}]. init_per_suite(Config) -> @@ -72,3 +74,44 @@ river() -> song. checks(Wanted) -> %% Must be one line to cause the unsafe optimization. {catch case river() of sheet -> begin +Wanted, if "da" -> Wanted end end end, catch case river() of sheet -> begin + Wanted, if "da" -> Wanted end end end}. + +unsafe_move_elimination(_Config) -> + {{left,right,false},false} = unsafe_move_elimination(left, right, false), + {{false,right,false},false} = unsafe_move_elimination(false, right, true), + {{true,right,right},right} = unsafe_move_elimination(true, right, true), + ok. + +unsafe_move_elimination(Left, Right, Simple0) -> + id(1), + + %% The move at label 29 would be removed by beam_jump, which is unsafe because + %% the two select_val instructions have different source registers. + %% + %% {select_val,{y,0},{f,25},{list,[{atom,true},{f,27},{atom,false},{f,29}]}}. + %% ^^^^^ ^^^^^^^^^^^^^^^^^^^ + %% {label,27}. + %% {kill,{y,0}}. + %% {move,{y,2},{x,0}}. + %% {line,...}. + %% {call,1,{f,31}}. + %% {select_val,{x,0},{f,33},{list,[{atom,true},{f,35},{atom,false},{f,29}]}}. + %% ^^^^^ ^^^^^^^^^^^^^^^^^^^ + %% {label,29}. + %% {move,{atom,false},{y,0}}. <=== REMOVED (unsafely). + %% {jump,{f,37}}. + + Simple = case case Simple0 of + false -> false; + true -> id(Left) + end + of + false -> + false; + true -> + id(Right) + end, + {id({Left,Right,Simple}),Simple}. + + +id(I) -> + I. diff --git a/lib/compiler/test/bs_match_SUITE.erl b/lib/compiler/test/bs_match_SUITE.erl index 8e105c6244..0c6db96081 100644 --- a/lib/compiler/test/bs_match_SUITE.erl +++ b/lib/compiler/test/bs_match_SUITE.erl @@ -43,7 +43,7 @@ beam_bsm/1,guard/1,is_ascii/1,non_opt_eq/1, expression_before_match/1,erl_689/1,restore_on_call/1, restore_after_catch/1,matches_on_parameter/1,big_positions/1, - matching_meets_apply/1]). + matching_meets_apply/1,bs_start_match2_defs/1]). -export([coverage_id/1,coverage_external_ignore/2]). @@ -78,7 +78,7 @@ groups() -> beam_bsm,guard,is_ascii,non_opt_eq, expression_before_match,erl_689,restore_on_call, matches_on_parameter,big_positions, - matching_meets_apply]}]. + matching_meets_apply,bs_start_match2_defs]}]. init_per_suite(Config) -> @@ -1799,6 +1799,19 @@ do_erl_689_2b(_, <<Length, Data/binary>>) -> {{Y, M, D}, Rest} end. +%% ERL-753 + +bs_start_match2_defs(_Config) -> + {<<"http://127.0.0.1:1234/vsaas/hello">>} = api_url(<<"hello">>, dummy), + {"https://127.0.0.1:4321/vsaas/hello"} = api_url({https, "hello"}, dummy). + +api_url(URL, Auth) -> + Header = [], + case URL of + <<_/binary>> -> {<<"http://127.0.0.1:1234/vsaas/",URL/binary>>}; + {https, [_|_] = URL1} -> {"https://127.0.0.1:4321/vsaas/"++URL1} + end. + check(F, R) -> R = F(). diff --git a/lib/crypto/src/crypto.erl b/lib/crypto/src/crypto.erl index 68c0bcef5e..9915a5a6f3 100644 --- a/lib/crypto/src/crypto.erl +++ b/lib/crypto/src/crypto.erl @@ -2153,7 +2153,7 @@ check_otp_test_engine(LibDir) -> case filelib:wildcard("otp_test_engine*", LibDir) of [] -> {error, notexist}; - [LibName] -> + [LibName|_] -> % In case of Valgrind there could be more than one LibPath = filename:join(LibDir,LibName), case filelib:is_file(LibPath) of true -> diff --git a/lib/crypto/test/engine_SUITE.erl b/lib/crypto/test/engine_SUITE.erl index 4b3ea10315..3f26493cd3 100644 --- a/lib/crypto/test/engine_SUITE.erl +++ b/lib/crypto/test/engine_SUITE.erl @@ -75,11 +75,18 @@ groups() -> init_per_suite(Config) -> - case crypto:info_lib() of - [{_,_, <<"OpenSSL 1.0.1s-freebsd 1 Mar 2016">>}] -> + case {os:type(), crypto:info_lib()} of + {_, [{_,_, <<"OpenSSL 1.0.1s-freebsd 1 Mar 2016">>}]} -> {skip, "Problem with engine on OpenSSL 1.0.1s-freebsd"}; - Res -> - ct:log("crypto:info_lib() -> ~p\n", [Res]), + + {{unix,darwin}, _} -> + {skip, "Engine unsupported on Darwin"}; + + {{win32,_}, _} -> + {skip, "Engine unsupported on Windows"}; + + {OS, Res} -> + ct:log("crypto:info_lib() -> ~p\nos:type() -> ~p", [Res,OS]), try crypto:start() of ok -> Config; diff --git a/lib/eldap/doc/src/notes.xml b/lib/eldap/doc/src/notes.xml index b390e0c047..bf9358c4d1 100644 --- a/lib/eldap/doc/src/notes.xml +++ b/lib/eldap/doc/src/notes.xml @@ -77,6 +77,22 @@ </section> +<section><title>Eldap 1.2.3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + A race condition at close could cause the eldap client to + exit with a badarg message as cause.</p> + <p> + Own Id: OTP-15342 Aux Id: ERIERL-242 </p> + </item> + </list> + </section> + +</section> + <section><title>Eldap 1.2.3</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -92,6 +108,22 @@ </section> +<section><title>Eldap 1.2.2.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + A race condition at close could cause the eldap client to + exit with a badarg message as cause.</p> + <p> + Own Id: OTP-15342 Aux Id: ERIERL-242 </p> + </item> + </list> + </section> + +</section> + <section><title>Eldap 1.2.2</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -368,4 +400,3 @@ <p>New application. </p> </section> </chapter> - diff --git a/lib/eldap/test/make_certs.erl b/lib/eldap/test/make_certs.erl index cfa43289e1..e8a13ae113 100644 --- a/lib/eldap/test/make_certs.erl +++ b/lib/eldap/test/make_certs.erl @@ -348,7 +348,7 @@ req_cnf(C) -> "default_bits = ", integer_to_list(C#config.default_bits), "\n" "RANDFILE = $ROOTDIR/RAND\n" "encrypt_key = no\n" - "default_md = md5\n" + "default_md = sha1\n" "#string_mask = pkix\n" "x509_extensions = ca_ext\n" "prompt = no\n" @@ -394,7 +394,7 @@ ca_cnf(C) -> ["crl_extensions = crl_ext\n" || C#config.v2_crls], "unique_subject = no\n" "default_days = 3600\n" - "default_md = md5\n" + "default_md = sha1\n" "preserve = no\n" "policy = policy_match\n" "\n" diff --git a/lib/hipe/main/hipe.erl b/lib/hipe/main/hipe.erl index 4a509c1ace..c8213e9b6d 100644 --- a/lib/hipe/main/hipe.erl +++ b/lib/hipe/main/hipe.erl @@ -542,7 +542,7 @@ file(File) -> | {'error', term()} when Mod :: mod(). file(File, Options) when is_atom(File) -> - case beam_lib:info(File) of + case beam_lib:info(atom_to_list(File)) of L when is_list(L) -> {module, Mod} = lists:keyfind(module, 1, L), case compile(Mod, File, Options) of diff --git a/lib/kernel/doc/src/inet.xml b/lib/kernel/doc/src/inet.xml index 127c110df4..87b08e4e36 100644 --- a/lib/kernel/doc/src/inet.xml +++ b/lib/kernel/doc/src/inet.xml @@ -1147,6 +1147,18 @@ setcap cap_sys_admin,cap_sys_ptrace,cap_dac_read_search+epi beam.smp</code> is turned on for the socket, which means that also small amounts of data are sent immediately.</p> </item> + <tag><c>{nopush, Boolean}</c>(TCP/IP sockets)</tag> + <item> + <p>This translates to <c>TCP_NOPUSH</c> on BSD and + to <c>TCP_CORK</c> on Linux.</p> + <p>If <c>Boolean == true</c>, the corresponding option + is turned on for the socket, which means that small + amounts of data are accumulated until a full MSS-worth + of data is available or this option is turned off.</p> + <p>Note that while <c>TCP_NOPUSH</c> socket option is available on OSX, its semantics + is very different (e.g., unsetting it does not cause immediate send + of accumulated data). Hence, <c>nopush</c> option is intentionally ignored on OSX.</p> + </item> <tag><c>{packet, PacketType}</c>(TCP/IP sockets)</tag> <item> <p><marker id="packet"/>Defines the type of packets to use for a socket. diff --git a/lib/kernel/doc/src/logger.xml b/lib/kernel/doc/src/logger.xml index 464c65ba76..2bcf137299 100644 --- a/lib/kernel/doc/src/logger.xml +++ b/lib/kernel/doc/src/logger.xml @@ -748,6 +748,14 @@ start(_, []) -> exists, its associated value will be changed to the given value. If it does not exist, it will be added.</p> + <p>If the value is incomplete, which for example can be the + case for the <c>config</c> key, it is up to the handler + implementation how the unspecified parts are set. For all + handlers in the Kernel application, unspecified data for + the <c>config</c> key is set to default values. To update + only specified data, and keep the existing configuration for + the rest, use <seealso marker="#update_handler_config-3"> + <c>update_handler_config/3</c></seealso>.</p> <p>See the definition of the <seealso marker="#type-handler_config"> <c>handler_config()</c></seealso> type for more @@ -933,6 +941,42 @@ logger:set_handler_config(HandlerId, maps:merge(Old, Config)). </func> <func> + <name name="update_handler_config" arity="3" clause_i="1"/> + <name name="update_handler_config" arity="3" clause_i="2"/> + <name name="update_handler_config" arity="3" clause_i="3"/> + <name name="update_handler_config" arity="3" clause_i="4"/> + <name name="update_handler_config" arity="3" clause_i="5"/> + <fsummary>Add or update configuration data for the specified + handler.</fsummary> + <type variable="HandlerId"/> + <type variable="Level" name_i="1"/> + <type variable="FilterDefault" name_i="2"/> + <type variable="Filters" name_i="3"/> + <type variable="Formatter" name_i="4"/> + <type variable="Config" name_i="5"/> + <type variable="Return"/> + <desc> + <p>Add or update configuration data for the specified + handler. If the given <c><anno>Key</anno></c> already + exists, its associated value will be changed + to the given value. If it does not exist, it will + be added.</p> + <p>If the value is incomplete, which for example can be the + case for the <c>config</c> key, it is up to the handler + implementation how the unspecified parts are set. For all + handlers in the Kernel application, unspecified data for + the <c>config</c> key is not changed. To reset unspecified + data to default values, + use <seealso marker="#set_handler_config-3"> + <c>set_handler_config/3</c></seealso>.</p> + <p>See the definition of + the <seealso marker="#type-handler_config"> + <c>handler_config()</c></seealso> type for more + information about the different parameters.</p> + </desc> + </func> + + <func> <name name="update_primary_config" arity="1"/> <fsummary>Update primary configuration data for Logger.</fsummary> <desc> @@ -1041,10 +1085,11 @@ logger:set_process_metadata(maps:merge(logger:get_process_metadata(), Meta)). </func> <func> - <name>HModule:changing_config(Config1, Config2) -> {ok, Config3} | {error, Reason}</name> + <name>HModule:changing_config(SetOrUpdate, OldConfig, NewConfig) -> {ok, Config} | {error, Reason}</name> <fsummary>The configuration for this handler is about to change.</fsummary> <type> - <v>Config1 = Config2 = Config3 = + <v>SetOrUpdate = set | update</v> + <v>OldConfig = NewConfig = Config = <seealso marker="#type-handler_config">handler_config()</seealso></v> <v>Reason = term()</v> </type> @@ -1053,18 +1098,51 @@ logger:set_process_metadata(maps:merge(logger:get_process_metadata(), Meta)). <p>The function is called on a temporary process when the configuration for a handler is about to change. The purpose is to verify and act on the new configuration.</p> - <p><c>Config1</c> is the existing configuration - and <c>Config2</c> is the new configuration.</p> + <p><c>OldConfig</c> is the existing configuration + and <c>NewConfig</c> is the new configuration.</p> <p>The handler identity is associated with the <c>id</c> key - in <c>Config1</c>.</p> + in <c>OldConfig</c>.</p> + <p><c>SetOrUpdate</c> has the value <c>set</c> if the + configuration change originates from a call to + <seealso marker="#set_handler_config-2"> + <c>set_handler_config/2,3</c></seealso>, and <c>update</c> + if it originates from <seealso marker="#update_handler_config-2"> + <c>update_handler_config/2,3</c></seealso>. The handler can + use this parameteter to decide how to update the value of + the <c>config</c> field, that is, the handler specific + configuration data. Typically, if <c>SetOrUpdate</c> + equals <c>set</c>, values that are not specified must be + given their default values. If <c>SetOrUpdate</c> + equals <c>update</c>, the values found in <c>OldConfig</c> + must be used instead.</p> <p>If everything succeeds, the callback function must return a - possibly adjusted configuration in <c>{ok,Config3}</c>.</p> + possibly adjusted configuration in <c>{ok,Config}</c>.</p> <p>If the configuration is faulty, the callback function must return <c>{error,Reason}</c>.</p> </desc> </func> <func> + <name>HModule:filter_config(Config) -> FilteredConfig</name> + <fsummary>Remove internal data from configuration.</fsummary> + <type> + <v>Config = FilteredConfig = + <seealso marker="#type-handler_config">handler_config()</seealso></v> + </type> + <desc> + <p>This callback function is optional.</p> + <p>The function is called when one of the Logger API functions + for fetching the handler configuration is called, for + example + <seealso marker="#get_handler_config-1"> + <c>logger:get_handler_config/1</c></seealso>.</p> + <p>It allows the handler to remove internal data fields from + its configuration data before it is returned to the + caller.</p> + </desc> + </func> + + <func> <name>HModule:log(LogEvent, Config) -> void()</name> <fsummary>Log the given log event.</fsummary> <type> @@ -1136,7 +1214,7 @@ logger:set_process_metadata(maps:merge(logger:get_process_metadata(), Meta)). <item><seealso marker="logger#set_handler_config-2"> <c>logger:set_handler_config/2,3</c></seealso></item> <item><seealso marker="logger#update_handler_config-2"> - <c>logger:updata_handler_config/2</c></seealso></item> + <c>logger:updata_handler_config/2,3</c></seealso></item> <item><seealso marker="logger#update_formatter_config-2"> <c>logger:update_formatter_config/2</c></seealso></item> </list> diff --git a/lib/kernel/doc/src/logger_chapter.xml b/lib/kernel/doc/src/logger_chapter.xml index fdb21fa24c..8fc52ce0a9 100644 --- a/lib/kernel/doc/src/logger_chapter.xml +++ b/lib/kernel/doc/src/logger_chapter.xml @@ -384,8 +384,8 @@ logger:debug(#{got => connection_request, id => Id, state => State}, <p>In addition to the mandatory callback function <c>log/2</c>, a handler module can export the optional callback - functions <c>adding_handler/1</c>, <c>changing_config/2</c> - and <c>removing_handler/1</c>. See + functions <c>adding_handler/1</c>, <c>changing_config/3</c>, + <c>filter_config/1</c>, and <c>removing_handler/1</c>. See section <seealso marker="logger#handler_callback_functions">Handler Callback Functions</seealso> in the logger(3) manual page for more information about these function.</p> @@ -555,7 +555,7 @@ logger:debug(#{got => connection_request, id => Id, state => State}, <item><seealso marker="logger#set_handler_config-2"> <c>set_handler_config/2,3</c></seealso></item> <item><seealso marker="logger#update_handler_config-2"> - <c>update_handler_config/2</c></seealso></item> + <c>update_handler_config/2,3</c></seealso></item> <item><seealso marker="logger#add_handler_filter-3"> <c>add_handler_filter/3</c></seealso></item> <item><seealso marker="logger#remove_handler_filter-2"> @@ -704,9 +704,13 @@ logger:debug(#{got => connection_request, id => Id, state => State}, <item> <p>If <c>HandlerId</c> is <c>default</c>, then this entry modifies the default handler, equivalent to calling</p> - <pre><seealso marker="logger#set_handler_config-2"> - logger:set_handler_config(default, Module, HandlerConfig) - </seealso></pre> + <pre><seealso marker="logger#remove_handler-1"> + logger:remove_handler(default) + </seealso></pre> + <p>followed by</p> + <pre><seealso marker="logger#add_handler-3"> + logger:add_handler(default, Module, HandlerConfig) + </seealso></pre> <p>For all other values of <c>HandlerId</c>, this entry adds a new handler, equivalent to calling</p> <pre><seealso marker="logger:add_handler/3"> @@ -1024,7 +1028,8 @@ ok</pre> <list> <item><c>adding_handler(Config)</c></item> <item><c>removing_handler(Config)</c></item> - <item><c>changing_config(OldConfig, NewConfig)</c></item> + <item><c>changing_config(SetOrUpdate, OldConfig, NewConfig)</c></item> + <item><c>filter_config(Config)</c></item> </list> <p>When a handler is added, by for example a call to <seealso marker="logger#add_handler-3"> @@ -1043,11 +1048,18 @@ ok</pre> <p>When <seealso marker="logger#set_handler_config-2"> <c>logger:set_handler_config/2,3</c></seealso> or <seealso marker="logger#update_handler_config/2"> - <c>logger:update_handler_config/2</c></seealso> is called, + <c>logger:update_handler_config/2,3</c></seealso> is called, Logger - calls <c>HModule:changing_config(OldConfig, NewConfig)</c>. If + calls <c>HModule:changing_config(SetOrUpdate, OldConfig, NewConfig)</c>. If this function returns <c>{ok,NewConfig1}</c>, Logger writes <c>NewConfig1</c> to the configuration database.</p> + <p>When <seealso marker="logger#get_config-0"> + <c>logger:get_config/0</c></seealso> or + <seealso marker="logger#get_handler_config-0"> + <c>logger:get_handler_config/0,1</c></seealso> is called, + Logger calls <c>HModule:filter_config(Config)</c>. This function + must return the handler configuration where internal data is + removed.</p> <p>A simple handler that prints to the terminal can be implemented as follows:</p> diff --git a/lib/kernel/doc/src/logger_disk_log_h.xml b/lib/kernel/doc/src/logger_disk_log_h.xml index dfe2ab3275..d9b941a0a9 100644 --- a/lib/kernel/doc/src/logger_disk_log_h.xml +++ b/lib/kernel/doc/src/logger_disk_log_h.xml @@ -66,6 +66,10 @@ corresponds to the <c>name</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> + <p>The value is set when the handler is added, and it can not + be changed in runtime.</p> + <p>Defaults to the same name as the handler identity, in the + current directory.</p> </item> <tag><c>type</c></tag> <item> @@ -73,6 +77,8 @@ corresponds to the <c>type</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> + <p>The value is set when the handler is added, and it can not + be changed in runtime.</p> <p>Defaults to <c>wrap</c>.</p> </item> <tag><c>max_no_files</c></tag> @@ -82,6 +88,8 @@ corresponds to the <c>MaxNoFiles</c> element in the <c>size</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> + <p>The value is set when the handler is added, and it can not + be changed in runtime.</p> <p>Defaults to <c>10</c>.</p> <p>The setting has no effect on a halt log.</p> </item> @@ -93,6 +101,8 @@ corresponds to the <c>MaxNoBytes</c> element in the <c>size</c> property in the <seealso marker="disk_log#open-1"><c>dlog_option()</c></seealso> datatype.</p> + <p>The value is set when the handler is added, and it can not + be changed in runtime.</p> <p>Defaults to <c>1048576</c> bytes for a wrap log, and <c>infinity</c> for a halt log.</p> </item> diff --git a/lib/kernel/doc/src/logger_std_h.xml b/lib/kernel/doc/src/logger_std_h.xml index fcd8189bae..e156f5719b 100644 --- a/lib/kernel/doc/src/logger_std_h.xml +++ b/lib/kernel/doc/src/logger_std_h.xml @@ -74,7 +74,9 @@ circular logging. Use the disk_log handler, <seealso marker="logger_disk_log_h"><c>logger_disk_log_h</c></seealso>, for this.</p> - <p> Defaults to <c>standard_io</c>.</p> + <p>The value is set when the handler is added, and it can not + be changed in runtime.</p> + <p>Defaults to <c>standard_io</c>.</p> </item> <tag><c>filesync_repeat_interval</c></tag> <item> diff --git a/lib/kernel/src/inet_int.hrl b/lib/kernel/src/inet_int.hrl index c8e09d18ad..f6525d7261 100644 --- a/lib/kernel/src/inet_int.hrl +++ b/lib/kernel/src/inet_int.hrl @@ -162,6 +162,7 @@ -define(INET_OPT_PKTOPTIONS, 45). -define(INET_OPT_TTL, 46). -define(INET_OPT_RECVTTL, 47). +-define(TCP_OPT_NOPUSH, 48). % Specific SCTP options: separate range: -define(SCTP_OPT_RTOINFO, 100). -define(SCTP_OPT_ASSOCINFO, 101). diff --git a/lib/kernel/src/logger.erl b/lib/kernel/src/logger.erl index 752dd8d493..6762998d4f 100644 --- a/lib/kernel/src/logger.erl +++ b/lib/kernel/src/logger.erl @@ -43,7 +43,8 @@ get_module_level/0, get_module_level/1, set_primary_config/1, set_primary_config/2, set_handler_config/2, set_handler_config/3, - update_primary_config/1, update_handler_config/2, + update_primary_config/1, + update_handler_config/2, update_handler_config/3, update_formatter_config/2, update_formatter_config/3, get_primary_config/0, get_handler_config/1, get_handler_config/0, get_handler_ids/0, get_config/0, @@ -423,6 +424,29 @@ set_handler_config(HandlerId,Config) -> update_primary_config(Config) -> logger_server:update_config(primary,Config). +-spec update_handler_config(HandlerId,level,Level) -> Return when + HandlerId :: handler_id(), + Level :: level() | all | none, + Return :: ok | {error,term()}; + (HandlerId,filter_default,FilterDefault) -> Return when + HandlerId :: handler_id(), + FilterDefault :: log | stop, + Return :: ok | {error,term()}; + (HandlerId,filters,Filters) -> Return when + HandlerId :: handler_id(), + Filters :: [{filter_id(),filter()}], + Return :: ok | {error,term()}; + (HandlerId,formatter,Formatter) -> Return when + HandlerId :: handler_id(), + Formatter :: {module(), formatter_config()}, + Return :: ok | {error,term()}; + (HandlerId,config,Config) -> Return when + HandlerId :: handler_id(), + Config :: term(), + Return :: ok | {error,term()}. +update_handler_config(HandlerId,Key,Value) -> + logger_server:update_config(HandlerId,Key,Value). + -spec update_handler_config(HandlerId,Config) -> ok | {error,term()} when HandlerId :: handler_id(), Config :: handler_config(). @@ -439,7 +463,14 @@ get_primary_config() -> HandlerId :: handler_id(), Config :: handler_config(). get_handler_config(HandlerId) -> - logger_config:get(?LOGGER_TABLE,HandlerId). + case logger_config:get(?LOGGER_TABLE,HandlerId) of + {ok,#{module:=Module}=Config} -> + {ok,try Module:filter_config(Config) + catch _:_ -> Config + end}; + Error -> + Error + end. -spec get_handler_config() -> [Config] when Config :: handler_config(). diff --git a/lib/kernel/src/logger_disk_log_h.erl b/lib/kernel/src/logger_disk_log_h.erl index a8f141f135..2a81458ec8 100644 --- a/lib/kernel/src/logger_disk_log_h.erl +++ b/lib/kernel/src/logger_disk_log_h.erl @@ -33,7 +33,8 @@ terminate/2, code_change/3]). %% logger callbacks --export([log/2, adding_handler/1, removing_handler/1, changing_config/2]). +-export([log/2, adding_handler/1, removing_handler/1, changing_config/3, + filter_config/1]). %% handler internal -export([log_handler_info/4]). @@ -114,9 +115,8 @@ reset(Name) -> %%% Handler being added adding_handler(#{id:=Name}=Config) -> case check_config(adding, Config) of - {ok, Config1} -> + {ok, #{config:=HConfig}=Config1} -> %% create initial handler state by merging defaults with config - HConfig = maps:get(config, Config1, #{}), HState = maps:merge(get_init_state(), HConfig), case logger_h_common:overload_levels_ok(HState) of true -> @@ -133,32 +133,40 @@ adding_handler(#{id:=Name}=Config) -> %%%----------------------------------------------------------------- %%% Updating handler config -changing_config(OldConfig = #{id:=Name, config:=OldHConfig}, - NewConfig = #{id:=Name, config:=NewHConfig}) -> - #{type:=Type, file:=File, max_no_files:=MaxFs, - max_no_bytes:=MaxBytes} = OldHConfig, - case NewHConfig of - #{type:=Type, file:=File, max_no_files:=MaxFs, - max_no_bytes:=MaxBytes} -> - changing_config1(OldConfig, NewConfig); - _ -> - {error,{illegal_config_change,OldConfig,NewConfig}} - end; -changing_config(OldConfig, NewConfig) -> - {error,{illegal_config_change,OldConfig,NewConfig}}. +changing_config(SetOrUpdate,OldConfig=#{config:=OldHConfig},NewConfig) -> + WriteOnce = maps:with([type,file,max_no_files,max_no_bytes],OldHConfig), + ReadOnly = maps:with([handler_pid,mode_tab],OldHConfig), + NewHConfig0 = maps:get(config, NewConfig, #{}), + Default = + case SetOrUpdate of + set -> + %% Do not reset write-once fields to defaults + maps:merge(get_default_config(),WriteOnce); + update -> + OldHConfig + end, -changing_config1(OldConfig=#{config:=OldHConfig}, NewConfig) -> + %% Allow (accidentially) included read-only fields - just overwrite them + NewHConfig = maps:merge(maps:merge(Default,NewHConfig0),ReadOnly), + + %% But fail if write-once fields are changed + case maps:with([type,file,max_no_files,max_no_bytes],NewHConfig) of + WriteOnce -> + changing_config1(maps:get(handler_pid,OldHConfig), + OldConfig, + NewConfig#{config=>NewHConfig}); + Other -> + {Old,New} = logger_server:diff_maps(WriteOnce,Other), + {error,{illegal_config_change,#{config=>Old},#{config=>New}}} + end. + +changing_config1(HPid, OldConfig, NewConfig) -> case check_config(changing, NewConfig) of - {ok,NewConfig1 = #{config:=NewHConfig}} -> - #{handler_pid:=HPid, - mode_tab:=ModeTab} = OldHConfig, - NewHConfig1 = NewHConfig#{handler_pid=>HPid, - mode_tab=>ModeTab}, - NewConfig2 = NewConfig1#{config=>NewHConfig1}, - try gen_server:call(HPid, {change_config,OldConfig,NewConfig2}, + Result = {ok,NewConfig1} -> + try gen_server:call(HPid, {change_config,OldConfig,NewConfig1}, ?DEFAULT_CALL_TIMEOUT) of - ok -> {ok,NewConfig2}; - HError -> HError + ok -> Result; + Error -> Error catch _:{timeout,_} -> {error,handler_busy} end; @@ -168,10 +176,12 @@ changing_config1(OldConfig=#{config:=OldHConfig}, NewConfig) -> check_config(adding, #{id:=Name}=Config) -> %% merge handler specific config data - HConfig = merge_default_logopts(Name, maps:get(config, Config, #{})), - case check_h_config(maps:to_list(HConfig)) of + HConfig1 = maps:get(config, Config, #{}), + HConfig2 = maps:merge(get_default_config(), HConfig1), + HConfig3 = merge_default_logopts(Name, HConfig2), + case check_h_config(maps:to_list(HConfig3)) of ok -> - {ok,Config#{config=>HConfig}}; + {ok,Config#{config=>HConfig3}}; Error -> Error end; @@ -238,6 +248,11 @@ log(LogEvent, Config = #{id := Name, Bin = logger_h_common:log_to_binary(LogEvent, Config), logger_h_common:call_cast_or_drop(Name, HPid, ModeTab, Bin). +%%%----------------------------------------------------------------- +%%% Remove internal fields from configuration +filter_config(#{config:=HConfig}=Config) -> + Config#{config=>maps:without([handler_pid,mode_tab],HConfig)}. + %%%=================================================================== %%% gen_server callbacks %%%=================================================================== @@ -438,7 +453,7 @@ code_change(_OldVsn, State, _Extra) -> %%%----------------------------------------------------------------- %%% -get_init_state() -> +get_default_config() -> #{sync_mode_qlen => ?SYNC_MODE_QLEN, drop_mode_qlen => ?DROP_MODE_QLEN, flush_qlen => ?FLUSH_QLEN, @@ -449,10 +464,12 @@ get_init_state() -> overload_kill_qlen => ?OVERLOAD_KILL_QLEN, overload_kill_mem_size => ?OVERLOAD_KILL_MEM_SIZE, overload_kill_restart_after => ?OVERLOAD_KILL_RESTART_AFTER, - dl_sync_int => ?CONTROLLER_SYNC_INTERVAL, - filesync_ok_qlen => ?FILESYNC_OK_QLEN, filesync_repeat_interval => ?FILESYNC_REPEAT_INTERVAL}. +get_init_state() -> + #{dl_sync_int => ?CONTROLLER_SYNC_INTERVAL, + filesync_ok_qlen => ?FILESYNC_OK_QLEN}. + %%%----------------------------------------------------------------- %%% Add a disk_log handler to the logger. %%% This starts a dedicated handler process which should always diff --git a/lib/kernel/src/logger_h_common.erl b/lib/kernel/src/logger_h_common.erl index 38ac7d8ffc..94c640cb92 100644 --- a/lib/kernel/src/logger_h_common.erl +++ b/lib/kernel/src/logger_h_common.erl @@ -306,8 +306,11 @@ stop_or_restart(Name, {shutdown,Reason={overloaded,_Name,_QLen,_Mem}}, exit(HandlerPid, kill) end, case ConfigResult of - {ok,#{module:=HMod}=HConfig} when is_integer(RestartAfter) -> + {ok,#{module:=HMod}=HConfig0} when is_integer(RestartAfter) -> _ = logger:remove_handler(Name), + HConfig = try HMod:filter_config(HConfig0) + catch _:_ -> HConfig0 + end, _ = timer:apply_after(RestartAfter, logger, add_handler, [Name,HMod,HConfig]); {ok,_} -> diff --git a/lib/kernel/src/logger_server.erl b/lib/kernel/src/logger_server.erl index a1d40f1123..b7735dbcf7 100644 --- a/lib/kernel/src/logger_server.erl +++ b/lib/kernel/src/logger_server.erl @@ -27,9 +27,13 @@ add_filter/2, remove_filter/2, set_module_level/2, unset_module_level/0, unset_module_level/1, cache_module_level/1, - set_config/2, set_config/3, update_config/2, + set_config/2, set_config/3, + update_config/2, update_config/3, update_formatter_config/2]). +%% Helper +-export([diff_maps/2]). + %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). @@ -105,12 +109,25 @@ cache_module_level(Module) -> gen_server:cast(?SERVER,{cache_module_level,Module}). set_config(Owner,Key,Value) -> - update_config(Owner,#{Key=>Value}). + case sanity_check(Owner,Key,Value) of + ok -> + call({change_config,set,Owner,Key,Value}); + Error -> + Error + end. set_config(Owner,Config) -> case sanity_check(Owner,Config) of ok -> - call({set_config,Owner,Config}); + call({change_config,set,Owner,Config}); + Error -> + Error + end. + +update_config(Owner,Key,Value) -> + case sanity_check(Owner,Key,Value) of + ok -> + call({change_config,update,Owner,Key,Value}); Error -> Error end. @@ -118,7 +135,7 @@ set_config(Owner,Config) -> update_config(Owner, Config) -> case sanity_check(Owner,Config) of ok -> - call({update_config,Owner,Config}); + call({change_config,update,Owner,Config}); Error -> Error end. @@ -204,46 +221,72 @@ handle_call({add_filter,Id,Filter}, _From,#state{tid=Tid}=State) -> handle_call({remove_filter,Id,FilterId}, _From, #state{tid=Tid}=State) -> Reply = do_remove_filter(Tid,Id,FilterId), {reply,Reply,State}; -handle_call({update_config,primary,NewConfig}, _From, #state{tid=Tid}=State) -> +handle_call({change_config,SetOrUpd,primary,Config0}, _From, + #state{tid=Tid}=State) -> + {ok,#{handlers:=Handlers}=OldConfig} = logger_config:get(Tid,primary), + Default = + case SetOrUpd of + set -> default_config(primary); + update -> OldConfig + end, + Config = maps:merge(Default,Config0), + Reply = logger_config:set(Tid,primary,Config#{handlers=>Handlers}), + {reply,Reply,State}; +handle_call({change_config,_SetOrUpd,primary,Key,Value}, _From, + #state{tid=Tid}=State) -> {ok,OldConfig} = logger_config:get(Tid,primary), - Config = maps:merge(OldConfig,NewConfig), - {reply,logger_config:set(Tid,primary,Config),State}; -handle_call({update_config,HandlerId,NewConfig}, From, #state{tid=Tid}=State) -> + Reply = logger_config:set(Tid,primary,OldConfig#{Key=>Value}), + {reply,Reply,State}; +handle_call({change_config,SetOrUpd,HandlerId,Config0}, From, + #state{tid=Tid}=State) -> case logger_config:get(Tid,HandlerId) of {ok,#{module:=Module}=OldConfig} -> - Config = maps:merge(OldConfig,NewConfig), - call_h_async( - fun() -> - call_h(Module,changing_config,[OldConfig,Config], - {ok,Config}) - end, - fun({ok,Config1}) -> - logger_config:set(Tid,HandlerId,Config1); - (Error) -> - Error - end,From,State); - Error -> - {reply,Error,State} + Default = + case SetOrUpd of + set -> default_config(HandlerId,Module); + update -> OldConfig + end, + Config = maps:merge(Default,Config0), + case check_config_change(OldConfig,Config) of + ok -> + call_h_async( + fun() -> + call_h(Module,changing_config, + [SetOrUpd,OldConfig,Config], + {ok,Config}) + end, + fun({ok,Config1}) -> + logger_config:set(Tid,HandlerId,Config1); + (Error) -> + Error + end,From,State); + Error -> + {reply,Error,State} + end; + _ -> + {reply,{error,{not_found,HandlerId}},State} end; -handle_call({set_config,primary,Config0}, _From, #state{tid=Tid}=State) -> - Config = maps:merge(default_config(primary),Config0), - {ok,#{handlers:=Handlers}} = logger_config:get(Tid,primary), - Reply = logger_config:set(Tid,primary,Config#{handlers=>Handlers}), - {reply,Reply,State}; -handle_call({set_config,HandlerId,Config0}, From, #state{tid=Tid}=State) -> +handle_call({change_config,SetOrUpd,HandlerId,Key,Value}, From, + #state{tid=Tid}=State) -> case logger_config:get(Tid,HandlerId) of {ok,#{module:=Module}=OldConfig} -> - Config = maps:merge(default_config(HandlerId,Module),Config0), - call_h_async( - fun() -> - call_h(Module,changing_config,[OldConfig,Config], - {ok,Config}) - end, - fun({ok,Config1}) -> - logger_config:set(Tid,HandlerId,Config1); - (Error) -> - Error - end,From,State); + Config = OldConfig#{Key=>Value}, + case check_config_change(OldConfig,Config) of + ok -> + call_h_async( + fun() -> + call_h(Module,changing_config, + [SetOrUpd,OldConfig,Config], + {ok,Config}) + end, + fun({ok,Config1}) -> + logger_config:set(Tid,HandlerId,Config1); + (Error) -> + Error + end,From,State); + Error -> + {reply,Error,State} + end; _ -> {reply,{error,{not_found,HandlerId}},State} end; @@ -320,7 +363,7 @@ call(Request) -> true when Action == add_handler; Action == remove_handler; Action == add_filter; Action == remove_filter; - Action == update_config; Action == set_config -> + Action == change_config -> {error,{attempting_syncronous_call_to_self,Request}}; _ -> gen_server:call(?SERVER,Request,?DEFAULT_LOGGER_CALL_TIMEOUT) @@ -458,6 +501,15 @@ check_formatter({Mod,Config}) -> check_formatter(Formatter) -> throw({invalid_formatter,Formatter}). +%% When changing configuration for a handler, the id and module fields +%% can not be changed. +check_config_change(#{id:=Id,module:=Module},#{id:=Id,module:=Module}) -> + ok; +check_config_change(OldConfig,NewConfig) -> + {Old,New} = logger_server:diff_maps(maps:with([id,module],OldConfig), + maps:with([id,module],NewConfig)), + {error,{illegal_config_change,Old,New}}. + call_h(Module, Function, Args, DefRet) -> %% Not calling code:ensure_loaded + erlang:function_exported here, %% since in some rare terminal cases, the code_server might not @@ -466,6 +518,11 @@ call_h(Module, Function, Args, DefRet) -> catch C:R:S -> case {C,R,S} of + {error,undef,[{Module,Function=changing_config,Args,_}|_]} + when length(Args)=:=3 -> + %% Backwards compatible call, if changing_config/3 + %% did not exist. + call_h(Module, Function, tl(Args), DefRet); {error,undef,[{Module,Function,Args,_}|_]} -> DefRet; _ -> @@ -525,3 +582,14 @@ call_h_reply(Unexpected,State) -> {process,?SERVER}, {message,Unexpected}]), {noreply,State}. + +%% Return two maps containing only the fields that differ. +diff_maps(M1,M2) -> + diffs(lists:sort(maps:to_list(M1)),lists:sort(maps:to_list(M2)),#{},#{}). + +diffs([H|T1],[H|T2],D1,D2) -> + diffs(T1,T2,D1,D2); +diffs([{K,V1}|T1],[{K,V2}|T2],D1,D2) -> + diffs(T1,T2,D1#{K=>V1},D2#{K=>V2}); +diffs([],[],D1,D2) -> + {D1,D2}. diff --git a/lib/kernel/src/logger_std_h.erl b/lib/kernel/src/logger_std_h.erl index 66fa6b6ab6..42e0f5caf4 100644 --- a/lib/kernel/src/logger_std_h.erl +++ b/lib/kernel/src/logger_std_h.erl @@ -35,7 +35,8 @@ terminate/2, code_change/3]). %% logger callbacks --export([log/2, adding_handler/1, removing_handler/1, changing_config/2]). +-export([log/2, adding_handler/1, removing_handler/1, changing_config/3, + filter_config/1]). %% handler internal -export([log_handler_info/4]). @@ -116,9 +117,8 @@ reset(Name) -> %%% Handler being added adding_handler(#{id:=Name}=Config) -> case check_config(adding, Config) of - {ok, Config1} -> + {ok, #{config:=HConfig}=Config1} -> %% create initial handler state by merging defaults with config - HConfig = maps:get(config, Config1, #{}), HState = maps:merge(get_init_state(), HConfig), case logger_h_common:overload_levels_ok(HState) of true -> @@ -135,22 +135,31 @@ adding_handler(#{id:=Name}=Config) -> %%%----------------------------------------------------------------- %%% Updating handler config -changing_config(OldConfig=#{id:=Name, config:=OldHConfig}, - NewConfig=#{id:=Name}) -> - #{type:=Type, handler_pid:=HPid, mode_tab:=ModeTab} = OldHConfig, - NewHConfig = maps:get(config, NewConfig, #{}), - case maps:get(type, NewHConfig, Type) of - Type -> - NewHConfig1 = NewHConfig#{type=>Type, - handler_pid=>HPid, - mode_tab=>ModeTab}, - changing_config1(HPid, OldConfig, - NewConfig#{config=>NewHConfig1}); - _ -> - {error,{illegal_config_change,OldConfig,NewConfig}} - end; -changing_config(OldConfig, NewConfig) -> - {error,{illegal_config_change,OldConfig,NewConfig}}. +changing_config(SetOrUpdate,OldConfig=#{config:=OldHConfig},NewConfig) -> + WriteOnce = maps:with([type],OldHConfig), + ReadOnly = maps:with([handler_pid,mode_tab],OldHConfig), + NewHConfig0 = maps:get(config, NewConfig, #{}), + Default = + case SetOrUpdate of + set -> + %% Do not reset write-once fields to defaults + maps:merge(get_default_config(),WriteOnce); + update -> + OldHConfig + end, + + %% Allow (accidentially) included read-only fields - just overwrite them + NewHConfig = maps:merge(maps:merge(Default, NewHConfig0),ReadOnly), + + %% But fail if write-once fields are changed + case maps:with([type],NewHConfig) of + WriteOnce -> + changing_config1(maps:get(handler_pid,OldHConfig), + OldConfig, + NewConfig#{config=>NewHConfig}); + Other -> + {error,{illegal_config_change,#{config=>WriteOnce},#{config=>Other}}} + end. changing_config1(HPid, OldConfig, NewConfig) -> case check_config(changing, NewConfig) of @@ -169,8 +178,7 @@ changing_config1(HPid, OldConfig, NewConfig) -> check_config(adding, Config) -> %% Merge in defaults on handler level HConfig0 = maps:get(config, Config, #{}), - HConfig = maps:merge(#{type => standard_io}, - HConfig0), + HConfig = maps:merge(get_default_config(),HConfig0), case check_h_config(maps:to_list(HConfig)) of ok -> {ok,Config#{config=>HConfig}}; @@ -223,6 +231,11 @@ log(LogEvent, Config = #{id := Name, Bin = logger_h_common:log_to_binary(LogEvent, Config), logger_h_common:call_cast_or_drop(Name, HPid, ModeTab, Bin). +%%%----------------------------------------------------------------- +%%% Remove internal fields from configuration +filter_config(#{config:=HConfig}=Config) -> + Config#{config=>maps:without([handler_pid,mode_tab],HConfig)}. + %%%=================================================================== %%% gen_server callbacks %%%=================================================================== @@ -428,8 +441,9 @@ code_change(_OldVsn, State, _Extra) -> %%%----------------------------------------------------------------- %%% -get_init_state() -> - #{sync_mode_qlen => ?SYNC_MODE_QLEN, +get_default_config() -> + #{type => standard_io, + sync_mode_qlen => ?SYNC_MODE_QLEN, drop_mode_qlen => ?DROP_MODE_QLEN, flush_qlen => ?FLUSH_QLEN, burst_limit_enable => ?BURST_LIMIT_ENABLE, @@ -439,10 +453,12 @@ get_init_state() -> overload_kill_qlen => ?OVERLOAD_KILL_QLEN, overload_kill_mem_size => ?OVERLOAD_KILL_MEM_SIZE, overload_kill_restart_after => ?OVERLOAD_KILL_RESTART_AFTER, - file_ctrl_sync_int => ?CONTROLLER_SYNC_INTERVAL, - filesync_ok_qlen => ?FILESYNC_OK_QLEN, filesync_repeat_interval => ?FILESYNC_REPEAT_INTERVAL}. +get_init_state() -> + #{file_ctrl_sync_int => ?CONTROLLER_SYNC_INTERVAL, + filesync_ok_qlen => ?FILESYNC_OK_QLEN}. + %%%----------------------------------------------------------------- %%% Add a standard handler to the logger. %%% This starts a dedicated handler process which should always @@ -825,4 +841,3 @@ sync_dev(Fd, DevName, PrevSyncResult, HandlerName) -> logger_h_common:error_notify({HandlerName,filesync,DevName,Error}), Error end. - diff --git a/lib/kernel/src/net_kernel.erl b/lib/kernel/src/net_kernel.erl index 3cf11fd7b1..01d809b566 100644 --- a/lib/kernel/src/net_kernel.erl +++ b/lib/kernel/src/net_kernel.erl @@ -1432,7 +1432,7 @@ validate_hostname([$@|HostPart] = Host) -> end. valid_name_head(Head) -> - {ok, MP} = re:compile("^[0-9A-Za-z_\\-]*$", [unicode]), + {ok, MP} = re:compile("^[0-9A-Za-z_\\-]+$", [unicode]), case re:run(Head, MP) of {match, _} -> true; diff --git a/lib/kernel/test/gen_tcp_misc_SUITE.erl b/lib/kernel/test/gen_tcp_misc_SUITE.erl index 194522c009..04c0c48e3a 100644 --- a/lib/kernel/test/gen_tcp_misc_SUITE.erl +++ b/lib/kernel/test/gen_tcp_misc_SUITE.erl @@ -52,7 +52,8 @@ several_accepts_in_one_go/1, accept_system_limit/1, active_once_closed/1, send_timeout/1, send_timeout_active/1, otp_7731/1, zombie_sockets/1, otp_7816/1, otp_8102/1, - wrapping_oct/0, wrapping_oct/1, otp_9389/1, otp_13939/1]). + wrapping_oct/0, wrapping_oct/1, otp_9389/1, otp_13939/1, + otp_12242/1]). %% Internal exports. -export([sender/3, not_owner/1, passive_sockets_server/2, priority_server/1, @@ -95,7 +96,8 @@ all() -> killing_multi_acceptors2, several_accepts_in_one_go, accept_system_limit, active_once_closed, send_timeout, send_timeout_active, otp_7731, wrapping_oct, - zombie_sockets, otp_7816, otp_8102, otp_9389]. + zombie_sockets, otp_7816, otp_8102, otp_9389, + otp_12242]. groups() -> []. @@ -3284,3 +3286,143 @@ otp_13939(Config) when is_list(Config) -> exit(Pid, normal), ct:fail("Server process blocked on send.") end. + +otp_12242(Config) when is_list(Config) -> + case os:type() of + {win32,_} -> + %% Even if we set sndbuf and recbuf to small sizes + %% Windows either happily accepts to send GBytes of data + %% in no time, so the second send below that is supposed + %% to time out just succedes, or the first send that + %% is supposed to fill the inet_drv I/O queue and + %% start waiting for when more data can be sent + %% instead sends all data but suffers a send + %% failure that closes the socket + {skipped,backpressure_broken_on_win32}; + _ -> + %% Find the IPv4 address of an up and running interface + %% that is not loopback nor pointtopoint + {ok,IFList} = inet:getifaddrs(), + ct:pal("IFList ~p~n", [IFList]), + case + lists:flatten( + [lists:filtermap( + fun ({addr,Addr}) when tuple_size(Addr) =:= 4 -> + {true,Addr}; + (_) -> + false + end, Opts) + || {_,Opts} <- IFList, + case lists:keyfind(flags, 1, Opts) of + {_,Flags} -> + lists:member(up, Flags) + andalso + lists:member(running, Flags) + andalso + not lists:member(loopback, Flags) + andalso + not lists:member(pointtopoint, Flags); + false -> + false + end]) + of + [Addr|_] -> + otp_12242(Addr); + Other -> + {skipped,{no_external_address,Other}} + end + end; +%% +otp_12242(Addr) when tuple_size(Addr) =:= 4 -> + ct:timetrap(30000), + ct:pal("Using address ~p~n", [Addr]), + Bufsize = 16 * 1024, + Datasize = 128 * 1024 * 1024, % At least 1 s on GBit interface + Blob = binary:copy(<<$x>>, Datasize), + LOpts = + [{backlog,4},{reuseaddr,true},{ip,Addr}, + binary,{active,false}, + {recbuf,Bufsize},{sndbuf,Bufsize},{buffer,Bufsize}], + COpts = + [binary,{active,false},{ip,Addr}, + {linger,{true,1}}, % 1 s + {send_timeout,500}, + {recbuf,Bufsize},{sndbuf,Bufsize},{buffer,Bufsize}], + Dir = filename:dirname(code:which(?MODULE)), + {ok,ListenerNode} = + test_server:start_node( + ?UNIQ_NODE_NAME, slave, [{args,"-pa " ++ Dir}]), + Tester = self(), + Listener = + spawn( + ListenerNode, + fun () -> + {ok,L} = gen_tcp:listen(0, LOpts), + {ok,LPort} = inet:port(L), + Tester ! {self(),port,LPort}, + {ok,A} = gen_tcp:accept(L), + ok = gen_tcp:close(L), + receive + {Tester,stop} -> + ok = gen_tcp:close(A) + end + end), + ListenerMref = monitor(process, Listener), + LPort = receive {Listener,port,P} -> P end, + {ok,C} = gen_tcp:connect(Addr, LPort, COpts, infinity), + {ok,ReadCOpts} = inet:getopts(C, [recbuf,sndbuf,buffer]), + ct:pal("ReadCOpts ~p~n", [ReadCOpts]), + %% + %% Fill the buffers + ct:pal("Sending ~p bytes~n", [Datasize]), + ok = gen_tcp:send(C, Blob), + ct:pal("Sent ~p bytes~n", [Datasize]), + %% Spawn the Closer, + %% try to ensure that the close call is in progress + %% before the owner proceeds with sending + Owner = self(), + {_Closer,CloserMref} = + spawn_opt( + fun () -> + Owner ! {tref, erlang:start_timer(50, Owner, closing)}, + ct:pal("Calling gen_tcp:close(C)~n"), + try gen_tcp:close(C) of + Result -> + ct:pal("gen_tcp:close(C) -> ~p~n", [Result]), + ok = Result + catch + Class:Reason:Stacktrace -> + ct:pal( + "gen_tcp:close(C) >< ~p:~p~n ~p~n", + [Class,Reason,Stacktrace]), + erlang:raise(Class, Reason, Stacktrace) + end + end, [link,monitor]), + receive + {tref,Tref} -> + receive {timeout,Tref,_} -> ok end, + ct:pal("Sending ~p bytes again~n", [Datasize]), + %% Now should the close be in progress... + %% All buffers are full, remote end is not reading, + %% and the send timeout is 1 s so this will timeout: + {error,timeout} = gen_tcp:send(C, Blob), + ct:pal("Sending ~p bytes again timed out~n", [Datasize]), + ok = inet:setopts(C, [{send_timeout,10000}]), + %% There is a hidden timeout here. Port close is sampled + %% every 5 s by prim_inet:send_recv_reply. + %% Linger is 3 s so the Closer will finish this send: + ct:pal("Sending ~p bytes with 10 s timeout~n", [Datasize]), + {error,closed} = gen_tcp:send(C, Blob), + ct:pal("Sending ~p bytes with 10 s timeout was closed~n", + [Datasize]), + normal = wait(CloserMref), + ct:pal("The Closer has exited~n"), + Listener ! {Tester,stop}, + receive {'DOWN',ListenerMref,_,_,_} -> ok end, + ct:pal("The Listener has exited~n"), + test_server:stop_node(ListenerNode), + ok + end. + +wait(Mref) -> + receive {'DOWN',Mref,_,_,Reason} -> Reason end. diff --git a/lib/kernel/test/inet_sockopt_SUITE.erl b/lib/kernel/test/inet_sockopt_SUITE.erl index ada9c2689c..27ff74e309 100644 --- a/lib/kernel/test/inet_sockopt_SUITE.erl +++ b/lib/kernel/test/inet_sockopt_SUITE.erl @@ -110,9 +110,14 @@ simple(Config) when is_list(Config) -> {S1,S2} = create_socketpair(Opt, Opt), {ok,Opt} = inet:getopts(S1,OptTags), {ok,Opt} = inet:getopts(S2,OptTags), - COpt = [{X,case X of nodelay -> false;_ -> Y end} || {X,Y} <- Opt], + NoPushOpt = case os:type() of + {unix, Osname} when Osname =:= linux; Osname =:= freebsd -> {nopush, true}; + {_,_} -> {nopush, false} + end, + COpt = [{X,case X of nodelay -> false;_ -> Y end} || {X,Y} <- [NoPushOpt|Opt]], + COptTags = [X || {X,_} <- COpt], inet:setopts(S1,COpt), - {ok,COpt} = inet:getopts(S1,OptTags), + {ok,COpt} = inet:getopts(S1,COptTags), {ok,Opt} = inet:getopts(S2,OptTags), gen_tcp:close(S1), gen_tcp:close(S2), diff --git a/lib/kernel/test/logger_SUITE.erl b/lib/kernel/test/logger_SUITE.erl index b7ccba8e70..d831d0d108 100644 --- a/lib/kernel/test/logger_SUITE.erl +++ b/lib/kernel/test/logger_SUITE.erl @@ -246,6 +246,18 @@ change_config(_Config) -> {ok,C4} = logger:get_handler_config(h1), C4 = C3#{custom:=new_custom}, + %% Change handler config: Id and module can not be changed + {error,{illegal_config_change,Old,New}} = + logger:set_handler_config(h1,id,newid), + %% Check that only the faulty field is included in return + [{id,h1}] = maps:to_list(Old), + [{id,newid}] = maps:to_list(New), + %% Check that both fields are included when both are changed + {error,{illegal_config_change, + #{id:=h1,module:=?MODULE}, + #{id:=newid,module:=newmodule}}} = + logger:set_handler_config(h1,#{id=>newid,module=>newmodule}), + %% Change primary config: Single key PConfig0 = logger:get_primary_config(), ok = logger:set_primary_config(level,warning), diff --git a/lib/kernel/test/logger_disk_log_h_SUITE.erl b/lib/kernel/test/logger_disk_log_h_SUITE.erl index a815db14e9..905c2c52c5 100644 --- a/lib/kernel/test/logger_disk_log_h_SUITE.erl +++ b/lib/kernel/test/logger_disk_log_h_SUITE.erl @@ -92,6 +92,7 @@ all() -> disk_log_opts, default_formatter, logging, + filter_config, errors, formatter_fail, config_fail, @@ -302,6 +303,20 @@ logging(cleanup, _Config) -> Name = list_to_atom(lists:concat([?FUNCTION_NAME,"_1"])), remove_and_stop(Name). +filter_config(_Config) -> + ok = logger:add_handler(?MODULE,logger_disk_log_h,#{}), + {ok,#{config:=HConfig}=Config} = logger:get_handler_config(?MODULE), + HConfig = maps:without([handler_pid,mode_tab],HConfig), + + FakeFullHConfig = HConfig#{handler_pid=>self(),mode_tab=>erlang:make_ref()}, + #{config:=HConfig} = + logger_disk_log_h:filter_config(Config#{config=>FakeFullHConfig}), + ok. + +filter_config(cleanup,_Config) -> + logger:remove_handler(?MODULE), + ok. + errors(Config) -> PrivDir = ?config(priv_dir,Config), Name1 = list_to_atom(lists:concat([?FUNCTION_NAME,"_1"])), @@ -316,13 +331,29 @@ errors(Config) -> %%! TODO: %%! Check how bad log_opts are handled! - {error,{illegal_config_change,_,_}} = - logger:set_handler_config(Name1, - config, - #{file=>LogFile1, - type=>halt}), - {error,{illegal_config_change,_,_}} = - logger:set_handler_config(Name1,id,new), + {error,{illegal_config_change, + #{config:=#{type:=wrap}}, + #{config:=#{type:=halt}}}} = + logger:update_handler_config(Name1, + config, + #{type=>halt, + file=>LogFile1}), + + {error,{illegal_config_change, + #{config:=#{file:=LogFile1}}, + #{config:=#{file:="newfilename"}}}} = + logger:update_handler_config(Name1, + config, + #{file=>"newfilename"}), + + %% Read-only fields may (accidentially) be included in the change, + %% but it won't take effect + {ok,C} = logger:get_handler_config(Name1), + ok = logger:set_handler_config(Name1,config, + #{handler_pid=>self(), + mode_tab=>erlang:make_ref()}), + {ok,C} = logger:get_handler_config(Name1), + ok = logger:remove_handler(Name1), {error,{not_found,Name1}} = logger:remove_handler(Name1), @@ -403,21 +434,21 @@ config_fail(_Config) -> formatter=>{?MODULE,self()}}), %% can't change the disk log options for a log already in use {error,{illegal_config_change,_,_}} = - logger:set_handler_config(?MODULE,config, - #{max_no_files=>2}), + logger:update_handler_config(?MODULE,config, + #{max_no_files=>2}), %% can't change name of an existing handler {error,{illegal_config_change,_,_}} = - logger:set_handler_config(?MODULE,id,bad), + logger:update_handler_config(?MODULE,id,bad), %% incorrect values of OP params {ok,#{config := HConfig}} = logger:get_handler_config(?MODULE), {error,{invalid_levels,_}} = - logger:set_handler_config(?MODULE,config, - HConfig#{sync_mode_qlen=>100, - flush_qlen=>99}), + logger:update_handler_config(?MODULE,config, + HConfig#{sync_mode_qlen=>100, + flush_qlen=>99}), %% invalid name of config parameter {error,{invalid_config,logger_disk_log_h,{filesync_rep_int,2000}}} = - logger:set_handler_config(?MODULE, config, - HConfig#{filesync_rep_int => 2000}), + logger:update_handler_config(?MODULE, config, + HConfig#{filesync_rep_int => 2000}), ok. config_fail(cleanup,_Config) -> logger:remove_handler(?MODULE). @@ -459,10 +490,26 @@ reconfig(Config) -> log_opts := #{type := ?DISK_LOG_TYPE, max_no_files := ?DISK_LOG_MAX_NO_FILES, max_no_bytes := ?DISK_LOG_MAX_NO_BYTES, - file := _DiskLogFile}} = + file := DiskLogFile}} = logger_disk_log_h:info(?MODULE), + {ok,#{config := + #{sync_mode_qlen := ?SYNC_MODE_QLEN, + drop_mode_qlen := ?DROP_MODE_QLEN, + flush_qlen := ?FLUSH_QLEN, + burst_limit_enable := ?BURST_LIMIT_ENABLE, + burst_limit_max_count := ?BURST_LIMIT_MAX_COUNT, + burst_limit_window_time := ?BURST_LIMIT_WINDOW_TIME, + overload_kill_enable := ?OVERLOAD_KILL_ENABLE, + overload_kill_qlen := ?OVERLOAD_KILL_QLEN, + overload_kill_mem_size := ?OVERLOAD_KILL_MEM_SIZE, + overload_kill_restart_after := ?OVERLOAD_KILL_RESTART_AFTER, + filesync_repeat_interval := ?FILESYNC_REPEAT_INTERVAL, + file := DiskLogFile, + max_no_files := ?DISK_LOG_MAX_NO_FILES, + max_no_bytes := ?DISK_LOG_MAX_NO_BYTES, + type := wrap} = HConfig0}} = + logger:get_handler_config(?MODULE), - {ok,#{config := HConfig0}} = logger:get_handler_config(?MODULE), HConfig1 = HConfig0#{sync_mode_qlen => 1, drop_mode_qlen => 2, flush_qlen => 3, @@ -488,6 +535,29 @@ reconfig(Config) -> overload_kill_restart_after := infinity, filesync_repeat_interval := no_repeat} = logger_disk_log_h:info(?MODULE), + {ok,#{config:=HConfig1}} = logger:get_handler_config(?MODULE), + + ok = logger:update_handler_config(?MODULE, config, + #{flush_qlen => ?FLUSH_QLEN}), + {ok,#{config:=C1}} = logger:get_handler_config(?MODULE), + ct:log("C1: ~p",[C1]), + C1 = HConfig1#{flush_qlen => ?FLUSH_QLEN}, + + ok = logger:set_handler_config(?MODULE, config, #{sync_mode_qlen => 1}), + {ok,#{config:=C2}} = logger:get_handler_config(?MODULE), + ct:log("C2: ~p",[C2]), + C2 = HConfig0#{sync_mode_qlen => 1}, + + ok = logger:set_handler_config(?MODULE, config, #{drop_mode_qlen => 100}), + {ok,#{config:=C3}} = logger:get_handler_config(?MODULE), + ct:log("C3: ~p",[C3]), + C3 = HConfig0#{drop_mode_qlen => 100}, + + ok = logger:update_handler_config(?MODULE, config, #{sync_mode_qlen => 1}), + {ok,#{config:=C4}} = logger:get_handler_config(?MODULE), + ct:log("C4: ~p",[C4]), + C4 = HConfig0#{sync_mode_qlen => 1, + drop_mode_qlen => 100}, ok = logger:remove_handler(?MODULE), @@ -507,6 +577,43 @@ reconfig(Config) -> max_no_bytes := 1024, file := File}} = logger_disk_log_h:info(?MODULE), + {ok,#{config := + #{type := halt, + max_no_files := 1, + max_no_bytes := 1024, + file := File}=HaltHConfig} = Config2} = + logger:get_handler_config(?MODULE), + + ok = logger:update_handler_config(?MODULE, level, notice), + {ok,C5} = logger:get_handler_config(?MODULE), + ct:log("C5: ~p",[C5]), + C5 = Config2#{level => notice}, + + ok = logger:set_handler_config(?MODULE, level, info), + {ok,C6} = logger:get_handler_config(?MODULE), + ct:log("C6: ~p",[C6]), + C6 = Config2#{level => info}, + + %% You are not allowed to actively set the write once fields + %% (type, max_no_files, max_no_bytes, file) in runtime. + {error, {illegal_config_change,_,_}} = + logger:set_handler_config(?MODULE,config,#{type=>wrap}), + {error, {illegal_config_change,_,_}} = + logger:set_handler_config(?MODULE,config,#{max_no_files=>2}), + {error, {illegal_config_change,_,_}} = + logger:set_handler_config(?MODULE,config,#{max_no_bytes=>2048}), + {error, {illegal_config_change,_,_}} = + logger:set_handler_config(?MODULE,config,#{file=>"otherfile.log"}), + {ok,C7} = logger:get_handler_config(?MODULE), + ct:log("C7: ~p",[C7]), + C7 = C6, + + %% ... but if you don't specify the write once fields, then + %% set_handler_config shall NOT reset them to their default value + ok = logger:set_handler_config(?MODULE,config,#{sync_mode_qlen=>1}), + {ok,#{config:=C8}} = logger:get_handler_config(?MODULE), + ct:log("C8: ~p",[C8]), + C8 = HaltHConfig#{sync_mode_qlen=>1}, ok. reconfig(cleanup, _Config) -> @@ -536,7 +643,7 @@ sync(Config) -> %% a disk_log_sync is still performed when handler goes idle {ok,#{config := HConfig}} = logger:get_handler_config(?MODULE), HConfig1 = HConfig#{filesync_repeat_interval => no_repeat}, - ok = logger:set_handler_config(?MODULE, config, HConfig1), + ok = logger:update_handler_config(?MODULE, config, HConfig1), no_repeat = maps:get(filesync_repeat_interval, logger_disk_log_h:info(?MODULE)), @@ -569,13 +676,13 @@ sync(Config) -> [OneSync || _ <- lists:seq(1, 1 + trunc(WaitT/SyncInt))]), HConfig2 = HConfig#{filesync_repeat_interval => SyncInt}, - ok = logger:set_handler_config(?MODULE, config, HConfig2), + ok = logger:update_handler_config(?MODULE, config, HConfig2), SyncInt = maps:get(filesync_repeat_interval, logger_disk_log_h:info(?MODULE)), timer:sleep(WaitT), HConfig3 = HConfig#{filesync_repeat_interval => no_repeat}, - ok = logger:set_handler_config(?MODULE, config, HConfig3), + ok = logger:update_handler_config(?MODULE, config, HConfig3), check_tracer(100), ok. sync(cleanup,_Config) -> @@ -797,7 +904,7 @@ sync_failure(Config) -> LogOpts = maps:get(log_opts, HState), SyncInt = 500, - ok = rpc:call(Node, logger, set_handler_config, + ok = rpc:call(Node, logger, update_handler_config, [?STANDARD_HANDLER, config, #{filesync_repeat_interval => SyncInt}]), Info = rpc:call(Node, logger_disk_log_h, info, [?STANDARD_HANDLER]), @@ -872,7 +979,7 @@ op_switch_to_sync(Config) -> drop_mode_qlen => NumOfReqs+1, flush_qlen => 2*NumOfReqs, burst_limit_enable => false}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), send_burst({n,NumOfReqs}, seq, {chars,79}, notice), Lines = count_lines(Log), NumOfReqs = Lines, @@ -897,7 +1004,7 @@ op_switch_to_drop(Config) -> drop_mode_qlen => 2, flush_qlen => Procs*NumOfReqs*Bursts, burst_limit_enable => false}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), %% It sometimes happens that the handler either gets %% the requests in a slow enough pace so that dropping %% never occurs. Therefore, lets generate a number of @@ -943,7 +1050,7 @@ op_switch_to_flush(Config) -> drop_mode_qlen => 300, flush_qlen => 300, burst_limit_enable => false}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), NumOfReqs = 1500, Procs = 10, Bursts = 10, @@ -985,7 +1092,7 @@ limit_burst_disabled(Config) -> burst_limit_window_time => 2000, drop_mode_qlen => 200, flush_qlen => 300}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), NumOfReqs = 100, send_burst({n,NumOfReqs}, seq, {chars,79}, notice), Logged = count_lines(Log), @@ -1005,7 +1112,7 @@ limit_burst_enabled_one(Config) -> burst_limit_window_time => 2000, drop_mode_qlen => 200, flush_qlen => 300}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), NumOfReqs = 100, send_burst({n,NumOfReqs}, seq, {chars,79}, notice), Logged = count_lines(Log), @@ -1026,7 +1133,7 @@ limit_burst_enabled_period(Config) -> burst_limit_window_time => BurstTWin, drop_mode_qlen => 20000, flush_qlen => 20001}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), Windows = 3, Sent = send_burst({t,BurstTWin*Windows}, seq, {chars,79}, notice), @@ -1046,7 +1153,7 @@ kill_disabled(Config) -> HConfig#{config=>DLHConfig#{overload_kill_enable=>false, overload_kill_qlen=>10, overload_kill_mem_size=>100}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), NumOfReqs = 100, send_burst({n,NumOfReqs}, seq, {chars,79}, notice), Logged = count_lines(Log), @@ -1068,7 +1175,7 @@ qlen_kill_new(Config) -> overload_kill_qlen=>10, overload_kill_mem_size=>Mem0+50000, overload_kill_restart_after=>RestartAfter}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), MRef = erlang:monitor(process, Pid0), NumOfReqs = 100, Procs = 4, @@ -1105,7 +1212,7 @@ mem_kill_new(Config) -> overload_kill_qlen=>50000, overload_kill_mem_size=>Mem0+500, overload_kill_restart_after=>RestartAfter}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), MRef = erlang:monitor(process, Pid0), NumOfReqs = 100, Procs = 4, @@ -1139,7 +1246,7 @@ restart_after(Config) -> HConfig#{config=>DLHConfig#{overload_kill_enable=>true, overload_kill_qlen=>10, overload_kill_restart_after=>infinity}}, - ok = logger:set_handler_config(?MODULE, NewHConfig1), + ok = logger:update_handler_config(?MODULE, NewHConfig1), MRef1 = erlang:monitor(process, whereis(h_proc_name())), %% kill handler send_burst({n,100}, {spawn,4,0}, {chars,79}, notice), @@ -1161,7 +1268,7 @@ restart_after(Config) -> HConfig#{config=>DLHConfig#{overload_kill_enable=>true, overload_kill_qlen=>10, overload_kill_restart_after=>RestartAfter}}, - ok = logger:set_handler_config(?MODULE, NewHConfig2), + ok = logger:update_handler_config(?MODULE, NewHConfig2), Pid0 = whereis(h_proc_name()), MRef2 = erlang:monitor(process, Pid0), %% kill handler @@ -1194,7 +1301,7 @@ handler_requests_under_load(Config) -> drop_mode_qlen => 1000, flush_qlen => 2000, burst_limit_enable => false}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), Pid = spawn_link(fun() -> send_requests(?MODULE, 1, [{filesync,[]}, {info,[]}, {reset,[]}, @@ -1227,9 +1334,9 @@ send_requests(HName, TO, Reqs = [{Req,Res}|Rs]) -> Result = case Req of change_config -> - logger:set_handler_config(HName, logger_disk_log_h, - #{overload_kill_enable => - false}); + logger:update_handler_config(HName, logger_disk_log_h, + #{overload_kill_enable => + false}); Func -> logger_disk_log_h:Func(HName) end, diff --git a/lib/kernel/test/logger_std_h_SUITE.erl b/lib/kernel/test/logger_std_h_SUITE.erl index 3426567bbf..b6a09f4980 100644 --- a/lib/kernel/test/logger_std_h_SUITE.erl +++ b/lib/kernel/test/logger_std_h_SUITE.erl @@ -108,6 +108,7 @@ all() -> add_remove_instance_file1, add_remove_instance_file2, default_formatter, + filter_config, errors, formatter_fail, config_fail, @@ -204,6 +205,20 @@ default_formatter(_Config) -> match = re:run(Msg,"=NOTICE REPORT====.*\n"++M1,[{capture,none}]), ok. +filter_config(_Config) -> + ok = logger:add_handler(?MODULE,logger_std_h,#{}), + {ok,#{config:=HConfig}=Config} = logger:get_handler_config(?MODULE), + HConfig = maps:without([handler_pid,mode_tab],HConfig), + + FakeFullHConfig = HConfig#{handler_pid=>self(),mode_tab=>erlang:make_ref()}, + #{config:=HConfig} = + logger_std_h:filter_config(Config#{config=>FakeFullHConfig}), + ok. + +filter_config(cleanup,_Config) -> + logger:remove_handler(?MODULE), + ok. + errors(Config) -> Dir = ?config(priv_dir,Config), Log = filename:join(Dir,?FUNCTION_NAME), @@ -319,11 +334,10 @@ config_fail(_Config) -> ok = logger:add_handler(?MODULE,logger_std_h, #{filter_default=>log, formatter=>{?MODULE,self()}}), - {error,{illegal_config_change,_,_}} = + {error,{illegal_config_change,#{config:=#{type:=_}},#{config:=#{type:=_}}}} = logger:set_handler_config(?MODULE,config, #{type=>{file,"file"}}), - {error,{illegal_config_change,_,_}} = - logger:set_handler_config(?MODULE,id,bad), + {error,{invalid_levels,_}} = logger:set_handler_config(?MODULE,config, #{sync_mode_qlen=>100, @@ -331,6 +345,15 @@ config_fail(_Config) -> {error,{invalid_config,logger_std_h,{filesync_rep_int,2000}}} = logger:set_handler_config(?MODULE, config, #{filesync_rep_int => 2000}), + + %% Read-only fields may (accidentially) be included in the change, + %% but it won't take effect + {ok,C} = logger:get_handler_config(?MODULE), + ok = logger:set_handler_config(?MODULE,config, + #{handler_pid=>self(), + mode_tab=>erlang:make_ref()}), + {ok,C} = logger:get_handler_config(?MODULE), + ok. config_fail(cleanup,_Config) -> @@ -457,9 +480,26 @@ reconfig(Config) -> overload_kill_qlen := ?OVERLOAD_KILL_QLEN, overload_kill_mem_size := ?OVERLOAD_KILL_MEM_SIZE, overload_kill_restart_after := ?OVERLOAD_KILL_RESTART_AFTER, - filesync_repeat_interval := ?FILESYNC_REPEAT_INTERVAL} = + filesync_repeat_interval := ?FILESYNC_REPEAT_INTERVAL} = DefaultInfo = logger_std_h:info(?MODULE), + {ok, + #{config:= + #{type := standard_io, + sync_mode_qlen := ?SYNC_MODE_QLEN, + drop_mode_qlen := ?DROP_MODE_QLEN, + flush_qlen := ?FLUSH_QLEN, + burst_limit_enable := ?BURST_LIMIT_ENABLE, + burst_limit_max_count := ?BURST_LIMIT_MAX_COUNT, + burst_limit_window_time := ?BURST_LIMIT_WINDOW_TIME, + overload_kill_enable := ?OVERLOAD_KILL_ENABLE, + overload_kill_qlen := ?OVERLOAD_KILL_QLEN, + overload_kill_mem_size := ?OVERLOAD_KILL_MEM_SIZE, + overload_kill_restart_after := ?OVERLOAD_KILL_RESTART_AFTER, + filesync_repeat_interval := ?FILESYNC_REPEAT_INTERVAL} = + DefaultHConf}} + = logger:get_handler_config(?MODULE), + ok = logger:set_handler_config(?MODULE, config, #{sync_mode_qlen => 1, drop_mode_qlen => 2, @@ -485,7 +525,77 @@ reconfig(Config) -> overload_kill_qlen := 100000, overload_kill_mem_size := 10000000, overload_kill_restart_after := infinity, - filesync_repeat_interval := no_repeat} = logger_std_h:info(?MODULE), + filesync_repeat_interval := no_repeat} = Info = logger_std_h:info(?MODULE), + + {ok,#{config := + #{type := standard_io, + sync_mode_qlen := 1, + drop_mode_qlen := 2, + flush_qlen := 3, + burst_limit_enable := false, + burst_limit_max_count := 10, + burst_limit_window_time := 10, + overload_kill_enable := true, + overload_kill_qlen := 100000, + overload_kill_mem_size := 10000000, + overload_kill_restart_after := infinity, + filesync_repeat_interval := no_repeat} = HConf}} = + logger:get_handler_config(?MODULE), + + ok = logger:update_handler_config(?MODULE, config, + #{flush_qlen => ?FLUSH_QLEN}), + {ok,#{config:=C1}} = logger:get_handler_config(?MODULE), + ct:log("C1: ~p",[C1]), + C1 = HConf#{flush_qlen => ?FLUSH_QLEN}, + + ok = logger:set_handler_config(?MODULE, config, #{sync_mode_qlen => 1}), + {ok,#{config:=C2}} = logger:get_handler_config(?MODULE), + ct:log("C2: ~p",[C2]), + C2 = DefaultHConf#{sync_mode_qlen => 1}, + + ok = logger:set_handler_config(?MODULE, config, #{drop_mode_qlen => 100}), + {ok,#{config:=C3}} = logger:get_handler_config(?MODULE), + ct:log("C3: ~p",[C3]), + C3 = DefaultHConf#{drop_mode_qlen => 100}, + + ok = logger:update_handler_config(?MODULE, config, #{sync_mode_qlen => 1}), + {ok,#{config:=C4}} = logger:get_handler_config(?MODULE), + ct:log("C4: ~p",[C4]), + C4 = DefaultHConf#{sync_mode_qlen => 1, + drop_mode_qlen => 100}, + + ok = logger:remove_handler(?MODULE), + + File = filename:join(Dir,lists:concat([?FUNCTION_NAME,".log"])), + ok = logger:add_handler(?MODULE, + logger_std_h, + #{config => #{type => {file,File}}, + filter_default=>log, + filters=>?DEFAULT_HANDLER_FILTERS([?MODULE]), + formatter=>{?MODULE,self()}}), + + {ok,#{config:=#{filesync_repeat_interval:=FSI}=FileHConfig}} = + logger:get_handler_config(?MODULE), + ok = logger:update_handler_config(?MODULE,config, + #{filesync_repeat_interval=>FSI+2000}), + {ok,#{config:=C5}} = logger:get_handler_config(?MODULE), + ct:log("C5: ~p",[C5]), + C5 = FileHConfig#{filesync_repeat_interval=>FSI+2000}, + + %% You are not allowed to actively set 'type' in runtime, since + %% this is a write once field. + {error, {illegal_config_change,_,_}} = + logger:set_handler_config(?MODULE,config,#{type=>standard_io}), + {ok,#{config:=C6}} = logger:get_handler_config(?MODULE), + ct:log("C6: ~p",[C6]), + C6 = C5, + + %% ... but if you don't specify 'type', then set_handler_config shall + %% NOT reset it to its default value + ok = logger:set_handler_config(?MODULE,config,#{sync_mode_qlen=>1}), + {ok,#{config:=C7}} = logger:get_handler_config(?MODULE), + ct:log("C7: ~p",[C7]), + C7 = FileHConfig#{sync_mode_qlen=>1}, ok. reconfig(cleanup, _Config) -> @@ -561,8 +671,8 @@ sync(Config) -> %% check that if there's no repeated filesync active, %% a filesync is still performed when handler goes idle - logger:set_handler_config(?MODULE, config, - #{filesync_repeat_interval => no_repeat}), + ok = logger:update_handler_config(?MODULE, config, + #{filesync_repeat_interval => no_repeat}), no_repeat = maps:get(filesync_repeat_interval, logger_std_h:info(?MODULE)), %% The following timer is to make sure the time from last log %% ("second") to next ("third") is long enough, so the a flush is @@ -592,12 +702,12 @@ sync(Config) -> start_tracer([{logger_std_h,handle_cast,2}], [OneSync || _ <- lists:seq(1, 1 + trunc(WaitT/SyncInt))]), - logger:set_handler_config(?MODULE, config, - #{filesync_repeat_interval => SyncInt}), + ok = logger:update_handler_config(?MODULE, config, + #{filesync_repeat_interval => SyncInt}), SyncInt = maps:get(filesync_repeat_interval, logger_std_h:info(?MODULE)), timer:sleep(WaitT), - logger:set_handler_config(?MODULE, config, - #{filesync_repeat_interval => no_repeat}), + ok = logger:update_handler_config(?MODULE, config, + #{filesync_repeat_interval => no_repeat}), check_tracer(100), ok. sync(cleanup, _Config) -> @@ -652,7 +762,7 @@ sync_failure(Config) -> rpc:call(Node, ?MODULE, set_result, [file_datasync,ok]), SyncInt = 500, - ok = rpc:call(Node, logger, set_handler_config, + ok = rpc:call(Node, logger, update_handler_config, [?STANDARD_HANDLER, config, #{filesync_repeat_interval => SyncInt}]), Info = rpc:call(Node, logger_std_h, info, [?STANDARD_HANDLER]), @@ -718,7 +828,7 @@ op_switch_to_sync_file(Config) -> drop_mode_qlen => NumOfReqs+1, flush_qlen => 2*NumOfReqs, burst_limit_enable => false}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), %% TRecvPid = start_op_trace(), send_burst({n,NumOfReqs}, seq, {chars,79}, notice), Lines = count_lines(Log), @@ -747,7 +857,7 @@ op_switch_to_sync_tty(Config) -> drop_mode_qlen => NumOfReqs+1, flush_qlen => 2*NumOfReqs, burst_limit_enable => false}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), send_burst({n,NumOfReqs}, seq, {chars,79}, notice), ok. op_switch_to_sync_tty(cleanup, _Config) -> @@ -770,7 +880,7 @@ op_switch_to_drop_file(Config) -> flush_qlen => Procs*NumOfReqs*Bursts, burst_limit_enable => false}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), %% It sometimes happens that the handler gets the %% requests in a slow enough pace so that dropping %% never occurs. Therefore, lets generate a number of @@ -807,7 +917,7 @@ op_switch_to_drop_tty(Config) -> flush_qlen => Procs*NumOfReqs+1, burst_limit_enable => false}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), send_burst({n,NumOfReqs}, {spawn,Procs,0}, {chars,79}, notice), ok. op_switch_to_drop_tty(cleanup, _Config) -> @@ -832,7 +942,7 @@ op_switch_to_flush_file(Config) -> drop_mode_qlen => 300, flush_qlen => 300, burst_limit_enable => false}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), NumOfReqs = 1500, Procs = 10, Bursts = 10, @@ -879,7 +989,7 @@ op_switch_to_flush_tty(Config) -> drop_mode_qlen => 100, flush_qlen => 100, burst_limit_enable => false}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), NumOfReqs = 1000, Procs = 100, send_burst({n,NumOfReqs}, {spawn,Procs,0}, {chars,79}, notice), @@ -895,7 +1005,7 @@ limit_burst_disabled(Config) -> burst_limit_window_time => 2000, drop_mode_qlen => 200, flush_qlen => 300}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), NumOfReqs = 100, send_burst({n,NumOfReqs}, seq, {chars,79}, notice), Logged = count_lines(Log), @@ -915,7 +1025,7 @@ limit_burst_enabled_one(Config) -> burst_limit_window_time => 2000, drop_mode_qlen => 200, flush_qlen => 300}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), NumOfReqs = 100, send_burst({n,NumOfReqs}, seq, {chars,79}, notice), Logged = count_lines(Log), @@ -936,7 +1046,7 @@ limit_burst_enabled_period(Config) -> burst_limit_window_time => BurstTWin, drop_mode_qlen => 20000, flush_qlen => 20001}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), Windows = 3, Sent = send_burst({t,BurstTWin*Windows}, seq, {chars,79}, notice), @@ -956,7 +1066,7 @@ kill_disabled(Config) -> HConfig#{config=>StdHConfig#{overload_kill_enable=>false, overload_kill_qlen=>10, overload_kill_mem_size=>100}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), NumOfReqs = 100, send_burst({n,NumOfReqs}, seq, {chars,79}, notice), Logged = count_lines(Log), @@ -977,7 +1087,7 @@ qlen_kill_new(Config) -> overload_kill_qlen=>10, overload_kill_mem_size=>Mem0+50000, overload_kill_restart_after=>RestartAfter}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), MRef = erlang:monitor(process, Pid0), NumOfReqs = 100, Procs = 4, @@ -1011,7 +1121,7 @@ qlen_kill_std(_Config) -> %% File = lists:concat([?MODULE,"_",?FUNCTION_NAME,".log"]), %% Log = filename:join(Dir, File), %% Node = start_std_h_on_new_node(Config, ?FUNCTION_NAME, Log), - %% ok = rpc:call(Node, logger, set_handler_config, + %% ok = rpc:call(Node, logger, update_handler_config, %% [?STANDARD_HANDLER, config, %% #{overload_kill_enable=>true, %% overload_kill_qlen=>10, @@ -1028,7 +1138,7 @@ mem_kill_new(Config) -> overload_kill_qlen=>50000, overload_kill_mem_size=>Mem0+500, overload_kill_restart_after=>RestartAfter}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), MRef = erlang:monitor(process, Pid0), NumOfReqs = 100, Procs = 4, @@ -1067,7 +1177,7 @@ restart_after(Config) -> HConfig#{config=>StdHConfig#{overload_kill_enable=>true, overload_kill_qlen=>10, overload_kill_restart_after=>infinity}}, - ok = logger:set_handler_config(?MODULE, NewHConfig1), + ok = logger:update_handler_config(?MODULE, NewHConfig1), MRef1 = erlang:monitor(process, whereis(h_proc_name())), %% kill handler send_burst({n,100}, {spawn,4,0}, {chars,79}, notice), @@ -1082,14 +1192,15 @@ restart_after(Config) -> ct:pal("Handler state = ~p", [Info1]), ct:fail("Handler not dead! It should not have survived this!") end, - + {Log,_,_} = start_handler(?MODULE, ?FUNCTION_NAME, Config), RestartAfter = ?OVERLOAD_KILL_RESTART_AFTER, + NewHConfig2 = HConfig#{config=>StdHConfig#{overload_kill_enable=>true, overload_kill_qlen=>10, overload_kill_restart_after=>RestartAfter}}, - ok = logger:set_handler_config(?MODULE, NewHConfig2), + ok = logger:update_handler_config(?MODULE, NewHConfig2), Pid0 = whereis(h_proc_name()), MRef2 = erlang:monitor(process, Pid0), %% kill handler @@ -1123,7 +1234,7 @@ handler_requests_under_load(Config) -> drop_mode_qlen => 1000, flush_qlen => 2000, burst_limit_enable => false}}, - ok = logger:set_handler_config(?MODULE, NewHConfig), + ok = logger:update_handler_config(?MODULE, NewHConfig), Pid = spawn_link(fun() -> send_requests(?MODULE, 1, [{filesync,[]}, {info,[]}, {reset,[]}, @@ -1155,9 +1266,9 @@ send_requests(HName, TO, Reqs = [{Req,Res}|Rs]) -> Result = case Req of change_config -> - logger:set_handler_config(HName, config, - #{overload_kill_enable => - false}); + logger:update_handler_config(HName, config, + #{overload_kill_enable => + false}); Func -> logger_std_h:Func(HName) end, diff --git a/lib/observer/src/crashdump_viewer.erl b/lib/observer/src/crashdump_viewer.erl index 14b086ff58..276b209ce5 100644 --- a/lib/observer/src/crashdump_viewer.erl +++ b/lib/observer/src/crashdump_viewer.erl @@ -1594,31 +1594,92 @@ read_heap(Fd,Pid,DecodeOpts,Dict0) -> Dict0 end. -read_heap(DecodeOpts,Dict0) -> - %% This function is never called if the dump is truncated in {?proc_heap,Pid} - case get(fd) of - end_of_heap -> +read_heap(DecodeOpts, Dict0) -> + %% This function is never called if the dump is truncated in + %% {?proc_heap,Pid}. + %% + %% It is not always possible to reconstruct the heap terms + %% in a single pass, especially if maps are involved. + %% See crashdump_helper:literal_map/0 for an example. + %% + %% Therefore, we need two passes. In the first pass + %% we collect all lines without parsing them, and in the + %% second pass we parse them. + %% + %% The first pass follows. + + Lines0 = read_heap_lines(), + + %% Save a map of all unprocessed lines so that deref_ptr() can + %% access any line when there are references to terms not yet + %% built. + + LineMap = maps:from_list(Lines0), + put(line_map, LineMap), + + %% Refc binaries (tag "Yc") must be processed before any sub + %% binaries (tag "Ys") referencing them, so we make sure to + %% process all the refc binaries first. + %% + %% The other lines can be processed in any order, but processing + %% them in the reverse order compared to how they are printed in + %% the crash dump seems to minimize the number of references to + %% terms that have not yet been built. That happens to be the + %% order of the line list as returned by read_heap_lines/0. + + RefcBins = [Refc || {_,<<"Yc",_/binary>>}=Refc <- Lines0], + Lines = RefcBins ++ Lines0, + + %% Second pass. + + init_progress("Processing terms", map_size(LineMap)), + Dict = parse_heap_terms(Lines, DecodeOpts, Dict0), + erase(line_map), + end_progress(), + Dict. + +read_heap_lines() -> + read_heap_lines_1(get(fd), []). + +read_heap_lines_1(Fd, Acc) -> + case bytes(Fd) of + "=" ++ _next_tag -> end_progress(), - Dict0; - Fd -> - case bytes(Fd) of - "=" ++ _next_tag -> - end_progress(), - put(fd, end_of_heap), - Dict0; - Line -> - update_progress(length(Line)+1), - Dict = parse(Line,DecodeOpts,Dict0), - read_heap(DecodeOpts,Dict) - end + put(fd, end_of_heap), + Acc; + Line0 -> + update_progress(length(Line0)+1), + {Addr,":"++Line1} = get_hex(Line0), + + %% Reduce the memory consumption by converting the + %% line to a binary. Measurements show that it may also + %% be benefical for performance, too, because it makes the + %% garbage collections cheaper. + + Line = list_to_binary(Line1), + read_heap_lines_1(Fd, [{Addr,Line}|Acc]) end. -parse(Line0, DecodeOpts, Dict0) -> - {Addr,":"++Line1} = get_hex(Line0), - {_Term,Line,Dict} = parse_heap_term(Line1, Addr, DecodeOpts, Dict0), - [] = skip_blanks(Line), +parse_heap_terms([{Addr,Line0}|T], DecodeOpts, Dict0) -> + case gb_trees:is_defined(Addr, Dict0) of + true -> + %% Already parsed (by a recursive call from do_deref_ptr() + %% to parse_line()). Nothing to do. + parse_heap_terms(T, DecodeOpts, Dict0); + false -> + %% Parse this previously unparsed term. + Dict = parse_line(Addr, Line0, DecodeOpts, Dict0), + parse_heap_terms(T, DecodeOpts, Dict) + end; +parse_heap_terms([], _DecodeOpts, Dict) -> Dict. +parse_line(Addr, Line0, DecodeOpts, Dict0) -> + update_progress(1), + Line1 = binary_to_list(Line0), + {_Term,Line,Dict} = parse_heap_term(Line1, Addr, DecodeOpts, Dict0), + [] = skip_blanks(Line), %Assertion. + Dict. %%----------------------------------------------------------------- %% Page with one port @@ -2871,16 +2932,18 @@ parse_atom_translation_table(N, Line0, As) -> deref_ptr(Ptr, Line, DecodeOpts, D) -> - Lookup = fun(D0) -> - gb_trees:lookup(Ptr, D0) - end, + Lookup0 = fun(D0) -> + gb_trees:lookup(Ptr, D0) + end, + Lookup = wrap_line_map(Ptr, Lookup0), do_deref_ptr(Lookup, Line, DecodeOpts, D). deref_bin(Binp0, Offset, Sz, Line, DecodeOpts, D) -> Binp = Binp0 bor DecodeOpts#dec_opts.bin_addr_adj, - Lookup = fun(D0) -> - lookup_binary(Binp, Offset, Sz, D0) - end, + Lookup0 = fun(D0) -> + lookup_binary(Binp, Offset, Sz, D0) + end, + Lookup = wrap_line_map(Binp, Lookup0), do_deref_ptr(Lookup, Line, DecodeOpts, D). lookup_binary(Binp, Offset, Sz, D) -> @@ -2899,26 +2962,36 @@ lookup_binary(Binp, Offset, Sz, D) -> end end. +wrap_line_map(Ptr, Lookup) -> + wrap_line_map_1(get(line_map), Ptr, Lookup). + +wrap_line_map_1(#{}=LineMap, Ptr, Lookup) -> + fun(D) -> + case Lookup(D) of + {value,_}=Res -> + Res; + none -> + case LineMap of + #{Ptr:=Line} -> + {line,Ptr,Line}; + #{} -> + none + end + end + end; +wrap_line_map_1(undefined, _Ptr, Lookup) -> + Lookup. + do_deref_ptr(Lookup, Line, DecodeOpts, D0) -> case Lookup(D0) of {value,Term} -> {Term,Line,D0}; none -> - case get(fd) of - end_of_heap -> - put(incomplete_heap,true), - {['#CDVIncompleteHeap'],Line,D0}; - Fd -> - case bytes(Fd) of - "="++_ -> - put(fd, end_of_heap), - do_deref_ptr(Lookup, Line, DecodeOpts, D0); - L -> - update_progress(length(L)+1), - D = parse(L, DecodeOpts, D0), - do_deref_ptr(Lookup, Line, DecodeOpts, D) - end - end + put(incomplete_heap, true), + {['#CDVIncompleteHeap'],Line,D0}; + {line,Addr,NewLine} -> + D = parse_line(Addr, NewLine, DecodeOpts, D0), + do_deref_ptr(Lookup, Line, DecodeOpts, D) end. get_hex(L) -> diff --git a/lib/observer/test/crashdump_helper.erl b/lib/observer/test/crashdump_helper.erl index 145ff56b71..d8f4e046ae 100644 --- a/lib/observer/test/crashdump_helper.erl +++ b/lib/observer/test/crashdump_helper.erl @@ -142,4 +142,23 @@ create_maps() -> Map3 = lists:foldl(fun(I, A) -> A#{I=>I*I} end, Map2, lists:seq(-10, 0)), - #{a=>Map0,b=>Map1,c=>Map2,d=>Map3,e=>#{}}. + #{a=>Map0,b=>Map1,c=>Map2,d=>Map3,e=>#{},literal=>literal_map()}. + +literal_map() -> + %% A literal map such as the one below will produce a heap dump + %% like this: + %% + %% Address1:t4:H<Address3>,H<Address4>,H<Address5>,H<Address6> + %% Address2:Mf4:H<Adress1>:I1,I2,I3,I4 + %% Address3: ... % "one" + %% Address4: ... % "two" + %% Address5: ... % "three" + %% Address6: ... % "four" + %% + %% The map cannot be reconstructed in a single sequential pass. + %% + %% To reconstruct the map, first the string keys "one" + %% through "four" must be reconstructed, then the tuple at + %% Adress1, then the map at Address2. + + #{"one"=>1,"two"=>2,"three"=>3,"four"=>4}. diff --git a/lib/public_key/asn1/OTP-PKIX.asn1 b/lib/public_key/asn1/OTP-PKIX.asn1 index 37196bb9bf..10a83555af 100644 --- a/lib/public_key/asn1/OTP-PKIX.asn1 +++ b/lib/public_key/asn1/OTP-PKIX.asn1 @@ -368,6 +368,13 @@ SupportedPublicKeyAlgorithms PUBLIC-KEY-ALGORITHM-CLASS ::= { ID id-dsaWithSHA1 TYPE DSAParams } + id-dsa-with-sha224 OBJECT IDENTIFIER ::= { + joint-iso-ccitt(2) country(16) us(840) organization(1) gov(101) + csor(3) algorithms(4) id-dsa-with-sha2(3) 1 } + + id-dsa-with-sha256 OBJECT IDENTIFIER ::= { + joint-iso-ccitt(2) country(16) us(840) organization(1) gov(101) + csor(3) algorithms(4) id-dsa-with-sha2(3) 2 } -- -- RSA Keys and Signatures -- diff --git a/lib/public_key/src/public_key.erl b/lib/public_key/src/public_key.erl index 3f609ce6c6..dca1e0766e 100644 --- a/lib/public_key/src/public_key.erl +++ b/lib/public_key/src/public_key.erl @@ -609,6 +609,10 @@ pkix_sign_types(?'id-dsa-with-sha1') -> {sha, dsa}; pkix_sign_types(?'id-dsaWithSHA1') -> {sha, dsa}; +pkix_sign_types(?'id-dsa-with-sha224') -> + {sha224, dsa}; +pkix_sign_types(?'id-dsa-with-sha256') -> + {sha256, dsa}; pkix_sign_types(?'ecdsa-with-SHA1') -> {sha, ecdsa}; pkix_sign_types(?'ecdsa-with-SHA256') -> diff --git a/lib/public_key/test/public_key_SUITE.erl b/lib/public_key/test/public_key_SUITE.erl index 1955e9e119..878489eb0f 100644 --- a/lib/public_key/test/public_key_SUITE.erl +++ b/lib/public_key/test/public_key_SUITE.erl @@ -44,7 +44,9 @@ all() -> encrypt_decrypt, {group, sign_verify}, pkix, pkix_countryname, pkix_emailaddress, pkix_path_validation, - pkix_iso_rsa_oid, pkix_iso_dsa_oid, pkix_crl, general_name, + pkix_iso_rsa_oid, pkix_iso_dsa_oid, + pkix_dsa_sha2_oid, + pkix_crl, general_name, pkix_verify_hostname_cn, pkix_verify_hostname_subjAltName, pkix_verify_hostname_subjAltName_IP, @@ -1114,6 +1116,13 @@ pkix_iso_dsa_oid(Config) when is_list(Config) -> {_, dsa} = public_key:pkix_sign_types(SigAlg#'SignatureAlgorithm'.algorithm). %%-------------------------------------------------------------------- +pkix_dsa_sha2_oid() -> + [{doc, "Test support dsa_sha2 oid"}]. +pkix_dsa_sha2_oid(Config) when is_list(Config) -> + {sha224, dsa} = public_key:pkix_sign_types(?'id-dsa-with-sha224'), + {sha256, dsa} = public_key:pkix_sign_types(?'id-dsa-with-sha256'). + +%%-------------------------------------------------------------------- pkix_crl() -> [{doc, "test pkix_crl_* functions"}]. diff --git a/lib/ssh/doc/src/Makefile b/lib/ssh/doc/src/Makefile index 77fa356092..4e32dd9976 100644 --- a/lib/ssh/doc/src/Makefile +++ b/lib/ssh/doc/src/Makefile @@ -45,6 +45,7 @@ XML_REF3_FILES = \ ssh_connection.xml \ ssh_server_channel.xml \ ssh_server_key_api.xml \ + ssh_file.xml \ ssh_sftp.xml \ ssh_sftpd.xml \ @@ -56,8 +57,8 @@ XML_CHAPTER_FILES = \ notes.xml \ introduction.xml \ using_ssh.xml \ + terminology.xml \ configure_algos.xml -# ssh_protocol.xml \ BOOK_FILES = book.xml diff --git a/lib/ssh/doc/src/ref_man.xml b/lib/ssh/doc/src/ref_man.xml index df37b0244f..60572b985b 100644 --- a/lib/ssh/doc/src/ref_man.xml +++ b/lib/ssh/doc/src/ref_man.xml @@ -40,6 +40,7 @@ <xi:include href="ssh_connection.xml"/> <xi:include href="ssh_client_key_api.xml"/> <xi:include href="ssh_server_key_api.xml"/> + <xi:include href="ssh_file.xml"/> <xi:include href="ssh_sftp.xml"/> <xi:include href="ssh_sftpd.xml"/> </application> diff --git a/lib/ssh/doc/src/specs.xml b/lib/ssh/doc/src/specs.xml index acdbe2ddfd..a6517f3660 100644 --- a/lib/ssh/doc/src/specs.xml +++ b/lib/ssh/doc/src/specs.xml @@ -6,6 +6,7 @@ <xi:include href="../specs/specs_ssh_connection.xml"/> <xi:include href="../specs/specs_ssh_server_channel.xml"/> <xi:include href="../specs/specs_ssh_server_key_api.xml"/> + <xi:include href="../specs/specs_ssh_file.xml"/> <xi:include href="../specs/specs_ssh_sftp.xml"/> <xi:include href="../specs/specs_ssh_sftpd.xml"/> </specs> diff --git a/lib/ssh/doc/src/ssh.xml b/lib/ssh/doc/src/ssh.xml index 3bc62073a2..93e52ce023 100644 --- a/lib/ssh/doc/src/ssh.xml +++ b/lib/ssh/doc/src/ssh.xml @@ -99,8 +99,8 @@ </p> <p>The paths could easily be changed by options: - <seealso marker="#type-user_dir_common_option"><c>user_dir</c></seealso> and - <seealso marker="#type-system_dir_daemon_option"><c>system_dir</c></seealso>. + <seealso marker="ssh_file#type-user_dir_common_option"><c>user_dir</c></seealso> and + <seealso marker="ssh_file#type-system_dir_daemon_option"><c>system_dir</c></seealso>. </p> <p>A completly different storage could be interfaced by writing call-back modules using the behaviours @@ -123,12 +123,12 @@ <item><c>ssh_host_ecdsa_key</c> and <c>ssh_host_ecdsa_key.pub</c></item> </list> <p>The host keys directory could be changed with the option - <seealso marker="#type-system_dir_daemon_option"><c>system_dir</c></seealso>.</p> + <seealso marker="ssh_file#type-system_dir_daemon_option"><c>system_dir</c></seealso>.</p> </item> <item>Optional: one or more <i>User's public key</i> in case of <c>publickey</c> authorization. Default is to store them concatenated in the file <c>.ssh/authorized_keys</c> in the user's home directory. <p>The user keys directory could be changed with the option - <seealso marker="#type-user_dir_common_option"><c>user_dir</c></seealso>.</p> + <seealso marker="ssh_file#type-user_dir_common_option"><c>user_dir</c></seealso>.</p> </item> </list> </section> @@ -138,7 +138,7 @@ <p>The keys and some other data are by default stored in files in the directory <c>.ssh</c> in the user's home directory.</p> <p>The directory could be changed with the option - <seealso marker="#type-user_dir_common_option"><c>user_dir</c></seealso>. + <seealso marker="ssh_file#type-user_dir_common_option"><c>user_dir</c></seealso>. </p> <list> <item>Optional: a list of <i>Host public key(s)</i> for previously connected hosts. This list @@ -192,22 +192,13 @@ <p>If there is no public key of a specified type available, the corresponding entry is ignored. Note that the available set is dependent on the underlying cryptolib and current user's public keys. </p> - <p>See also the option <seealso marker="#type-user_dir_common_option"><c>user_dir</c></seealso> + <p>See also the option <seealso marker="ssh_file#type-user_dir_common_option"><c>user_dir</c></seealso> for specifying the path to the user's keys. </p> </desc> </datatype> <datatype> - <name name="pubkey_passphrase_client_options"/> - <desc> - <p>If the user's DSA, RSA or ECDSA key is protected by a passphrase, it can be - supplied with thoose options. - </p> - </desc> - </datatype> - - <datatype> <name name="host_accepting_client_options"/> <name name="accept_hosts"/> <name name="fp_digest_alg"/> @@ -220,7 +211,7 @@ <p>This option guides the <c>connect</c> function on how to act when the connected server presents a Host Key that the client has not seen before. The default is to ask the user with a question on stdio of whether to accept or reject the new Host Key. - See the option <seealso marker="#type-user_dir_common_option"><c>user_dir</c></seealso> + See the option <seealso marker="ssh_file#type-user_dir_common_option"><c>user_dir</c></seealso> for specifying the path to the file <c>known_hosts</c> where previously accepted Host Keys are recorded. See also the option <seealso marker="#type-key_cb_common_option">key_cb</seealso> @@ -276,7 +267,7 @@ accept question the next time the same host is connected. If the option <seealso marker="#type-key_cb_common_option"><c>key_cb</c></seealso> is not present, the key is saved in the file "known_hosts". See option - <seealso marker="#type-user_dir_common_option"><c>user_dir</c></seealso> for + <seealso marker="ssh_file#type-user_dir_common_option"><c>user_dir</c></seealso> for the location of that file. </p> <p>If <c>false</c>, the key is not saved and the key will still be unknown @@ -478,18 +469,6 @@ <name name="pwdfun_4"/> <desc> <taglist> - <tag><marker id="type-system_dir_daemon_option"/><c>system_dir</c></tag> - <item> - <p>Sets the system directory, containing the host key files - that identify the host keys for <c>ssh</c>. Defaults to - <c>/etc/ssh</c>.</p> - <p>For security reasons, this directory is normally accessible only to the root user.</p> - <p>See also the option - <seealso marker="#type-key_cb_common_option">key_cb</seealso> - for the general way to handle keys. - </p> - </item> - <tag><c>auth_method_kb_interactive_data</c></tag> <item> <p>Sets the text strings that the daemon sends to the client for presentation to the user when @@ -502,7 +481,7 @@ </p> </item> - <tag><c>user_passwords</c></tag> + <tag><marker id="option-user_passwords"/><c>user_passwords</c></tag> <item> <p>Provides passwords for password authentication. The passwords are used when someone tries to connect to the server and public key user-authentication fails. The option provides @@ -510,7 +489,7 @@ </p> </item> - <tag><c>password</c></tag> + <tag><marker id="option-password"/><c>password</c></tag> <item> <p>Provides a global password that authenticates any user.</p> <warning> @@ -519,7 +498,9 @@ </warning> </item> - <tag><c>pwdfun</c> with <c>pwdfun_4()</c></tag> + <tag><marker id="option-pwdfun"/><c>pwdfun</c> with + <seealso marker="#type-pwdfun_4"><c>pwdfun_4()</c></seealso> + </tag> <item> <p>Provides a function for password validation. This could used for calling an external system or handeling passwords stored as hash values. @@ -546,7 +527,9 @@ can be used for this. The return value <c>disconnect</c> is useful for this.</p> </item> - <tag><c>pwdfun</c> with <c>pwdfun_2()</c></tag> + <tag><c>pwdfun</c> with + <seealso marker="#type-pwdfun_2"><c>pwdfun_2()</c></seealso> + </tag> <item> <p>Provides a function for password validation. This function is called with user and password as strings, and returns:</p> @@ -725,21 +708,6 @@ </datatype> <datatype> - <name name="user_dir_common_option"/> - <desc> - <p>Sets the user directory. That is, the directory containing <c>ssh</c> configuration - files for the user, such as - <c>known_hosts</c>, <c>id_rsa</c>, <c>id_dsa</c>>, <c>id_ecdsa</c> and <c>authorized_key</c>. - Defaults to the directory normally referred to as <c>~/.ssh</c>. - </p> - <p>See also the option - <seealso marker="#type-key_cb_common_option">key_cb</seealso> - for the general way to handle keys. - </p> - </desc> - </datatype> - - <datatype> <name name="profile_common_option"/> <desc> <p>Used together with <c>ip-address</c> and <c>port</c> to @@ -795,7 +763,8 @@ </p> <p>The <c>Opts</c> defaults to <c>[]</c> when only the <c>Module</c> is specified. </p> - <p>The default value of this option is <c>{ssh_file, []}</c>. + <p>The default value of this option is <c>{ssh_file, []}</c>. See also the manpage of + <seealso marker="ssh:ssh_file">ssh_file</seealso>. </p> <p>A call to the call-back function <c>F</c> will be</p> <code> @@ -804,7 +773,10 @@ <p>where <c>...</c> are arguments to <c>F</c> as in <seealso marker="ssh_client_key_api">ssh_client_key_api</seealso> and/or <seealso marker="ssh_server_key_api">ssh_server_key_api</seealso>. - The <c>UserOptions</c> are the options given to <c>ssh:connect</c>, <c>ssh:shell</c> or <c>ssh:daemon</c>. + The <c>UserOptions</c> are the options given to + <seealso marker="ssh:ssh#connect-3">ssh:connect</seealso>, + <seealso marker="ssh:ssh#shell-1">ssh:shell</seealso> or + <seealso marker="ssh:ssh#daemon-2">ssh:daemon</seealso>. </p> </desc> diff --git a/lib/ssh/doc/src/ssh_app.xml b/lib/ssh/doc/src/ssh_app.xml index e80bb1853d..eb804e67dc 100644 --- a/lib/ssh/doc/src/ssh_app.xml +++ b/lib/ssh/doc/src/ssh_app.xml @@ -74,13 +74,18 @@ <c>id_ecdsa_key</c>, <c>known_hosts</c>, and <c>authorized_keys</c> in ~/.ssh, and for the host key files in <c>/etc/ssh</c>. These locations can be changed - by the options <c>user_dir</c> and <c>system_dir</c>. + by the options + <seealso marker="ssh_file#type-user_dir_common_option"><c>user_dir</c></seealso> and + <seealso marker="ssh_file#type-system_dir_daemon_option"><c>system_dir</c></seealso>. </p> <p>Public key handling can also be customized through a callback module that implements the behaviors <seealso marker="ssh_client_key_api">ssh_client_key_api</seealso> and <seealso marker="ssh_server_key_api">ssh_server_key_api</seealso>. </p> + <p>See also the default callback module documentation in + <seealso marker="ssh_file">ssh_file</seealso>. + </p> </section> <section> diff --git a/lib/ssh/doc/src/ssh_file.xml b/lib/ssh/doc/src/ssh_file.xml new file mode 100644 index 0000000000..ae6ba2e1d9 --- /dev/null +++ b/lib/ssh/doc/src/ssh_file.xml @@ -0,0 +1,275 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2018</year><year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>ssh_file</title> + <prepared></prepared> + <docno></docno> + <date></date> + <rev></rev> + </header> + <module>ssh_file</module> + <modulesummary>Default callback module for the client's and server's database operations in the ssh application</modulesummary> + <description> + <p>This module is the default callback handler for the client's and the server's user and host "database" operations. + All data, for instance key pairs, are stored in files in the normal file system. This page documents the files, where they + are stored and configuration options for this callback module. + </p> + <p>The intention is to be compatible with the + <url href="http://www.openssh.com">OpenSSH</url> + storage in files. Therefore it mimics directories and filenames of + <url href="http://www.openssh.com">OpenSSH</url>. + </p> + + <p>Ssh_file implements the <seealso marker="ssh:ssh_server_key_api">ssh_server_key_api</seealso> and + the <seealso marker="ssh:ssh_client_key_api">ssh_client_key_api</seealso>. + This enables the user to make an own interface using for example a database handler. + </p> + <p>Such another callback module could be used by setting the option + <seealso marker="ssh:ssh#type-key_cb_common_option"><c>key_cb</c></seealso> + when starting a client or a server (with for example + <seealso marker="ssh:ssh#connect-3">ssh:connect</seealso>, + <seealso marker="ssh:ssh#daemon-2">ssh:daemon</seealso> of + <seealso marker="ssh:ssh#shell-1">ssh:shell</seealso> + ). + </p> + + <note> + <p>The functions are <i>Callbacks</i> for the SSH app. They are not intended to be called from the user's code! + </p> + </note> + </description> + + <section> + <title>Files, directories and who uses them</title> + <section> + <title>Daemons</title> + <p>Daemons uses all files stored in the <seealso marker="#SYSDIR">SYSDIR</seealso> directory. + </p> + <p>Optionaly, in case of <c>publickey</c> authorization, one or more of the remote user's public keys + in the <seealso marker="#USERDIR">USERDIR</seealso> directory are used. + See the files + <seealso marker="#USERDIR-authorized_keys"><c>USERDIR/authorized_keys</c></seealso> and + <seealso marker="#USERDIR-authorized_keys2"><c>USERDIR/authorized_keys2</c></seealso>. + </p> + </section> + + <section> + <title>Clients</title> + <p>Clients uses all files stored in the <seealso marker="#USERDIR">USERDIR</seealso> directory. + </p> + </section> + + <section> + <title>Directory contents</title> + <taglist> + <tag><marker id="LOCALUSER"/>LOCALUSER</tag> + <item><p>The user name of the OS process running the Erlang virtual machine (emulator).</p> + </item> + + <tag><marker id="SYSDIR"/>SYSDIR</tag> + <item><p>This is the directory holding the server's files:</p> + <list> + <item><marker id="SYSDIR-ssh_host_dsa_key"/><c>ssh_host_dsa_key</c> - private dss host key (optional)</item> + <item><marker id="SYSDIR-ssh_host_rsa_key"/><c>ssh_host_rsa_key</c> - private rsa host key (optional)</item> + <item><marker id="SYSDIR-ssh_host_ecdsa_key"/><c>ssh_host_ecdsa_key</c> - private ecdsa host key (optional)</item> + </list> + <p>At least one host key must be defined. The default value of SYSDIR is <marker id="#/etc/ssh"/><c>/etc/ssh</c>. + </p> + <p>For security reasons, this directory is normally accessible only to the root user. + </p> + <p>To change the SYSDIR, see the <seealso marker="#type-system_dir_daemon_option">system_dir</seealso> option. + </p> + </item> + + <tag><marker id="USERDIR"/>USERDIR</tag> + <item><p>This is the directory holding the files:</p> + <list> + <item><marker id="USERDIR-authorized_keys"/><c>authorized_keys</c> + and, as second alternative + <marker id="USERDIR-authorized_keys2"/><c>authorized_keys2</c> - + the user's public keys are stored concatenated in one of those files. + </item> + <item><marker id="USERDIR-known_hosts"/><c>known_hosts</c> - host keys from hosts visited + concatenated. The file is created and used by the client.</item> + <item><marker id="USERDIR-id_dsa"/><c>id_dsa</c> - private dss user key (optional)</item> + <item><marker id="USERDIR-id_rsa"/><c>id_rsa</c> - private rsa user key (optional)</item> + <item><marker id="USERDIR-id_ecdsa"/><c>id_ecdsa</c> - private ecdsa user key (optional)</item> + </list> + <p>The default value of USERDIR is <c>/home/</c><seealso marker="#LOCALUSER"><c>LOCALUSER</c></seealso><c>/.ssh</c>. + </p> + <p>To change the USERDIR, see the <seealso marker="#type-user_dir_common_option">user_dir</seealso> option + </p> + </item> + </taglist> + </section> + </section> + + <datatypes> + <datatype_title>Options for the default ssh_file callback module</datatype_title> + <datatype> + <name name="user_dir_common_option"/> + <desc> + <p>Sets the <seealso marker="#USERDIR">user directory</seealso>.</p> + </desc> + </datatype> + + <datatype> + <name name="user_dir_fun_common_option"/> + <name name="user2dir"/> + <desc> + <p>Sets the <seealso marker="#USERDIR">user directory</seealso> dynamically + by evaluating the <c>user2dir</c> function. + </p> + </desc> + </datatype> + + <datatype> + <name name="system_dir_daemon_option"/> + <desc> + <p>Sets the <seealso marker="#SYSDIR">system directory</seealso>.</p> + </desc> + </datatype> + + <datatype> + <name name="pubkey_passphrase_client_options"/> + <desc> + <p>If the user's DSA, RSA or ECDSA key is protected by a passphrase, it can be + supplied with thoose options. + </p> + </desc> + </datatype> + + </datatypes> + + <funcs> + <func> + <name>host_key(Algorithm, DaemonOptions) -> {ok, Key} | {error, Reason}</name> + <fsummary></fsummary> + <desc> + <p><strong>Types and description</strong></p> + <p>See the api description in + <seealso marker="ssh:ssh_server_key_api#Module:host_key-2">ssh_server_key_api, Module:host_key/2</seealso>. + </p> + <p><strong>Options</strong></p> + <list> + <item><seealso marker="#type-system_dir_daemon_option">system_dir</seealso></item> + <!-- item>dsa_pass_phrase</item --> + <!-- item>rsa_pass_phrase</item --> + <!-- item>ecdsa_pass_phrase</item --> + </list> + <p><strong>Files</strong></p> + <list> + <item><seealso marker="#SYSDIR-ssh_host_rsa_key"><c>SYSDIR/ssh_host_rsa_key</c></seealso></item> + <item><seealso marker="#SYSDIR-ssh_host_dsa_key"><c>SYSDIR/ssh_host_dsa_key</c></seealso></item> + <item><seealso marker="#SYSDIR-ssh_host_ecdsa_key"><c>SYSDIR/ssh_host_ecdsa_key</c></seealso></item> + </list> + </desc> + </func> + + <func> + <name>is_auth_key(PublicUserKey, User, DaemonOptions) -> Result</name> + <fsummary></fsummary> + <desc> + <p><strong>Types and description</strong></p> + <p>See the api description in + <seealso marker="ssh:ssh_server_key_api#Module:is_auth_key-3">ssh_server_key_api: Module:is_auth_key/3</seealso>. + </p> + <p><strong>Options</strong></p> + <list> + <item><seealso marker="#type-user_dir_fun_common_option">user_dir_fun</seealso></item> + <item><seealso marker="#type-user_dir_common_option">user_dir</seealso></item> + </list> + <p><strong>Files</strong></p> + <list> + <item><seealso marker="#USERDIR-authorized_keys"><c>USERDIR/authorized_keys</c></seealso></item> + <item><seealso marker="#USERDIR-authorized_keys2"><c>USERDIR/authorized_keys2</c></seealso></item> + </list> + </desc> + </func> + + <func> + <name>add_host_key(HostNames, PublicHostKey, ConnectOptions) -> ok | {error, Reason}</name> + <fsummary></fsummary> + <desc> + <p><strong>Types and description</strong></p> + <p>See the api description in + <seealso marker="ssh:ssh_client_key_api#Module:add_host_key-3">ssh_client_key_api, Module:add_host_key/3</seealso>. + </p> + <p><strong>Option</strong></p> + <list> + <item><seealso marker="#type-user_dir_common_option">user_dir</seealso></item> + </list> + <p><strong>File</strong></p> + <list> + <item><seealso marker="#USERDIR-known_hosts"><c>USERDIR/known_hosts</c></seealso></item> + </list> + </desc> + </func> + + <func> + <name>is_host_key(Key, Host, Algorithm, ConnectOptions) -> Result</name> + <fsummary></fsummary> + <desc> + <p><strong>Types and description</strong></p> + <p>See the api description in + <seealso marker="ssh:ssh_client_key_api#Module:is_host_key-4">ssh_client_key_api, Module:is_host_key/4</seealso>. + </p> + <p><strong>Option</strong></p> + <list> + <item><seealso marker="#type-user_dir_common_option">user_dir</seealso></item> + </list> + <p><strong>File</strong></p> + <list> + <item><seealso marker="#USERDIR-known_hosts"><c>USERDIR/known_hosts</c></seealso></item> + </list> + </desc> + </func> + + <func> + <name>user_key(Algorithm, ConnectOptions) -> {ok, PrivateKey} | {error, Reason}</name> + <fsummary></fsummary> + <desc> + <p><strong>Types and description</strong></p> + <p>See the api description in + <seealso marker="ssh:ssh_client_key_api#Module:user_key-2">ssh_client_key_api, Module:user_key/2</seealso>. + </p> + <p><strong>Options</strong></p> + <list> + <item><seealso marker="#type-user_dir_common_option">user_dir</seealso></item> + <item><seealso marker="#type-pubkey_passphrase_client_options">dsa_pass_phrase</seealso></item> + <item><seealso marker="#type-pubkey_passphrase_client_options">rsa_pass_phrase</seealso></item> + <item><seealso marker="#type-pubkey_passphrase_client_options">ecdsa_pass_phrase</seealso></item> + </list> + <p><strong>Files</strong></p> + <list> + <item><seealso marker="#USERDIR-id_dsa"><c>USERDIR/id_dsa</c></seealso></item> + <item><seealso marker="#USERDIR-id_rsa"><c>USERDIR/id_rsa</c></seealso></item> + <item><seealso marker="#USERDIR-id_ecdsa"><c>USERDIR/id_ecdsa</c></seealso></item> + </list> + </desc> + </func> + + </funcs> + +</erlref> diff --git a/lib/ssh/doc/src/terminology.xml b/lib/ssh/doc/src/terminology.xml new file mode 100644 index 0000000000..db1e08970d --- /dev/null +++ b/lib/ssh/doc/src/terminology.xml @@ -0,0 +1,185 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2018</year> + <year>2018</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>Terminology</title> + <prepared></prepared> + <docno></docno> + <approved></approved> + <date></date> + <rev></rev> + <file>terminology.xml</file> + </header> + + <section> + <title>General Information</title> + <p>In the following terms that may cause confusion are explained. + </p> + </section> + + <section> + <title>The term "user"</title> + <p>A "user" is a term that everyone understands intuitively. However, the understandings may differ which can + cause confusion. + </p> + <p>The term is used differently in <url href="http://www.openssh.com">OpenSSH</url> and SSH in Erlang/OTP. + The reason is the different environments and use cases that are not immediatly obvious. + </p> + <p>This chapter aims at explaining the differences and giving a rationale for why Erlang/OTP handles "user" as + it does. + </p> + + <section> + <title>In OpenSSH</title> + <p>Many have been in contact with the command 'ssh' on a Linux machine (or similar) to remotly log in on + another machine. One types + </p> + <code>ssh host</code> + <p>to log in on the machine named <c>host</c>. The command prompts for your password on the remote <c>host</c> and + then you can read, write and execute as your <i>user name</i> has rights on the remote <c>host</c>. There are + stronger variants with pre-distributed keys or certificates, but that are for now just details in the + authentication process. + </p> + <p>You could log in as the user <c>anotheruser</c> with + </p> + <code>ssh anotheruser@host</code> + <p>and you will then be enabled to act as <c>anotheruser</c> on the <c>host</c> if authorized correctly. + </p> + <p>So what does <i>"your user name has rights"</i> mean? In a UNIX/Linux/etc context it is exactly as that context: + The <i>user</i> could read, write and execute programs according to the OS rules. + In addition, the user has a home directory (<c>$HOME</c>) and there is a <c>$HOME/.ssh/</c> directory + with ssh-specific files. + </p> + <section> + <title>SSH password authentication</title> + <p>When SSH tries to log in to a host, the ssh protocol communicates the user name (as a string) and a password. + The remote ssh server checks that there is such a user defined and that the provided password is acceptable. + </p> + <p>If so, the user is authorized. + </p> + </section> + <section> + <title>SSH public key authentication</title> + <p>This is a stronger method where the ssh protocol brings the user name, the user's public key and some + cryptographic information which we could ignore here. + </p> + <p>The ssh server on the remote host checks: + </p> + <list> + <item>That the <i>user</i> has a home directory,</item> + <item>that home directory contains a .ssh/ directory and</item> + <item>the .ssh/ directory contains the public key just received in the <c>authorized_keys</c> file</item> + </list> + <p>if so, the user is authorized. + </p> + </section> + <section> + <title>The SSH server on UNIX/Linux/etc after a succesful authentication</title> + <p>After a succesful incoming authentication, a new process runs as the just authenticated user.</p> + <p>Next step is to start a service according to the ssh request. In case of a request of a shell, + a new one is started which handles the OS-commands that arrives from the client (that's "you"). + </p> + <p>In case of a sftp request, an sftp server is started in with the user's rights. So it could read, write or delete + files if allowed for that user. + </p> + </section> + </section> + + <section> + <title>In Erlang/OTP SSH</title> + <p>For the Erlang/OTP SSH server the situation is different. The server executes in an Erlang process + in the Erlang emulator which in turn executes in an OS process. The emulator does not try to change its + user when authenticated over the SSH protocol. + So the remote user name is only for authentication purposes in the Erlang/OTP SSH application. + </p> + <section> + <title>Password authentication in Erlang SSH</title> + <p>The Erlang/OTP SSH server checks the user name and password in the following order: + </p> + <list type="ordered"> + <item>If a + <seealso marker="ssh:ssh#option-pwdfun"><c>pwdfun</c></seealso> + is defined, that one is called and the returned boolean is the authentication result. + </item> + <item>Else, if the + <seealso marker="ssh:ssh#option-user_passwords"><c>user_passwords</c></seealso> + option is defined and the username and the password matches, the authentication is a success. + </item> + <item>Else, if the option + <seealso marker="ssh:ssh#option-password"><c>password</c></seealso> + is defined and matches the password the authentication is a success. + Note that the use of this option is not recommended in non-test code. + </item> + </list> + </section> + <section> + <title>Public key authentication in Erlang SSH</title> + <p>The user name, public key and cryptographic data (a signature) that is sent by the client, are used as follows + (some steps left out for clearity): + </p> + <list type="ordered"> + <item>A callback module is selected using the options + <seealso marker="ssh:ssh#type-key_cb_common_option"><c>key_cb</c></seealso>. + </item> + <item>The callback module is used to check that the provided public key is one of the user's pre-stored. + In case of the default callback module, the files <c>authorized_keys</c> and <c>authorized_keys2</c> + are searched in a directory found in the following order: + <list> + <item>If the option + <seealso marker="ssh:ssh_file#type-user_dir_fun_common_option"><c>user_dir_fun</c></seealso> + is defined, that fun is called and the returned directory is used, + </item> + <item>Else, If the option + <seealso marker="ssh:ssh_file#type-user_dir_common_option"><c>user_dir</c></seealso> + is defined, that directory is used, + </item> + <item>Else the subdirectory <c>.ssh</c> in the home directory of the user executing + the OS process of the Erlang emulator is used. + </item> + </list> + If the provided public key is not found, the authentication fails. + </item> + <item>Finally, if the provided public key is found, the signature provided by the client is checked with + the public key. + </item> + </list> + </section> + <section> + <title>The Erlang/OTP SSH server after a succesful authentication</title> + <p>After a successful authentication an <i>Erlang process</i> is handling the service request from the remote + ssh client. The rights of that process are those of the user of the OS process running the Erlang emulator. + </p> + <p>If a shell service request arrives to the server, an <i>Erlang shell</i> is opened in the server's emulator. + The rights in that shell is independent of the just authenticated user. + </p> + <p>In case of an sftp request, an sftp server is started with the rights of the user of the Erlang emulator's OS + process. So with sftp the authenticated user does not influence the rights. + </p> + <p>So after an authentication, the user name is not used anymore and has no influence. + </p> + </section> + </section> + </section> +</chapter> + diff --git a/lib/ssh/doc/src/usersguide.xml b/lib/ssh/doc/src/usersguide.xml index 38ffa48cde..8a4df208d8 100644 --- a/lib/ssh/doc/src/usersguide.xml +++ b/lib/ssh/doc/src/usersguide.xml @@ -36,5 +36,6 @@ </description> <xi:include href="introduction.xml"/> <xi:include href="using_ssh.xml"/> + <xi:include href="terminology.xml"/> <xi:include href="configure_algos.xml"/> </part> diff --git a/lib/ssh/doc/src/using_ssh.xml b/lib/ssh/doc/src/using_ssh.xml index 80662e9a70..4455d5ecc5 100644 --- a/lib/ssh/doc/src/using_ssh.xml +++ b/lib/ssh/doc/src/using_ssh.xml @@ -74,16 +74,17 @@ <marker id="Running an Erlang ssh Daemon"></marker> <title>Running an Erlang ssh Daemon</title> - <p>The <c>system_dir</c> option must be a directory containing a host - key file and it defaults to <c>/etc/ssh</c>. For details, see Section - Configuration Files in <seealso - marker="SSH_app">ssh(6)</seealso>. + <p>The + <seealso marker="ssh_file#type-system_dir_daemon_option"><c>system_dir</c></seealso> + option must be a directory containing a host key file and it defaults to <c>/etc/ssh</c>. + For details, see Section Configuration Files in <seealso marker="SSH_app">ssh(6)</seealso>. </p> <note><p>Normally, the <c>/etc/ssh</c> directory is only readable by root.</p> </note> - <p>The option <c>user_dir</c> defaults to directory <c>users ~/.ssh</c>.</p> + <p>The option <seealso marker="ssh_file#type-user_dir_common_option"><c>user_dir</c></seealso> + defaults to directory <c>users ~/.ssh</c>.</p> <p><em>Step 1.</em> To run the example without root privileges, generate new keys and host keys:</p> diff --git a/lib/ssh/src/ssh.hrl b/lib/ssh/src/ssh.hrl index 94b9f3a196..3ac74c4925 100644 --- a/lib/ssh/src/ssh.hrl +++ b/lib/ssh/src/ssh.hrl @@ -173,7 +173,7 @@ -type common_options() :: [ common_option() ]. -type common_option() :: - user_dir_common_option() + ssh_file:user_dir_common_option() | profile_common_option() | max_idle_time_common_option() | key_cb_common_option() @@ -191,8 +191,6 @@ -define(COMMON_OPTION, common_option()). - --type user_dir_common_option() :: {user_dir, false | string()}. -type profile_common_option() :: {profile, atom() }. -type max_idle_time_common_option() :: {idle_time, timeout()}. -type rekey_limit_common_option() :: {rekey_limit, Bytes::limit_bytes() | @@ -223,14 +221,14 @@ {transport, {atom(),atom(),atom()} } | {vsn, {non_neg_integer(),non_neg_integer()} } | {tstflg, list(term())} - | {user_dir_fun, fun()} + | ssh_file:user_dir_fun_common_option() | {max_random_length_padding, non_neg_integer()} . -type client_option() :: pref_public_key_algs_client_option() - | pubkey_passphrase_client_options() + | ssh_file:pubkey_passphrase_client_options() | host_accepting_client_options() | authentication_client_options() | diffie_hellman_group_exchange_client_option() @@ -246,10 +244,6 @@ -type pref_public_key_algs_client_option() :: {pref_public_key_algs, [pubkey_alg()] } . --type pubkey_passphrase_client_options() :: {dsa_pass_phrase, string()} - | {rsa_pass_phrase, string()} - | {ecdsa_pass_phrase, string()} . - -type host_accepting_client_options() :: {silently_accept_hosts, accept_hosts()} | {user_interaction, boolean()} @@ -311,7 +305,7 @@ -type send_ext_info_daemon_option() :: {send_ext_info, boolean()} . -type authentication_daemon_options() :: - {system_dir, string()} + ssh_file:system_dir_daemon_option() | {auth_method_kb_interactive_data, prompt_texts() } | {user_passwords, [{UserName::string(),Pwd::string()}]} | {password, string()} diff --git a/lib/ssh/src/ssh_file.erl b/lib/ssh/src/ssh_file.erl index 832952ed52..669b0f9be2 100644 --- a/lib/ssh/src/ssh_file.erl +++ b/lib/ssh/src/ssh_file.erl @@ -39,6 +39,23 @@ is_auth_key/3]). +-export_type([system_dir_daemon_option/0, + user_dir_common_option/0, + user_dir_fun_common_option/0, + pubkey_passphrase_client_options/0 + ]). + +-type system_dir_daemon_option() :: {system_dir, string()}. +-type user_dir_common_option() :: {user_dir, string()}. +-type user_dir_fun_common_option() :: {user_dir_fun, user2dir()}. +-type user2dir() :: fun((RemoteUserName::string()) -> UserDir :: string()) . + +-type pubkey_passphrase_client_options() :: {dsa_pass_phrase, string()} + | {rsa_pass_phrase, string()} + | {ecdsa_pass_phrase, string()} . + + + -define(PERM_700, 8#700). -define(PERM_644, 8#644). diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml index 6efa022a79..ef99ace351 100644 --- a/lib/ssl/doc/src/ssl.xml +++ b/lib/ssl/doc/src/ssl.xml @@ -209,7 +209,24 @@ elliptic_curves => [oid] | undefined, sni => string() | undefined} }</c></p></item> - + + <tag><c>signature_scheme() =</c></tag> + <item> + <p><c>rsa_pkcs1_sha256</c></p> + <p><c>| rsa_pkcs1_sha384</c></p> + <p><c>| rsa_pkcs1_sha512</c></p> + <p><c>| ecdsa_secp256r1_sha256</c></p> + <p><c>| ecdsa_secp384r1_sha384</c></p> + <p><c>| ecdsa_secp521r1_sha512</c></p> + <p><c>| rsa_pss_rsae_sha256</c></p> + <p><c>| rsa_pss_rsae_sha384</c></p> + <p><c>| rsa_pss_rsae_sha512</c></p> + <p><c>| rsa_pss_pss_sha256</c></p> + <p><c>| rsa_pss_pss_sha384</c></p> + <p><c>| rsa_pss_pss_sha512</c></p> + <p><c>| rsa_pkcs1_sha1</c></p> + <p><c>| ecdsa_sha1</c></p> + </item> </taglist> </section> @@ -709,6 +726,26 @@ fun(srp, Username :: string(), UserState :: term()) -> that may be selected. Default support for {md5, rsa} removed in ssl-8.0 </p> </item> + <tag><marker id="signature_algs_cert"/><c>{signature_algs_cert, [signature_scheme()]}</c></tag> + <item> + <p> + In addition to the signature_algorithms extension from TLS 1.2, + <url href="http://www.ietf.org/rfc/rfc8446.txt#section-4.2.3">TLS 1.3 + (RFC 5246 Section 4.2.3)</url>adds the signature_algorithms_cert extension + which enables having special requirements on the signatures used in the + certificates that differs from the requirements on digital signatures as a whole. + If this is not required this extension is not needed. + </p> + <p> + The client will send a signature_algorithms_cert extension (ClientHello), + if TLS version 1.3 or later is used, and the signature_algs_cert option is + explicitly specified. By default, only the signature_algs extension is sent. + </p> + <p> + The signature schemes shall be ordered according to the client's preference + (favorite choice first). + </p> + </item> </taglist> </section> @@ -860,7 +897,6 @@ fun(srp, Username :: string(), UserState :: term()) -> negotiation, introduced in TLS-1.2. The algorithms will also be offered to the client if a client certificate is requested. For more details see the <seealso marker="#client_signature_algs">corresponding client option</seealso>. </p> </item> - </taglist> </section> diff --git a/lib/ssl/src/Makefile b/lib/ssl/src/Makefile index af36d04a99..d4cb5350d1 100644 --- a/lib/ssl/src/Makefile +++ b/lib/ssl/src/Makefile @@ -83,6 +83,7 @@ MODULES= \ tls_socket \ dtls_socket \ tls_record \ + tls_record_1_3\ dtls_record \ ssl_record \ ssl_v3 \ @@ -95,7 +96,7 @@ INTERNAL_HRL_FILES = \ tls_connection.hrl dtls_connection.hrl ssl_connection.hrl \ ssl_handshake.hrl tls_handshake.hrl tls_handshake_1_3.hrl dtls_handshake.hrl \ ssl_api.hrl ssl_internal.hrl \ - ssl_record.hrl tls_record.hrl dtls_record.hrl ssl_srp.hrl + ssl_record.hrl tls_record.hrl tls_record_1_3.hrl dtls_record.hrl ssl_srp.hrl ERL_FILES= \ $(MODULES:%=%.erl) \ diff --git a/lib/ssl/src/dtls_handshake.erl b/lib/ssl/src/dtls_handshake.erl index c8daa11433..55aa8174a3 100644 --- a/lib/ssl/src/dtls_handshake.erl +++ b/lib/ssl/src/dtls_handshake.erl @@ -194,7 +194,7 @@ handle_client_hello(Version, ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY); _ -> #{key_exchange := KeyExAlg} = ssl_cipher_format:suite_definition(CipherSuite), - case ssl_handshake:select_hashsign(ClientHashSigns, Cert, KeyExAlg, + case ssl_handshake:select_hashsign({ClientHashSigns, undefined}, Cert, KeyExAlg, SupportedHashSigns, TLSVersion) of #alert{} = Alert -> Alert; diff --git a/lib/ssl/src/dtls_record.erl b/lib/ssl/src/dtls_record.erl index 9eb0d8e2d7..b7346d3ec8 100644 --- a/lib/ssl/src/dtls_record.erl +++ b/lib/ssl/src/dtls_record.erl @@ -499,23 +499,22 @@ encode_dtls_cipher_text(Type, {MajVer, MinVer}, Fragment, WriteState#{sequence_number => Seq + 1}}. encode_plain_text(Type, Version, Data, #{compression_state := CompS0, + cipher_state := CipherS0, epoch := Epoch, sequence_number := Seq, - cipher_state := CipherS0, security_parameters := #security_parameters{ cipher_type = ?AEAD, - bulk_cipher_algorithm = - BulkCipherAlgo, + bulk_cipher_algorithm = BCAlg, compression_algorithm = CompAlg} } = WriteState0) -> {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), - AAD = calc_aad(Type, Version, Epoch, Seq), + AAD = start_additional_data(Type, Version, Epoch, Seq), + CipherS = ssl_record:nonce_seed(BCAlg, <<?UINT16(Epoch), ?UINT48(Seq)>>, CipherS0), + WriteState = WriteState0#{compression_state => CompS1, + cipher_state => CipherS}, TLSVersion = dtls_v1:corresponding_tls_version(Version), - {CipherFragment, CipherS1} = - ssl_cipher:cipher_aead(BulkCipherAlgo, CipherS0, Seq, AAD, Comp, TLSVersion), - {CipherFragment, WriteState0#{compression_state => CompS1, - cipher_state => CipherS1}}; + ssl_record:cipher_aead(TLSVersion, Comp, WriteState, AAD); encode_plain_text(Type, Version, Fragment, #{compression_state := CompS0, epoch := Epoch, sequence_number := Seq, @@ -547,9 +546,10 @@ decode_cipher_text(#ssl_tls{type = Type, version = Version, BulkCipherAlgo, compression_algorithm = CompAlg}} = ReadState0, ConnnectionStates0) -> - AAD = calc_aad(Type, Version, Epoch, Seq), + AAD = start_additional_data(Type, Version, Epoch, Seq), + CipherS1 = ssl_record:nonce_seed(BulkCipherAlgo, <<?UINT16(Epoch), ?UINT48(Seq)>>, CipherS0), TLSVersion = dtls_v1:corresponding_tls_version(Version), - case ssl_cipher:decipher_aead(BulkCipherAlgo, CipherS0, Seq, AAD, CipherFragment, TLSVersion) of + case ssl_record:decipher_aead(BulkCipherAlgo, CipherS1, AAD, CipherFragment, TLSVersion) of {PlainFragment, CipherState} -> {Plain, CompressionS1} = ssl_record:uncompress(CompAlg, PlainFragment, CompressionS0), @@ -600,7 +600,7 @@ mac_hash({Major, Minor}, MacAlg, MacSecret, Epoch, SeqNo, Type, Length, Fragment Fragment], dtls_v1:hmac_hash(MacAlg, MacSecret, Value). -calc_aad(Type, {MajVer, MinVer}, Epoch, SeqNo) -> +start_additional_data(Type, {MajVer, MinVer}, Epoch, SeqNo) -> <<?UINT16(Epoch), ?UINT48(SeqNo), ?BYTE(Type), ?BYTE(MajVer), ?BYTE(MinVer)>>. %%-------------------------------------------------------------------- diff --git a/lib/ssl/src/ssl.app.src b/lib/ssl/src/ssl.app.src index 17173d7c79..0d92c7b447 100644 --- a/lib/ssl/src/ssl.app.src +++ b/lib/ssl/src/ssl.app.src @@ -7,6 +7,7 @@ tls_handshake, tls_handshake_1_3, tls_record, + tls_record_1_3, tls_socket, tls_v1, ssl_v3, diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl index 18d94adc18..3319aadd68 100644 --- a/lib/ssl/src/ssl.erl +++ b/lib/ssl/src/ssl.erl @@ -994,10 +994,7 @@ handle_options(Opts0, Role, Host) -> proplists:get_value( signature_algs_cert, Opts, - default_option_role(server, - tls_v1:default_signature_schemes(HighestVersion), - Role - )), + undefined), %% Do not send by default tls_version(HighestVersion)), %% Server side option reuse_session = handle_option(reuse_session, Opts, ReuseSessionFun), @@ -1060,8 +1057,8 @@ handle_options(Opts0, Role, Host) -> alpn_preferred_protocols, next_protocols_advertised, client_preferred_next_protocols, log_alert, log_level, server_name_indication, honor_cipher_order, padding_check, crl_check, crl_cache, - fallback, signature_algs, eccs, honor_ecc_order, beast_mitigation, - max_handshake_size, handshake, customize_hostname_check], + fallback, signature_algs, signature_algs_cert, eccs, honor_ecc_order, + beast_mitigation, max_handshake_size, handshake, customize_hostname_check], SockOpts = lists:foldl(fun(Key, PropList) -> proplists:delete(Key, PropList) end, Opts, SslOptions), @@ -1345,8 +1342,6 @@ handle_signature_algorithms_option(Value, Version) when is_list(Value) _ -> Value end; -handle_signature_algorithms_option(_, Version) when Version >= {3, 4} -> - handle_signature_algorithms_option(tls_v1:default_signature_schemes(Version), Version); handle_signature_algorithms_option(_, _Version) -> undefined. @@ -1664,6 +1659,14 @@ new_ssl_options([{signature_algs, Value} | Rest], #ssl_options{} = Opts, RecordC handle_hashsigns_option(Value, tls_version(RecordCB:highest_protocol_version()))}, RecordCB); +new_ssl_options([{signature_algs_cert, Value} | Rest], #ssl_options{} = Opts, RecordCB) -> + new_ssl_options( + Rest, + Opts#ssl_options{signature_algs_cert = + handle_signature_algorithms_option( + Value, + tls_version(RecordCB:highest_protocol_version()))}, + RecordCB); new_ssl_options([{protocol, dtls = Value} | Rest], #ssl_options{} = Opts, dtls_record = RecordCB) -> new_ssl_options(Rest, Opts#ssl_options{protocol = Value}, RecordCB); new_ssl_options([{protocol, tls = Value} | Rest], #ssl_options{} = Opts, tls_record = RecordCB) -> diff --git a/lib/ssl/src/ssl_cipher.erl b/lib/ssl/src/ssl_cipher.erl index 9bb2beaebd..cfcdcb8ac5 100644 --- a/lib/ssl/src/ssl_cipher.erl +++ b/lib/ssl/src/ssl_cipher.erl @@ -35,7 +35,7 @@ -include_lib("public_key/include/public_key.hrl"). -export([security_parameters/2, security_parameters/3, - cipher_init/3, decipher/6, cipher/5, decipher_aead/6, cipher_aead/6, + cipher_init/3, nonce_seed/2, decipher/6, cipher/5, aead_encrypt/5, aead_decrypt/6, suites/1, all_suites/1, crypto_support_filters/0, chacha_suites/1, anonymous_suites/1, psk_suites/1, psk_suites_anon/1, srp_suites/0, srp_suites_anon/0, @@ -50,6 +50,8 @@ -type cipher_enum() :: integer(). +-export_type([cipher_enum/0]). + %%-------------------------------------------------------------------- -spec security_parameters(ssl_cipher_format:cipher_suite(), #security_parameters{}) -> #security_parameters{}. @@ -93,10 +95,15 @@ cipher_init(?RC4, IV, Key) -> #cipher_state{iv = IV, key = Key, state = State}; cipher_init(?AES_GCM, IV, Key) -> <<Nonce:64>> = random_bytes(8), - #cipher_state{iv = IV, key = Key, nonce = Nonce}; + #cipher_state{iv = IV, key = Key, nonce = Nonce, tag_len = 16}; +cipher_init(?CHACHA20_POLY1305, IV, Key) -> + #cipher_state{iv = IV, key = Key, tag_len = 16}; cipher_init(_BCA, IV, Key) -> #cipher_state{iv = IV, key = Key}. +nonce_seed(Seed, CipherState) -> + CipherState#cipher_state{nonce = Seed}. + %%-------------------------------------------------------------------- -spec cipher(cipher_enum(), #cipher_state{}, binary(), iodata(), ssl_record:ssl_version()) -> {binary(), #cipher_state{}}. @@ -128,32 +135,16 @@ cipher(?AES_CBC, CipherState, Mac, Fragment, Version) -> crypto:block_encrypt(aes_cbc256, Key, IV, T) end, block_size(aes_128_cbc), CipherState, Mac, Fragment, Version). -%%-------------------------------------------------------------------- --spec cipher_aead(cipher_enum(), #cipher_state{}, integer(), binary(), iodata(), ssl_record:ssl_version()) -> - {binary(), #cipher_state{}}. -%% -%% Description: Encrypts the data and protects associated data (AAD) using chipher -%% described by cipher_enum() and updating the cipher state -%% Use for suites that use authenticated encryption with associated data (AEAD) -%%------------------------------------------------------------------- -cipher_aead(?AES_GCM, CipherState, SeqNo, AAD, Fragment, Version) -> - aead_cipher(aes_gcm, CipherState, SeqNo, AAD, Fragment, Version); -cipher_aead(?CHACHA20_POLY1305, CipherState, SeqNo, AAD, Fragment, Version) -> - aead_cipher(chacha20_poly1305, CipherState, SeqNo, AAD, Fragment, Version). - -aead_cipher(chacha20_poly1305, #cipher_state{key=Key} = CipherState, SeqNo, AAD0, Fragment, _Version) -> - CipherLen = erlang:iolist_size(Fragment), - AAD = <<AAD0/binary, ?UINT16(CipherLen)>>, - Nonce = ?uint64(SeqNo), - {Content, CipherTag} = crypto:block_encrypt(chacha20_poly1305, Key, Nonce, {AAD, Fragment}), - {<<Content/binary, CipherTag/binary>>, CipherState}; -aead_cipher(Type, #cipher_state{key=Key, iv = IV0, nonce = Nonce} = CipherState, _SeqNo, AAD0, Fragment, _Version) -> - CipherLen = erlang:iolist_size(Fragment), - AAD = <<AAD0/binary, ?UINT16(CipherLen)>>, - <<Salt:4/bytes, _/binary>> = IV0, - IV = <<Salt/binary, Nonce:64/integer>>, - {Content, CipherTag} = crypto:block_encrypt(Type, Key, IV, {AAD, Fragment}), - {<<Nonce:64/integer, Content/binary, CipherTag/binary>>, CipherState#cipher_state{nonce = Nonce + 1}}. +aead_encrypt(Type, Key, Nonce, Fragment, AdditionalData) -> + crypto:block_encrypt(aead_type(Type), Key, Nonce, {AdditionalData, Fragment}). + +aead_decrypt(Type, Key, Nonce, CipherText, CipherTag, AdditionalData) -> + crypto:block_decrypt(aead_type(Type), Key, Nonce, {AdditionalData, CipherText, CipherTag}). + +aead_type(?AES_GCM) -> + aes_gcm; +aead_type(?CHACHA20_POLY1305) -> + chacha20_poly1305. build_cipher_block(BlockSz, Mac, Fragment) -> TotSz = byte_size(Mac) + erlang:iolist_size(Fragment) + 1, @@ -220,19 +211,6 @@ decipher(?AES_CBC, HashSz, CipherState, Fragment, Version, PaddingCheck) -> crypto:block_decrypt(aes_cbc256, Key, IV, T) end, CipherState, HashSz, Fragment, Version, PaddingCheck). -%%-------------------------------------------------------------------- --spec decipher_aead(cipher_enum(), #cipher_state{}, integer(), binary(), binary(), ssl_record:ssl_version()) -> - {binary(), #cipher_state{}} | #alert{}. -%% -%% Description: Decrypts the data and checks the associated data (AAD) MAC using -%% cipher described by cipher_enum() and updating the cipher state. -%% Use for suites that use authenticated encryption with associated data (AEAD) -%%------------------------------------------------------------------- -decipher_aead(?AES_GCM, CipherState, SeqNo, AAD, Fragment, Version) -> - aead_decipher(aes_gcm, CipherState, SeqNo, AAD, Fragment, Version); -decipher_aead(?CHACHA20_POLY1305, CipherState, SeqNo, AAD, Fragment, Version) -> - aead_decipher(chacha20_poly1305, CipherState, SeqNo, AAD, Fragment, Version). - block_decipher(Fun, #cipher_state{key=Key, iv=IV} = CipherState0, HashSz, Fragment, Version, PaddingCheck) -> try @@ -263,34 +241,6 @@ block_decipher(Fun, #cipher_state{key=Key, iv=IV} = CipherState0, ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC, decryption_failed) end. -aead_ciphertext_to_state(chacha20_poly1305, SeqNo, _IV, AAD0, Fragment, _Version) -> - CipherLen = size(Fragment) - 16, - <<CipherText:CipherLen/bytes, CipherTag:16/bytes>> = Fragment, - AAD = <<AAD0/binary, ?UINT16(CipherLen)>>, - Nonce = ?uint64(SeqNo), - {Nonce, AAD, CipherText, CipherTag}; -aead_ciphertext_to_state(_, _SeqNo, <<Salt:4/bytes, _/binary>>, AAD0, Fragment, _Version) -> - CipherLen = size(Fragment) - 24, - <<ExplicitNonce:8/bytes, CipherText:CipherLen/bytes, CipherTag:16/bytes>> = Fragment, - AAD = <<AAD0/binary, ?UINT16(CipherLen)>>, - Nonce = <<Salt/binary, ExplicitNonce/binary>>, - {Nonce, AAD, CipherText, CipherTag}. - -aead_decipher(Type, #cipher_state{key = Key, iv = IV} = CipherState, - SeqNo, AAD0, Fragment, Version) -> - try - {Nonce, AAD, CipherText, CipherTag} = aead_ciphertext_to_state(Type, SeqNo, IV, AAD0, Fragment, Version), - case crypto:block_decrypt(Type, Key, Nonce, {AAD, CipherText, CipherTag}) of - Content when is_binary(Content) -> - {Content, CipherState}; - _ -> - ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC, decryption_failed) - end - catch - _:_ -> - ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC, decryption_failed) - end. - %%-------------------------------------------------------------------- -spec suites(ssl_record:ssl_version()) -> [ssl_cipher_format:cipher_suite()]. %% diff --git a/lib/ssl/src/ssl_cipher.hrl b/lib/ssl/src/ssl_cipher.hrl index 1febc52e43..5891f3a7cc 100644 --- a/lib/ssl/src/ssl_cipher.hrl +++ b/lib/ssl/src/ssl_cipher.hrl @@ -48,7 +48,8 @@ iv, key, state, - nonce + nonce, + tag_len }). %%% TLS_NULL_WITH_NULL_NULL is specified and is the initial state of a diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl index 3eb3b0a980..6e602eac23 100644 --- a/lib/ssl/src/ssl_connection.erl +++ b/lib/ssl/src/ssl_connection.erl @@ -848,7 +848,9 @@ certify(internal, #certificate_request{} = CertRequest, role = client, ssl_options = #ssl_options{signature_algs = SupportedHashSigns}, negotiated_version = Version} = State0, Connection) -> - case ssl_handshake:select_hashsign(CertRequest, Cert, SupportedHashSigns, ssl:tls_version(Version)) of + case ssl_handshake:select_hashsign(CertRequest, Cert, + SupportedHashSigns, + ssl:tls_version(Version)) of #alert {} = Alert -> handle_own_alert(Alert, Version, ?FUNCTION_NAME, State0); NegotiatedHashSign -> diff --git a/lib/ssl/src/ssl_handshake.erl b/lib/ssl/src/ssl_handshake.erl index ba0b670091..1e57dfd710 100644 --- a/lib/ssl/src/ssl_handshake.erl +++ b/lib/ssl/src/ssl_handshake.erl @@ -1015,12 +1015,17 @@ client_hello_extensions(Version, CipherSuites, {3,4} -> HelloExtensions#{client_hello_versions => #client_hello_versions{versions = Versions}, - signature_algs_cert => - #signature_scheme_list{signature_scheme_list = SignatureSchemes}}; + signature_algs_cert => + signature_scheme_list(SignatureSchemes)}; _Else -> HelloExtensions end. +signature_scheme_list(undefined) -> + undefined; +signature_scheme_list(SignatureSchemes) -> + #signature_scheme_list{signature_scheme_list = SignatureSchemes}. + handle_client_hello_extensions(RecordCB, Random, ClientCipherSuites, Exts, Version, #ssl_options{secure_renegotiate = SecureRenegotation, @@ -1128,26 +1133,50 @@ select_hashsign(_, _, KeyExAlgo, _, _Version) when KeyExAlgo == dh_anon; {null, anon}; %% The signature_algorithms extension was introduced with TLS 1.2. Ignore it if we have %% negotiated a lower version. -select_hashsign(HashSigns, Cert, KeyExAlgo, - undefined, {Major, Minor} = Version) when Major >= 3 andalso Minor >= 3-> - select_hashsign(HashSigns, Cert, KeyExAlgo, tls_v1:default_signature_algs(Version), Version); -select_hashsign(#hash_sign_algos{hash_sign_algos = HashSigns}, Cert, KeyExAlgo, SupportedHashSigns, - {Major, Minor}) when Major >= 3 andalso Minor >= 3 -> - #'OTPCertificate'{tbsCertificate = TBSCert} = public_key:pkix_decode_cert(Cert, otp), - #'OTPSubjectPublicKeyInfo'{algorithm = {_, SubjAlgo, _}} = - TBSCert#'OTPTBSCertificate'.subjectPublicKeyInfo, - - SubSign = sign_algo(SubjAlgo), - - case lists:filter(fun({_, S} = Algos) when S == SubSign -> - is_acceptable_hash_sign(Algos, KeyExAlgo, SupportedHashSigns); - (_) -> - false - end, HashSigns) of - [] -> - ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_signature_algorithm); - [HashSign | _] -> - HashSign +select_hashsign({ClientHashSigns, ClientSignatureSchemes}, + Cert, KeyExAlgo, undefined, {Major, Minor} = Version) + when Major >= 3 andalso Minor >= 3-> + select_hashsign({ClientHashSigns, ClientSignatureSchemes}, Cert, KeyExAlgo, + tls_v1:default_signature_algs(Version), Version); +select_hashsign({#hash_sign_algos{hash_sign_algos = ClientHashSigns}, + ClientSignatureSchemes0}, + Cert, KeyExAlgo, SupportedHashSigns, {Major, Minor}) + when Major >= 3 andalso Minor >= 3 -> + ClientSignatureSchemes = get_signature_scheme(ClientSignatureSchemes0), + {SignAlgo0, Param, PublicKeyAlgo0} = get_cert_params(Cert), + SignAlgo = sign_algo(SignAlgo0), + PublicKeyAlgo = public_key_algo(PublicKeyAlgo0), + + %% RFC 5246 (TLS 1.2) + %% If the client provided a "signature_algorithms" extension, then all + %% certificates provided by the server MUST be signed by a + %% hash/signature algorithm pair that appears in that extension. + %% + %% RFC 8446 (TLS 1.3) + %% TLS 1.3 provides two extensions for indicating which signature + %% algorithms may be used in digital signatures. The + %% "signature_algorithms_cert" extension applies to signatures in + %% certificates and the "signature_algorithms" extension, which + %% originally appeared in TLS 1.2, applies to signatures in + %% CertificateVerify messages. + %% + %% If no "signature_algorithms_cert" extension is + %% present, then the "signature_algorithms" extension also applies to + %% signatures appearing in certificates. + case is_supported_sign(SignAlgo, Param, ClientHashSigns, ClientSignatureSchemes) of + true -> + case lists:filter(fun({_, S} = Algos) when S == PublicKeyAlgo -> + is_acceptable_hash_sign(Algos, KeyExAlgo, SupportedHashSigns); + (_) -> + false + end, ClientHashSigns) of + [] -> + ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_signature_algorithm); + [HashSign | _] -> + HashSign + end; + false -> + ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_signature_algorithm) end; select_hashsign(_, Cert, _, _, Version) -> #'OTPCertificate'{tbsCertificate = TBSCert} = public_key:pkix_decode_cert(Cert, otp), @@ -1161,21 +1190,23 @@ select_hashsign(_, Cert, _, _, Version) -> %% %% Description: Handles signature algorithms selection for certificate requests (client) %%-------------------------------------------------------------------- -select_hashsign(#certificate_request{hashsign_algorithms = #hash_sign_algos{hash_sign_algos = HashSigns}, - certificate_types = Types}, Cert, SupportedHashSigns, +select_hashsign(#certificate_request{ + hashsign_algorithms = #hash_sign_algos{ + hash_sign_algos = HashSigns}, + certificate_types = Types}, + Cert, + SupportedHashSigns, {Major, Minor}) when Major >= 3 andalso Minor >= 3-> - #'OTPCertificate'{tbsCertificate = TBSCert} = public_key:pkix_decode_cert(Cert, otp), - #'OTPCertificate'{tbsCertificate = TBSCert, - signatureAlgorithm = {_,SignAlgo, _}} = public_key:pkix_decode_cert(Cert, otp), - #'OTPSubjectPublicKeyInfo'{algorithm = {_, SubjAlgo, _}} = - TBSCert#'OTPTBSCertificate'.subjectPublicKeyInfo, - - Sign = sign_algo(SignAlgo), - SubSign = sign_algo(SubjAlgo), - - case is_acceptable_cert_type(SubSign, HashSigns, Types) andalso is_supported_sign(Sign, HashSigns) of + {SignAlgo0, Param, PublicKeyAlgo0} = get_cert_params(Cert), + SignAlgo = sign_algo(SignAlgo0), + PublicKeyAlgo = public_key_algo(PublicKeyAlgo0), + + case is_acceptable_cert_type(PublicKeyAlgo, Types) andalso + %% certificate_request has no "signature_algorithms_cert" + %% extension in TLS 1.2. + is_supported_sign(SignAlgo, Param, HashSigns, undefined) of true -> - case lists:filter(fun({_, S} = Algos) when S == SubSign -> + case lists:filter(fun({_, S} = Algos) when S == PublicKeyAlgo -> is_acceptable_hash_sign(Algos, SupportedHashSigns); (_) -> false @@ -1188,8 +1219,38 @@ select_hashsign(#certificate_request{hashsign_algorithms = #hash_sign_algos{hash false -> ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_signature_algorithm) end; -select_hashsign(#certificate_request{}, Cert, _, Version) -> - select_hashsign(undefined, Cert, undefined, [], Version). +select_hashsign(#certificate_request{certificate_types = Types}, Cert, _, Version) -> + {_, _, PublicKeyAlgo0} = get_cert_params(Cert), + PublicKeyAlgo = public_key_algo(PublicKeyAlgo0), + + %% Check cert even for TLS 1.0/1.1 + case is_acceptable_cert_type(PublicKeyAlgo, Types) of + true -> + select_hashsign(undefined, Cert, undefined, [], Version); + false -> + ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_signature_algorithm) + end. + + +%% Gets the relevant parameters of a certificate: +%% - signature algorithm +%% - parameters of the signature algorithm +%% - public key algorithm (key type) +get_cert_params(Cert) -> + #'OTPCertificate'{tbsCertificate = TBSCert, + signatureAlgorithm = + {_,SignAlgo, Param}} = public_key:pkix_decode_cert(Cert, otp), + #'OTPSubjectPublicKeyInfo'{algorithm = {_, PublicKeyAlgo, _}} = + TBSCert#'OTPTBSCertificate'.subjectPublicKeyInfo, + {SignAlgo, Param, PublicKeyAlgo}. + + +get_signature_scheme(undefined) -> + undefined; +get_signature_scheme(#signature_scheme_list{ + signature_scheme_list = ClientSignatureSchemes}) -> + ClientSignatureSchemes. + %%-------------------------------------------------------------------- -spec select_hashsign_algs({atom(), atom()}| undefined, oid(), ssl_record:ssl_version()) -> @@ -1258,33 +1319,30 @@ int_to_bin(I) -> L = (length(integer_to_list(I, 16)) + 1) div 2, <<I:(L*8)>>. -certificate_types(_, {N, M}) when N >= 3 andalso M >= 3 -> - case proplists:get_bool(ecdsa, - proplists:get_value(public_keys, crypto:supports())) of - true -> - <<?BYTE(?ECDSA_SIGN), ?BYTE(?RSA_SIGN), ?BYTE(?DSS_SIGN)>>; - false -> - <<?BYTE(?RSA_SIGN), ?BYTE(?DSS_SIGN)>> - end; - -certificate_types(#{key_exchange := KeyExchange}, _) when KeyExchange == rsa; - KeyExchange == dh_rsa; - KeyExchange == dhe_rsa; - KeyExchange == ecdhe_rsa -> - <<?BYTE(?RSA_SIGN)>>; - -certificate_types(#{key_exchange := KeyExchange}, _) when KeyExchange == dh_dss; - KeyExchange == dhe_dss; - KeyExchange == srp_dss -> - <<?BYTE(?DSS_SIGN)>>; - -certificate_types(#{key_exchange := KeyExchange}, _) when KeyExchange == dh_ecdsa; - KeyExchange == dhe_ecdsa; - KeyExchange == ecdh_ecdsa; - KeyExchange == ecdhe_ecdsa -> - <<?BYTE(?ECDSA_SIGN)>>; +%% TLS 1.0+ +%% The end-entity certificate provided by the client MUST contain a +%% key that is compatible with certificate_types. +certificate_types(_, {N, M}) when N >= 3 andalso M >= 1 -> + ECDSA = supported_cert_type_or_empty(ecdsa, ?ECDSA_SIGN), + RSA = supported_cert_type_or_empty(rsa, ?RSA_SIGN), + DSS = supported_cert_type_or_empty(dss, ?DSS_SIGN), + <<ECDSA/binary,RSA/binary,DSS/binary>>; +%% SSL 3.0 certificate_types(_, _) -> - <<?BYTE(?RSA_SIGN)>>. + RSA = supported_cert_type_or_empty(rsa, ?RSA_SIGN), + DSS = supported_cert_type_or_empty(dss, ?DSS_SIGN), + <<RSA/binary,DSS/binary>>. + +%% Returns encoded certificate_type if algorithm is supported +supported_cert_type_or_empty(Algo, Type) -> + case proplists:get_bool( + Algo, + proplists:get_value(public_keys, crypto:supports())) of + true -> + <<?BYTE(Type)>>; + false -> + <<>> + end. certificate_authorities(CertDbHandle, CertDbRef) -> Authorities = certificate_authorities_from_db(CertDbHandle, CertDbRef), @@ -2355,17 +2413,6 @@ handle_srp_extension(undefined, Session) -> handle_srp_extension(#srp{username = Username}, Session) -> Session#session{srp_username = Username}. - -sign_algo(?rsaEncryption) -> - rsa; -sign_algo(?'id-ecPublicKey') -> - ecdsa; -sign_algo(?'id-dsa') -> - dsa; -sign_algo(Alg) -> - {_, Sign} =public_key:pkix_sign_types(Alg), - Sign. - is_acceptable_hash_sign( _, KeyExAlgo, _) when KeyExAlgo == psk; KeyExAlgo == dhe_psk; @@ -2381,15 +2428,80 @@ is_acceptable_hash_sign(Algos,_, SupportedHashSigns) -> is_acceptable_hash_sign(Algos, SupportedHashSigns) -> lists:member(Algos, SupportedHashSigns). -is_acceptable_cert_type(Sign, _HashSigns, Types) -> +is_acceptable_cert_type(Sign, Types) -> lists:member(sign_type(Sign), binary_to_list(Types)). -is_supported_sign(Sign, HashSigns) -> - [] =/= lists:dropwhile(fun({_, S}) when S =/= Sign -> - true; - (_)-> - false - end, HashSigns). +%% signature_algorithms_cert = undefined +is_supported_sign(SignAlgo, _, HashSigns, undefined) -> + lists:member(SignAlgo, HashSigns); + +%% {'SignatureAlgorithm',{1,2,840,113549,1,1,11},'NULL'} +is_supported_sign({Hash, Sign}, 'NULL', _, SignatureSchemes) -> + Fun = fun (Scheme, Acc) -> + {H0, S0, _} = ssl_cipher:scheme_to_components(Scheme), + S1 = case S0 of + rsa_pkcs1 -> rsa; + S -> S + end, + H1 = case H0 of + sha1 -> sha; + H -> H + end, + Acc orelse (Sign =:= S1 andalso + Hash =:= H1) + end, + lists:foldl(Fun, false, SignatureSchemes); + +%% TODO: Implement validation for the curve used in the signature +%% RFC 3279 - 2.2.3 ECDSA Signature Algorithm +%% When the ecdsa-with-SHA1 algorithm identifier appears as the +%% algorithm field in an AlgorithmIdentifier, the encoding MUST omit the +%% parameters field. That is, the AlgorithmIdentifier SHALL be a +%% SEQUENCE of one component: the OBJECT IDENTIFIER ecdsa-with-SHA1. +%% +%% The elliptic curve parameters in the subjectPublicKeyInfo field of +%% the certificate of the issuer SHALL apply to the verification of the +%% signature. +is_supported_sign({Hash, Sign}, _Param, _, SignatureSchemes) -> + Fun = fun (Scheme, Acc) -> + {H0, S0, _} = ssl_cipher:scheme_to_components(Scheme), + S1 = case S0 of + rsa_pkcs1 -> rsa; + S -> S + end, + H1 = case H0 of + sha1 -> sha; + H -> H + end, + Acc orelse (Sign =:= S1 andalso + Hash =:= H1) + end, + lists:foldl(Fun, false, SignatureSchemes). + +%% SupportedPublicKeyAlgorithms PUBLIC-KEY-ALGORITHM-CLASS ::= { +%% dsa | rsa-encryption | dh | kea | ec-public-key } +public_key_algo(?rsaEncryption) -> + rsa; +public_key_algo(?'id-ecPublicKey') -> + ecdsa; +public_key_algo(?'id-dsa') -> + dsa. + +%% SupportedSignatureAlgorithms SIGNATURE-ALGORITHM-CLASS ::= { +%% dsa-with-sha1 | dsaWithSHA1 | md2-with-rsa-encryption | +%% md5-with-rsa-encryption | sha1-with-rsa-encryption | sha-1with-rsa-encryption | +%% sha224-with-rsa-encryption | +%% sha256-with-rsa-encryption | +%% sha384-with-rsa-encryption | +%% sha512-with-rsa-encryption | +%% ecdsa-with-sha1 | +%% ecdsa-with-sha224 | +%% ecdsa-with-sha256 | +%% ecdsa-with-sha384 | +%% ecdsa-with-sha512 } +sign_algo(Alg) -> + public_key:pkix_sign_types(Alg). + sign_type(rsa) -> ?RSA_SIGN; sign_type(dsa) -> diff --git a/lib/ssl/src/ssl_handshake.hrl b/lib/ssl/src/ssl_handshake.hrl index 36aefd5e22..b14bc21862 100644 --- a/lib/ssl/src/ssl_handshake.hrl +++ b/lib/ssl/src/ssl_handshake.hrl @@ -401,7 +401,7 @@ %% Datagram Transport Layer Security (DTLS) Heartbeat Extension %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Not supported --define(HEARTBEAT, 15). +-define(HS_HEARTBEAT, 15). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% RFC 6962 Certificate Transparency diff --git a/lib/ssl/src/ssl_record.erl b/lib/ssl/src/ssl_record.erl index 446bb6c56a..ddc83821b4 100644 --- a/lib/ssl/src/ssl_record.erl +++ b/lib/ssl/src/ssl_record.erl @@ -45,7 +45,7 @@ -export([compress/3, uncompress/3, compressions/0]). %% Payload encryption/decryption --export([cipher/4, decipher/4, cipher_aead/4, is_correct_mac/2]). +-export([cipher/4, decipher/4, cipher_aead/4, decipher_aead/5, is_correct_mac/2, nonce_seed/3]). -export_type([ssl_version/0, ssl_atom_version/0, connection_states/0, connection_state/0]). @@ -305,22 +305,20 @@ cipher(Version, Fragment, {CipherFragment, CipherS1} = ssl_cipher:cipher(BulkCipherAlgo, CipherS0, MacHash, Fragment, Version), {CipherFragment, WriteState0#{cipher_state => CipherS1}}. -%% %%-------------------------------------------------------------------- -%% -spec cipher_aead(ssl_version(), iodata(), connection_state(), MacHash::binary()) -> -%% {CipherFragment::binary(), connection_state()}. -%% %% -%% %% Description: Payload encryption +%%-------------------------------------------------------------------- +-spec cipher_aead(ssl_version(), iodata(), connection_state(), AAD::binary()) -> + {CipherFragment::binary(), connection_state()}. + +%% Description: Payload encryption %% %%-------------------------------------------------------------------- cipher_aead(Version, Fragment, #{cipher_state := CipherS0, - sequence_number := SeqNo, security_parameters := #security_parameters{bulk_cipher_algorithm = BulkCipherAlgo} } = WriteState0, AAD) -> - {CipherFragment, CipherS1} = - ssl_cipher:cipher_aead(BulkCipherAlgo, CipherS0, SeqNo, AAD, Fragment, Version), + cipher_aead(BulkCipherAlgo, CipherS0, AAD, Fragment, Version), {CipherFragment, WriteState0#{cipher_state => CipherS1}}. %%-------------------------------------------------------------------- @@ -343,10 +341,39 @@ decipher(Version, CipherFragment, #alert{} = Alert -> Alert end. +%%-------------------------------------------------------------------- +-spec decipher_aead(ssl_cipher:cipher_enum(), #cipher_state{}, + binary(), binary(), ssl_record:ssl_version()) -> + {binary(), #cipher_state{}} | #alert{}. +%% +%% Description: Decrypts the data and checks the associated data (AAD) MAC using +%% cipher described by cipher_enum() and updating the cipher state. +%% Use for suites that use authenticated encryption with associated data (AEAD) +%%------------------------------------------------------------------- +decipher_aead(Type, #cipher_state{key = Key} = CipherState, AAD0, CipherFragment, _) -> + try + Nonce = decrypt_nonce(Type, CipherState, CipherFragment), + {AAD, CipherText, CipherTag} = aead_ciphertext_split(Type, CipherState, CipherFragment, AAD0), + case ssl_cipher:aead_decrypt(Type, Key, Nonce, CipherText, CipherTag, AAD) of + Content when is_binary(Content) -> + {Content, CipherState}; + _ -> + ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC, decryption_failed) + end + catch + _:_ -> + ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC, decryption_failed) + end. + +nonce_seed(?CHACHA20_POLY1305, Seed, CipherState) -> + ssl_cipher:nonce_seed(Seed, CipherState); +nonce_seed(_,_, CipherState) -> + CipherState. %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- + empty_connection_state(ConnectionEnd, BeastMitigation) -> SecParams = empty_security_params(ConnectionEnd), #{security_parameters => SecParams, @@ -399,3 +426,37 @@ initial_security_params(ConnectionEnd) -> compression_algorithm = ?NULL}, ssl_cipher:security_parameters(?TLS_NULL_WITH_NULL_NULL, SecParams). +cipher_aead(?CHACHA20_POLY1305 = Type, #cipher_state{key=Key} = CipherState, AAD0, Fragment, _Version) -> + AAD = end_additional_data(AAD0, erlang:iolist_size(Fragment)), + Nonce = encrypt_nonce(Type, CipherState), + {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD), + {<<Content/binary, CipherTag/binary>>, CipherState}; +cipher_aead(Type, #cipher_state{key=Key, nonce = ExplicitNonce} = CipherState, AAD0, Fragment, _Version) -> + AAD = end_additional_data(AAD0, erlang:iolist_size(Fragment)), + Nonce = encrypt_nonce(Type, CipherState), + {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD), + {<<ExplicitNonce:64/integer, Content/binary, CipherTag/binary>>, CipherState#cipher_state{nonce = ExplicitNonce + 1}}. + +encrypt_nonce(?CHACHA20_POLY1305, #cipher_state{nonce = Nonce, iv = IV}) -> + crypto:exor(<<?UINT32(0), Nonce/binary>>, IV); +encrypt_nonce(?AES_GCM, #cipher_state{iv = IV, nonce = ExplicitNonce}) -> + <<Salt:4/bytes, _/binary>> = IV, + <<Salt/binary, ExplicitNonce:64/integer>>. + +decrypt_nonce(?CHACHA20_POLY1305, #cipher_state{nonce = Nonce, iv = IV}, _) -> + crypto:exor(<<Nonce:96/unsigned-big-integer>>, IV); +decrypt_nonce(?AES_GCM, #cipher_state{iv = <<Salt:4/bytes, _/binary>>}, <<ExplicitNonce:8/bytes, _/binary>>) -> + <<Salt/binary, ExplicitNonce/binary>>. + +aead_ciphertext_split(?CHACHA20_POLY1305, #cipher_state{tag_len = Len}, CipherTextFragment, AAD) -> + CipherLen = size(CipherTextFragment) - Len, + <<CipherText:CipherLen/bytes, CipherTag:Len/bytes>> = CipherTextFragment, + {end_additional_data(AAD, CipherLen), CipherText, CipherTag}; +aead_ciphertext_split(?AES_GCM, #cipher_state{tag_len = Len}, CipherTextFragment, AAD) -> + CipherLen = size(CipherTextFragment) - (Len + 8), %% 8 is length of explicit Nonce + << _:8/bytes, CipherText:CipherLen/bytes, CipherTag:Len/bytes>> = CipherTextFragment, + {end_additional_data(AAD, CipherLen), CipherText, CipherTag}. + +end_additional_data(AAD, Len) -> + <<AAD/binary, ?UINT16(Len)>>. + diff --git a/lib/ssl/src/ssl_record.hrl b/lib/ssl/src/ssl_record.hrl index e8ce50040f..4cb19d9d0d 100644 --- a/lib/ssl/src/ssl_record.hrl +++ b/lib/ssl/src/ssl_record.hrl @@ -140,6 +140,7 @@ -define(ALERT, 21). -define(HANDSHAKE, 22). -define(APPLICATION_DATA, 23). +-define(HEARTBEAT, 24). -define(MAX_PLAIN_TEXT_LENGTH, 16384). -define(MAX_COMPRESSED_LENGTH, (?MAX_PLAIN_TEXT_LENGTH+1024)). -define(MAX_CIPHER_TEXT_LENGTH, (?MAX_PLAIN_TEXT_LENGTH+2048)). diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl index 8ded2cbff7..af59dda442 100644 --- a/lib/ssl/src/tls_connection.erl +++ b/lib/ssl/src/tls_connection.erl @@ -152,9 +152,10 @@ next_record(#state{protocol_buffers = #protocol_buffers{tls_packets = [], tls_cipher_texts = [CT | Rest]} = Buffers, connection_states = ConnStates0, + negotiated_version = Version, ssl_options = #ssl_options{padding_check = Check}} = State) -> - case tls_record:decode_cipher_text(CT, ConnStates0, Check) of + case tls_record:decode_cipher_text(Version, CT, ConnStates0, Check) of {Plain, ConnStates} -> {Plain, State#state{protocol_buffers = Buffers#protocol_buffers{tls_cipher_texts = Rest}, diff --git a/lib/ssl/src/tls_handshake.erl b/lib/ssl/src/tls_handshake.erl index 050b4be870..b39a7732e7 100644 --- a/lib/ssl/src/tls_handshake.erl +++ b/lib/ssl/src/tls_handshake.erl @@ -276,6 +276,7 @@ handle_client_hello(Version, true -> Curves = maps:get(elliptic_curves, HelloExt, undefined), ClientHashSigns = maps:get(signature_algs, HelloExt, undefined), + ClientSignatureSchemes = maps:get(signature_algs_cert, HelloExt, undefined), AvailableHashSigns = ssl_handshake:available_signature_algs( ClientHashSigns, SupportedHashSigns, Cert, Version), ECCCurve = ssl_handshake:select_curve(Curves, SupportedECCs, ECCOrder), @@ -289,8 +290,10 @@ handle_client_hello(Version, ?ALERT_REC(?FATAL, ?INSUFFICIENT_SECURITY, no_suitable_ciphers); _ -> #{key_exchange := KeyExAlg} = ssl_cipher_format:suite_definition(CipherSuite), - case ssl_handshake:select_hashsign(ClientHashSigns, Cert, KeyExAlg, - SupportedHashSigns, Version) of + case ssl_handshake:select_hashsign({ClientHashSigns, ClientSignatureSchemes}, + Cert, KeyExAlg, + SupportedHashSigns, + Version) of #alert{} = Alert -> Alert; HashSign -> diff --git a/lib/ssl/src/tls_record.erl b/lib/ssl/src/tls_record.erl index 444759aafa..7debac7d37 100644 --- a/lib/ssl/src/tls_record.erl +++ b/lib/ssl/src/tls_record.erl @@ -41,7 +41,7 @@ -export([encode_plain_text/4]). %% Decoding --export([decode_cipher_text/3]). +-export([decode_cipher_text/4]). %% Protocol version handling -export([protocol_version/1, lowest_protocol_version/1, lowest_protocol_version/2, @@ -106,6 +106,8 @@ get_tls_records(Data, Versions, Buffer, SslOpts) -> % %% Description: Encodes a handshake message to send on the ssl-socket. %%-------------------------------------------------------------------- +encode_handshake(Frag, {3, 4}, ConnectionStates) -> + tls_record_1_3:encode_handshake(Frag, ConnectionStates); encode_handshake(Frag, Version, #{current_write := #{beast_mitigation := BeastMitigation, @@ -126,6 +128,8 @@ encode_handshake(Frag, Version, %% %% Description: Encodes an alert message to send on the ssl-socket. %%-------------------------------------------------------------------- +encode_alert_record(Alert, {3, 4}, ConnectionStates) -> + tls_record_1_3:encode_handshake(Alert, ConnectionStates); encode_alert_record(#alert{level = Level, description = Description}, Version, ConnectionStates) -> encode_plain_text(?ALERT, Version, <<?BYTE(Level), ?BYTE(Description)>>, @@ -146,6 +150,8 @@ encode_change_cipher_spec(Version, ConnectionStates) -> %% %% Description: Encodes data to send on the ssl-socket. %%-------------------------------------------------------------------- +encode_data(Data, {3, 4}, ConnectionStates) -> + tls_record_1_3:encode_data(Data, ConnectionStates); encode_data(Frag, Version, #{current_write := #{beast_mitigation := BeastMitigation, security_parameters := @@ -159,12 +165,14 @@ encode_data(Frag, Version, %%==================================================================== %%-------------------------------------------------------------------- --spec decode_cipher_text(#ssl_tls{}, ssl_record:connection_states(), boolean()) -> +-spec decode_cipher_text(tls_version(), #ssl_tls{}, ssl_record:connection_states(), boolean()) -> {#ssl_tls{}, ssl_record:connection_states()}| #alert{}. %% %% Description: Decode cipher text %%-------------------------------------------------------------------- -decode_cipher_text(#ssl_tls{type = Type, version = Version, +decode_cipher_text({3,4}, CipherTextRecord, ConnectionStates, _) -> + tls_record_1_3:decode_cipher_text(CipherTextRecord, ConnectionStates); +decode_cipher_text(_, #ssl_tls{type = Type, version = Version, fragment = CipherFragment} = CipherText, #{current_read := #{compression_state := CompressionS0, @@ -177,14 +185,15 @@ decode_cipher_text(#ssl_tls{type = Type, version = Version, BulkCipherAlgo, compression_algorithm = CompAlg} } = ReadState0} = ConnnectionStates0, _) -> - AAD = calc_aad(Type, Version, ReadState0), - case ssl_cipher:decipher_aead(BulkCipherAlgo, CipherS0, Seq, AAD, CipherFragment, Version) of - {PlainFragment, CipherS1} -> + AAD = start_additional_data(Type, Version, ReadState0), + CipherS1 = ssl_record:nonce_seed(BulkCipherAlgo, <<?UINT64(Seq)>>, CipherS0), + case ssl_record:decipher_aead(BulkCipherAlgo, CipherS1, AAD, CipherFragment, Version) of + {PlainFragment, CipherState} -> {Plain, CompressionS1} = ssl_record:uncompress(CompAlg, PlainFragment, CompressionS0), ConnnectionStates = ConnnectionStates0#{ current_read => ReadState0#{ - cipher_state => CipherS1, + cipher_state => CipherState, sequence_number => Seq + 1, compression_state => CompressionS1}}, {CipherText#ssl_tls{fragment = Plain}, ConnnectionStates}; @@ -192,7 +201,7 @@ decode_cipher_text(#ssl_tls{type = Type, version = Version, Alert end; -decode_cipher_text(#ssl_tls{type = Type, version = Version, +decode_cipher_text(_, #ssl_tls{type = Type, version = Version, fragment = CipherFragment} = CipherText, #{current_read := #{compression_state := CompressionS0, @@ -486,15 +495,20 @@ encode_iolist(Type, Data, Version, ConnectionStates0) -> {lists:reverse(EncodedMsg), ConnectionStates}. %%-------------------------------------------------------------------- do_encode_plain_text(Type, Version, Data, #{compression_state := CompS0, - security_parameters := + cipher_state := CipherS0, + sequence_number := Seq, + security_parameters := #security_parameters{ cipher_type = ?AEAD, + bulk_cipher_algorithm = BCAlg, compression_algorithm = CompAlg} } = WriteState0) -> {Comp, CompS1} = ssl_record:compress(CompAlg, Data, CompS0), - WriteState1 = WriteState0#{compression_state => CompS1}, - AAD = calc_aad(Type, Version, WriteState1), - ssl_record:cipher_aead(Version, Comp, WriteState1, AAD); + CipherS = ssl_record:nonce_seed(BCAlg, <<?UINT64(Seq)>>, CipherS0), + WriteState = WriteState0#{compression_state => CompS1, + cipher_state => CipherS}, + AAD = start_additional_data(Type, Version, WriteState), + ssl_record:cipher_aead(Version, Comp, WriteState, AAD); do_encode_plain_text(Type, Version, Data, #{compression_state := CompS0, security_parameters := #security_parameters{compression_algorithm = CompAlg} @@ -506,7 +520,7 @@ do_encode_plain_text(Type, Version, Data, #{compression_state := CompS0, do_encode_plain_text(_,_,_,CS) -> exit({cs, CS}). %%-------------------------------------------------------------------- -calc_aad(Type, {MajVer, MinVer}, +start_additional_data(Type, {MajVer, MinVer}, #{sequence_number := SeqNo}) -> <<?UINT64(SeqNo), ?BYTE(Type), ?BYTE(MajVer), ?BYTE(MinVer)>>. diff --git a/lib/ssl/src/tls_record_1_3.erl b/lib/ssl/src/tls_record_1_3.erl new file mode 100644 index 0000000000..ff198a09bf --- /dev/null +++ b/lib/ssl/src/tls_record_1_3.erl @@ -0,0 +1,260 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2007-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% + +-module(tls_record_1_3). + +-include("tls_record.hrl"). +-include("tls_record_1_3.hrl"). +-include("ssl_internal.hrl"). +-include("ssl_alert.hrl"). +-include("ssl_cipher.hrl"). + +%% Encoding +-export([encode_handshake/2, encode_alert_record/2, + encode_data/2]). +-export([encode_plain_text/3]). + +%% Decoding +-export([decode_cipher_text/2]). + +%%==================================================================== +%% Encoding +%%==================================================================== + +%%-------------------------------------------------------------------- +-spec encode_handshake(iolist(), ssl_record:connection_states()) -> + {iolist(), ssl_record:connection_states()}. +% +%% Description: Encodes a handshake message to send on the tls-1.3-socket. +%%-------------------------------------------------------------------- +encode_handshake(Frag, ConnectionStates) -> + case iolist_size(Frag) of + N when N > ?MAX_PLAIN_TEXT_LENGTH -> + %% TODO: Consider padding here + Data = split_bin(iolist_to_binary(Frag), ?MAX_PLAIN_TEXT_LENGTH), + encode_iolist(?HANDSHAKE, Data, ConnectionStates); + _ -> + encode_plain_text(?HANDSHAKE, Frag, ConnectionStates) + end. + +%%-------------------------------------------------------------------- +-spec encode_alert_record(#alert{}, ssl_record:connection_states()) -> + {iolist(), ssl_record:connection_states()}. +%% +%% Description: Encodes an alert message to send on the ssl-socket. +%%-------------------------------------------------------------------- +encode_alert_record(#alert{level = Level, description = Description}, + ConnectionStates) -> + encode_plain_text(?ALERT, <<?BYTE(Level), ?BYTE(Description)>>, + ConnectionStates). +%%-------------------------------------------------------------------- +-spec encode_data(binary(), ssl_record:connection_states()) -> + {iolist(), ssl_record:connection_states()}. +%% +%% Description: Encodes data to send on the ssl-socket. +%%-------------------------------------------------------------------- +encode_data(Frag, ConnectionStates) -> + Data = split_bin(Frag, ?MAX_PLAIN_TEXT_LENGTH, {3,4}), + encode_iolist(?APPLICATION_DATA, Data, ConnectionStates). + +encode_plain_text(Type, Data0, #{current_write := Write0} = ConnectionStates) -> + PadLen = 0, %% TODO where to specify PadLen? + Data = inner_plaintext(Type, Data0, PadLen), + {CipherFragment, Write1} = encode_plain_text(Data, Write0), + {CipherText, Write} = encode_tls_cipher_text(CipherFragment, Write1), + {CipherText, ConnectionStates#{current_write => Write}}. + +encode_iolist(Type, Data, ConnectionStates0) -> + {ConnectionStates, EncodedMsg} = + lists:foldl(fun(Text, {CS0, Encoded}) -> + {Enc, CS1} = + encode_plain_text(Type, Text, CS0), + {CS1, [Enc | Encoded]} + end, {ConnectionStates0, []}, Data), + {lists:reverse(EncodedMsg), ConnectionStates}. + +%%==================================================================== +%% Decoding +%%==================================================================== + +%%-------------------------------------------------------------------- +-spec decode_cipher_text(#ssl_tls{}, ssl_record:connection_states()) -> + {#ssl_tls{}, ssl_record:connection_states()}| #alert{}. +%% +%% Description: Decode cipher text, use legacy type ssl_tls instead of tls_cipher_text +%% in decoding context so that we can reuse the code from erlier versions. +%%-------------------------------------------------------------------- +decode_cipher_text(#ssl_tls{type = ?OPAQUE_TYPE, + version = ?LEGACY_VERSION, + fragment = CipherFragment}, + #{current_read := + #{sequence_number := Seq, + cipher_state := CipherS0, + security_parameters := + #security_parameters{ + cipher_type = ?AEAD, + bulk_cipher_algorithm = + BulkCipherAlgo} + } = ReadState0} = ConnnectionStates0) -> + AAD = start_additional_data(), + CipherS1 = ssl_cipher:nonce_seed(<<?UINT64(Seq)>>, CipherS0), + case decipher_aead(BulkCipherAlgo, CipherS1, AAD, CipherFragment) of + {PlainFragment, CipherS1} -> + ConnnectionStates = + ConnnectionStates0#{current_read => + ReadState0#{cipher_state => CipherS1, + sequence_number => Seq + 1}}, + decode_inner_plaintext(PlainFragment, ConnnectionStates); + #alert{} = Alert -> + Alert + end; +decode_cipher_text(#ssl_tls{type = Type}, _) -> + %% Version mismatch is already asserted + ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC, {record_typ_mismatch, Type}). + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- +split_bin(Bin, ChunkSize) -> + split_bin(Bin, ChunkSize, []). +split_bin(Bin, ChunkSize, _) -> + do_split_bin(Bin, ChunkSize, []). + +do_split_bin(<<>>, _, Acc) -> + lists:reverse(Acc); +do_split_bin(Bin, ChunkSize, Acc) -> + case Bin of + <<Chunk:ChunkSize/binary, Rest/binary>> -> + do_split_bin(Rest, ChunkSize, [Chunk | Acc]); + _ -> + lists:reverse(Acc, [Bin]) + end. + +inner_plaintext(Type, Data, Length) -> + #inner_plaintext{ + content = Data, + type = Type, + zeros = zero_padding(Length) + }. +zero_padding(Length)-> + binary:copy(<<?BYTE(0)>>, Length). + +encode_plain_text(#inner_plaintext{ + content = Data, + type = Type, + zeros = Zeros + }, #{cipher_state := CipherS0, + sequence_number := Seq, + security_parameters := + #security_parameters{ + cipher_type = ?AEAD} + } = WriteState0) -> + PlainText = <<Data/binary, ?BYTE(Type), Zeros/binary>>, + AAD = start_additional_data(), + CipherS1 = ssl_cipher:nonce_seed(<<?UINT64(Seq)>>, CipherS0), + {Encoded, WriteState} = cipher_aead(PlainText, WriteState0#{cipher_state => CipherS1}, AAD), + {#tls_cipher_text{encoded_record = Encoded}, WriteState}; +encode_plain_text(_, CS) -> + exit({cs, CS}). + +start_additional_data() -> + {MajVer, MinVer} = ?LEGACY_VERSION, + <<?BYTE(?OPAQUE_TYPE), ?BYTE(MajVer), ?BYTE(MinVer)>>. + +end_additional_data(AAD, Len) -> + <<AAD/binary, ?UINT16(Len)>>. + +nonce(#cipher_state{nonce = Nonce, iv = IV}) -> + Len = size(IV), + crypto:exor(<<Nonce:Len/bytes>>, IV). + +cipher_aead(Fragment, + #{cipher_state := CipherS0, + security_parameters := + #security_parameters{bulk_cipher_algorithm = + BulkCipherAlgo} + } = WriteState0, AAD) -> + {CipherFragment, CipherS1} = + cipher_aead(BulkCipherAlgo, CipherS0, AAD, Fragment), + {CipherFragment, WriteState0#{cipher_state => CipherS1}}. + +cipher_aead(Type, #cipher_state{key=Key} = CipherState, AAD0, Fragment) -> + AAD = end_additional_data(AAD0, erlang:iolist_size(Fragment)), + Nonce = nonce(CipherState), + {Content, CipherTag} = ssl_cipher:aead_encrypt(Type, Key, Nonce, Fragment, AAD), + {<<Content/binary, CipherTag/binary>>, CipherState}. + +encode_tls_cipher_text(#tls_cipher_text{opaque_type = Type, + legacy_version = {MajVer, MinVer}, + encoded_record = Encoded}, #{sequence_number := Seq} = Write) -> + Length = erlang:iolist_size(Encoded), + {[<<?BYTE(Type), ?BYTE(MajVer), ?BYTE(MinVer), ?UINT16(Length)>>, Encoded], + Write#{sequence_number => Seq +1}}. + +decipher_aead(Type, #cipher_state{key = Key} = CipherState, AAD0, CipherFragment) -> + try + Nonce = nonce(CipherState), + {AAD, CipherText, CipherTag} = aead_ciphertext_split(CipherState, CipherFragment, AAD0), + case ssl_cipher:aead_decrypt(Type, Key, Nonce, CipherText, CipherTag, AAD) of + Content when is_binary(Content) -> + {Content, CipherState}; + _ -> + ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC, decryption_failed) + end + catch + _:_ -> + ?ALERT_REC(?FATAL, ?BAD_RECORD_MAC, decryption_failed) + end. + +aead_ciphertext_split(#cipher_state{tag_len = Len}, CipherTextFragment, AAD) -> + CipherLen = size(CipherTextFragment) - Len, + <<CipherText:CipherLen/bytes, CipherTag:Len/bytes>> = CipherTextFragment, + {end_additional_data(AAD, CipherLen), CipherText, CipherTag}. + +decode_inner_plaintext(PlainText, ConnnectionStates) -> + case remove_padding(PlainText) of + #alert{} = Alert -> + Alert; + {Data, Type} -> + {#ssl_tls{type = Type, + version = {3,4}, %% Internally use real version + fragment = Data}, ConnnectionStates} + end. + +remove_padding(PlainText)-> + case binary:split(PlainText, <<0>>, [global, trim]) of + [] -> + ?ALERT_REC(?FATAL, ?UNEXPECTED_MESSAGE, padding_error); + [Content] -> + Type = binary:last(Content), + split_content(Type, Content, erlang:byte_size(Content) - 1) + end. + +split_content(?HANDSHAKE, _, 0) -> + ?ALERT_REC(?FATAL, ?UNEXPECTED_MESSAGE, empty_handshake); +split_content(?ALERT, _, 0) -> + ?ALERT_REC(?FATAL, ?UNEXPECTED_MESSAGE, empty_alert); +%% For special middlebox compatible case! +split_content(?CHANGE_CIPHER_SPEC, _, 0) -> + ?ALERT_REC(?FATAL, ?UNEXPECTED_MESSAGE, empty_change_cipher_spec); +split_content(?APPLICATION_DATA = Type, _, 0) -> + {Type, <<>>}; +split_content(Type, Content, N) -> + <<Data:N/bytes, ?BYTE(Type)>> = Content, + {Type, Data}. diff --git a/lib/ssl/src/tls_record_1_3.hrl b/lib/ssl/src/tls_record_1_3.hrl new file mode 100644 index 0000000000..273427a34e --- /dev/null +++ b/lib/ssl/src/tls_record_1_3.hrl @@ -0,0 +1,58 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%%---------------------------------------------------------------------- +%% Purpose: Record and constant defenitions for the TLS-1.3-record protocol +%% see RFC 8446 not present in earlier versions +%%---------------------------------------------------------------------- + +-ifndef(tls_record_1_3). +-define(tls_record_1_3, true). + +%% enum { +%% invalid(0), +%% %% defined in ssl_record.hrl +%% change_cipher_spec(20), +%% alert(21), +%% handshake(22), +%% application_data(23), +%% heartbeat(24), /* RFC 6520 */ +%% (255) +%% } ContentType; + +-define(INVALID, 0). +-define(LEGACY_VERSION, {3,3}). +-define(OPAQUE_TYPE, 23). + +-record(inner_plaintext, { + content, %% data + type, %% Contentype + zeros %% padding "uint8 zeros[length_of_padding]" + }). +-record(tls_cipher_text, { %% Equivalent of encrypted version of #ssl_tls from previous versions + %% decrypted version will still use #ssl_tls for code reuse purposes + %% with real values for content type and version + opaque_type = ?OPAQUE_TYPE, + legacy_version = ?LEGACY_VERSION, + encoded_record + }). + +-endif. % -ifdef(tls_record_1_3). diff --git a/lib/ssl/test/make_certs.erl b/lib/ssl/test/make_certs.erl index 8fe7c54549..578f6a731a 100644 --- a/lib/ssl/test/make_certs.erl +++ b/lib/ssl/test/make_certs.erl @@ -365,7 +365,7 @@ req_cnf(Root, C) -> "default_bits = ", integer_to_list(C#config.default_bits), "\n" "RANDFILE = $ROOTDIR/RAND\n" "encrypt_key = no\n" - "default_md = md5\n" + "default_md = sha1\n" "#string_mask = pkix\n" "x509_extensions = ca_ext\n" "prompt = no\n" @@ -415,7 +415,7 @@ ca_cnf( ["crl_extensions = crl_ext\n" || C#config.v2_crls], "unique_subject = no\n" "default_days = 3600\n" - "default_md = md5\n" + "default_md = sha1\n" "preserve = no\n" "policy = policy_match\n" "\n" @@ -499,7 +499,7 @@ ca_cnf( ["crl_extensions = crl_ext\n" || C#config.v2_crls], "unique_subject = no\n" "default_days = 3600\n" - "default_md = md5\n" + "default_md = sha1\n" "preserve = no\n" "policy = policy_match\n" "\n" diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl index a0cc9f5c08..97b794c3e2 100644 --- a/lib/ssl/test/ssl_basic_SUITE.erl +++ b/lib/ssl/test/ssl_basic_SUITE.erl @@ -3570,14 +3570,14 @@ conf_signature_algs(Config) when is_list(Config) -> ssl_test_lib:start_server([{node, ServerNode}, {port, 0}, {from, self()}, {mfa, {ssl_test_lib, send_recv_result, []}}, - {options, [{active, false}, {signature_algs, [{sha256, rsa}]} | ServerOpts]}]), + {options, [{active, false}, {signature_algs, [{sha, rsa}]} | ServerOpts]}]), Port = ssl_test_lib:inet_port(Server), Client = ssl_test_lib:start_client([{node, ClientNode}, {port, Port}, {host, Hostname}, {from, self()}, {mfa, {ssl_test_lib, send_recv_result, []}}, - {options, [{active, false}, {signature_algs, [{sha256, rsa}]} | ClientOpts]}]), + {options, [{active, false}, {signature_algs, [{sha, rsa}]} | ClientOpts]}]), ct:log("Testcase ~p, Client ~p Server ~p ~n", [self(), Client, Server]), diff --git a/lib/ssl/test/ssl_handshake_SUITE.erl b/lib/ssl/test/ssl_handshake_SUITE.erl index ef1f6be286..861fecc554 100644 --- a/lib/ssl/test/ssl_handshake_SUITE.erl +++ b/lib/ssl/test/ssl_handshake_SUITE.erl @@ -25,6 +25,7 @@ -compile(export_all). -include_lib("common_test/include/ct.hrl"). +-include("ssl_alert.hrl"). -include("ssl_internal.hrl"). -include("tls_handshake.hrl"). -include_lib("public_key/include/public_key.hrl"). @@ -41,7 +42,7 @@ all() -> [decode_hello_handshake, decode_empty_server_sni_correctly, select_proper_tls_1_2_rsa_default_hashsign, ignore_hassign_extension_pre_tls_1_2, - unorded_chain]. + unorded_chain, signature_algorithms]. %%-------------------------------------------------------------------- init_per_suite(Config) -> @@ -55,7 +56,9 @@ init_per_group(_GroupName, Config) -> end_per_group(_,Config) -> Config. -init_per_testcase(ignore_hassign_extension_pre_tls_1_2, Config0) -> +init_per_testcase(TC, Config0) when + TC =:= ignore_hassign_extension_pre_tls_1_2 orelse + TC =:= signature_algorithms -> catch crypto:stop(), try crypto:start() of ok -> @@ -163,11 +166,11 @@ ignore_hassign_extension_pre_tls_1_2(Config) -> Opts = proplists:get_value(server_opts, Config), CertFile = proplists:get_value(certfile, Opts), [{_, Cert, _}] = ssl_test_lib:pem_to_der(CertFile), - HashSigns = #hash_sign_algos{hash_sign_algos = [{sha512, rsa}, {sha, dsa}]}, - {sha512, rsa} = ssl_handshake:select_hashsign(HashSigns, Cert, ecdhe_rsa, tls_v1:default_signature_algs({3,3}), {3,3}), + HashSigns = #hash_sign_algos{hash_sign_algos = [{sha512, rsa}, {sha, dsa}, {sha, rsa}]}, + {sha512, rsa} = ssl_handshake:select_hashsign({HashSigns, undefined}, Cert, ecdhe_rsa, tls_v1:default_signature_algs({3,3}), {3,3}), %%% Ignore - {md5sha, rsa} = ssl_handshake:select_hashsign(HashSigns, Cert, ecdhe_rsa, tls_v1:default_signature_algs({3,2}), {3,2}), - {md5sha, rsa} = ssl_handshake:select_hashsign(HashSigns, Cert, ecdhe_rsa, tls_v1:default_signature_algs({3,0}), {3,0}). + {md5sha, rsa} = ssl_handshake:select_hashsign({HashSigns, undefined}, Cert, ecdhe_rsa, tls_v1:default_signature_algs({3,2}), {3,2}), + {md5sha, rsa} = ssl_handshake:select_hashsign({HashSigns, undefined}, Cert, ecdhe_rsa, tls_v1:default_signature_algs({3,0}), {3,0}). unorded_chain(Config) when is_list(Config) -> DefConf = ssl_test_lib:default_cert_chain_conf(), @@ -188,6 +191,55 @@ unorded_chain(Config) when is_list(Config) -> ssl_certificate:certificate_chain(PeerCert, ets:new(foo, []), ExtractedCerts, UnordedChain). +signature_algorithms(Config) -> + Opts = proplists:get_value(server_opts, Config), + CertFile = proplists:get_value(certfile, Opts), + io:format("Cert = ~p~n", [CertFile]), + [{_, Cert, _}] = ssl_test_lib:pem_to_der(CertFile), + HashSigns0 = #hash_sign_algos{ + hash_sign_algos = [{sha512, rsa}, + {sha, dsa}, + {sha, rsa}]}, + Schemes0 = #signature_scheme_list{ + signature_scheme_list = [rsa_pkcs1_sha1, + ecdsa_sha1]}, + {sha512, rsa} = ssl_handshake:select_hashsign( + {HashSigns0, Schemes0}, + Cert, ecdhe_rsa, + tls_v1:default_signature_algs({3,3}), + {3,3}), + HashSigns1 = #hash_sign_algos{ + hash_sign_algos = [{sha, dsa}, + {sha, rsa}]}, + {sha, rsa} = ssl_handshake:select_hashsign( + {HashSigns1, Schemes0}, + Cert, ecdhe_rsa, + tls_v1:default_signature_algs({3,3}), + {3,3}), + Schemes1 = #signature_scheme_list{ + signature_scheme_list = [rsa_pkcs1_sha256, + ecdsa_sha1]}, + %% Signature not supported + #alert{} = ssl_handshake:select_hashsign( + {HashSigns1, Schemes1}, + Cert, ecdhe_rsa, + tls_v1:default_signature_algs({3,3}), + {3,3}), + %% No scheme, hashsign is used + {sha, rsa} = ssl_handshake:select_hashsign( + {HashSigns1, undefined}, + Cert, ecdhe_rsa, + tls_v1:default_signature_algs({3,3}), + {3,3}), + HashSigns2 = #hash_sign_algos{ + hash_sign_algos = [{sha, dsa}]}, + %% Signature not supported + #alert{} = ssl_handshake:select_hashsign( + {HashSigns2, Schemes1}, + Cert, ecdhe_rsa, + tls_v1:default_signature_algs({3,3}), + {3,3}). + %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- diff --git a/lib/stdlib/doc/src/assert_hrl.xml b/lib/stdlib/doc/src/assert_hrl.xml index 4dc7299609..fb27954235 100644 --- a/lib/stdlib/doc/src/assert_hrl.xml +++ b/lib/stdlib/doc/src/assert_hrl.xml @@ -46,7 +46,7 @@ is the macro name, for example, <c>assertEqual</c>. <c>Info</c> is a list of tagged values, such as <c>[{module, M}, {line, L}, ...]</c>, which gives more information about the location and cause of the exception. All - entries in the <c>Info</c> list are optional; do not rely programatically + entries in the <c>Info</c> list are optional; do not rely programmatically on any of them being present.</p> <p>Each assert macro has a corresponding version with an extra argument, diff --git a/lib/stdlib/doc/src/beam_lib.xml b/lib/stdlib/doc/src/beam_lib.xml index 26d0724aaf..213170df7f 100644 --- a/lib/stdlib/doc/src/beam_lib.xml +++ b/lib/stdlib/doc/src/beam_lib.xml @@ -180,8 +180,8 @@ io:fwrite("~s~n", [erl_prettypr:format(erl_syntax:form_list(AC))]).</code> <name name="beam"/> <desc> <p>Each of the functions described below accept either the - module name, the filename, or a binary containing the BEAM - module.</p> + filename (as a string) or a binary containing the BEAM + module.</p> </desc> </datatype> <datatype> diff --git a/lib/stdlib/src/beam_lib.erl b/lib/stdlib/src/beam_lib.erl index 01181b1097..3386cfcbe6 100644 --- a/lib/stdlib/src/beam_lib.erl +++ b/lib/stdlib/src/beam_lib.erl @@ -53,7 +53,7 @@ %%------------------------------------------------------------------------- --type beam() :: module() | file:filename() | binary(). +-type beam() :: file:filename() | binary(). -type debug_info() :: {DbgiVersion :: atom(), Backend :: module(), Data :: term()} | 'no_debug_info'. -type forms() :: [erl_parse:abstract_form() | erl_parse:form_info()]. diff --git a/lib/stdlib/test/gen_fsm_SUITE.erl b/lib/stdlib/test/gen_fsm_SUITE.erl index 41ee3246f5..a8264e5a84 100644 --- a/lib/stdlib/test/gen_fsm_SUITE.erl +++ b/lib/stdlib/test/gen_fsm_SUITE.erl @@ -124,8 +124,10 @@ start2(Config) when is_list(Config) -> {ok, Pid0} = gen_fsm:start(gen_fsm_SUITE, [], []), ok = do_func_test(Pid0), ok = do_sync_func_test(Pid0), + MRef = monitor(process,Pid0), shutdown_stopped = gen_fsm:sync_send_all_state_event(Pid0, stop_shutdown), + receive {'DOWN',MRef,_,_,shutdown} -> ok end, {'EXIT', {noproc,_}} = (catch gen_fsm:sync_send_event(Pid0, hej)), diff --git a/lib/stdlib/test/sys_SUITE.erl b/lib/stdlib/test/sys_SUITE.erl index 3278eb0eb0..fcc4419569 100644 --- a/lib/stdlib/test/sys_SUITE.erl +++ b/lib/stdlib/test/sys_SUITE.erl @@ -219,7 +219,7 @@ spec_proc(Mod) -> {Mod,system_get_state},{throw,fail}},_}} -> ok end, - ok = sys:terminate(Mod, normal), + ok = sync_terminate(Mod), {ok,_} = Mod:start_link(4), ok = case catch sys:replace_state(Mod, fun(_) -> {} end) of {} -> @@ -228,7 +228,7 @@ spec_proc(Mod) -> {Mod,system_replace_state},{throw,fail}},_}} -> ok end, - ok = sys:terminate(Mod, normal), + ok = sync_terminate(Mod), {ok,_} = Mod:start_link(4), StateFun = fun(_) -> error(fail) end, ok = case catch sys:replace_state(Mod, StateFun) of @@ -240,7 +240,18 @@ spec_proc(Mod) -> {'EXIT',{{callback_failed,StateFun,{error,fail}},_}} -> ok end, - ok = sys:terminate(Mod, normal). + ok = sync_terminate(Mod). + +sync_terminate(Mod) -> + P = whereis(Mod), + MRef = erlang:monitor(process,P), + ok = sys:terminate(Mod, normal), + receive + {'DOWN',MRef,_,_,normal} -> + ok + end, + undefined = whereis(Mod), + ok. %%%%%%%%%%%%%%%%%%%% %% Dummy server diff --git a/make/configure.in b/make/configure.in index 987e02ddc1..bf3fd0751f 100644 --- a/make/configure.in +++ b/make/configure.in @@ -189,6 +189,10 @@ fi AC_PROG_LN_S AC_PROG_RANLIB +LM_PROG_PERL5 +if test "$ac_cv_path_PERL" = false; then + AC_MSG_ERROR([Perl version 5 is required!]) +fi # # Get erts version from erts/vsn.mk diff --git a/make/fixup_development_runtime_dependencies b/make/fixup_development_runtime_dependencies new file mode 100755 index 0000000000..e06bd5faca --- /dev/null +++ b/make/fixup_development_runtime_dependencies @@ -0,0 +1,111 @@ +#!/usr/bin/env perl + +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2018. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# + +# +# Replaces runtime_dependencies pointing to future not yet +# resolved versions in the maint and master branches while under +# development. Such dependencies may exist in .app files on the +# form (<app name>-@<ticket>(:<ticket>)*@) and will be replaced +# with the current version of the application in the source tree. +# This in order not to break tests looking at runtime_dependencies. +# + +use strict; +use File::Basename; + +my $usage_text = <<"HERE"; + usage: $0 <ERL_TOP> +HERE + +my %app_vsn; +my $exit_status = 0; + +@ARGV == 1 or die $usage_text; +my $erl_top = shift @ARGV; + +chdir $erl_top or die "Failed to change directory into '$erl_top'"; + +print "Fixup of development runtime dependencies\n"; + +# +# Determine versions of all applications in the source tree... +# +foreach my $vsn_mk (<lib/*/vsn.mk>, <erts/vsn.mk>) { + my $app_dir = dirname($vsn_mk); + my $app = basename($app_dir); + + if (!open(VSN, $vsn_mk)) { + $exit_status = 1; + print STDERR "ERROR: Failed to open '$vsn_mk' for reading: $!\n"; + } + else { + my $vsn = ''; + while (<VSN>) { + if (/VSN\s*=\s*(\S+)/) { + $vsn = $1; + last; + } + } + close VSN; + if (!$vsn) { + $exit_status = 1; + print STDERR "ERROR: No version found in '$vsn_mk'\n" + } + else { + $app_vsn{$app} = "$app-$vsn"; + } + } +} + +my $valid_apps = join('|', keys %app_vsn); + +# +# Replace all <app name>-@<ticket>(:<ticket>)*@ versions +# in all *.app files with the versions currently used... +# +foreach my $app_file (<lib/*/ebin/*.app>, <erts/preloaded/ebin/erts.app>) { + if (!open(IN, "<", $app_file)) { + $exit_status = 1; + print STDERR "ERROR: Failed to open '$app_file' for reading: $!"; + } + else { + local $/; + my $file = <IN>; + close IN; + my $old_file = $file; + + $file =~ s/($valid_apps)-\@OTP-\d{4,5}(?::OTP-\d{4,5})*\@/$app_vsn{$1}/g; + + if ($file ne $old_file) { + if (!open(OUT, ">", $app_file)) { + $exit_status = 1; + print STDERR "ERROR: Failed to open '$app_file' for writing: $!"; + } + else { + print OUT $file; + close OUT; + } + } + } +} + +exit $exit_status; diff --git a/make/otp_patch_solve_forward_merge_version b/make/otp_patch_solve_forward_merge_version new file mode 100644 index 0000000000..b8626c4cff --- /dev/null +++ b/make/otp_patch_solve_forward_merge_version @@ -0,0 +1 @@ +4 diff --git a/make/otp_version_tickets b/make/otp_version_tickets index e69de29bb2..b8220e1a87 100644 --- a/make/otp_version_tickets +++ b/make/otp_version_tickets @@ -0,0 +1 @@ +DEVELOPMENT diff --git a/otp_versions.table b/otp_versions.table index c6544dfc90..a2e78cab01 100644 --- a/otp_versions.table +++ b/otp_versions.table @@ -10,6 +10,7 @@ OTP-21.0.3 : erts-10.0.3 # asn1-5.0.6 common_test-1.16 compiler-7.2.2 crypto-4.3 OTP-21.0.2 : compiler-7.2.2 erts-10.0.2 public_key-1.6.1 stdlib-3.5.1 # asn1-5.0.6 common_test-1.16 crypto-4.3 debugger-4.2.5 dialyzer-3.3 diameter-2.1.5 edoc-0.9.3 eldap-1.2.4 erl_docgen-0.8 erl_interface-3.10.3 et-1.6.2 eunit-2.3.6 ftp-1.0 hipe-3.18 inets-7.0 jinterface-1.9 kernel-6.0 megaco-3.18.3 mnesia-4.15.4 observer-2.8 odbc-2.12.1 os_mon-2.4.5 otp_mibs-1.2 parsetools-2.1.7 reltool-0.7.6 runtime_tools-1.13 sasl-3.2 snmp-5.2.11 ssh-4.7 ssl-9.0 syntax_tools-2.1.5 tftp-1.0 tools-3.0 wx-1.8.4 xmerl-1.3.17 : OTP-21.0.1 : compiler-7.2.1 erts-10.0.1 # asn1-5.0.6 common_test-1.16 crypto-4.3 debugger-4.2.5 dialyzer-3.3 diameter-2.1.5 edoc-0.9.3 eldap-1.2.4 erl_docgen-0.8 erl_interface-3.10.3 et-1.6.2 eunit-2.3.6 ftp-1.0 hipe-3.18 inets-7.0 jinterface-1.9 kernel-6.0 megaco-3.18.3 mnesia-4.15.4 observer-2.8 odbc-2.12.1 os_mon-2.4.5 otp_mibs-1.2 parsetools-2.1.7 public_key-1.6 reltool-0.7.6 runtime_tools-1.13 sasl-3.2 snmp-5.2.11 ssh-4.7 ssl-9.0 stdlib-3.5 syntax_tools-2.1.5 tftp-1.0 tools-3.0 wx-1.8.4 xmerl-1.3.17 : OTP-21.0 : asn1-5.0.6 common_test-1.16 compiler-7.2 crypto-4.3 debugger-4.2.5 dialyzer-3.3 diameter-2.1.5 edoc-0.9.3 eldap-1.2.4 erl_docgen-0.8 erl_interface-3.10.3 erts-10.0 et-1.6.2 eunit-2.3.6 ftp-1.0 hipe-3.18 inets-7.0 jinterface-1.9 kernel-6.0 mnesia-4.15.4 observer-2.8 os_mon-2.4.5 otp_mibs-1.2 parsetools-2.1.7 public_key-1.6 reltool-0.7.6 runtime_tools-1.13 sasl-3.2 ssh-4.7 ssl-9.0 stdlib-3.5 syntax_tools-2.1.5 tftp-1.0 tools-3.0 wx-1.8.4 xmerl-1.3.17 # megaco-3.18.3 odbc-2.12.1 snmp-5.2.11 : +OTP-20.3.8.10 : eldap-1.2.3.1 erts-9.3.3.4 # asn1-5.0.5.1 common_test-1.15.4 compiler-7.1.5.2 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2.2.2 debugger-4.2.4 dialyzer-3.2.4 diameter-2.1.4 edoc-0.9.2 erl_docgen-0.7.3 erl_interface-3.10.2.1 et-1.6.1 eunit-2.3.5 hipe-3.17.1 ic-4.4.4.2 inets-6.5.2.4 jinterface-1.8.1 kernel-5.4.3.2 megaco-3.18.3 mnesia-4.15.3.2 observer-2.7 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.5 sasl-3.1.2 snmp-5.2.11 ssh-4.6.9.1 ssl-8.2.6.2 stdlib-3.4.5 syntax_tools-2.1.4.1 tools-2.11.2 wx-1.8.3 xmerl-1.3.16 : OTP-20.3.8.9 : compiler-7.1.5.2 # asn1-5.0.5.1 common_test-1.15.4 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2.2.2 debugger-4.2.4 dialyzer-3.2.4 diameter-2.1.4 edoc-0.9.2 eldap-1.2.3 erl_docgen-0.7.3 erl_interface-3.10.2.1 erts-9.3.3.3 et-1.6.1 eunit-2.3.5 hipe-3.17.1 ic-4.4.4.2 inets-6.5.2.4 jinterface-1.8.1 kernel-5.4.3.2 megaco-3.18.3 mnesia-4.15.3.2 observer-2.7 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.5 sasl-3.1.2 snmp-5.2.11 ssh-4.6.9.1 ssl-8.2.6.2 stdlib-3.4.5 syntax_tools-2.1.4.1 tools-2.11.2 wx-1.8.3 xmerl-1.3.16 : OTP-20.3.8.8 : inets-6.5.2.4 # asn1-5.0.5.1 common_test-1.15.4 compiler-7.1.5.1 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 crypto-4.2.2.2 debugger-4.2.4 dialyzer-3.2.4 diameter-2.1.4 edoc-0.9.2 eldap-1.2.3 erl_docgen-0.7.3 erl_interface-3.10.2.1 erts-9.3.3.3 et-1.6.1 eunit-2.3.5 hipe-3.17.1 ic-4.4.4.2 jinterface-1.8.1 kernel-5.4.3.2 megaco-3.18.3 mnesia-4.15.3.2 observer-2.7 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.5 sasl-3.1.2 snmp-5.2.11 ssh-4.6.9.1 ssl-8.2.6.2 stdlib-3.4.5 syntax_tools-2.1.4.1 tools-2.11.2 wx-1.8.3 xmerl-1.3.16 : OTP-20.3.8.7 : crypto-4.2.2.2 mnesia-4.15.3.2 # asn1-5.0.5.1 common_test-1.15.4 compiler-7.1.5.1 cosEvent-2.2.2 cosEventDomain-1.2.2 cosFileTransfer-1.2.2 cosNotification-1.2.3 cosProperty-1.2.3 cosTime-1.2.3 cosTransactions-1.3.3 debugger-4.2.4 dialyzer-3.2.4 diameter-2.1.4 edoc-0.9.2 eldap-1.2.3 erl_docgen-0.7.3 erl_interface-3.10.2.1 erts-9.3.3.3 et-1.6.1 eunit-2.3.5 hipe-3.17.1 ic-4.4.4.2 inets-6.5.2.3 jinterface-1.8.1 kernel-5.4.3.2 megaco-3.18.3 observer-2.7 odbc-2.12.1 orber-3.8.4 os_mon-2.4.4 otp_mibs-1.1.2 parsetools-2.1.6 public_key-1.5.2 reltool-0.7.5 runtime_tools-1.12.5 sasl-3.1.2 snmp-5.2.11 ssh-4.6.9.1 ssl-8.2.6.2 stdlib-3.4.5 syntax_tools-2.1.4.1 tools-2.11.2 wx-1.8.3 xmerl-1.3.16 : @@ -48,6 +49,7 @@ OTP-20.0.3 : asn1-5.0.2 compiler-7.1.1 erts-9.0.3 ssh-4.5.1 # common_test-1.15.1 OTP-20.0.2 : asn1-5.0.1 erts-9.0.2 kernel-5.3.1 # common_test-1.15.1 compiler-7.1 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.2 cosTime-1.2.2 cosTransactions-1.3.2 crypto-4.0 debugger-4.2.2 dialyzer-3.2 diameter-2.0 edoc-0.9 eldap-1.2.2 erl_docgen-0.7 erl_interface-3.10 et-1.6 eunit-2.3.3 hipe-3.16 ic-4.4.2 inets-6.4 jinterface-1.8 megaco-3.18.2 mnesia-4.15 observer-2.4 odbc-2.12 orber-3.8.3 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.5 public_key-1.4.1 reltool-0.7.4 runtime_tools-1.12.1 sasl-3.0.4 snmp-5.2.6 ssh-4.5 ssl-8.2 stdlib-3.4.1 syntax_tools-2.1.2 tools-2.10.1 wx-1.8.1 xmerl-1.3.15 : OTP-20.0.1 : common_test-1.15.1 erts-9.0.1 runtime_tools-1.12.1 stdlib-3.4.1 tools-2.10.1 # asn1-5.0 compiler-7.1 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.2 cosTime-1.2.2 cosTransactions-1.3.2 crypto-4.0 debugger-4.2.2 dialyzer-3.2 diameter-2.0 edoc-0.9 eldap-1.2.2 erl_docgen-0.7 erl_interface-3.10 et-1.6 eunit-2.3.3 hipe-3.16 ic-4.4.2 inets-6.4 jinterface-1.8 kernel-5.3 megaco-3.18.2 mnesia-4.15 observer-2.4 odbc-2.12 orber-3.8.3 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.5 public_key-1.4.1 reltool-0.7.4 sasl-3.0.4 snmp-5.2.6 ssh-4.5 ssl-8.2 syntax_tools-2.1.2 wx-1.8.1 xmerl-1.3.15 : OTP-20.0 : asn1-5.0 common_test-1.15 compiler-7.1 cosProperty-1.2.2 crypto-4.0 debugger-4.2.2 dialyzer-3.2 diameter-2.0 edoc-0.9 erl_docgen-0.7 erl_interface-3.10 erts-9.0 eunit-2.3.3 hipe-3.16 inets-6.4 jinterface-1.8 kernel-5.3 megaco-3.18.2 mnesia-4.15 observer-2.4 orber-3.8.3 parsetools-2.1.5 public_key-1.4.1 reltool-0.7.4 runtime_tools-1.12 sasl-3.0.4 snmp-5.2.6 ssh-4.5 ssl-8.2 stdlib-3.4 syntax_tools-2.1.2 tools-2.10 wx-1.8.1 xmerl-1.3.15 # cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosTime-1.2.2 cosTransactions-1.3.2 eldap-1.2.2 et-1.6 ic-4.4.2 odbc-2.12 os_mon-2.4.2 otp_mibs-1.1.1 : +OTP-19.3.6.12 : eldap-1.2.2.1 # asn1-4.0.4 common_test-1.14 compiler-7.0.4.1 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.4 debugger-4.2.1 dialyzer-3.1.1 diameter-1.12.2 edoc-0.8.1 erl_docgen-0.6.1 erl_interface-3.9.3 erts-8.3.5.6 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 inets-6.3.9 jinterface-1.7.1 kernel-5.2.0.1 megaco-3.18.1 mnesia-4.14.3.1 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssh-4.4.2.4 ssl-8.1.3.1.1 stdlib-3.3 syntax_tools-2.1.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.14 : OTP-19.3.6.11 : erts-8.3.5.6 # asn1-4.0.4 common_test-1.14 compiler-7.0.4.1 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.4 debugger-4.2.1 dialyzer-3.1.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 inets-6.3.9 jinterface-1.7.1 kernel-5.2.0.1 megaco-3.18.1 mnesia-4.14.3.1 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssh-4.4.2.4 ssl-8.1.3.1.1 stdlib-3.3 syntax_tools-2.1.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.14 : OTP-19.3.6.10 : erts-8.3.5.5 syntax_tools-2.1.1.1 # asn1-4.0.4 common_test-1.14 compiler-7.0.4.1 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.4 debugger-4.2.1 dialyzer-3.1.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 inets-6.3.9 jinterface-1.7.1 kernel-5.2.0.1 megaco-3.18.1 mnesia-4.14.3.1 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssh-4.4.2.4 ssl-8.1.3.1.1 stdlib-3.3 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.14 : OTP-19.3.6.9 : ssh-4.4.2.4 # asn1-4.0.4 common_test-1.14 compiler-7.0.4.1 cosEvent-2.2.1 cosEventDomain-1.2.1 cosFileTransfer-1.2.1 cosNotification-1.2.2 cosProperty-1.2.1 cosTime-1.2.2 cosTransactions-1.3.2 crypto-3.7.4 debugger-4.2.1 dialyzer-3.1.1 diameter-1.12.2 edoc-0.8.1 eldap-1.2.2 erl_docgen-0.6.1 erl_interface-3.9.3 erts-8.3.5.4 et-1.6 eunit-2.3.2 gs-1.6.2 hipe-3.15.4 ic-4.4.2 inets-6.3.9 jinterface-1.7.1 kernel-5.2.0.1 megaco-3.18.1 mnesia-4.14.3.1 observer-2.3.1 odbc-2.12 orber-3.8.2 os_mon-2.4.2 otp_mibs-1.1.1 parsetools-2.1.4 percept-0.9 public_key-1.4 reltool-0.7.3 runtime_tools-1.11.1 sasl-3.0.3 snmp-5.2.5 ssl-8.1.3.1.1 stdlib-3.3 syntax_tools-2.1.1 tools-2.9.1 typer-0.9.12 wx-1.8 xmerl-1.3.14 : |