From 9b1f2439845e5627a00fb816244af06d77c2d70a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20H=C3=B6gberg?= Date: Wed, 1 Nov 2017 16:53:51 +0100 Subject: Only apply EOS behaviors if there's pending data --- erts/emulator/nifs/common/zlib_nif.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'erts') diff --git a/erts/emulator/nifs/common/zlib_nif.c b/erts/emulator/nifs/common/zlib_nif.c index fa29b4fb71..104e52754a 100644 --- a/erts/emulator/nifs/common/zlib_nif.c +++ b/erts/emulator/nifs/common/zlib_nif.c @@ -929,7 +929,7 @@ static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar return enif_raise_exception(env, am_not_initialized); } - if(d->eos_seen) { + if(d->eos_seen && enif_ioq_size(d->input_queue) > 0) { int res; switch(d->eos_behavior) { @@ -943,11 +943,10 @@ static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar } d->eos_seen = 0; + break; case EOS_BEHAVIOR_CUT: zlib_reset_input(d); - - return enif_make_tuple2(env, am_finished, enif_make_list(env, 0)); } } -- cgit v1.2.3 From b1ce8325624d22fa34b76407932947a543754962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20H=C3=B6gberg?= Date: Tue, 7 Nov 2017 07:16:31 +0100 Subject: Emasculate writable binaries on entering an iovec The lack of this caused serious data corruption when a binary was altered after entering the queue. This went unnoticed because it was never used without erlang:iolist_to_iovec, which always emasculates binaries. --- erts/emulator/beam/erl_nif.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'erts') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index ac4ecd77e5..8342070521 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3488,6 +3488,10 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) { if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) { ProcBin *pb = (ProcBin*)parent_header; + if (pb->flags & (PB_IS_WRITABLE | PB_ACTIVE_WRITER)) { + erts_emasculate_writable_binary(pb); + } + ASSERT(pb->val != NULL); ASSERT(byte_offset < pb->size); ASSERT(&pb->bytes[byte_offset] >= (byte*)(pb->val)->orig_bytes); -- cgit v1.2.3 From c221ada91c0027bbd74e3e5bff2865df7765a2ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20H=C3=B6gberg?= Date: Tue, 7 Nov 2017 11:30:27 +0100 Subject: Ignore empty binaries in enif_inspect_iovec --- erts/emulator/beam/erl_nif.c | 49 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 24 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 8342070521..3ade17b10d 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -3426,36 +3426,38 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul size = binary_size(binary); binary_header = binary_val(binary); - /* If we're a sub-binary we'll need to check our underlying binary to - * determine whether we're on-heap or not. */ - if(thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) { - ErlSubBin *sb = (ErlSubBin*)binary_header; - - /* Reject bitstrings */ - if((sb->bitoffs + sb->bitsize) > 0) { - return 0; + if (size > 0) { + /* If we're a sub-binary we'll need to check our underlying binary + * to determine whether we're on-heap or not. */ + if (thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) { + ErlSubBin *sb = (ErlSubBin*)binary_header; + + /* Reject bitstrings */ + if((sb->bitoffs + sb->bitsize) > 0) { + return 0; + } + + ASSERT(size <= binary_size(sb->orig)); + binary_header = binary_val(sb->orig); } - ASSERT(size <= binary_size(sb->orig)); - binary_header = binary_val(sb->orig); - } - - if(thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) { - ASSERT(size <= ERL_ONHEAP_BIN_LIMIT); + if (thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) { + ASSERT(size <= ERL_ONHEAP_BIN_LIMIT); - result->iovec_len += 1; - result->onheap_size += size; - } else { - ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG); + result->iovec_len += 1; + result->onheap_size += size; + } else { + ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG); - result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN; - result->offheap_size += size; + result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN; + result->offheap_size += size; + } } result->sublist_length += 1; lookahead = CDR(cell); - if(result->sublist_length >= max_length) { + if (result->sublist_length >= max_length) { break; } } @@ -3535,7 +3537,7 @@ static int fill_iovec_with_slice(ErlNifEnv *env, /* If this isn't a refc binary, copy its contents to the onheap buffer * and reference that instead. */ - if (raw_data.ref_bin == NULL) { + if (raw_data.size > 0 && raw_data.ref_bin == NULL) { ASSERT(onheap_offset < onheap_data.size); ASSERT(slice->onheap_size > 0); @@ -3546,12 +3548,11 @@ static int fill_iovec_with_slice(ErlNifEnv *env, raw_data.ref_bin = onheap_data.ref_bin; } - ASSERT(raw_data.ref_bin != NULL); - while (raw_data.size > 0) { UWord chunk_len = MIN(raw_data.size, MAX_SYSIOVEC_IOVLEN); ASSERT(iovec_idx < iovec->iovcnt); + ASSERT(raw_data.ref_bin != NULL); iovec->iov[iovec_idx].iov_base = raw_data.data; iovec->iov[iovec_idx].iov_len = chunk_len; -- cgit v1.2.3 From 8cfe9496873fa36eccda939e56bf7a922e945ca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20H=C3=B6gberg?= Date: Tue, 7 Nov 2017 17:24:55 +0100 Subject: Fix deflateParams on zlib 1.2.11 1.2.11 started bailing when avail_out==0 regardless of whether there's anything to flush or not, and there's no point in adapting the old method since it was vulnerable to bugs in other zlib versions which updated the deflate parameters even on failure. The api_deflateParams test has been expanded accordingly, and two white-box cases in zip_usage has been updated to make fewer assumptions about the output; the validity of the compressed data is what matters, not whether it's exactly the same as the test vector. --- erts/emulator/nifs/common/zlib_nif.c | 25 +++++++++++++++++++++---- erts/preloaded/ebin/zlib.beam | Bin 19784 -> 19744 bytes erts/preloaded/src/zlib.erl | 13 ++++++------- 3 files changed, 27 insertions(+), 11 deletions(-) (limited to 'erts') diff --git a/erts/emulator/nifs/common/zlib_nif.c b/erts/emulator/nifs/common/zlib_nif.c index fa29b4fb71..9565a5a059 100644 --- a/erts/emulator/nifs/common/zlib_nif.c +++ b/erts/emulator/nifs/common/zlib_nif.c @@ -717,7 +717,9 @@ static ERL_NIF_TERM zlib_deflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { zlib_data_t *d; + int res, level, strategy; + Bytef dummy_buffer; if(argc != 3 || !get_zlib_data(env, argv[0], &d) || !enif_get_int(env, argv[1], &level) @@ -729,12 +731,27 @@ static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_T return enif_raise_exception(env, am_not_initialized); } - /* deflateParams will flush everything currently in the stream, corrupting - * the heap unless it's empty. We therefore pretend to have a full output - * buffer, forcing a Z_BUF_ERROR if there's anything left to be flushed. */ - d->s.avail_out = 0; + /* This is a bit of a hack; deflateParams flushes with Z_BLOCK which won't + * stop at a byte boundary, so we can't split this operation up, and we + * can't allocate a buffer large enough to fit it in one go since we have + * to support zlib versions that lack deflatePending. + * + * We therefore flush everything prior to this call to ensure that we are + * stopped on a byte boundary and have no pending data. We then hand it a + * dummy buffer to detect when this assumption doesn't hold (Hopefully + * never), and to smooth over an issue with zlib 1.2.11 which always + * returns Z_BUF_ERROR when d->s.avail_out is 0, regardless of whether + * there's any pending data or not. */ + + d->s.next_out = &dummy_buffer; + d->s.avail_out = 1; + res = deflateParams(&d->s, level, strategy); + if(d->s.avail_out == 0) { + return zlib_return(env, Z_STREAM_ERROR); + } + return zlib_return(env, res); } diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam index f388bc723a..4ad5f37434 100644 Binary files a/erts/preloaded/ebin/zlib.beam and b/erts/preloaded/ebin/zlib.beam differ diff --git a/erts/preloaded/src/zlib.erl b/erts/preloaded/src/zlib.erl index 03c9ae38a1..a4ef42204d 100644 --- a/erts/preloaded/src/zlib.erl +++ b/erts/preloaded/src/zlib.erl @@ -188,14 +188,13 @@ deflateReset_nif(_Z) -> deflateParams(Z, Level0, Strategy0) -> Level = arg_level(Level0), Strategy = arg_strategy(Strategy0), + Progress = deflate(Z, <<>>, sync), case deflateParams_nif(Z, Level, Strategy) of - buf_error -> - %% We had data left in the pipe; flush everything and stash it away - %% for the next deflate call before trying again. - Output = deflate(Z, <<>>, full), - save_progress(Z, deflate, Output), - deflateParams_nif(Z, Level, Strategy); - Any -> Any + ok -> + save_progress(Z, deflate, Progress), + ok; + Other -> + Other end. deflateParams_nif(_Z, _Level, _Strategy) -> erlang:nif_error(undef). -- cgit v1.2.3 From de930e60f1a6352491ac543b5d50d13122c10745 Mon Sep 17 00:00:00 2001 From: Erlang/OTP Date: Thu, 9 Nov 2017 16:15:43 +0100 Subject: Update version numbers --- erts/vsn.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'erts') diff --git a/erts/vsn.mk b/erts/vsn.mk index a788b2e491..220c3b5f6c 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% # -VSN = 9.1.3 +VSN = 9.1.4 # Port number 4365 in 4.2 # Port number 4366 in 4.3 -- cgit v1.2.3 From 2ad665902380f1d18d6ad0b175b00b9e66dec77c Mon Sep 17 00:00:00 2001 From: Erlang/OTP Date: Thu, 9 Nov 2017 16:15:57 +0100 Subject: Update release notes --- erts/doc/src/notes.xml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) (limited to 'erts') diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 2507451533..1d309f8f86 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -31,6 +31,49 @@

This document describes the changes made to the ERTS application.

+
Erts 9.1.4 + +
Fixed Bugs and Malfunctions + + +

Microstate accounting sometimes produced incorrect + results for dirty schedulers.

+

+ Own Id: OTP-14707

+ + +

Fixed a regression in zlib:gunzip/1 that + prevented it from working when the decompressed size was + a perfect multiple of 16384. This regression was + introduced in 20.1.1

+

+ Own Id: OTP-14730 Aux Id: ERL-507

+ + +

Fixed a memory corruption bug in + enif_inspect_iovec; writable binaries stayed + writable after entering the iovec.

+

+ Own Id: OTP-14745

+ + +

Fixed a crash in enif_inspect_iovec on + encountering empty binaries.

+

+ Own Id: OTP-14750

+ + +

zlib:deflateParams/3 will no longer return + buf_error when called after zlib:deflate/2 + with zlib 1.2.11.

+

+ Own Id: OTP-14751

+ + +
+ +
+
Erts 9.1.3
Fixed Bugs and Malfunctions -- cgit v1.2.3