Merge branch 'maint-20' into maint

* maint-20:
  Updated OTP version
  Update release notes
  Update version numbers
  inets: Prepare for release
  inets: Add missing guard
  Avoid WindowBits=8 as per the manual
  Fix deflateParams on zlib 1.2.11
  Ignore empty binaries in enif_inspect_iovec
  Emasculate writable binaries on entering an iovec
  Only apply EOS behaviors if there's pending data
  Stop assuming that all schedulers are managed when updating msacc

8 files changed, 133 insertions, 56 deletions

diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml
index e4c33f8bd3..b1d2fd7be7 100644
--- a/erts/doc/src/notes.xml
+++ b/erts/doc/src/notes.xml
@@ -31,6 +31,49 @@
   </header>
   <p>This document describes the changes made to the ERTS application.</p>
 
+<section><title>Erts 9.1.4</title>
+
+    <section><title>Fixed Bugs and Malfunctions</title>
+      <list>
+        <item>
+	    <p>Microstate accounting sometimes produced incorrect
+	    results for dirty schedulers.</p>
+          <p>
+	    Own Id: OTP-14707</p>
+        </item>
+        <item>
+	    <p>Fixed a regression in <c>zlib:gunzip/1</c> that
+	    prevented it from working when the decompressed size was
+	    a perfect multiple of 16384. This regression was
+	    introduced in 20.1.1</p>
+          <p>
+	    Own Id: OTP-14730 Aux Id: ERL-507 </p>
+        </item>
+        <item>
+	    <p>Fixed a memory corruption bug in
+	    <c>enif_inspect_iovec</c>; writable binaries stayed
+	    writable after entering the iovec.</p>
+          <p>
+	    Own Id: OTP-14745</p>
+        </item>
+        <item>
+	    <p>Fixed a crash in <c>enif_inspect_iovec</c> on
+	    encountering empty binaries.</p>
+          <p>
+	    Own Id: OTP-14750</p>
+        </item>
+        <item>
+	    <p><c>zlib:deflateParams/3</c> will no longer return
+	    <c>buf_error</c> when called after <c>zlib:deflate/2</c>
+	    with zlib <c>1.2.11</c>.</p>
+          <p>
+	    Own Id: OTP-14751</p>
+        </item>
+      </list>
+    </section>
+
+</section>
+
 <section><title>Erts 9.1.3</title>
 
     <section><title>Fixed Bugs and Malfunctions</title>
diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
index d64ef8c8b9..3f6273f214 100644
--- a/erts/emulator/beam/erl_msacc.h
+++ b/erts/emulator/beam/erl_msacc.h
@@ -279,18 +279,32 @@ void erts_msacc_init_thread(char *type, int id, int liberty);
 #define ERTS_MSACC_PUSH_STATE_M()                         \
     ERTS_MSACC_DECLARE_CACHE();                           \
     ERTS_MSACC_PUSH_STATE_CACHED_M()
-#define ERTS_MSACC_PUSH_STATE_CACHED_M()                                \
-    __erts_msacc_state = ERTS_MSACC_IS_ENABLED_CACHED() ?    \
-        erts_msacc_get_state_m__(__erts_msacc_cache) : ERTS_MSACC_STATE_OTHER
+#define ERTS_MSACC_PUSH_STATE_CACHED_M() \
+    do { \
+        if (ERTS_MSACC_IS_ENABLED_CACHED()) { \
+            ASSERT(!__erts_msacc_cache->unmanaged); \
+            __erts_msacc_state = erts_msacc_get_state_m__(__erts_msacc_cache); \
+        } else { \
+            __erts_msacc_state = ERTS_MSACC_STATE_OTHER; \
+        } \
+    } while(0)
 #define ERTS_MSACC_SET_STATE_M(state)                   \
     ERTS_MSACC_DECLARE_CACHE();                         \
     ERTS_MSACC_SET_STATE_CACHED_M(state)
-#define ERTS_MSACC_SET_STATE_CACHED_M(state)            \
-    if (ERTS_MSACC_IS_ENABLED_CACHED())      \
-        erts_msacc_set_state_m__(__erts_msacc_cache, state, 1)
-#define ERTS_MSACC_POP_STATE_M()                                  \
-    if (ERTS_MSACC_IS_ENABLED_CACHED())                      \
-        erts_msacc_set_state_m__(__erts_msacc_cache, __erts_msacc_state, 0)
+#define ERTS_MSACC_SET_STATE_CACHED_M(state) \
+    do { \
+        if (ERTS_MSACC_IS_ENABLED_CACHED()) { \
+            ASSERT(!__erts_msacc_cache->unmanaged); \
+            erts_msacc_set_state_m__(__erts_msacc_cache, state, 1); \
+        } \
+    } while(0)
+#define ERTS_MSACC_POP_STATE_M() \
+    do { \
+        if (ERTS_MSACC_IS_ENABLED_CACHED()) { \
+            ASSERT(!__erts_msacc_cache->unmanaged); \
+            erts_msacc_set_state_m__(__erts_msacc_cache, __erts_msacc_state, 0); \
+        } \
+    } while(0)
 #define ERTS_MSACC_PUSH_AND_SET_STATE_M(state)                    \
     ERTS_MSACC_PUSH_STATE_M(); ERTS_MSACC_SET_STATE_CACHED_M(state)
 
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index ac4ecd77e5..3ade17b10d 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3426,36 +3426,38 @@ static int examine_iovec_term(Eterm list, UWord max_length, iovec_slice_t *resul
         size = binary_size(binary);
         binary_header = binary_val(binary);
 
-        /* If we're a sub-binary we'll need to check our underlying binary to
-         * determine whether we're on-heap or not. */
-        if(thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) {
-            ErlSubBin *sb = (ErlSubBin*)binary_header;
-
-            /* Reject bitstrings */
-            if((sb->bitoffs + sb->bitsize) > 0) {
-                return 0;
+        if (size > 0) {
+            /* If we're a sub-binary we'll need to check our underlying binary
+             * to determine whether we're on-heap or not. */
+            if (thing_subtag(*binary_header) == SUB_BINARY_SUBTAG) {
+                ErlSubBin *sb = (ErlSubBin*)binary_header;
+
+                /* Reject bitstrings */
+                if((sb->bitoffs + sb->bitsize) > 0) {
+                    return 0;
+                }
+
+                ASSERT(size <= binary_size(sb->orig));
+                binary_header = binary_val(sb->orig);
             }
 
-            ASSERT(size <= binary_size(sb->orig));
-            binary_header = binary_val(sb->orig);
-        }
-
-        if(thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) {
-            ASSERT(size <= ERL_ONHEAP_BIN_LIMIT);
+            if (thing_subtag(*binary_header) == HEAP_BINARY_SUBTAG) {
+                ASSERT(size <= ERL_ONHEAP_BIN_LIMIT);
 
-            result->iovec_len += 1;
-            result->onheap_size += size;
-        } else {
-            ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG);
+                result->iovec_len += 1;
+                result->onheap_size += size;
+            } else {
+                ASSERT(thing_subtag(*binary_header) == REFC_BINARY_SUBTAG);
 
-            result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN;
-            result->offheap_size += size;
+                result->iovec_len += 1 + size / MAX_SYSIOVEC_IOVLEN;
+                result->offheap_size += size;
+            }
         }
 
         result->sublist_length += 1;
         lookahead = CDR(cell);
 
-        if(result->sublist_length >= max_length) {
+        if (result->sublist_length >= max_length) {
             break;
         }
     }
@@ -3488,6 +3490,10 @@ static void inspect_raw_binary_data(Eterm binary, ErlNifBinary *result) {
     if (thing_subtag(*parent_header) == REFC_BINARY_SUBTAG) {
         ProcBin *pb = (ProcBin*)parent_header;
 
+        if (pb->flags & (PB_IS_WRITABLE | PB_ACTIVE_WRITER)) {
+            erts_emasculate_writable_binary(pb);
+        }
+
         ASSERT(pb->val != NULL);
         ASSERT(byte_offset < pb->size);
         ASSERT(&pb->bytes[byte_offset] >= (byte*)(pb->val)->orig_bytes);
@@ -3531,7 +3537,7 @@ static int fill_iovec_with_slice(ErlNifEnv *env,
 
         /* If this isn't a refc binary, copy its contents to the onheap buffer
          * and reference that instead. */
-        if (raw_data.ref_bin == NULL) {
+        if (raw_data.size > 0 && raw_data.ref_bin == NULL) {
             ASSERT(onheap_offset < onheap_data.size);
             ASSERT(slice->onheap_size > 0);
 
@@ -3542,12 +3548,11 @@ static int fill_iovec_with_slice(ErlNifEnv *env,
             raw_data.ref_bin = onheap_data.ref_bin;
         }
 
-        ASSERT(raw_data.ref_bin != NULL);
-
         while (raw_data.size > 0) {
             UWord chunk_len = MIN(raw_data.size, MAX_SYSIOVEC_IOVLEN);
 
             ASSERT(iovec_idx < iovec->iovcnt);
+            ASSERT(raw_data.ref_bin != NULL);
 
             iovec->iov[iovec_idx].iov_base = raw_data.data;
             iovec->iov[iovec_idx].iov_len = chunk_len;
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1f8a756305..63e9275ac1 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -3417,7 +3417,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
     int thr_prgr_active = 1;
     erts_aint32_t flgs;
 #endif
-    ERTS_MSACC_PUSH_STATE_M();
+    ERTS_MSACC_PUSH_STATE();
 #ifdef ERTS_SMP
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
@@ -3542,9 +3542,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 								     - 1) + 1;
 				} else
 				    timeout = -1;
-                                ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP);
+                                ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP);
 				res = erts_tse_twait(ssi->event, timeout);
-                                ERTS_MSACC_POP_STATE_M();
+                                ERTS_MSACC_POP_STATE();
 				current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 :
 				    erts_get_monotonic_time(esdp);
 			    } while (res == EINTR);
@@ -3743,12 +3743,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	ASSERT(!erts_port_task_have_outstanding_io_tasks());
 
-	ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO);
+	ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_CHECK_IO);
         LTTNG2(scheduler_poll, esdp->no, 0);
 
 	erl_sys_schedule(0);
 
-	ERTS_MSACC_POP_STATE_M();
+	ERTS_MSACC_POP_STATE();
 
 	if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
 	    ErtsMonotonicTime current_time = erts_get_monotonic_time(esdp);
@@ -10350,7 +10350,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
                                      | ERTS_PROC_LOCK_STATUS
                                      | ERTS_PROC_LOCK_TRACE));
 
-            ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER);
+            ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER);
 
 #ifdef ERTS_SMP
             if (state & ERTS_PSFLG_FREE) {
@@ -10642,7 +10642,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    case 0:			/* No process at all */
 	    default:
 		ASSERT(qmask == 0);
-                ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER);
+                ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER);
 		goto check_activities_to_run;
 	    }
 
@@ -10766,7 +10766,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	}
 
-        ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR);
+        ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_EMULATOR);
 
 #ifdef ERTS_SMP
 
diff --git a/erts/emulator/nifs/common/zlib_nif.c b/erts/emulator/nifs/common/zlib_nif.c
index fa29b4fb71..b709ed5a6f 100644
--- a/erts/emulator/nifs/common/zlib_nif.c
+++ b/erts/emulator/nifs/common/zlib_nif.c
@@ -717,7 +717,9 @@ static ERL_NIF_TERM zlib_deflateEnd(ErlNifEnv *env, int argc, const ERL_NIF_TERM
 
 static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
     zlib_data_t *d;
+
     int res, level, strategy;
+    Bytef dummy_buffer;
 
     if(argc != 3 || !get_zlib_data(env, argv[0], &d)
                  || !enif_get_int(env, argv[1], &level)
@@ -729,12 +731,27 @@ static ERL_NIF_TERM zlib_deflateParams(ErlNifEnv *env, int argc, const ERL_NIF_T
         return enif_raise_exception(env, am_not_initialized);
     }
 
-    /* deflateParams will flush everything currently in the stream, corrupting
-     * the heap unless it's empty. We therefore pretend to have a full output
-     * buffer, forcing a Z_BUF_ERROR if there's anything left to be flushed. */
-    d->s.avail_out = 0;
+    /* This is a bit of a hack; deflateParams flushes with Z_BLOCK which won't
+     * stop at a byte boundary, so we can't split this operation up, and we
+     * can't allocate a buffer large enough to fit it in one go since we have
+     * to support zlib versions that lack deflatePending.
+     *
+     * We therefore flush everything prior to this call to ensure that we are
+     * stopped on a byte boundary and have no pending data. We then hand it a
+     * dummy buffer to detect when this assumption doesn't hold (Hopefully
+     * never), and to smooth over an issue with zlib 1.2.11 which always
+     * returns Z_BUF_ERROR when d->s.avail_out is 0, regardless of whether
+     * there's any pending data or not. */
+
+    d->s.next_out = &dummy_buffer;
+    d->s.avail_out = 1;
+
     res = deflateParams(&d->s, level, strategy);
 
+    if(d->s.avail_out == 0) {
+        return zlib_return(env, Z_STREAM_ERROR);
+    }
+
     return zlib_return(env, res);
 }
 
@@ -929,7 +946,7 @@ static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar
         return enif_raise_exception(env, am_not_initialized);
     }
 
-    if(d->eos_seen) {
+    if(d->eos_seen && enif_ioq_size(d->input_queue) > 0) {
         int res;
 
         switch(d->eos_behavior) {
@@ -943,11 +960,10 @@ static ERL_NIF_TERM zlib_inflate(ErlNifEnv *env, int argc, const ERL_NIF_TERM ar
             }
 
             d->eos_seen = 0;
+
             break;
         case EOS_BEHAVIOR_CUT:
             zlib_reset_input(d);
-
-            return enif_make_tuple2(env, am_finished, enif_make_list(env, 0));
         }
     }
 
diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam
index f388bc723a..4ad5f37434 100644
--- a/erts/preloaded/ebin/zlib.beam
+++ b/erts/preloaded/ebin/zlib.beam
diff --git a/erts/preloaded/src/zlib.erl b/erts/preloaded/src/zlib.erl
index 03c9ae38a1..a4ef42204d 100644
--- a/erts/preloaded/src/zlib.erl
+++ b/erts/preloaded/src/zlib.erl
@@ -188,14 +188,13 @@ deflateReset_nif(_Z) ->
 deflateParams(Z, Level0, Strategy0) ->
     Level = arg_level(Level0),
     Strategy = arg_strategy(Strategy0),
+    Progress = deflate(Z, <<>>, sync),
     case deflateParams_nif(Z, Level, Strategy) of
-        buf_error ->
-            %% We had data left in the pipe; flush everything and stash it away
-            %% for the next deflate call before trying again.
-            Output = deflate(Z, <<>>, full),
-            save_progress(Z, deflate, Output),
-            deflateParams_nif(Z, Level, Strategy);
-        Any -> Any
+        ok ->
+            save_progress(Z, deflate, Progress),
+            ok;
+        Other ->
+            Other
     end.
 deflateParams_nif(_Z, _Level, _Strategy) ->
     erlang:nif_error(undef).
diff --git a/erts/vsn.mk b/erts/vsn.mk
index a788b2e491..220c3b5f6c 100644
--- a/erts/vsn.mk
+++ b/erts/vsn.mk
@@ -18,7 +18,7 @@
 # %CopyrightEnd%
 # 
 
-VSN = 9.1.3
+VSN = 9.1.4
 
 # Port number 4365 in 4.2
 # Port number 4366 in 4.3