Replace the zlib driver with a NIF

All operations will now yield appropriately, allowing them to be used
freely in concurrent applications. This commit also deprecates the
functions listed below, although they won't raise deprecation
warnings until OTP 21:

zlib:adler32
zlib:crc32
zlib:inflateChunk
zlib:getBufSize
zlib:setBufSize

The behavior of throwing an error when a dictionary is required for
decompression has also been deprecated.

1 files changed, 171 insertions, 48 deletions

diff --git a/erts/doc/src/zlib.xml b/erts/doc/src/zlib.xml
index 1d272c4c18..9c3e97d814 100644
--- a/erts/doc/src/zlib.xml
+++ b/erts/doc/src/zlib.xml
@@ -90,6 +90,9 @@ list_to_binary([Compressed|Last])</pre>
       <name name="zlevel"/>
     </datatype>
     <datatype>
+      <name name="zflush"/>
+    </datatype>
+    <datatype>
       <name name="zmemlevel"/>
     </datatype>
     <datatype>
@@ -112,6 +115,11 @@ list_to_binary([Compressed|Last])</pre>
       <fsummary>Calculate the Adler checksum.</fsummary>
       <desc>
         <p>Calculates the Adler-32 checksum for <c><anno>Data</anno></c>.</p>
+        <warning>
+          <p>This function is deprecated and will be removed in a future
+             release. Use <seealso marker="erts:erlang#adler32/1">
+             <c>erlang:adler32/1</c></seealso> instead.</p>
+        </warning>
       </desc>
     </func>
 
@@ -127,6 +135,11 @@ list_to_binary([Compressed|Last])</pre>
 Crc = lists:foldl(fun(Data,Crc0) ->
                       zlib:adler32(Z, Crc0, Data),
                   end, zlib:adler32(Z,&lt;&lt; &gt;&gt;), Datas)</pre>
+        <warning>
+          <p>This function is deprecated and will be removed in a future
+             release. Use <seealso marker="erts:erlang#adler32/2">
+             <c>erlang:adler32/2</c></seealso> instead.</p>
+        </warning>
       </desc>
     </func>
 
@@ -141,6 +154,11 @@ Crc = lists:foldl(fun(Data,Crc0) ->
         <p>This function returns the <c><anno>Adler</anno></c> checksum of
           <c>[Data1,Data2]</c>, requiring only <c><anno>Adler1</anno></c>,
           <c><anno>Adler2</anno></c>, and <c><anno>Size2</anno></c>.</p>
+        <warning>
+          <p>This function is deprecated and will be removed in a future
+             release. Use <seealso marker="erts:erlang#adler32_combine/3">
+             <c>erlang:adler32_combine/3</c></seealso> instead.</p>
+        </warning>
       </desc>
     </func>
 
@@ -165,6 +183,12 @@ Crc = lists:foldl(fun(Data,Crc0) ->
       <fsummary>Get current CRC.</fsummary>
       <desc>
         <p>Gets the current calculated CRC checksum.</p>
+        <warning>
+          <p>This function is deprecated and will be removed in a future
+             release. Use <seealso marker="erts:erlang#crc32/1">
+             <c>erlang:crc32/1</c></seealso> on the uncompressed data
+             instead.</p>
+        </warning>
       </desc>
     </func>
 
@@ -173,6 +197,11 @@ Crc = lists:foldl(fun(Data,Crc0) ->
       <fsummary>Calculate CRC.</fsummary>
       <desc>
         <p>Calculates the CRC checksum for <c><anno>Data</anno></c>.</p>
+        <warning>
+          <p>This function is deprecated and will be removed in a future
+             release. Use <seealso marker="erts:erlang#crc32/1">
+             <c>erlang:crc32/1</c></seealso> instead.</p>
+        </warning>
       </desc>
     </func>
 
@@ -188,6 +217,11 @@ Crc = lists:foldl(fun(Data,Crc0) ->
 Crc = lists:foldl(fun(Data,Crc0) ->
                       zlib:crc32(Z, Crc0, Data),
                   end, zlib:crc32(Z,&lt;&lt; &gt;&gt;), Datas)</pre>
+        <warning>
+          <p>This function is deprecated and will be removed in a future
+             release. Use <seealso marker="erts:erlang#crc32/2">
+             <c>erlang:crc32/2</c></seealso> instead.</p>
+        </warning>
       </desc>
     </func>
 
@@ -202,6 +236,11 @@ Crc = lists:foldl(fun(Data,Crc0) ->
         <p>This function returns the <c><anno>CRC</anno></c> checksum of
           <c>[Data1,Data2]</c>, requiring only <c><anno>CRC1</anno></c>,
           <c><anno>CRC2</anno></c>, and <c><anno>Size2</anno></c>.</p>
+        <warning>
+          <p>This function is deprecated and will be removed in a future
+             release. Use <seealso marker="erts:erlang#crc32_combine/3">
+             <c>erlang:crc32_combine/3</c></seealso> instead.</p>
+        </warning>
       </desc>
     </func>
 
@@ -407,8 +446,8 @@ list_to_binary([B1,B2])</pre>
           <seealso marker="#deflateInit/1"><c>deflateInit/1,2,6</c></seealso> or
           <seealso marker="#deflateReset/1"><c>deflateReset/1</c></seealso>,
           before any call of
-          <seealso marker="#deflate/3"><c>deflate/3</c></seealso>.
-          The compressor and decompressor must use the same dictionary (see
+          <seealso marker="#deflate/3"><c>deflate/3</c></seealso>.</p>
+        <p>The compressor and decompressor must use the same dictionary (see
           <seealso marker="#inflateSetDictionary/2">
           <c>inflateSetDictionary/2</c></seealso>).</p>
         <p>The Adler checksum of the dictionary is returned.</p>
@@ -420,6 +459,10 @@ list_to_binary([B1,B2])</pre>
       <fsummary>Get buffer size.</fsummary>
       <desc>
         <p>Gets the size of the intermediate buffer.</p>
+        <warning>
+          <p>This function is deprecated and will be removed in a future
+             release.</p>
+        </warning>
       </desc>
     </func>
 
@@ -443,14 +486,31 @@ list_to_binary([B1,B2])</pre>
       <name name="inflate" arity="2"/>
       <fsummary>Decompress data.</fsummary>
       <desc>
-        <p>Decompresses as much data as possible.
-          It can introduce some output latency (reading
-          input without producing any output).</p>
-        <p>If a preset dictionary is needed at this point (see
-          <seealso marker="#inflateSetDictionary/2">
-          <c>inflateSetDictionary/2</c></seealso>), <c>inflate/2</c> throws a
-          <c>{need_dictionary,Adler}</c> exception, where <c>Adler</c> is
-          the Adler-32 checksum of the dictionary chosen by the compressor.</p>
+        <p>Equivalent to
+           <seealso marker="#inflate/3"><c>inflate(Z, Data, [])</c></seealso>
+           </p>
+      </desc>
+    </func>
+
+    <func>
+      <name name="inflate" arity="3"/>
+      <fsummary>Decompress data.</fsummary>
+      <desc>
+        <p>Decompresses as much data as possible. It can introduce some output
+           latency (reading input without producing any output).</p>
+        <p>Currently the only available option is
+           <c>{exception_on_need_dict,boolean()}</c> which controls whether the
+           function should throw an exception when a preset dictionary is
+           required for decompression. When set to false, a
+           <c>need_dictionary</c> tuple will be returned instead. See
+           <seealso marker="#inflateSetDictionary/2">
+           <c>inflateSetDictionary/2</c></seealso> for details.</p>
+        <warning>
+           <p>This option defaults to <c>true</c> for backwards compatibility
+              but we intend to remove the exception behavior in a future
+              release. New code that needs to handle dictionaries manually
+              should always specify <c>{exception_on_need_dict,false}</c>.</p>
+        </warning>
       </desc>
     </func>
 
@@ -458,6 +518,11 @@ list_to_binary([B1,B2])</pre>
       <name name="inflateChunk" arity="1"/>
       <fsummary>Read next uncompressed chunk.</fsummary>
       <desc>
+        <warning>
+          <p>This function is deprecated and will be removed in a future
+             release. Use <seealso marker="#safeInflate/2"><c>safeInflate/2</c>
+             </seealso> instead.</p>
+        </warning>
         <p>Reads the next chunk of uncompressed data, initialized by
           <seealso marker="#inflateChunk/2"><c>inflateChunk/2</c></seealso>.</p>
         <p>This function is to be repeatedly called, while it returns
@@ -469,23 +534,27 @@ list_to_binary([B1,B2])</pre>
       <name name="inflateChunk" arity="2"/>
       <fsummary>Decompress data with limited output size.</fsummary>
       <desc>
+        <warning>
+          <p>This function is deprecated and will be removed in a future
+             release. Use <seealso marker="#safeInflate/2"><c>safeInflate/2</c>
+             </seealso> instead.</p>
+        </warning>
         <p>Like <seealso marker="#inflate/2"><c>inflate/2</c></seealso>,
-          but decompresses no more data than will fit in the buffer configured
-          through <seealso marker="#setBufSize/2"><c>setBufSize/2</c></seealso>.
-          Is is useful when decompressing a stream with a high compression
-          ratio, such that a small amount of compressed input can expand up to
-          1000 times.</p>
+           but decompresses no more data than will fit in the buffer configured
+           through <seealso marker="#setBufSize/2"><c>setBufSize/2</c>
+           </seealso>. Is is useful when decompressing a stream with a high
+           compression ratio, such that a small amount of compressed input can
+           expand up to 1000 times.</p>
         <p>This function returns <c>{more, Decompressed}</c>, when there is
-          more output available, and
-          <seealso marker="#inflateChunk/1"><c>inflateChunk/1</c></seealso>
-          is to be used to read it.</p>
-        <p>This function can introduce some output latency (reading
-          input without producing any output).</p>
-        <p>If a preset dictionary is needed at this point (see
-          <seealso marker="#inflateSetDictionary/2">
-          <c>inflateSetDictionary/2</c></seealso>), this function throws a
-          <c>{need_dictionary,Adler}</c> exception, where <c>Adler</c> is
-          the Adler-32 checksum of the dictionary chosen by the compressor.</p>
+           more output available, and
+           <seealso marker="#inflateChunk/1"><c>inflateChunk/1</c></seealso>
+           is to be used to read it.</p>
+        <p>This function can introduce some output latency (reading input
+           without producing any output).</p>
+        <p>An exception will be thrown if a preset dictionary is required for
+           further decompression. See
+           <seealso marker="#inflateSetDictionary/2">
+           <c>inflateSetDictionary/2</c></seealso> for details.</p>
         <p>Example:</p>
         <pre>
 walk(Compressed, Handler) ->
@@ -517,6 +586,18 @@ loop(Z, Handler, Uncompressed) ->
     </func>
 
     <func>
+      <name name="inflateGetDictionary" arity="1"/>
+      <fsummary>Return the decompression dictionary.</fsummary>
+      <desc>
+        <p>Returns the decompression dictionary currently in use
+          by the stream. This function must be called between
+          <seealso marker="#inflateInit/1"><c>inflateInit/1,2</c></seealso>
+          and <seealso marker="#inflateEnd/1"><c>inflateEnd</c></seealso>.</p>
+        <p>Only supported if ERTS was compiled with zlib >= 1.2.8.</p>
+      </desc>
+    </func>
+
+    <func>
       <name name="inflateInit" arity="1"/>
       <fsummary>Initialize a session for decompression.</fsummary>
       <desc>
@@ -562,45 +643,83 @@ loop(Z, Handler, Uncompressed) ->
       <fsummary>Initialize the decompression dictionary.</fsummary>
       <desc>
         <p>Initializes the decompression dictionary from the specified
-          uncompressed byte sequence. This function must be called
-          immediately after a call of
-          <seealso marker="#inflate/2"><c>inflate/2</c></seealso>
-          if this call threw a <c>{need_dictionary,Adler}</c> exception.
-          The dictionary chosen by the compressor can be determined from the
-          Adler value thrown by the call to <c>inflate/2</c>.
-          The compressor and decompressor must use the same dictionary (see
-          <seealso marker="#deflateSetDictionary/2">
-          <c>deflateSetDictionary/2</c></seealso>).</p>
+           uncompressed byte sequence. This function must be called as a
+           response to an inflate operation (eg.
+           <seealso marker="#safeInflate/2"><c>safeInflate/2</c></seealso>)
+           returning <c>{need_dictionary,Adler,Output}</c> or in the case of
+           deprecated functions, throwing an
+           <c>{'EXIT',{{need_dictionary,Adler},_StackTrace}}</c> exception.</p>
+        <p>The dictionary chosen by the compressor can be determined from the
+           Adler value returned or thrown by the call to the inflate function.
+           The compressor and decompressor must use the same dictionary (See
+           <seealso marker="#deflateSetDictionary/2">
+           <c>deflateSetDictionary/2</c></seealso>).</p>
+        <p>After setting the dictionary the inflate operation should be
+           retried without new input.</p>
         <p>Example:</p>
         <pre>
-unpack(Z, Compressed, Dict) ->
+deprecated_unpack(Z, Compressed, Dict) ->
      case catch zlib:inflate(Z, Compressed) of
-          {'EXIT',{{need_dictionary,DictID},_}} ->
-                   zlib:inflateSetDictionary(Z, Dict),
+          {'EXIT',{{need_dictionary,_DictID},_}} ->
+                 ok = zlib:inflateSetDictionary(Z, Dict),
                  Uncompressed = zlib:inflate(Z, []);
           Uncompressed ->
                  Uncompressed
-     end.</pre>
+     end.
+
+new_unpack(Z, Compressed, Dict) ->
+    case zlib:inflate(Z, Compressed, [{exception_on_need_dict, false}]) of
+        {need_dictionary, _DictId, Output} ->
+            ok = zlib:inflateSetDictionary(Z, Dict),
+            [Output | zlib:inflate(Z, [])];
+        Uncompressed ->
+            Uncompressed
+    end.</pre>
       </desc>
     </func>
 
     <func>
-      <name name="inflateGetDictionary" arity="1"/>
-      <fsummary>Return the decompression dictionary.</fsummary>
+      <name name="open" arity="0"/>
+      <fsummary>Open a stream and return a stream reference.</fsummary>
       <desc>
-        <p>Returns the decompression dictionary currently in use
-          by the stream. This function must be called between
-          <seealso marker="#inflateInit/1"><c>inflateInit/1,2</c></seealso>
-          and <seealso marker="#inflateEnd/1"><c>inflateEnd</c></seealso>.</p>
-        <p>Only supported if ERTS was compiled with zlib >= 1.2.8.</p>
+        <p>Opens a zlib stream.</p>
       </desc>
     </func>
 
     <func>
-      <name name="open" arity="0"/>
-      <fsummary>Open a stream and return a stream reference.</fsummary>
+      <name name="safeInflate" arity="2"/>
+      <fsummary>Decompress data with limited output size.</fsummary>
       <desc>
-        <p>Opens a zlib stream.</p>
+        <p>Like <seealso marker="#inflate/2"><c>inflate/2</c></seealso>,
+           but returns once it has expanded beyond a small 
+           implementation-defined threshold. It's useful when decompressing
+           untrusted input which could have been maliciously crafted to expand
+           until the system runs out of memory.</p>
+        <p>This function returns <c>{continue | finished, Output}</c>, where
+           <anno>Output</anno> is the data that was decompressed in this call.
+           New input can be queued up on each call if desired, and the function
+           will return <c>{finished, Output}</c> once all queued data has been
+           decompressed.</p>
+        <p>This function can introduce some output latency (reading
+           input without producing any output).</p>
+        <p>If a preset dictionary is required for further decompression, this
+           function returns a <c>need_dictionary</c> tuple. See
+           <seealso marker="#inflateSetDictionary/2">
+           <c>inflateSetDictionary/2</c></seealso>) for details.</p>
+        <p>Example:</p>
+        <pre>
+walk(Compressed, Handler) ->
+    Z = zlib:open(),
+    zlib:inflateInit(Z),
+    loop(Z, Handler, zlib:safeInflate(Z, Compressed)),
+    zlib:inflateEnd(Z),
+    zlib:close(Z).
+
+loop(Z, Handler, {continue, Output}) ->
+    Handler(Output),
+    loop(Z, Handler, zlib:safeInflate(Z, []));
+loop(Z, Handler, {finished, Output}) ->
+    Handler(Output).</pre>
       </desc>
     </func>
 
@@ -609,6 +728,10 @@ unpack(Z, Compressed, Dict) ->
       <fsummary>Set buffer size.</fsummary>
       <desc>
         <p>Sets the intermediate buffer size.</p>
+        <warning>
+          <p>This function is deprecated and will be removed in a future
+             release.</p>
+        </warning>
       </desc>
     </func>