From 948ee9b34f34a55cb4b70b7077a849c7dc7a5f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20H=C3=B6gberg?= Date: Fri, 21 Jul 2017 14:20:16 +0200 Subject: Replace the zlib driver with a NIF All operations will now yield appropriately, allowing them to be used freely in concurrent applications. This commit also deprecates the functions listed below, although they won't raise deprecation warnings until OTP 21: zlib:adler32 zlib:crc32 zlib:inflateChunk zlib:getBufSize zlib:setBufSize The behavior of throwing an error when a dictionary is required for decompression has also been deprecated. --- erts/doc/src/zlib.xml | 219 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 171 insertions(+), 48 deletions(-) (limited to 'erts/doc') diff --git a/erts/doc/src/zlib.xml b/erts/doc/src/zlib.xml index 1d272c4c18..9c3e97d814 100644 --- a/erts/doc/src/zlib.xml +++ b/erts/doc/src/zlib.xml @@ -89,6 +89,9 @@ list_to_binary([Compressed|Last]) + + + @@ -112,6 +115,11 @@ list_to_binary([Compressed|Last]) Calculate the Adler checksum.

Calculates the Adler-32 checksum for Data.

+ +

This function is deprecated and will be removed in a future + release. Use + erlang:adler32/1 instead.

+ @@ -127,6 +135,11 @@ list_to_binary([Compressed|Last]) Crc = lists:foldl(fun(Data,Crc0) -> zlib:adler32(Z, Crc0, Data), end, zlib:adler32(Z,<< >>), Datas) + +

This function is deprecated and will be removed in a future + release. Use + erlang:adler32/2 instead.

+ @@ -141,6 +154,11 @@ Crc = lists:foldl(fun(Data,Crc0) ->

This function returns the Adler checksum of [Data1,Data2], requiring only Adler1, Adler2, and Size2.

+ +

This function is deprecated and will be removed in a future + release. Use + erlang:adler32_combine/3 instead.

+ @@ -165,6 +183,12 @@ Crc = lists:foldl(fun(Data,Crc0) -> Get current CRC.

Gets the current calculated CRC checksum.

+ +

This function is deprecated and will be removed in a future + release. Use + erlang:crc32/1 on the uncompressed data + instead.

+ @@ -173,6 +197,11 @@ Crc = lists:foldl(fun(Data,Crc0) -> Calculate CRC.

Calculates the CRC checksum for Data.

+ +

This function is deprecated and will be removed in a future + release. Use + erlang:crc32/1 instead.

+ @@ -188,6 +217,11 @@ Crc = lists:foldl(fun(Data,Crc0) -> Crc = lists:foldl(fun(Data,Crc0) -> zlib:crc32(Z, Crc0, Data), end, zlib:crc32(Z,<< >>), Datas) + +

This function is deprecated and will be removed in a future + release. Use + erlang:crc32/2 instead.

+ @@ -202,6 +236,11 @@ Crc = lists:foldl(fun(Data,Crc0) ->

This function returns the CRC checksum of [Data1,Data2], requiring only CRC1, CRC2, and Size2.

+ +

This function is deprecated and will be removed in a future + release. Use + erlang:crc32_combine/3 instead.

+ @@ -407,8 +446,8 @@ list_to_binary([B1,B2]) deflateInit/1,2,6 or deflateReset/1, before any call of - deflate/3. - The compressor and decompressor must use the same dictionary (see + deflate/3.

+

The compressor and decompressor must use the same dictionary (see inflateSetDictionary/2).

The Adler checksum of the dictionary is returned.

@@ -420,6 +459,10 @@ list_to_binary([B1,B2]) Get buffer size.

Gets the size of the intermediate buffer.

+ +

This function is deprecated and will be removed in a future + release.

+ @@ -443,14 +486,31 @@ list_to_binary([B1,B2]) Decompress data. -

Decompresses as much data as possible. - It can introduce some output latency (reading - input without producing any output).

-

If a preset dictionary is needed at this point (see - - inflateSetDictionary/2), inflate/2 throws a - {need_dictionary,Adler} exception, where Adler is - the Adler-32 checksum of the dictionary chosen by the compressor.

+

Equivalent to + inflate(Z, Data, []) +

+ + + + + + Decompress data. + +

Decompresses as much data as possible. It can introduce some output + latency (reading input without producing any output).

+

Currently the only available option is + {exception_on_need_dict,boolean()} which controls whether the + function should throw an exception when a preset dictionary is + required for decompression. When set to false, a + need_dictionary tuple will be returned instead. See + + inflateSetDictionary/2 for details.

+ +

This option defaults to true for backwards compatibility + but we intend to remove the exception behavior in a future + release. New code that needs to handle dictionaries manually + should always specify {exception_on_need_dict,false}.

+ @@ -458,6 +518,11 @@ list_to_binary([B1,B2]) Read next uncompressed chunk. + +

This function is deprecated and will be removed in a future + release. Use safeInflate/2 + instead.

+

Reads the next chunk of uncompressed data, initialized by inflateChunk/2.

This function is to be repeatedly called, while it returns @@ -469,23 +534,27 @@ list_to_binary([B1,B2]) Decompress data with limited output size. + +

This function is deprecated and will be removed in a future + release. Use safeInflate/2 + instead.

+

Like inflate/2, - but decompresses no more data than will fit in the buffer configured - through setBufSize/2. - Is is useful when decompressing a stream with a high compression - ratio, such that a small amount of compressed input can expand up to - 1000 times.

+ but decompresses no more data than will fit in the buffer configured + through setBufSize/2 + . Is is useful when decompressing a stream with a high + compression ratio, such that a small amount of compressed input can + expand up to 1000 times.

This function returns {more, Decompressed}, when there is - more output available, and - inflateChunk/1 - is to be used to read it.

-

This function can introduce some output latency (reading - input without producing any output).

-

If a preset dictionary is needed at this point (see - - inflateSetDictionary/2), this function throws a - {need_dictionary,Adler} exception, where Adler is - the Adler-32 checksum of the dictionary chosen by the compressor.

+ more output available, and + inflateChunk/1 + is to be used to read it.

+

This function can introduce some output latency (reading input + without producing any output).

+

An exception will be thrown if a preset dictionary is required for + further decompression. See + + inflateSetDictionary/2 for details.

Example:

 walk(Compressed, Handler) ->
@@ -516,6 +585,18 @@ loop(Z, Handler, Uncompressed) ->
       
     
 
+    
+      
+      Return the decompression dictionary.
+      
+        
Returns the decompression dictionary currently in use
+          by the stream. This function must be called between
+          inflateInit/1,2
+          and inflateEnd.

+        
Only supported if ERTS was compiled with zlib >= 1.2.8.

+      
+    
+
     
       
       Initialize a session for decompression.
@@ -562,45 +643,83 @@ loop(Z, Handler, Uncompressed) ->
       Initialize the decompression dictionary.
       
         
Initializes the decompression dictionary from the specified
-          uncompressed byte sequence. This function must be called
-          immediately after a call of
-          inflate/2
-          if this call threw a {need_dictionary,Adler} exception.
-          The dictionary chosen by the compressor can be determined from the
-          Adler value thrown by the call to inflate/2.
-          The compressor and decompressor must use the same dictionary (see
-          
-          deflateSetDictionary/2).

+           uncompressed byte sequence. This function must be called as a
+           response to an inflate operation (eg.
+           safeInflate/2)
+           returning {need_dictionary,Adler,Output} or in the case of
+           deprecated functions, throwing an
+           {'EXIT',{{need_dictionary,Adler},_StackTrace}} exception.

+        
The dictionary chosen by the compressor can be determined from the
+           Adler value returned or thrown by the call to the inflate function.
+           The compressor and decompressor must use the same dictionary (See
+           
+           deflateSetDictionary/2).

+        
After setting the dictionary the inflate operation should be
+           retried without new input.

         
Example:

         
-unpack(Z, Compressed, Dict) ->
+deprecated_unpack(Z, Compressed, Dict) ->
      case catch zlib:inflate(Z, Compressed) of
-          {'EXIT',{{need_dictionary,DictID},_}} ->
-                   zlib:inflateSetDictionary(Z, Dict),
+          {'EXIT',{{need_dictionary,_DictID},_}} ->
+                 ok = zlib:inflateSetDictionary(Z, Dict),
                  Uncompressed = zlib:inflate(Z, []);
           Uncompressed ->
                  Uncompressed
-     end.

+     end.
+
+new_unpack(Z, Compressed, Dict) ->
+    case zlib:inflate(Z, Compressed, [{exception_on_need_dict, false}]) of
+        {need_dictionary, _DictId, Output} ->
+            ok = zlib:inflateSetDictionary(Z, Dict),
+            [Output | zlib:inflate(Z, [])];
+        Uncompressed ->
+            Uncompressed
+    end.
 - - Return the decompression dictionary. + + Open a stream and return a stream reference. -

Returns the decompression dictionary currently in use - by the stream. This function must be called between - inflateInit/1,2 - and inflateEnd.

-

Only supported if ERTS was compiled with zlib >= 1.2.8.

+

Opens a zlib stream.

 - - Open a stream and return a stream reference. + + Decompress data with limited output size. -

Opens a zlib stream.

+

Like inflate/2, + but returns once it has expanded beyond a small + implementation-defined threshold. It's useful when decompressing + untrusted input which could have been maliciously crafted to expand + until the system runs out of memory.

+

This function returns {continue | finished, Output}, where + Output is the data that was decompressed in this call. + New input can be queued up on each call if desired, and the function + will return {finished, Output} once all queued data has been + decompressed.

+

This function can introduce some output latency (reading + input without producing any output).

+

If a preset dictionary is required for further decompression, this + function returns a need_dictionary tuple. See + + inflateSetDictionary/2) for details.

+

Example:

+ 
+walk(Compressed, Handler) ->
+    Z = zlib:open(),
+    zlib:inflateInit(Z),
+    loop(Z, Handler, zlib:safeInflate(Z, Compressed)),
+    zlib:inflateEnd(Z),
+    zlib:close(Z).
+
+loop(Z, Handler, {continue, Output}) ->
+    Handler(Output),
+    loop(Z, Handler, zlib:safeInflate(Z, []));
+loop(Z, Handler, {finished, Output}) ->
+    Handler(Output).
 @@ -609,6 +728,10 @@ unpack(Z, Compressed, Dict) -> Set buffer size.

Sets the intermediate buffer size.

+ +

This function is deprecated and will be removed in a future + release.

+ -- cgit v1.2.3 From 8e8a7f16acb479fc2737ff73e5e0b5569a7ab71f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20H=C3=B6gberg?= Date: Thu, 10 Aug 2017 14:25:34 +0200 Subject: Improve zlib error messages and update test suite to fit OTP-14527 --- erts/doc/src/zlib.xml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'erts/doc') diff --git a/erts/doc/src/zlib.xml b/erts/doc/src/zlib.xml index 9c3e97d814..f5cc1b1e64 100644 --- a/erts/doc/src/zlib.xml +++ b/erts/doc/src/zlib.xml @@ -65,13 +65,17 @@ list_to_binary([Compressed|Last]) badarg Bad argument. + not_initialized + The stream hasn't been initialized, eg. if + inflateInit/1 wasn't + called prior to a call to + inflate/2. + data_error The data contains errors. stream_error Inconsistent stream state. - einval - Bad value or wrong function called. {need_dictionary,Adler32} See inflate/2. -- cgit v1.2.3