Rework and improve the decompress stream handler

The read buffer was changed into an iovec to avoid doing
too many binary concatenations and allocations.

Decompression happens transparently: when decoding gzip,
the content-encoding header is removed (we only decode
when "gzip" is the only encoding so nothing remains).

We always add a content_decoded key to the Req object.
This key contains a list of codings that were decoded,
in the reverse order in which they were. Currently it
can only be empty or contain <<"gzip">> but future
improvements or user handlers may see it contain more
values.

The option to disable decompression was renamed to
decompress_enabled and defaults to true.

It is no longer possible to enable/disable decompression
in the middle of reading the body: this ensures that the
data we pass forward is always valid.

Various smaller improvements were made to the code,
tests and manual pages.

1 files changed, 179 insertions, 90 deletions

diff --git a/test/decompress_SUITE.erl b/test/decompress_SUITE.erl
index 6fd0a73..3614427 100644
--- a/test/decompress_SUITE.erl
+++ b/test/decompress_SUITE.erl
@@ -1,3 +1,17 @@
+%% Copyright (c) 2024, jdamanalo <[email protected]>
+%%
+%% Permission to use, copy, modify, and/or distribute this software for any
+%% purpose with or without fee is hereby granted, provided that the above
+%% copyright notice and this permission notice appear in all copies.
+%%
+%% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+%% WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+%% MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+%% ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+%% WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+%% ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+%% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
 -module(decompress_SUITE).
 -compile(export_all).
 -compile(nowarn_export_all).
@@ -50,14 +64,8 @@ init_compress_opts(Config) ->
 
 init_routes(_) ->
 	[{'_', [
-		{"/echo/:what", decompress_h, []},
-		{"/header", decompress_h, header_command},
-		{"/invalid-header", decompress_h, invalid_header},
-		{"/accept-identity", decompress_h, accept_identity},
-		{"/reject-explicit-header", decompress_h, reject_explicit_header},
-		{"/reject-implicit-header", decompress_h, reject_implicit_header},
-		{"/accept-explicit-header", decompress_h, accept_explicit_header},
-		{"/accept-implicit-header", decompress_h, accept_implicit_header}
+		{"/echo/:what", decompress_h, echo},
+		{"/test/:what", decompress_h, test}
 	]}].
 
 %% Internal.
@@ -92,49 +100,91 @@ do_create_gzip_bomb(Z, N) ->
 %% Tests.
 
 content_encoding_none(Config) ->
-	doc("Send no content-encoding; get echo."),
+	doc("Requests without content-encoding are processed normally."),
 	Body = <<"test">>,
-	{200, _, Body} = do_post("/echo/normal",
-		[{<<"content-encoding">>, <<";">>}], Body, Config),
+	{200, _, Body} = do_post("/echo/normal", [], Body, Config),
+	%% The content-encoding header would be propagated,
+	%% but there was no content-encoding header to propagate.
+	{200, _, <<"undefined">>} = do_post("/test/content-encoding", [], Body, Config),
+	%% The content_decoded list is empty.
+	{200, _, <<"[]">>} = do_post("/test/content-decoded", [], Body, Config),
 	ok.
 
 content_encoding_malformed(Config) ->
-	doc("Send malformed content-encoding; get echo."),
+	doc("Requests with a malformed content-encoding are processed "
+		"as if no content-encoding was sent."),
 	Body = <<"test">>,
 	{200, _, Body} = do_post("/echo/normal",
 		[{<<"content-encoding">>, <<";">>}], Body, Config),
+	%% The content-encoding header is propagated.
+	{200, _, <<";">>} = do_post("/test/content-encoding",
+		[{<<"content-encoding">>, <<";">>}], Body, Config),
+	%% The content_decoded list is empty.
+	{200, _, <<"[]">>} = do_post("/test/content-decoded",
+		[{<<"content-encoding">>, <<";">>}], Body, Config),
 	ok.
 
 content_encoding_not_supported(Config) ->
-	doc("Send content-encoding: compress (unsupported by Cowboy); get echo."),
+	doc("Requests with an unsupported content-encoding are processed "
+		"as if no content-encoding was sent."),
 	Body = <<"test">>,
 	{200, _, Body} = do_post("/echo/normal",
 		[{<<"content-encoding">>, <<"compress">>}], Body, Config),
+	%% The content-encoding header is propagated.
+	{200, _, <<"compress">>} = do_post("/test/content-encoding",
+		[{<<"content-encoding">>, <<"compress">>}], Body, Config),
+	%% The content_decoded list is empty.
+	{200, _, <<"[]">>} = do_post("/test/content-decoded",
+		[{<<"content-encoding">>, <<"compress">>}], Body, Config),
 	ok.
 
-content_encoding_wrong(Config) ->
-	doc("Send content-encoding and unencoded body; get 400."),
+content_encoding_multiple(Config) ->
+	doc("Requests with multiple content-encoding values are processed "
+		"as if no content-encoding was sent."),
 	Body = <<"test">>,
-	{400, _, _} = do_post("/echo/normal",
-		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
+	{200, _, Body} = do_post("/echo/normal",
+		[{<<"content-encoding">>, <<"gzip, compress">>}], Body, Config),
+	%% The content-encoding header is propagated.
+	{200, _, <<"gzip, compress">>} = do_post("/test/content-encoding",
+		[{<<"content-encoding">>, <<"gzip, compress">>}], Body, Config),
+	%% The content_decoded list is empty.
+	{200, _, <<"[]">>} = do_post("/test/content-decoded",
+		[{<<"content-encoding">>, <<"gzip, compress">>}], Body, Config),
 	ok.
 
 decompress(Config) ->
-	doc("Send content-encoding and encoded body; get decompressed response."),
+	doc("Requests with content-encoding set to gzip and gzipped data "
+		"are transparently decompressed."),
 	Data = <<"test">>,
 	Body = zlib:gzip(Data),
 	{200, _, Data} = do_post("/echo/normal",
 		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
+	%% The content-encoding header is NOT propagated.
+	{200, _, <<"undefined">>} = do_post("/test/content-encoding",
+		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
+	%% The content_decoded list contains <<"gzip">>.
+	{200, _, <<"[<<\"gzip\">>]">>} = do_post("/test/content-decoded",
+		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
+	ok.
+
+decompress_error(Config) ->
+	doc("Requests with content-encoding set to gzip but the data "
+		"cannot be decoded are rejected with a 400 Bad Request error."),
+	Body = <<"test">>,
+	{400, _, _} = do_post("/echo/normal",
+		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
 	ok.
 
 decompress_stream(Config) ->
-	doc("Stream encoded body; get decompressed response."),
+	doc("Requests with content-encoding set to gzip and gzipped data "
+		"are transparently decompressed, even when the data is streamed."),
 	%% Handler read length 1KB. Compressing 3KB should be enough to trigger more.
 	Data = crypto:strong_rand_bytes(3000),
 	Body = zlib:gzip(Data),
 	Size = byte_size(Body),
 	ConnPid = gun_open(Config),
-	Ref = gun:post(ConnPid, "/echo/normal", [{<<"content-encoding">>, <<"gzip">>}]),
+	Ref = gun:post(ConnPid, "/echo/normal",
+		[{<<"content-encoding">>, <<"gzip">>}]),
 	gun:data(ConnPid, Ref, nofin, binary:part(Body, 0, Size div 2)),
 	timer:sleep(1000),
 	gun:data(ConnPid, Ref, fin, binary:part(Body, Size div 2, Size div 2 + Size rem 2)),
@@ -144,12 +194,23 @@ decompress_stream(Config) ->
 		fin -> {ok, <<>>}
 	end,
 	gun:close(ConnPid),
-	{200, _, Data} = do_post("/echo/normal",
-		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
-	ok.
-
-opts_decompress_ignore(Config0) ->
-	doc("Confirm that the decompress_ignore option can be set."),
+	%% The content-encoding header is NOT propagated.
+	ConnPid2 = gun_open(Config),
+	Ref2 = gun:post(ConnPid2, "/test/content-encoding",
+		[{<<"content-encoding">>, <<"gzip">>}]),
+	{response, nofin, 200, _} = gun:await(ConnPid2, Ref2),
+	{ok, <<"undefined">>} = gun:await_body(ConnPid2, Ref2),
+	gun:close(ConnPid2),
+	%% The content_decoded list contains <<"gzip">>.
+	ConnPid3 = gun_open(Config),
+	Ref3 = gun:post(ConnPid3, "/test/content-decoded",
+		[{<<"content-encoding">>, <<"gzip">>}]),
+	{response, nofin, 200, _} = gun:await(ConnPid3, Ref3),
+	{ok, <<"[<<\"gzip\">>]">>} = gun:await_body(ConnPid3, Ref3),
+	gun:close(ConnPid3).
+
+opts_decompress_enabled_false(Config0) ->
+	doc("Confirm that the decompress_enabled option can be set."),
 	Fun = case config(ref, Config0) of
 		HTTPS when HTTPS =:= https_compress; HTTPS =:= https -> init_https;
 		H2 when H2 =:= h2_compress; H2 =:= h2 -> init_http2;
@@ -158,28 +219,75 @@ opts_decompress_ignore(Config0) ->
 	Config = cowboy_test:Fun(?FUNCTION_NAME, #{
 		env => #{dispatch => cowboy_router:compile(init_routes(Config0))},
 		stream_handlers => [cowboy_decompress_h, cowboy_stream_h],
-		decompress_ignore => true
+		decompress_enabled => false
 	}, Config0),
 	Data = <<"test">>,
 	Body = zlib:gzip(Data),
 	try
-		{200, _, Body} = do_post("/echo/normal",
-			[{<<"content-encoding">>, <<"gzip">>}], Body, Config)
+		{200, Headers, Body} = do_post("/echo/normal",
+			[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
+		%% We do not set accept-encoding when we are disabled.
+		false = lists:keyfind(<<"accept-encoding">>, 1, Headers)
 	after
 		cowboy:stop_listener(?FUNCTION_NAME)
 	end.
 
-set_options_decompress_ignore(Config) ->
-	doc("Confirm that the decompress_ignore option can be dynamically
-		set to true and the data received is not decompressed."),
+set_options_decompress_enabled_false(Config) ->
+	doc("Confirm that the decompress_enabled option can be dynamically "
+		"set to false and the data received is not decompressed."),
 	Data = <<"test">>,
 	Body = zlib:gzip(Data),
-	{200, _, Body} = do_post("/echo/decompress_ignore",
+	{200, Headers, Body} = do_post("/echo/decompress_disable",
 		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
+	%% We do not set accept-encoding when we are disabled.
+	false = lists:keyfind(<<"accept-encoding">>, 1, Headers),
 	ok.
 
+set_options_decompress_disable_in_the_middle(Config) ->
+	doc("Confirm that setting the decompress_enabled option dynamically "
+		"to false after starting to read the body does not disable decompression "
+		"and the data received is decompressed."),
+	Data = rand:bytes(1000000),
+	Body = zlib:gzip(Data),
+	%% Since we were not ignoring before starting to read,
+	%% we receive the entire body decompressed.
+	{200, Headers, Data} = do_post("/test/disable-in-the-middle",
+		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
+	%% We do set accept-encoding when we are enabled,
+	%% even if an attempt to disable in the middle is ignored.
+	{_, _} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
+	ok.
+
+set_options_decompress_enable_in_the_middle(Config0) ->
+	doc("Confirm that setting the decompress_enabled option dynamically "
+		"to true after starting to read the body does not enable decompression "
+		"and the data received is not decompressed."),
+	Fun = case config(ref, Config0) of
+		HTTPS when HTTPS =:= https_compress; HTTPS =:= https -> init_https;
+		H2 when H2 =:= h2_compress; H2 =:= h2 -> init_http2;
+		_ -> init_http
+	end,
+	Config = cowboy_test:Fun(?FUNCTION_NAME, #{
+		env => #{dispatch => cowboy_router:compile(init_routes(Config0))},
+		stream_handlers => [cowboy_decompress_h, cowboy_stream_h],
+		decompress_enabled => false
+	}, Config0),
+	Data = rand:bytes(1000000),
+	Body = zlib:gzip(Data),
+	try
+		%% Since we were ignoring before starting to read,
+		%% we receive the entire body compressed.
+		{200, Headers, Body} = do_post("/test/enable-in-the-middle",
+			[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
+		%% We do not set accept-encoding when we are disabled,
+		%% even if an attempt to enable in the middle is ignored.
+		false = lists:keyfind(<<"accept-encoding">>, 1, Headers)
+	after
+		cowboy:stop_listener(?FUNCTION_NAME)
+	end.
+
 opts_decompress_ratio_limit(Config0) ->
-	doc("Confirm that the decompress_ignore option can be set"),
+	doc("Confirm that the decompress_ratio_limit option can be set."),
 	Fun = case config(ref, Config0) of
 		HTTPS when HTTPS =:= https_compress; HTTPS =:= https -> init_https;
 		H2 when H2 =:= h2_compress; H2 =:= h2 -> init_http2;
@@ -190,7 +298,8 @@ opts_decompress_ratio_limit(Config0) ->
 		stream_handlers => [cowboy_decompress_h, cowboy_stream_h],
 		decompress_ratio_limit => 1
 	}, Config0),
-	%% Data must be big enough for compression to be effective, so that ratio_limit=1 will fail.
+	%% Data must be big enough for compression to be effective,
+	%% so that ratio_limit=1 will fail.
 	Data = <<0:800>>,
 	Body = zlib:gzip(Data),
 	try
@@ -202,7 +311,8 @@ opts_decompress_ratio_limit(Config0) ->
 
 set_options_decompress_ratio_limit(Config) ->
 	doc("Confirm that the decompress_ratio_limit option can be dynamically set."),
-	%% Data must be big enough for compression to be effective, so that ratio_limit=1 will fail.
+	%% Data must be big enough for compression to be effective,
+	%% so that ratio_limit=1 will fail.
 	Data = <<0:800>>,
 	Body = zlib:gzip(Data),
 	{413, _, _} = do_post("/echo/decompress_ratio_limit",
@@ -210,19 +320,16 @@ set_options_decompress_ratio_limit(Config) ->
 	ok.
 
 gzip_bomb(Config) ->
-	doc("Send body compressed with suspiciously large ratio; get 413."),
+	doc("Confirm that requests are rejected with a 413 Payload Too Large "
+		"error when the ratio limit is exceeded."),
 	Body = create_gzip_bomb(),
 	{413, _, _} = do_post("/echo/normal",
 		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
 	ok.
 
-%% RFC 9110. Section 12.5.3. 3. When sent by a server in a response,
-%% Accept-Encoding provides information about which content codings are
-%% preferred in the content of a subsequent request to the same resource.
-%%
-%% Set or add gzip
 set_accept_encoding_response(Config) ->
-	doc("Header accept-encoding must be set on valid response command."),
+	doc("Header accept-encoding must be set on valid response command. "
+		"(RFC9110 12.5.3)"),
 	Data = <<"test">>,
 	Body = zlib:gzip(Data),
 	{200, Headers, Data} = do_post("/echo/normal",
@@ -231,97 +338,79 @@ set_accept_encoding_response(Config) ->
 	ok.
 
 set_accept_encoding_header(Config) ->
-	doc("Header accept-encoding must be set on valid header command."),
+	doc("Header accept-encoding must be set on valid header command. "
+		"(RFC9110 12.5.3)"),
 	Data = <<"test">>,
 	Body = zlib:gzip(Data),
-	{200, Headers, Data} = do_post("/header",
+	{200, Headers, Data} = do_post("/test/header-command",
 		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
 	{_, <<"gzip">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
 	ok.
 
 add_accept_encoding_header_valid(Config) ->
-	doc("Header accept-encoding must be added on valid accept-encoding."),
+	doc("Supported content codings must be added to the accept-encoding "
+		"header if it already exists. (RFC9110 12.5.3)"),
 	Data = <<"test">>,
 	Body = zlib:gzip(Data),
-	{200, Headers, Data} = do_post("/accept-identity",
+	{200, Headers, Data} = do_post("/test/accept-identity",
 		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
 	{_, <<"identity, gzip">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
 	ok.
 
 override_accept_encoding_header_invalid(Config) ->
-	doc("Header accept-encoding must override invalid accept-encoding."),
+	doc("When the stream handler cannot parse the accept-encoding header "
+		"found in the response, it overrides it."),
 	Data = <<"test">>,
 	Body = zlib:gzip(Data),
-	{200, Headers, Data} = do_post("/invalid-header",
+	{200, Headers, Data} = do_post("/test/invalid-header",
 		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
 	{_, <<"gzip">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
 	ok.
 
-%% RFC 9110. Section 12.5.3. 10.3. If the representation's content coding is
-%% one of the content codings listed in the Accept-Encoding field value, then
-%% it is acceptable unless it is accompanied by a qvalue of 0.
-%%
-%% gzip must not have a qvalue of 0 when the handler is used. Set to 1.
 override_accept_encoding_excluded(Config) ->
-	doc("Header accept-encoding must override when explicitly excluded."),
+	doc("The stream handler must ensure that the content encodings "
+		"it supports are not marked as unsupported in response headers. "
+		"The stream handler enables gzip when explicitly excluded. "
+		"(RFC9110 12.5.3)"),
 	Data = <<"test">>,
 	Body = zlib:gzip(Data),
-	{200, Headers, Data} = do_post("/reject-explicit-header",
+	{200, Headers, Data} = do_post("/test/reject-explicit-header",
 		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
 	{_, <<"identity;q=1, gzip;q=1">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
 	ok.
 
-%% RFC 9110. Section 12.5.3. 10.2. If the representation has no content coding,
-%% then it is acceptable by default unless specifically excluded by the
-%% Accept-Encoding field stating either "identity;q=0" or "*;q=0" wihout a more
-%% specific entry for "identity".
-%%
-%% *;q=0 will reject codings that are not listed. Specific entry gzip must
-%% always be listed when the handler is used. Add gzip.
+%% *;q=0 will reject codings that are not listed. Supported codings
+%% must always be enabled when the handler is used.
 add_accept_encoding_excluded(Config) ->
-	doc("Header accept-encoding must added when implicitly excluded."),
+	doc("The stream handler must ensure that the content encodings "
+		"it supports are not marked as unsupported in response headers. "
+		"The stream handler enables gzip when implicitly excluded (*;q=0). "
+		"(RFC9110 12.5.3)"),
 	Data = <<"test">>,
 	Body = zlib:gzip(Data),
-	{200, Headers, Data} = do_post("/reject-implicit-header",
+	{200, Headers, Data} = do_post("/test/reject-implicit-header",
 		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
 	{_, <<"gzip;q=1, identity;q=1, *;q=0">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
 	ok.
 
 no_override_accept_coding_set_explicit(Config) ->
-	doc("Confirm that accept-encoding is not overridden when explicitly set."),
+	doc("Confirm that accept-encoding is not overridden when the "
+		"content encodings it supports are explicitly set. "
+		"(RFC9110 12.5.3)"),
 	Data = <<"test">>,
 	Body = zlib:gzip(Data),
-	{200, Headers, Data} = do_post("/accept-explicit-header",
+	{200, Headers, Data} = do_post("/test/accept-explicit-header",
 		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
 	{_, <<"identity, gzip;q=0.5">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
 	ok.
 
 no_override_accept_coding_set_implicit(Config) ->
-	doc("Confirm that accept-encoding is not overridden when implicitly set."),
+	doc("Confirm that accept-encoding is not overridden when the "
+		"content encodings it supports are implicitly set. "
+		"(RFC9110 12.5.3)"),
 	Data = <<"test">>,
 	Body = zlib:gzip(Data),
-	{200, Headers, Data} = do_post("/accept-implicit-header",
+	{200, Headers, Data} = do_post("/test/accept-implicit-header",
 		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
 	{_, <<"identity, *;q=0.5">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers),
 	ok.
-
-%% RFC 9110. Section 12.5.3. 10.1. If no Accept-Encoding header field is in the
-%% request, any content coding is considered acceptable by the user agent.
-%%
-%% Don't add anything on error or when that handler is not used.
-no_set_accept_encoding(Config) ->
-	doc("No header accept-encoding on invalid responses."),
-	Body = <<"test">>,
-	{400, Headers, _} = do_post("/echo/normal",
-		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
-	false = lists:keyfind(<<"accept-encoding">>, 1, Headers),
-	ok.
-
-no_set_accept_encoding_ignore(Config) ->
-	doc("Confirm that no accept-encoding is set when stream is ignored."),
-	Data = <<"test">>,
-	Body = zlib:gzip(Data),
-	{200, Headers, Body} = do_post("/echo/decompress_ignore",
-		[{<<"content-encoding">>, <<"gzip">>}], Body, Config),
-	false = lists:keyfind(<<"accept-encoding">>, 1, Headers),
-	ok.