diff options
| -rw-r--r-- | doc/src/guide/streams.asciidoc | 6 | ||||
| -rw-r--r-- | doc/src/manual/cowboy_app.asciidoc | 1 | ||||
| -rw-r--r-- | doc/src/manual/cowboy_decompress_h.asciidoc | 34 | ||||
| -rw-r--r-- | src/cowboy_decompress_h.erl | 146 | ||||
| -rw-r--r-- | test/decompress_SUITE.erl | 269 | ||||
| -rw-r--r-- | test/handlers/decompress_h.erl | 98 |
6 files changed, 356 insertions, 198 deletions
diff --git a/doc/src/guide/streams.asciidoc b/doc/src/guide/streams.asciidoc index b6e4d34..e8ddae0 100644 --- a/doc/src/guide/streams.asciidoc +++ b/doc/src/guide/streams.asciidoc @@ -66,9 +66,9 @@ enabled by default. It is a good example for writing your own handlers that will modify responses. link:man:cowboy_decompress_h(3)[cowboy_decompress_h] will -automatically decompress requests when possible. It is not -enabled by default. It is a good example for writing your -own handlers that will modify requests. +automatically decompress request bodies when possible. +It is not enabled by default. It is a good example for +writing your own handlers that will modify requests. link:man:cowboy_metrics_h(3)[cowboy_metrics_h] gathers metrics about a stream then passes them to a configurable diff --git a/doc/src/manual/cowboy_app.asciidoc b/doc/src/manual/cowboy_app.asciidoc index 8db84b1..fd833be 100644 --- a/doc/src/manual/cowboy_app.asciidoc +++ b/doc/src/manual/cowboy_app.asciidoc @@ -36,6 +36,7 @@ Stream handlers: * link:man:cowboy_stream_h(3)[cowboy_stream_h(3)] - Default stream handler * link:man:cowboy_compress_h(3)[cowboy_compress_h(3)] - Compress stream handler +* link:man:cowboy_decompress_h(3)[cowboy_decompress_h(3)] - Decompress stream handler * link:man:cowboy_metrics_h(3)[cowboy_metrics_h(3)] - Metrics stream handler * link:man:cowboy_tracer_h(3)[cowboy_tracer_h(3)] - Tracer stream handler diff --git a/doc/src/manual/cowboy_decompress_h.asciidoc b/doc/src/manual/cowboy_decompress_h.asciidoc index 63ee0ee..8598ae4 100644 --- a/doc/src/manual/cowboy_decompress_h.asciidoc +++ b/doc/src/manual/cowboy_decompress_h.asciidoc @@ -7,18 +7,26 @@ cowboy_decompress_h - Decompress stream handler == Description The module `cowboy_decompress_h` decompresses request bodies -automatically when the server supports it. Requests will -only be decompressed when their compression ratio is lower -than the configured limit. Mismatch of the content and -`content-encoding` is rejected with `400 Bad Request`. +automatically when the server supports it. + +The only compression algorithm currently supported is the +gzip algorithm. Another limitation is that decompression +is only attempted when gzip is the only content-encoding +in the request. + +This stream handler always adds a field to the Req object +with the name `content_decoded` which is treated as a +list of decoded content-encoding values. Currently this +list may only contain the `<<"gzip">>` binary if content +was decoded; or be empty otherwise. == Options [source,erlang] ---- opts() :: #{ - decompress_ratio_limit => non_neg_integer(), - decompress_ignore => boolean() + decompress_enabled => boolean(), + decompress_ratio_limit => non_neg_integer() } ---- @@ -28,17 +36,21 @@ The default value is given next to the option name: decompress_ratio_limit (20):: The max ratio of the compressed and decompressed body -before it is rejected with `413 Payload Too Large`. +before it is rejected with a `413 Payload Too Large` +error response. + This option can be updated at any time using the `set_options` stream handler command. -decompress_ignore (false):: +decompress_enabled (true):: -Whether the handler will be ignored. +Whether the handler is enabled by default. + -This option can be updated at any time using the -`set_options` stream handler command. +This option can be updated using the `set_options` +stream handler command. This allows disabling +decompression for the current stream. Attempts +to enable or disable decompression after starting +to read the body will be ignored. == Events diff --git a/src/cowboy_decompress_h.erl b/src/cowboy_decompress_h.erl index ffbec25..d13601b 100644 --- a/src/cowboy_decompress_h.erl +++ b/src/cowboy_decompress_h.erl @@ -1,3 +1,18 @@ +%% Copyright (c) 2024, jdamanalo <[email protected]> +%% Copyright (c) 2024, Loïc Hoguin <[email protected]> +%% +%% Permission to use, copy, modify, and/or distribute this software for any +%% purpose with or without fee is hereby granted, provided that the above +%% copyright notice and this permission notice appear in all copies. +%% +%% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +%% WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +%% MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +%% ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +%% WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +%% ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +%% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + -module(cowboy_decompress_h). -behavior(cowboy_stream). @@ -9,21 +24,27 @@ -record(state, { next :: any(), + enabled :: boolean(), ratio_limit :: non_neg_integer() | undefined, - ignore = false :: boolean(), compress = undefined :: undefined | gzip, inflate = undefined :: undefined | zlib:zstream(), is_reading = false :: boolean(), - read_body_buffer = <<>> :: binary(), + + %% We use a list of binaries to avoid doing unnecessary + %% memory allocations when inflating. We convert to binary + %% when we propagate the data. The data must be reversed + %% before converting to binary or inflating: this is done + %% via the buffer_to_binary/buffer_to_iovec functions. + read_body_buffer = [] :: [binary()], read_body_is_fin = nofin :: nofin | {fin, non_neg_integer()} }). -spec init(cowboy_stream:streamid(), cowboy_req:req(), cowboy:opts()) -> {cowboy_stream:commands(), #state{}}. -init(StreamID, Req, Opts) -> +init(StreamID, Req0, Opts) -> + Enabled = maps:get(decompress_enabled, Opts, true), RatioLimit = maps:get(decompress_ratio_limit, Opts, 20), - Ignore = maps:get(decompress_ignore, Opts, false), - State = check_req(Req), + {Req, State} = check_and_update_req(Req0), Inflate = case State#state.compress of undefined -> undefined; @@ -33,48 +54,46 @@ init(StreamID, Req, Opts) -> Z end, {Commands, Next} = cowboy_stream:init(StreamID, Req, Opts), - fold(Commands, State#state{next=Next, ratio_limit=RatioLimit, ignore=Ignore, - inflate=Inflate}). + fold(Commands, State#state{next=Next, enabled=Enabled, + ratio_limit=RatioLimit, inflate=Inflate}). -spec data(cowboy_stream:streamid(), cowboy_stream:fin(), cowboy_req:resp_body(), State) -> {cowboy_stream:commands(), State} when State::#state{}. data(StreamID, IsFin, Data, State=#state{next=Next0, inflate=undefined}) -> {Commands, Next} = cowboy_stream:data(StreamID, IsFin, Data, Next0), fold(Commands, State#state{next=Next, read_body_is_fin=IsFin}); -data(StreamID, IsFin, Data, State=#state{next=Next0, ignore=true, read_body_buffer=Buffer}) -> +data(StreamID, IsFin, Data, State=#state{next=Next0, enabled=false, read_body_buffer=Buffer}) -> {Commands, Next} = cowboy_stream:data(StreamID, IsFin, - << Buffer/binary, Data/binary >>, Next0), + buffer_to_binary([Data|Buffer]), Next0), fold(Commands, State#state{next=Next, read_body_is_fin=IsFin}); data(StreamID, IsFin, Data, State0=#state{next=Next0, ratio_limit=RatioLimit, - inflate=Z, is_reading=true, read_body_buffer=Buffer0}) -> - Buffer = << Buffer0/binary, Data/binary >>, - case inflate(Z, RatioLimit, Buffer) of - {error, Type} -> - Status = case Type of - data -> 400; - size -> 413 + inflate=Z, is_reading=true, read_body_buffer=Buffer}) -> + case inflate(Z, RatioLimit, buffer_to_iovec([Data|Buffer])) of + {error, ErrorType} -> + zlib:close(Z), + Status = case ErrorType of + data_error -> 400; + size_error -> 413 end, Commands = [ {error_response, Status, #{<<"content-length">> => <<"0">>}, <<>>}, stop ], - fold(Commands, State0#state{inflate=undefined}); + fold(Commands, State0#state{inflate=undefined, read_body_buffer=[]}); {ok, Inflated} -> State = case IsFin of nofin -> State0; fin -> - zlib:inflateEnd(Z), zlib:close(Z), State0#state{inflate=undefined} end, {Commands, Next} = cowboy_stream:data(StreamID, IsFin, Inflated, Next0), - fold(Commands, State#state{next=Next, read_body_buffer= <<>>, + fold(Commands, State#state{next=Next, read_body_buffer=[], read_body_is_fin=IsFin}) end; -data(_, IsFin, Data, State=#state{read_body_buffer=Buffer0}) -> - Buffer = << Buffer0/binary, Data/binary >>, - {[], State#state{read_body_buffer=Buffer, read_body_is_fin=IsFin}}. +data(_, IsFin, Data, State=#state{read_body_buffer=Buffer}) -> + {[], State#state{read_body_buffer=[Data|Buffer], read_body_is_fin=IsFin}}. -spec info(cowboy_stream:streamid(), any(), State) -> {cowboy_stream:commands(), State} when State::#state{}. @@ -86,12 +105,19 @@ info(StreamID, Info={CommandTag, _, _, _, _}, State=#state{next=Next0, read_body {Commands0, Next1} = cowboy_stream:info(StreamID, Info, Next0), {Commands, Next} = data(StreamID, IsFin, <<>>, State#state{next=Next1, is_reading=true}), fold(Commands ++ Commands0, Next); -info(StreamID, Info={set_options, Opts}, State=#state{next=Next0, - ignore=Ignore0, ratio_limit=RatioLimit0}) -> - Ignore = maps:get(decompress_ignore, Opts, Ignore0), +info(StreamID, Info={set_options, Opts}, State0=#state{next=Next0, + enabled=Enabled0, ratio_limit=RatioLimit0, is_reading=IsReading}) -> + Enabled = maps:get(decompress_enabled, Opts, Enabled0), RatioLimit = maps:get(decompress_ratio_limit, Opts, RatioLimit0), {Commands, Next} = cowboy_stream:info(StreamID, Info, Next0), - fold(Commands, State#state{next=Next, ignore=Ignore, ratio_limit=RatioLimit}); + %% We can't change the enabled setting after we start reading, + %% otherwise the data becomes garbage. Changing the setting + %% is not treated as an error, it is just ignored. + State = case IsReading of + true -> State0; + false -> State0#state{enabled=Enabled} + end, + fold(Commands, State#state{next=Next, ratio_limit=RatioLimit}); info(StreamID, Info, State=#state{next=Next0}) -> {Commands, Next} = cowboy_stream:info(StreamID, Info, Next0), fold(Commands, State#state{next=Next}). @@ -112,31 +138,49 @@ early_error(StreamID, Reason, PartialReq, Resp, Opts) -> %% Internal. -check_req(Req) -> +%% Check whether the request needs content decoding, and if it does +%% whether it fits our criteria for decoding. We also update the +%% Req to indicate whether content was decoded. +%% +%% We always set the content_decoded value in the Req because it +%% indicates whether content decoding was attempted. +%% +%% A malformed content-encoding header results in no decoding. +check_and_update_req(Req=#{headers := Headers}) -> + ContentDecoded = maps:get(content_decoded, Req, []), try cowboy_req:parse_header(<<"content-encoding">>, Req) of - undefined -> - #state{compress=undefined}; - Encodings -> - case [E || E=(<<"gzip">>) <- Encodings] of - [] -> - #state{compress=undefined}; - _ -> - #state{compress=gzip} - end - catch - _:_ -> - #state{compress=undefined} + %% We only automatically decompress when gzip is the only + %% encoding used. Since it's the only encoding used, we + %% can remove the header entirely before passing the Req + %% forward. + [<<"gzip">>] -> + {Req#{ + headers => maps:remove(<<"content-encoding">>, Headers), + content_decoded => [<<"gzip">>|ContentDecoded] + }, #state{compress=gzip}}; + _ -> + {Req#{content_decoded => ContentDecoded}, + #state{compress=undefined}} + catch _:_ -> + {Req#{content_decoded => ContentDecoded}, + #state{compress=undefined}} end. +buffer_to_iovec(Buffer) -> + lists:reverse(Buffer). + +buffer_to_binary(Buffer) -> + iolist_to_binary(lists:reverse(Buffer)). + fold(Commands, State) -> fold(Commands, State, []). fold([], State, Acc) -> {lists:reverse(Acc), State}; -fold([{response, Status, Headers0, Body}|Tail], State=#state{ignore=false}, Acc) -> +fold([{response, Status, Headers0, Body}|Tail], State=#state{enabled=true}, Acc) -> Headers = add_accept_encoding(Headers0), fold(Tail, State, [{response, Status, Headers, Body}|Acc]); -fold([{headers, Status, Headers0} | Tail], State=#state{ignore=false}, Acc) -> +fold([{headers, Status, Headers0} | Tail], State=#state{enabled=true}, Acc) -> Headers = add_accept_encoding(Headers0), fold(Tail, State, [{headers, Status, Headers}|Acc]); fold([Command|Tail], State, Acc) -> @@ -146,7 +190,7 @@ add_accept_encoding(Headers=#{<<"accept-encoding">> := AcceptEncoding}) -> try cow_http_hd:parse_accept_encoding(iolist_to_binary(AcceptEncoding)) of List -> case lists:keyfind(<<"gzip">>, 1, List) of - %% gzip is excluded but this handler is not ignored; we replace. + %% gzip is excluded but this handler is enabled; we replace. {_, 0} -> Replaced = lists:keyreplace(<<"gzip">>, 1, List, {<<"gzip">>, 1000}), Codings = build_accept_encoding(Replaced), @@ -167,18 +211,20 @@ add_accept_encoding(Headers=#{<<"accept-encoding">> := AcceptEncoding}) -> end end catch _:_ -> + %% The accept-encoding header is invalid. Probably empty. We replace it with ours. Headers#{<<"accept-encoding">> => <<"gzip">>} end; add_accept_encoding(Headers) -> Headers#{<<"accept-encoding">> => <<"gzip">>}. -%% From cowlib, maybe expose? +%% @todo From cowlib, maybe expose? qvalue_to_iodata(0) -> <<"0">>; qvalue_to_iodata(Q) when Q < 10 -> [<<"0.00">>, integer_to_binary(Q)]; qvalue_to_iodata(Q) when Q < 100 -> [<<"0.0">>, integer_to_binary(Q)]; qvalue_to_iodata(Q) when Q < 1000 -> [<<"0.">>, integer_to_binary(Q)]; qvalue_to_iodata(1000) -> <<"1">>. +%% @todo Should be added to Cowlib. build_accept_encoding([{ContentCoding, Q}|Tail]) -> Weight = iolist_to_binary(qvalue_to_iodata(Q)), Acc = <<ContentCoding/binary, ";q=", Weight/binary>>, @@ -195,20 +241,14 @@ inflate(Z, RatioLimit, Data) -> try {Status, Output} = zlib:safeInflate(Z, Data), Size = iolist_size(Output), - do_inflate(Z, Size, byte_size(Data) * RatioLimit, Status, [Output]) + do_inflate(Z, Size, iolist_size(Data) * RatioLimit, Status, [Output]) catch error:data_error -> - zlib:close(Z), - {error, data} + {error, data_error} end. -do_inflate(Z, Size, Limit, Status, _) when Size > Limit -> - case Status of - continue -> ok; - finished -> zlib:inflateEnd(Z) - end, - zlib:close(Z), - {error, size}; +do_inflate(_, Size, Limit, _, _) when Size > Limit -> + {error, size_error}; do_inflate(Z, Size0, Limit, continue, Acc) -> {Status, Output} = zlib:safeInflate(Z, []), Size = Size0 + iolist_size(Output), diff --git a/test/decompress_SUITE.erl b/test/decompress_SUITE.erl index 6fd0a73..3614427 100644 --- a/test/decompress_SUITE.erl +++ b/test/decompress_SUITE.erl @@ -1,3 +1,17 @@ +%% Copyright (c) 2024, jdamanalo <[email protected]> +%% +%% Permission to use, copy, modify, and/or distribute this software for any +%% purpose with or without fee is hereby granted, provided that the above +%% copyright notice and this permission notice appear in all copies. +%% +%% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +%% WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +%% MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +%% ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +%% WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +%% ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +%% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + -module(decompress_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -50,14 +64,8 @@ init_compress_opts(Config) -> init_routes(_) -> [{'_', [ - {"/echo/:what", decompress_h, []}, - {"/header", decompress_h, header_command}, - {"/invalid-header", decompress_h, invalid_header}, - {"/accept-identity", decompress_h, accept_identity}, - {"/reject-explicit-header", decompress_h, reject_explicit_header}, - {"/reject-implicit-header", decompress_h, reject_implicit_header}, - {"/accept-explicit-header", decompress_h, accept_explicit_header}, - {"/accept-implicit-header", decompress_h, accept_implicit_header} + {"/echo/:what", decompress_h, echo}, + {"/test/:what", decompress_h, test} ]}]. %% Internal. @@ -92,49 +100,91 @@ do_create_gzip_bomb(Z, N) -> %% Tests. content_encoding_none(Config) -> - doc("Send no content-encoding; get echo."), + doc("Requests without content-encoding are processed normally."), Body = <<"test">>, - {200, _, Body} = do_post("/echo/normal", - [{<<"content-encoding">>, <<";">>}], Body, Config), + {200, _, Body} = do_post("/echo/normal", [], Body, Config), + %% The content-encoding header would be propagated, + %% but there was no content-encoding header to propagate. + {200, _, <<"undefined">>} = do_post("/test/content-encoding", [], Body, Config), + %% The content_decoded list is empty. + {200, _, <<"[]">>} = do_post("/test/content-decoded", [], Body, Config), ok. content_encoding_malformed(Config) -> - doc("Send malformed content-encoding; get echo."), + doc("Requests with a malformed content-encoding are processed " + "as if no content-encoding was sent."), Body = <<"test">>, {200, _, Body} = do_post("/echo/normal", [{<<"content-encoding">>, <<";">>}], Body, Config), + %% The content-encoding header is propagated. + {200, _, <<";">>} = do_post("/test/content-encoding", + [{<<"content-encoding">>, <<";">>}], Body, Config), + %% The content_decoded list is empty. + {200, _, <<"[]">>} = do_post("/test/content-decoded", + [{<<"content-encoding">>, <<";">>}], Body, Config), ok. content_encoding_not_supported(Config) -> - doc("Send content-encoding: compress (unsupported by Cowboy); get echo."), + doc("Requests with an unsupported content-encoding are processed " + "as if no content-encoding was sent."), Body = <<"test">>, {200, _, Body} = do_post("/echo/normal", [{<<"content-encoding">>, <<"compress">>}], Body, Config), + %% The content-encoding header is propagated. + {200, _, <<"compress">>} = do_post("/test/content-encoding", + [{<<"content-encoding">>, <<"compress">>}], Body, Config), + %% The content_decoded list is empty. + {200, _, <<"[]">>} = do_post("/test/content-decoded", + [{<<"content-encoding">>, <<"compress">>}], Body, Config), ok. -content_encoding_wrong(Config) -> - doc("Send content-encoding and unencoded body; get 400."), +content_encoding_multiple(Config) -> + doc("Requests with multiple content-encoding values are processed " + "as if no content-encoding was sent."), Body = <<"test">>, - {400, _, _} = do_post("/echo/normal", - [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + {200, _, Body} = do_post("/echo/normal", + [{<<"content-encoding">>, <<"gzip, compress">>}], Body, Config), + %% The content-encoding header is propagated. + {200, _, <<"gzip, compress">>} = do_post("/test/content-encoding", + [{<<"content-encoding">>, <<"gzip, compress">>}], Body, Config), + %% The content_decoded list is empty. + {200, _, <<"[]">>} = do_post("/test/content-decoded", + [{<<"content-encoding">>, <<"gzip, compress">>}], Body, Config), ok. decompress(Config) -> - doc("Send content-encoding and encoded body; get decompressed response."), + doc("Requests with content-encoding set to gzip and gzipped data " + "are transparently decompressed."), Data = <<"test">>, Body = zlib:gzip(Data), {200, _, Data} = do_post("/echo/normal", [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + %% The content-encoding header is NOT propagated. + {200, _, <<"undefined">>} = do_post("/test/content-encoding", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + %% The content_decoded list contains <<"gzip">>. + {200, _, <<"[<<\"gzip\">>]">>} = do_post("/test/content-decoded", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + ok. + +decompress_error(Config) -> + doc("Requests with content-encoding set to gzip but the data " + "cannot be decoded are rejected with a 400 Bad Request error."), + Body = <<"test">>, + {400, _, _} = do_post("/echo/normal", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), ok. decompress_stream(Config) -> - doc("Stream encoded body; get decompressed response."), + doc("Requests with content-encoding set to gzip and gzipped data " + "are transparently decompressed, even when the data is streamed."), %% Handler read length 1KB. Compressing 3KB should be enough to trigger more. Data = crypto:strong_rand_bytes(3000), Body = zlib:gzip(Data), Size = byte_size(Body), ConnPid = gun_open(Config), - Ref = gun:post(ConnPid, "/echo/normal", [{<<"content-encoding">>, <<"gzip">>}]), + Ref = gun:post(ConnPid, "/echo/normal", + [{<<"content-encoding">>, <<"gzip">>}]), gun:data(ConnPid, Ref, nofin, binary:part(Body, 0, Size div 2)), timer:sleep(1000), gun:data(ConnPid, Ref, fin, binary:part(Body, Size div 2, Size div 2 + Size rem 2)), @@ -144,12 +194,23 @@ decompress_stream(Config) -> fin -> {ok, <<>>} end, gun:close(ConnPid), - {200, _, Data} = do_post("/echo/normal", - [{<<"content-encoding">>, <<"gzip">>}], Body, Config), - ok. - -opts_decompress_ignore(Config0) -> - doc("Confirm that the decompress_ignore option can be set."), + %% The content-encoding header is NOT propagated. + ConnPid2 = gun_open(Config), + Ref2 = gun:post(ConnPid2, "/test/content-encoding", + [{<<"content-encoding">>, <<"gzip">>}]), + {response, nofin, 200, _} = gun:await(ConnPid2, Ref2), + {ok, <<"undefined">>} = gun:await_body(ConnPid2, Ref2), + gun:close(ConnPid2), + %% The content_decoded list contains <<"gzip">>. + ConnPid3 = gun_open(Config), + Ref3 = gun:post(ConnPid3, "/test/content-decoded", + [{<<"content-encoding">>, <<"gzip">>}]), + {response, nofin, 200, _} = gun:await(ConnPid3, Ref3), + {ok, <<"[<<\"gzip\">>]">>} = gun:await_body(ConnPid3, Ref3), + gun:close(ConnPid3). + +opts_decompress_enabled_false(Config0) -> + doc("Confirm that the decompress_enabled option can be set."), Fun = case config(ref, Config0) of HTTPS when HTTPS =:= https_compress; HTTPS =:= https -> init_https; H2 when H2 =:= h2_compress; H2 =:= h2 -> init_http2; @@ -158,28 +219,75 @@ opts_decompress_ignore(Config0) -> Config = cowboy_test:Fun(?FUNCTION_NAME, #{ env => #{dispatch => cowboy_router:compile(init_routes(Config0))}, stream_handlers => [cowboy_decompress_h, cowboy_stream_h], - decompress_ignore => true + decompress_enabled => false }, Config0), Data = <<"test">>, Body = zlib:gzip(Data), try - {200, _, Body} = do_post("/echo/normal", - [{<<"content-encoding">>, <<"gzip">>}], Body, Config) + {200, Headers, Body} = do_post("/echo/normal", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + %% We do not set accept-encoding when we are disabled. + false = lists:keyfind(<<"accept-encoding">>, 1, Headers) after cowboy:stop_listener(?FUNCTION_NAME) end. -set_options_decompress_ignore(Config) -> - doc("Confirm that the decompress_ignore option can be dynamically - set to true and the data received is not decompressed."), +set_options_decompress_enabled_false(Config) -> + doc("Confirm that the decompress_enabled option can be dynamically " + "set to false and the data received is not decompressed."), Data = <<"test">>, Body = zlib:gzip(Data), - {200, _, Body} = do_post("/echo/decompress_ignore", + {200, Headers, Body} = do_post("/echo/decompress_disable", [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + %% We do not set accept-encoding when we are disabled. + false = lists:keyfind(<<"accept-encoding">>, 1, Headers), ok. +set_options_decompress_disable_in_the_middle(Config) -> + doc("Confirm that setting the decompress_enabled option dynamically " + "to false after starting to read the body does not disable decompression " + "and the data received is decompressed."), + Data = rand:bytes(1000000), + Body = zlib:gzip(Data), + %% Since we were not ignoring before starting to read, + %% we receive the entire body decompressed. + {200, Headers, Data} = do_post("/test/disable-in-the-middle", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + %% We do set accept-encoding when we are enabled, + %% even if an attempt to disable in the middle is ignored. + {_, _} = lists:keyfind(<<"accept-encoding">>, 1, Headers), + ok. + +set_options_decompress_enable_in_the_middle(Config0) -> + doc("Confirm that setting the decompress_enabled option dynamically " + "to true after starting to read the body does not enable decompression " + "and the data received is not decompressed."), + Fun = case config(ref, Config0) of + HTTPS when HTTPS =:= https_compress; HTTPS =:= https -> init_https; + H2 when H2 =:= h2_compress; H2 =:= h2 -> init_http2; + _ -> init_http + end, + Config = cowboy_test:Fun(?FUNCTION_NAME, #{ + env => #{dispatch => cowboy_router:compile(init_routes(Config0))}, + stream_handlers => [cowboy_decompress_h, cowboy_stream_h], + decompress_enabled => false + }, Config0), + Data = rand:bytes(1000000), + Body = zlib:gzip(Data), + try + %% Since we were ignoring before starting to read, + %% we receive the entire body compressed. + {200, Headers, Body} = do_post("/test/enable-in-the-middle", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + %% We do not set accept-encoding when we are disabled, + %% even if an attempt to enable in the middle is ignored. + false = lists:keyfind(<<"accept-encoding">>, 1, Headers) + after + cowboy:stop_listener(?FUNCTION_NAME) + end. + opts_decompress_ratio_limit(Config0) -> - doc("Confirm that the decompress_ignore option can be set"), + doc("Confirm that the decompress_ratio_limit option can be set."), Fun = case config(ref, Config0) of HTTPS when HTTPS =:= https_compress; HTTPS =:= https -> init_https; H2 when H2 =:= h2_compress; H2 =:= h2 -> init_http2; @@ -190,7 +298,8 @@ opts_decompress_ratio_limit(Config0) -> stream_handlers => [cowboy_decompress_h, cowboy_stream_h], decompress_ratio_limit => 1 }, Config0), - %% Data must be big enough for compression to be effective, so that ratio_limit=1 will fail. + %% Data must be big enough for compression to be effective, + %% so that ratio_limit=1 will fail. Data = <<0:800>>, Body = zlib:gzip(Data), try @@ -202,7 +311,8 @@ opts_decompress_ratio_limit(Config0) -> set_options_decompress_ratio_limit(Config) -> doc("Confirm that the decompress_ratio_limit option can be dynamically set."), - %% Data must be big enough for compression to be effective, so that ratio_limit=1 will fail. + %% Data must be big enough for compression to be effective, + %% so that ratio_limit=1 will fail. Data = <<0:800>>, Body = zlib:gzip(Data), {413, _, _} = do_post("/echo/decompress_ratio_limit", @@ -210,19 +320,16 @@ set_options_decompress_ratio_limit(Config) -> ok. gzip_bomb(Config) -> - doc("Send body compressed with suspiciously large ratio; get 413."), + doc("Confirm that requests are rejected with a 413 Payload Too Large " + "error when the ratio limit is exceeded."), Body = create_gzip_bomb(), {413, _, _} = do_post("/echo/normal", [{<<"content-encoding">>, <<"gzip">>}], Body, Config), ok. -%% RFC 9110. Section 12.5.3. 3. When sent by a server in a response, -%% Accept-Encoding provides information about which content codings are -%% preferred in the content of a subsequent request to the same resource. -%% -%% Set or add gzip set_accept_encoding_response(Config) -> - doc("Header accept-encoding must be set on valid response command."), + doc("Header accept-encoding must be set on valid response command. " + "(RFC9110 12.5.3)"), Data = <<"test">>, Body = zlib:gzip(Data), {200, Headers, Data} = do_post("/echo/normal", @@ -231,97 +338,79 @@ set_accept_encoding_response(Config) -> ok. set_accept_encoding_header(Config) -> - doc("Header accept-encoding must be set on valid header command."), + doc("Header accept-encoding must be set on valid header command. " + "(RFC9110 12.5.3)"), Data = <<"test">>, Body = zlib:gzip(Data), - {200, Headers, Data} = do_post("/header", + {200, Headers, Data} = do_post("/test/header-command", [{<<"content-encoding">>, <<"gzip">>}], Body, Config), {_, <<"gzip">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), ok. add_accept_encoding_header_valid(Config) -> - doc("Header accept-encoding must be added on valid accept-encoding."), + doc("Supported content codings must be added to the accept-encoding " + "header if it already exists. (RFC9110 12.5.3)"), Data = <<"test">>, Body = zlib:gzip(Data), - {200, Headers, Data} = do_post("/accept-identity", + {200, Headers, Data} = do_post("/test/accept-identity", [{<<"content-encoding">>, <<"gzip">>}], Body, Config), {_, <<"identity, gzip">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), ok. override_accept_encoding_header_invalid(Config) -> - doc("Header accept-encoding must override invalid accept-encoding."), + doc("When the stream handler cannot parse the accept-encoding header " + "found in the response, it overrides it."), Data = <<"test">>, Body = zlib:gzip(Data), - {200, Headers, Data} = do_post("/invalid-header", + {200, Headers, Data} = do_post("/test/invalid-header", [{<<"content-encoding">>, <<"gzip">>}], Body, Config), {_, <<"gzip">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), ok. -%% RFC 9110. Section 12.5.3. 10.3. If the representation's content coding is -%% one of the content codings listed in the Accept-Encoding field value, then -%% it is acceptable unless it is accompanied by a qvalue of 0. -%% -%% gzip must not have a qvalue of 0 when the handler is used. Set to 1. override_accept_encoding_excluded(Config) -> - doc("Header accept-encoding must override when explicitly excluded."), + doc("The stream handler must ensure that the content encodings " + "it supports are not marked as unsupported in response headers. " + "The stream handler enables gzip when explicitly excluded. " + "(RFC9110 12.5.3)"), Data = <<"test">>, Body = zlib:gzip(Data), - {200, Headers, Data} = do_post("/reject-explicit-header", + {200, Headers, Data} = do_post("/test/reject-explicit-header", [{<<"content-encoding">>, <<"gzip">>}], Body, Config), {_, <<"identity;q=1, gzip;q=1">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), ok. -%% RFC 9110. Section 12.5.3. 10.2. If the representation has no content coding, -%% then it is acceptable by default unless specifically excluded by the -%% Accept-Encoding field stating either "identity;q=0" or "*;q=0" wihout a more -%% specific entry for "identity". -%% -%% *;q=0 will reject codings that are not listed. Specific entry gzip must -%% always be listed when the handler is used. Add gzip. +%% *;q=0 will reject codings that are not listed. Supported codings +%% must always be enabled when the handler is used. add_accept_encoding_excluded(Config) -> - doc("Header accept-encoding must added when implicitly excluded."), + doc("The stream handler must ensure that the content encodings " + "it supports are not marked as unsupported in response headers. " + "The stream handler enables gzip when implicitly excluded (*;q=0). " + "(RFC9110 12.5.3)"), Data = <<"test">>, Body = zlib:gzip(Data), - {200, Headers, Data} = do_post("/reject-implicit-header", + {200, Headers, Data} = do_post("/test/reject-implicit-header", [{<<"content-encoding">>, <<"gzip">>}], Body, Config), {_, <<"gzip;q=1, identity;q=1, *;q=0">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), ok. no_override_accept_coding_set_explicit(Config) -> - doc("Confirm that accept-encoding is not overridden when explicitly set."), + doc("Confirm that accept-encoding is not overridden when the " + "content encodings it supports are explicitly set. " + "(RFC9110 12.5.3)"), Data = <<"test">>, Body = zlib:gzip(Data), - {200, Headers, Data} = do_post("/accept-explicit-header", + {200, Headers, Data} = do_post("/test/accept-explicit-header", [{<<"content-encoding">>, <<"gzip">>}], Body, Config), {_, <<"identity, gzip;q=0.5">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), ok. no_override_accept_coding_set_implicit(Config) -> - doc("Confirm that accept-encoding is not overridden when implicitly set."), + doc("Confirm that accept-encoding is not overridden when the " + "content encodings it supports are implicitly set. " + "(RFC9110 12.5.3)"), Data = <<"test">>, Body = zlib:gzip(Data), - {200, Headers, Data} = do_post("/accept-implicit-header", + {200, Headers, Data} = do_post("/test/accept-implicit-header", [{<<"content-encoding">>, <<"gzip">>}], Body, Config), {_, <<"identity, *;q=0.5">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), ok. - -%% RFC 9110. Section 12.5.3. 10.1. If no Accept-Encoding header field is in the -%% request, any content coding is considered acceptable by the user agent. -%% -%% Don't add anything on error or when that handler is not used. -no_set_accept_encoding(Config) -> - doc("No header accept-encoding on invalid responses."), - Body = <<"test">>, - {400, Headers, _} = do_post("/echo/normal", - [{<<"content-encoding">>, <<"gzip">>}], Body, Config), - false = lists:keyfind(<<"accept-encoding">>, 1, Headers), - ok. - -no_set_accept_encoding_ignore(Config) -> - doc("Confirm that no accept-encoding is set when stream is ignored."), - Data = <<"test">>, - Body = zlib:gzip(Data), - {200, Headers, Body} = do_post("/echo/decompress_ignore", - [{<<"content-encoding">>, <<"gzip">>}], Body, Config), - false = lists:keyfind(<<"accept-encoding">>, 1, Headers), - ok. diff --git a/test/handlers/decompress_h.erl b/test/handlers/decompress_h.erl index 346200b..deb6de0 100644 --- a/test/handlers/decompress_h.erl +++ b/test/handlers/decompress_h.erl @@ -5,10 +5,10 @@ -export([init/2]). -init(Req0, State=[]) -> +init(Req0, State=echo) -> case cowboy_req:binding(what, Req0) of - <<"decompress_ignore">> -> - cowboy_req:cast({set_options, #{decompress_ignore => true}}, Req0); + <<"decompress_disable">> -> + cowboy_req:cast({set_options, #{decompress_enabled => false}}, Req0); <<"decompress_ratio_limit">> -> cowboy_req:cast({set_options, #{decompress_ratio_limit => 0.5}}, Req0); <<"normal">> -> ok @@ -16,46 +16,62 @@ init(Req0, State=[]) -> {ok, Body, Req1} = read_body(Req0), Req = cowboy_req:reply(200, #{}, Body, Req1), {ok, Req, State}; +init(Req0, State=test) -> + Req = test(Req0, cowboy_req:binding(what, Req0)), + {ok, Req, State}. -init(Req0, State=header_command) -> - {ok, Body, Req1} = read_body(Req0), - Req2 = cowboy_req:stream_reply(200, #{}, Req1), - Req = cowboy_req:stream_body(Body, fin, Req2), - {ok, Req, State}; - -init(Req0, State=accept_identity) -> - {ok, Body, Req1} = read_body(Req0), - Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity">>}, Body, Req1), - {ok, Req, State}; - -init(Req0, State=invalid_header) -> - {ok, Body, Req1} = read_body(Req0), - Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<";">>}, Body, Req1), - {ok, Req, State}; - -init(Req0, State=reject_explicit_header) -> - {ok, Body, Req1} = read_body(Req0), - Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity, gzip;q=0">>}, - Body, Req1), - {ok, Req, State}; - -init(Req0, State=reject_implicit_header) -> - {ok, Body, Req1} = read_body(Req0), - Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity, *;q=0">>}, - Body, Req1), - {ok, Req, State}; - -init(Req0, State=accept_explicit_header) -> - {ok, Body, Req1} = read_body(Req0), - Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity, gzip;q=0.5">>}, - Body, Req1), - {ok, Req, State}; - -init(Req0, State=accept_implicit_header) -> +test(Req, <<"content-encoding">>) -> + cowboy_req:reply(200, #{}, + cowboy_req:header(<<"content-encoding">>, Req, <<"undefined">>), + Req); +test(Req, <<"content-decoded">>) -> + cowboy_req:reply(200, #{}, + io_lib:format("~0p", [maps:get(content_decoded, Req, undefined)]), + Req); +test(Req0, <<"disable-in-the-middle">>) -> + {Status, Data, Req1} = cowboy_req:read_body(Req0, #{length => 1000}), + cowboy_req:cast({set_options, #{decompress_enabled => false}}, Req1), + {ok, Body, Req} = do_read_body(Status, Req1, Data), + cowboy_req:reply(200, #{}, Body, Req); +test(Req0, <<"enable-in-the-middle">>) -> + {Status, Data, Req1} = cowboy_req:read_body(Req0, #{length => 1000}), + cowboy_req:cast({set_options, #{decompress_enabled => true}}, Req1), + {ok, Body, Req} = do_read_body(Status, Req1, Data), + cowboy_req:reply(200, #{}, Body, Req); +test(Req0, <<"header-command">>) -> {ok, Body, Req1} = read_body(Req0), - Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity, *;q=0.5">>}, - Body, Req1), - {ok, Req, State}. + Req = cowboy_req:stream_reply(200, #{}, Req1), + cowboy_req:stream_body(Body, fin, Req); +test(Req0, <<"accept-identity">>) -> + {ok, Body, Req} = read_body(Req0), + cowboy_req:reply(200, + #{<<"accept-encoding">> => <<"identity">>}, + Body, Req); +test(Req0, <<"invalid-header">>) -> + {ok, Body, Req} = read_body(Req0), + cowboy_req:reply(200, + #{<<"accept-encoding">> => <<";">>}, + Body, Req); +test(Req0, <<"reject-explicit-header">>) -> + {ok, Body, Req} = read_body(Req0), + cowboy_req:reply(200, + #{<<"accept-encoding">> => <<"identity, gzip;q=0">>}, + Body, Req); +test(Req0, <<"reject-implicit-header">>) -> + {ok, Body, Req} = read_body(Req0), + cowboy_req:reply(200, + #{<<"accept-encoding">> => <<"identity, *;q=0">>}, + Body, Req); +test(Req0, <<"accept-explicit-header">>) -> + {ok, Body, Req} = read_body(Req0), + cowboy_req:reply(200, + #{<<"accept-encoding">> => <<"identity, gzip;q=0.5">>}, + Body, Req); +test(Req0, <<"accept-implicit-header">>) -> + {ok, Body, Req} = read_body(Req0), + cowboy_req:reply(200, + #{<<"accept-encoding">> => <<"identity, *;q=0.5">>}, + Body, Req). read_body(Req0) -> {Status, Data, Req} = cowboy_req:read_body(Req0, #{length => 1000}), |