diff options
author | jdamanalo <[email protected]> | 2023-03-31 15:56:23 +0800 |
---|---|---|
committer | Loïc Hoguin <[email protected]> | 2023-12-21 15:39:08 +0100 |
commit | 3ed1b24dd6ef6cd3e78a2fa6d600cce082b6984a (patch) | |
tree | 741d1845f7c3b04c198ac2476abf34aa4757d6ba | |
parent | ffbcdf534c7bdcca545e245443cc48056bcd6944 (diff) | |
download | cowboy-3ed1b24dd6ef6cd3e78a2fa6d600cce082b6984a.tar.gz cowboy-3ed1b24dd6ef6cd3e78a2fa6d600cce082b6984a.tar.bz2 cowboy-3ed1b24dd6ef6cd3e78a2fa6d600cce082b6984a.zip |
Add cowboy_decompress_h stream handler
-rw-r--r-- | doc/src/guide/streams.asciidoc | 5 | ||||
-rw-r--r-- | doc/src/manual/cowboy_compress_h.asciidoc | 1 | ||||
-rw-r--r-- | doc/src/manual/cowboy_decompress_h.asciidoc | 58 | ||||
-rw-r--r-- | doc/src/manual/cowboy_metrics_h.asciidoc | 1 | ||||
-rw-r--r-- | doc/src/manual/cowboy_stream_h.asciidoc | 1 | ||||
-rw-r--r-- | doc/src/manual/cowboy_tracer_h.asciidoc | 1 | ||||
-rw-r--r-- | ebin/cowboy.app | 2 | ||||
-rw-r--r-- | src/cowboy_decompress_h.erl | 217 | ||||
-rw-r--r-- | test/decompress_SUITE.erl | 327 | ||||
-rw-r--r-- | test/handlers/decompress_h.erl | 68 |
10 files changed, 680 insertions, 1 deletions
diff --git a/doc/src/guide/streams.asciidoc b/doc/src/guide/streams.asciidoc index 0ac84ce..b6e4d34 100644 --- a/doc/src/guide/streams.asciidoc +++ b/doc/src/guide/streams.asciidoc @@ -65,6 +65,11 @@ automatically compress responses when possible. It is not enabled by default. It is a good example for writing your own handlers that will modify responses. +link:man:cowboy_decompress_h(3)[cowboy_decompress_h] will +automatically decompress requests when possible. It is not +enabled by default. It is a good example for writing your +own handlers that will modify requests. + link:man:cowboy_metrics_h(3)[cowboy_metrics_h] gathers metrics about a stream then passes them to a configurable function. It is not enabled by default. diff --git a/doc/src/manual/cowboy_compress_h.asciidoc b/doc/src/manual/cowboy_compress_h.asciidoc index 31a9162..f6a45a9 100644 --- a/doc/src/manual/cowboy_compress_h.asciidoc +++ b/doc/src/manual/cowboy_compress_h.asciidoc @@ -63,6 +63,7 @@ The compress stream handler does not produce any event. link:man:cowboy(7)[cowboy(7)], link:man:cowboy_stream(3)[cowboy_stream(3)], +link:man:cowboy_decompress_h(3)[cowboy_decompress_h(3)], link:man:cowboy_metrics_h(3)[cowboy_metrics_h(3)], link:man:cowboy_stream_h(3)[cowboy_stream_h(3)], link:man:cowboy_tracer_h(3)[cowboy_tracer_h(3)] diff --git a/doc/src/manual/cowboy_decompress_h.asciidoc b/doc/src/manual/cowboy_decompress_h.asciidoc new file mode 100644 index 0000000..63ee0ee --- /dev/null +++ b/doc/src/manual/cowboy_decompress_h.asciidoc @@ -0,0 +1,58 @@ += cowboy_decompress_h(3) + +== Name + +cowboy_decompress_h - Decompress stream handler + +== Description + +The module `cowboy_decompress_h` decompresses request bodies +automatically when the server supports it. Requests will +only be decompressed when their compression ratio is lower +than the configured limit. Mismatch of the content and +`content-encoding` is rejected with `400 Bad Request`. + +== Options + +[source,erlang] +---- +opts() :: #{ + decompress_ratio_limit => non_neg_integer(), + decompress_ignore => boolean() +} +---- + +Configuration for the decompress stream handler. + +The default value is given next to the option name: + +decompress_ratio_limit (20):: +The max ratio of the compressed and decompressed body +before it is rejected with `413 Payload Too Large`. ++ +This option can be updated at any time using the +`set_options` stream handler command. + +decompress_ignore (false):: + +Whether the handler will be ignored. ++ +This option can be updated at any time using the +`set_options` stream handler command. + +== Events + +The decompress stream handler does not produce any event. + +== Changelog + +* *2.11*: Module introduced. + +== See also + +link:man:cowboy(7)[cowboy(7)], +link:man:cowboy_stream(3)[cowboy_stream(3)], +link:man:cowboy_compress_h(3)[cowboy_compress_h(3)], +link:man:cowboy_metrics_h(3)[cowboy_metrics_h(3)], +link:man:cowboy_stream_h(3)[cowboy_stream_h(3)], +link:man:cowboy_tracer_h(3)[cowboy_tracer_h(3)] diff --git a/doc/src/manual/cowboy_metrics_h.asciidoc b/doc/src/manual/cowboy_metrics_h.asciidoc index c871d57..801bdbb 100644 --- a/doc/src/manual/cowboy_metrics_h.asciidoc +++ b/doc/src/manual/cowboy_metrics_h.asciidoc @@ -160,5 +160,6 @@ The metrics stream handler does not produce any event. link:man:cowboy(7)[cowboy(7)], link:man:cowboy_stream(3)[cowboy_stream(3)], link:man:cowboy_compress_h(3)[cowboy_compress_h(3)], +link:man:cowboy_decompress_h(3)[cowboy_decompress_h(3)], link:man:cowboy_stream_h(3)[cowboy_stream_h(3)], link:man:cowboy_tracer_h(3)[cowboy_tracer_h(3)] diff --git a/doc/src/manual/cowboy_stream_h.asciidoc b/doc/src/manual/cowboy_stream_h.asciidoc index c25aa3d..588346e 100644 --- a/doc/src/manual/cowboy_stream_h.asciidoc +++ b/doc/src/manual/cowboy_stream_h.asciidoc @@ -69,5 +69,6 @@ may not work properly if they are executed link:man:cowboy(7)[cowboy(7)], link:man:cowboy_stream(3)[cowboy_stream(3)], link:man:cowboy_compress_h(3)[cowboy_compress_h(3)], +link:man:cowboy_decompress_h(3)[cowboy_decompress_h(3)], link:man:cowboy_metrics_h(3)[cowboy_metrics_h(3)], link:man:cowboy_tracer_h(3)[cowboy_tracer_h(3)] diff --git a/doc/src/manual/cowboy_tracer_h.asciidoc b/doc/src/manual/cowboy_tracer_h.asciidoc index e3592e4..4f4e9bc 100644 --- a/doc/src/manual/cowboy_tracer_h.asciidoc +++ b/doc/src/manual/cowboy_tracer_h.asciidoc @@ -84,5 +84,6 @@ The tracer stream handler does not produce any event. link:man:cowboy(7)[cowboy(7)], link:man:cowboy_stream(3)[cowboy_stream(3)], link:man:cowboy_compress_h(3)[cowboy_compress_h(3)], +link:man:cowboy_decompress_h(3)[cowboy_decompress_h(3)], link:man:cowboy_metrics_h(3)[cowboy_metrics_h(3)], link:man:cowboy_stream_h(3)[cowboy_stream_h(3)] diff --git a/ebin/cowboy.app b/ebin/cowboy.app index cb3047e..9f3e1cb 100644 --- a/ebin/cowboy.app +++ b/ebin/cowboy.app @@ -1,7 +1,7 @@ {application, 'cowboy', [ {description, "Small, fast, modern HTTP server."}, {vsn, "2.10.0"}, - {modules, ['cowboy','cowboy_app','cowboy_bstr','cowboy_children','cowboy_clear','cowboy_clock','cowboy_compress_h','cowboy_constraints','cowboy_handler','cowboy_http','cowboy_http2','cowboy_loop','cowboy_metrics_h','cowboy_middleware','cowboy_req','cowboy_rest','cowboy_router','cowboy_static','cowboy_stream','cowboy_stream_h','cowboy_sub_protocol','cowboy_sup','cowboy_tls','cowboy_tracer_h','cowboy_websocket']}, + {modules, ['cowboy','cowboy_app','cowboy_bstr','cowboy_children','cowboy_clear','cowboy_clock','cowboy_compress_h','cowboy_constraints','cowboy_decompress_h','cowboy_handler','cowboy_http','cowboy_http2','cowboy_loop','cowboy_metrics_h','cowboy_middleware','cowboy_req','cowboy_rest','cowboy_router','cowboy_static','cowboy_stream','cowboy_stream_h','cowboy_sub_protocol','cowboy_sup','cowboy_tls','cowboy_tracer_h','cowboy_websocket']}, {registered, [cowboy_sup,cowboy_clock]}, {applications, [kernel,stdlib,crypto,cowlib,ranch]}, {optional_applications, []}, diff --git a/src/cowboy_decompress_h.erl b/src/cowboy_decompress_h.erl new file mode 100644 index 0000000..ffbec25 --- /dev/null +++ b/src/cowboy_decompress_h.erl @@ -0,0 +1,217 @@ +-module(cowboy_decompress_h). +-behavior(cowboy_stream). + +-export([init/3]). +-export([data/4]). +-export([info/3]). +-export([terminate/3]). +-export([early_error/5]). + +-record(state, { + next :: any(), + ratio_limit :: non_neg_integer() | undefined, + ignore = false :: boolean(), + compress = undefined :: undefined | gzip, + inflate = undefined :: undefined | zlib:zstream(), + is_reading = false :: boolean(), + read_body_buffer = <<>> :: binary(), + read_body_is_fin = nofin :: nofin | {fin, non_neg_integer()} +}). + +-spec init(cowboy_stream:streamid(), cowboy_req:req(), cowboy:opts()) + -> {cowboy_stream:commands(), #state{}}. +init(StreamID, Req, Opts) -> + RatioLimit = maps:get(decompress_ratio_limit, Opts, 20), + Ignore = maps:get(decompress_ignore, Opts, false), + State = check_req(Req), + Inflate = case State#state.compress of + undefined -> + undefined; + gzip -> + Z = zlib:open(), + zlib:inflateInit(Z, 31), + Z + end, + {Commands, Next} = cowboy_stream:init(StreamID, Req, Opts), + fold(Commands, State#state{next=Next, ratio_limit=RatioLimit, ignore=Ignore, + inflate=Inflate}). + +-spec data(cowboy_stream:streamid(), cowboy_stream:fin(), cowboy_req:resp_body(), State) + -> {cowboy_stream:commands(), State} when State::#state{}. +data(StreamID, IsFin, Data, State=#state{next=Next0, inflate=undefined}) -> + {Commands, Next} = cowboy_stream:data(StreamID, IsFin, Data, Next0), + fold(Commands, State#state{next=Next, read_body_is_fin=IsFin}); +data(StreamID, IsFin, Data, State=#state{next=Next0, ignore=true, read_body_buffer=Buffer}) -> + {Commands, Next} = cowboy_stream:data(StreamID, IsFin, + << Buffer/binary, Data/binary >>, Next0), + fold(Commands, State#state{next=Next, read_body_is_fin=IsFin}); +data(StreamID, IsFin, Data, State0=#state{next=Next0, ratio_limit=RatioLimit, + inflate=Z, is_reading=true, read_body_buffer=Buffer0}) -> + Buffer = << Buffer0/binary, Data/binary >>, + case inflate(Z, RatioLimit, Buffer) of + {error, Type} -> + Status = case Type of + data -> 400; + size -> 413 + end, + Commands = [ + {error_response, Status, #{<<"content-length">> => <<"0">>}, <<>>}, + stop + ], + fold(Commands, State0#state{inflate=undefined}); + {ok, Inflated} -> + State = case IsFin of + nofin -> + State0; + fin -> + zlib:inflateEnd(Z), + zlib:close(Z), + State0#state{inflate=undefined} + end, + {Commands, Next} = cowboy_stream:data(StreamID, IsFin, Inflated, Next0), + fold(Commands, State#state{next=Next, read_body_buffer= <<>>, + read_body_is_fin=IsFin}) + end; +data(_, IsFin, Data, State=#state{read_body_buffer=Buffer0}) -> + Buffer = << Buffer0/binary, Data/binary >>, + {[], State#state{read_body_buffer=Buffer, read_body_is_fin=IsFin}}. + +-spec info(cowboy_stream:streamid(), any(), State) + -> {cowboy_stream:commands(), State} when State::#state{}. +info(StreamID, Info, State=#state{next=Next0, inflate=undefined}) -> + {Commands, Next} = cowboy_stream:info(StreamID, Info, Next0), + fold(Commands, State#state{next=Next}); +info(StreamID, Info={CommandTag, _, _, _, _}, State=#state{next=Next0, read_body_is_fin=IsFin}) + when CommandTag =:= read_body; CommandTag =:= read_body_timeout -> + {Commands0, Next1} = cowboy_stream:info(StreamID, Info, Next0), + {Commands, Next} = data(StreamID, IsFin, <<>>, State#state{next=Next1, is_reading=true}), + fold(Commands ++ Commands0, Next); +info(StreamID, Info={set_options, Opts}, State=#state{next=Next0, + ignore=Ignore0, ratio_limit=RatioLimit0}) -> + Ignore = maps:get(decompress_ignore, Opts, Ignore0), + RatioLimit = maps:get(decompress_ratio_limit, Opts, RatioLimit0), + {Commands, Next} = cowboy_stream:info(StreamID, Info, Next0), + fold(Commands, State#state{next=Next, ignore=Ignore, ratio_limit=RatioLimit}); +info(StreamID, Info, State=#state{next=Next0}) -> + {Commands, Next} = cowboy_stream:info(StreamID, Info, Next0), + fold(Commands, State#state{next=Next}). + +-spec terminate(cowboy_stream:streamid(), cowboy_stream:reason(), #state{}) -> any(). +terminate(StreamID, Reason, #state{next=Next, inflate=Z}) -> + case Z of + undefined -> ok; + _ -> zlib:close(Z) + end, + cowboy_stream:terminate(StreamID, Reason, Next). + +-spec early_error(cowboy_stream:streamid(), cowboy_stream:reason(), + cowboy_stream:partial_req(), Resp, cowboy:opts()) -> Resp + when Resp::cowboy_stream:resp_command(). +early_error(StreamID, Reason, PartialReq, Resp, Opts) -> + cowboy_stream:early_error(StreamID, Reason, PartialReq, Resp, Opts). + +%% Internal. + +check_req(Req) -> + try cowboy_req:parse_header(<<"content-encoding">>, Req) of + undefined -> + #state{compress=undefined}; + Encodings -> + case [E || E=(<<"gzip">>) <- Encodings] of + [] -> + #state{compress=undefined}; + _ -> + #state{compress=gzip} + end + catch + _:_ -> + #state{compress=undefined} + end. + +fold(Commands, State) -> + fold(Commands, State, []). + +fold([], State, Acc) -> + {lists:reverse(Acc), State}; +fold([{response, Status, Headers0, Body}|Tail], State=#state{ignore=false}, Acc) -> + Headers = add_accept_encoding(Headers0), + fold(Tail, State, [{response, Status, Headers, Body}|Acc]); +fold([{headers, Status, Headers0} | Tail], State=#state{ignore=false}, Acc) -> + Headers = add_accept_encoding(Headers0), + fold(Tail, State, [{headers, Status, Headers}|Acc]); +fold([Command|Tail], State, Acc) -> + fold(Tail, State, [Command|Acc]). + +add_accept_encoding(Headers=#{<<"accept-encoding">> := AcceptEncoding}) -> + try cow_http_hd:parse_accept_encoding(iolist_to_binary(AcceptEncoding)) of + List -> + case lists:keyfind(<<"gzip">>, 1, List) of + %% gzip is excluded but this handler is not ignored; we replace. + {_, 0} -> + Replaced = lists:keyreplace(<<"gzip">>, 1, List, {<<"gzip">>, 1000}), + Codings = build_accept_encoding(Replaced), + Headers#{<<"accept-encoding">> => Codings}; + {_, _} -> + Headers; + false -> + case lists:keyfind(<<"*">>, 1, List) of + %% Others are excluded along with gzip; we add. + {_, 0} -> + WithGzip = [{<<"gzip">>, 1000} | List], + Codings = build_accept_encoding(WithGzip), + Headers#{<<"accept-encoding">> => Codings}; + {_, _} -> + Headers; + false -> + Headers#{<<"accept-encoding">> => [AcceptEncoding, <<", gzip">>]} + end + end + catch _:_ -> + Headers#{<<"accept-encoding">> => <<"gzip">>} + end; +add_accept_encoding(Headers) -> + Headers#{<<"accept-encoding">> => <<"gzip">>}. + +%% From cowlib, maybe expose? +qvalue_to_iodata(0) -> <<"0">>; +qvalue_to_iodata(Q) when Q < 10 -> [<<"0.00">>, integer_to_binary(Q)]; +qvalue_to_iodata(Q) when Q < 100 -> [<<"0.0">>, integer_to_binary(Q)]; +qvalue_to_iodata(Q) when Q < 1000 -> [<<"0.">>, integer_to_binary(Q)]; +qvalue_to_iodata(1000) -> <<"1">>. + +build_accept_encoding([{ContentCoding, Q}|Tail]) -> + Weight = iolist_to_binary(qvalue_to_iodata(Q)), + Acc = <<ContentCoding/binary, ";q=", Weight/binary>>, + do_build_accept_encoding(Tail, Acc). + +do_build_accept_encoding([{ContentCoding, Q}|Tail], Acc0) -> + Weight = iolist_to_binary(qvalue_to_iodata(Q)), + Acc = <<Acc0/binary, ", ", ContentCoding/binary, ";q=", Weight/binary>>, + do_build_accept_encoding(Tail, Acc); +do_build_accept_encoding([], Acc) -> + Acc. + +inflate(Z, RatioLimit, Data) -> + try + {Status, Output} = zlib:safeInflate(Z, Data), + Size = iolist_size(Output), + do_inflate(Z, Size, byte_size(Data) * RatioLimit, Status, [Output]) + catch + error:data_error -> + zlib:close(Z), + {error, data} + end. + +do_inflate(Z, Size, Limit, Status, _) when Size > Limit -> + case Status of + continue -> ok; + finished -> zlib:inflateEnd(Z) + end, + zlib:close(Z), + {error, size}; +do_inflate(Z, Size0, Limit, continue, Acc) -> + {Status, Output} = zlib:safeInflate(Z, []), + Size = Size0 + iolist_size(Output), + do_inflate(Z, Size, Limit, Status, [Output | Acc]); +do_inflate(_, _, _, finished, Acc) -> + {ok, iolist_to_binary(lists:reverse(Acc))}. diff --git a/test/decompress_SUITE.erl b/test/decompress_SUITE.erl new file mode 100644 index 0000000..6fd0a73 --- /dev/null +++ b/test/decompress_SUITE.erl @@ -0,0 +1,327 @@ +-module(decompress_SUITE). +-compile(export_all). +-compile(nowarn_export_all). + +-import(ct_helper, [config/2]). +-import(ct_helper, [doc/1]). +-import(cowboy_test, [gun_open/1]). + +%% ct. + +all() -> + cowboy_test:common_all(). + +groups() -> + cowboy_test:common_groups(ct_helper:all(?MODULE)). + +init_per_group(Name = http, Config) -> + cowboy_test:init_http(Name, init_plain_opts(Config), Config); +init_per_group(Name = https, Config) -> + cowboy_test:init_http(Name, init_plain_opts(Config), Config); +init_per_group(Name = h2, Config) -> + cowboy_test:init_http2(Name, init_plain_opts(Config), Config); +init_per_group(Name = h2c, Config) -> + Config1 = cowboy_test:init_http(Name, init_plain_opts(Config), Config), + lists:keyreplace(protocol, 1, Config1, {protocol, http2}); +init_per_group(Name = http_compress, Config) -> + cowboy_test:init_http(Name, init_compress_opts(Config), Config); +init_per_group(Name = https_compress, Config) -> + cowboy_test:init_http(Name, init_compress_opts(Config), Config); +init_per_group(Name = h2_compress, Config) -> + cowboy_test:init_http2(Name, init_compress_opts(Config), Config); +init_per_group(Name = h2c_compress, Config) -> + Config1 = cowboy_test:init_http(Name, init_compress_opts(Config), Config), + lists:keyreplace(protocol, 1, Config1, {protocol, http2}). + +end_per_group(Name, _) -> + cowboy:stop_listener(Name). + +init_plain_opts(Config) -> + #{ + env => #{dispatch => cowboy_router:compile(init_routes(Config))}, + stream_handlers => [cowboy_decompress_h, cowboy_stream_h] + }. + +init_compress_opts(Config) -> + #{ + env => #{dispatch => cowboy_router:compile(init_routes(Config))}, + stream_handlers => [cowboy_decompress_h, cowboy_compress_h, cowboy_stream_h] + }. + +init_routes(_) -> + [{'_', [ + {"/echo/:what", decompress_h, []}, + {"/header", decompress_h, header_command}, + {"/invalid-header", decompress_h, invalid_header}, + {"/accept-identity", decompress_h, accept_identity}, + {"/reject-explicit-header", decompress_h, reject_explicit_header}, + {"/reject-implicit-header", decompress_h, reject_implicit_header}, + {"/accept-explicit-header", decompress_h, accept_explicit_header}, + {"/accept-implicit-header", decompress_h, accept_implicit_header} + ]}]. + +%% Internal. + +do_post(Path, ReqHeaders, Body, Config) -> + ConnPid = gun_open(Config), + Ref = gun:post(ConnPid, Path, ReqHeaders, Body), + {response, IsFin, Status, RespHeaders} = gun:await(ConnPid, Ref), + {ok, ResponseBody} = case IsFin of + nofin -> gun:await_body(ConnPid, Ref); + fin -> {ok, <<>>} + end, + gun:close(ConnPid), + {Status, RespHeaders, ResponseBody}. + +create_gzip_bomb() -> + Z = zlib:open(), + zlib:deflateInit(Z, 9, deflated, 31, 8, default), + %% 1000 chunks of 100000 zeroes (100MB). + Bomb = do_create_gzip_bomb(Z, 1000), + zlib:deflateEnd(Z), + zlib:close(Z), + iolist_to_binary(Bomb). + +do_create_gzip_bomb(Z, 0) -> + zlib:deflate(Z, << >>, finish); +do_create_gzip_bomb(Z, N) -> + Data = <<0:800000>>, + Deflate = zlib:deflate(Z, Data), + [Deflate | do_create_gzip_bomb(Z, N - 1)]. + +%% Tests. + +content_encoding_none(Config) -> + doc("Send no content-encoding; get echo."), + Body = <<"test">>, + {200, _, Body} = do_post("/echo/normal", + [{<<"content-encoding">>, <<";">>}], Body, Config), + ok. + +content_encoding_malformed(Config) -> + doc("Send malformed content-encoding; get echo."), + Body = <<"test">>, + {200, _, Body} = do_post("/echo/normal", + [{<<"content-encoding">>, <<";">>}], Body, Config), + ok. + +content_encoding_not_supported(Config) -> + doc("Send content-encoding: compress (unsupported by Cowboy); get echo."), + Body = <<"test">>, + {200, _, Body} = do_post("/echo/normal", + [{<<"content-encoding">>, <<"compress">>}], Body, Config), + ok. + +content_encoding_wrong(Config) -> + doc("Send content-encoding and unencoded body; get 400."), + Body = <<"test">>, + {400, _, _} = do_post("/echo/normal", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + ok. + +decompress(Config) -> + doc("Send content-encoding and encoded body; get decompressed response."), + Data = <<"test">>, + Body = zlib:gzip(Data), + {200, _, Data} = do_post("/echo/normal", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + ok. + +decompress_stream(Config) -> + doc("Stream encoded body; get decompressed response."), + %% Handler read length 1KB. Compressing 3KB should be enough to trigger more. + Data = crypto:strong_rand_bytes(3000), + Body = zlib:gzip(Data), + Size = byte_size(Body), + ConnPid = gun_open(Config), + Ref = gun:post(ConnPid, "/echo/normal", [{<<"content-encoding">>, <<"gzip">>}]), + gun:data(ConnPid, Ref, nofin, binary:part(Body, 0, Size div 2)), + timer:sleep(1000), + gun:data(ConnPid, Ref, fin, binary:part(Body, Size div 2, Size div 2 + Size rem 2)), + {response, IsFin, 200, _} = gun:await(ConnPid, Ref), + {ok, Data} = case IsFin of + nofin -> gun:await_body(ConnPid, Ref); + fin -> {ok, <<>>} + end, + gun:close(ConnPid), + {200, _, Data} = do_post("/echo/normal", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + ok. + +opts_decompress_ignore(Config0) -> + doc("Confirm that the decompress_ignore option can be set."), + Fun = case config(ref, Config0) of + HTTPS when HTTPS =:= https_compress; HTTPS =:= https -> init_https; + H2 when H2 =:= h2_compress; H2 =:= h2 -> init_http2; + _ -> init_http + end, + Config = cowboy_test:Fun(?FUNCTION_NAME, #{ + env => #{dispatch => cowboy_router:compile(init_routes(Config0))}, + stream_handlers => [cowboy_decompress_h, cowboy_stream_h], + decompress_ignore => true + }, Config0), + Data = <<"test">>, + Body = zlib:gzip(Data), + try + {200, _, Body} = do_post("/echo/normal", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config) + after + cowboy:stop_listener(?FUNCTION_NAME) + end. + +set_options_decompress_ignore(Config) -> + doc("Confirm that the decompress_ignore option can be dynamically + set to true and the data received is not decompressed."), + Data = <<"test">>, + Body = zlib:gzip(Data), + {200, _, Body} = do_post("/echo/decompress_ignore", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + ok. + +opts_decompress_ratio_limit(Config0) -> + doc("Confirm that the decompress_ignore option can be set"), + Fun = case config(ref, Config0) of + HTTPS when HTTPS =:= https_compress; HTTPS =:= https -> init_https; + H2 when H2 =:= h2_compress; H2 =:= h2 -> init_http2; + _ -> init_http + end, + Config = cowboy_test:Fun(?FUNCTION_NAME, #{ + env => #{dispatch => cowboy_router:compile(init_routes(Config0))}, + stream_handlers => [cowboy_decompress_h, cowboy_stream_h], + decompress_ratio_limit => 1 + }, Config0), + %% Data must be big enough for compression to be effective, so that ratio_limit=1 will fail. + Data = <<0:800>>, + Body = zlib:gzip(Data), + try + {413, _, _} = do_post("/echo/normal", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config) + after + cowboy:stop_listener(?FUNCTION_NAME) + end. + +set_options_decompress_ratio_limit(Config) -> + doc("Confirm that the decompress_ratio_limit option can be dynamically set."), + %% Data must be big enough for compression to be effective, so that ratio_limit=1 will fail. + Data = <<0:800>>, + Body = zlib:gzip(Data), + {413, _, _} = do_post("/echo/decompress_ratio_limit", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + ok. + +gzip_bomb(Config) -> + doc("Send body compressed with suspiciously large ratio; get 413."), + Body = create_gzip_bomb(), + {413, _, _} = do_post("/echo/normal", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + ok. + +%% RFC 9110. Section 12.5.3. 3. When sent by a server in a response, +%% Accept-Encoding provides information about which content codings are +%% preferred in the content of a subsequent request to the same resource. +%% +%% Set or add gzip +set_accept_encoding_response(Config) -> + doc("Header accept-encoding must be set on valid response command."), + Data = <<"test">>, + Body = zlib:gzip(Data), + {200, Headers, Data} = do_post("/echo/normal", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + {_, <<"gzip">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), + ok. + +set_accept_encoding_header(Config) -> + doc("Header accept-encoding must be set on valid header command."), + Data = <<"test">>, + Body = zlib:gzip(Data), + {200, Headers, Data} = do_post("/header", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + {_, <<"gzip">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), + ok. + +add_accept_encoding_header_valid(Config) -> + doc("Header accept-encoding must be added on valid accept-encoding."), + Data = <<"test">>, + Body = zlib:gzip(Data), + {200, Headers, Data} = do_post("/accept-identity", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + {_, <<"identity, gzip">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), + ok. + +override_accept_encoding_header_invalid(Config) -> + doc("Header accept-encoding must override invalid accept-encoding."), + Data = <<"test">>, + Body = zlib:gzip(Data), + {200, Headers, Data} = do_post("/invalid-header", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + {_, <<"gzip">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), + ok. + +%% RFC 9110. Section 12.5.3. 10.3. If the representation's content coding is +%% one of the content codings listed in the Accept-Encoding field value, then +%% it is acceptable unless it is accompanied by a qvalue of 0. +%% +%% gzip must not have a qvalue of 0 when the handler is used. Set to 1. +override_accept_encoding_excluded(Config) -> + doc("Header accept-encoding must override when explicitly excluded."), + Data = <<"test">>, + Body = zlib:gzip(Data), + {200, Headers, Data} = do_post("/reject-explicit-header", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + {_, <<"identity;q=1, gzip;q=1">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), + ok. + +%% RFC 9110. Section 12.5.3. 10.2. If the representation has no content coding, +%% then it is acceptable by default unless specifically excluded by the +%% Accept-Encoding field stating either "identity;q=0" or "*;q=0" wihout a more +%% specific entry for "identity". +%% +%% *;q=0 will reject codings that are not listed. Specific entry gzip must +%% always be listed when the handler is used. Add gzip. +add_accept_encoding_excluded(Config) -> + doc("Header accept-encoding must added when implicitly excluded."), + Data = <<"test">>, + Body = zlib:gzip(Data), + {200, Headers, Data} = do_post("/reject-implicit-header", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + {_, <<"gzip;q=1, identity;q=1, *;q=0">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), + ok. + +no_override_accept_coding_set_explicit(Config) -> + doc("Confirm that accept-encoding is not overridden when explicitly set."), + Data = <<"test">>, + Body = zlib:gzip(Data), + {200, Headers, Data} = do_post("/accept-explicit-header", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + {_, <<"identity, gzip;q=0.5">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), + ok. + +no_override_accept_coding_set_implicit(Config) -> + doc("Confirm that accept-encoding is not overridden when implicitly set."), + Data = <<"test">>, + Body = zlib:gzip(Data), + {200, Headers, Data} = do_post("/accept-implicit-header", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + {_, <<"identity, *;q=0.5">>} = lists:keyfind(<<"accept-encoding">>, 1, Headers), + ok. + +%% RFC 9110. Section 12.5.3. 10.1. If no Accept-Encoding header field is in the +%% request, any content coding is considered acceptable by the user agent. +%% +%% Don't add anything on error or when that handler is not used. +no_set_accept_encoding(Config) -> + doc("No header accept-encoding on invalid responses."), + Body = <<"test">>, + {400, Headers, _} = do_post("/echo/normal", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + false = lists:keyfind(<<"accept-encoding">>, 1, Headers), + ok. + +no_set_accept_encoding_ignore(Config) -> + doc("Confirm that no accept-encoding is set when stream is ignored."), + Data = <<"test">>, + Body = zlib:gzip(Data), + {200, Headers, Body} = do_post("/echo/decompress_ignore", + [{<<"content-encoding">>, <<"gzip">>}], Body, Config), + false = lists:keyfind(<<"accept-encoding">>, 1, Headers), + ok. diff --git a/test/handlers/decompress_h.erl b/test/handlers/decompress_h.erl new file mode 100644 index 0000000..346200b --- /dev/null +++ b/test/handlers/decompress_h.erl @@ -0,0 +1,68 @@ +%% This module echoes a request body of to test +%% the cowboy_decompress_h stream handler. + +-module(decompress_h). + +-export([init/2]). + +init(Req0, State=[]) -> + case cowboy_req:binding(what, Req0) of + <<"decompress_ignore">> -> + cowboy_req:cast({set_options, #{decompress_ignore => true}}, Req0); + <<"decompress_ratio_limit">> -> + cowboy_req:cast({set_options, #{decompress_ratio_limit => 0.5}}, Req0); + <<"normal">> -> ok + end, + {ok, Body, Req1} = read_body(Req0), + Req = cowboy_req:reply(200, #{}, Body, Req1), + {ok, Req, State}; + +init(Req0, State=header_command) -> + {ok, Body, Req1} = read_body(Req0), + Req2 = cowboy_req:stream_reply(200, #{}, Req1), + Req = cowboy_req:stream_body(Body, fin, Req2), + {ok, Req, State}; + +init(Req0, State=accept_identity) -> + {ok, Body, Req1} = read_body(Req0), + Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity">>}, Body, Req1), + {ok, Req, State}; + +init(Req0, State=invalid_header) -> + {ok, Body, Req1} = read_body(Req0), + Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<";">>}, Body, Req1), + {ok, Req, State}; + +init(Req0, State=reject_explicit_header) -> + {ok, Body, Req1} = read_body(Req0), + Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity, gzip;q=0">>}, + Body, Req1), + {ok, Req, State}; + +init(Req0, State=reject_implicit_header) -> + {ok, Body, Req1} = read_body(Req0), + Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity, *;q=0">>}, + Body, Req1), + {ok, Req, State}; + +init(Req0, State=accept_explicit_header) -> + {ok, Body, Req1} = read_body(Req0), + Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity, gzip;q=0.5">>}, + Body, Req1), + {ok, Req, State}; + +init(Req0, State=accept_implicit_header) -> + {ok, Body, Req1} = read_body(Req0), + Req = cowboy_req:reply(200, #{<<"accept-encoding">> => <<"identity, *;q=0.5">>}, + Body, Req1), + {ok, Req, State}. + +read_body(Req0) -> + {Status, Data, Req} = cowboy_req:read_body(Req0, #{length => 1000}), + do_read_body(Status, Req, Data). + +do_read_body(more, Req0, Acc) -> + {Status, Data, Req} = cowboy_req:read_body(Req0), + do_read_body(Status, Req, << Acc/binary, Data/binary >>); +do_read_body(ok, Req, Acc) -> + {ok, Acc, Req}. |