From fbfec873f6026b858c3604d74d88470ce45f4296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Thu, 15 Nov 2018 10:11:36 +0100 Subject: Add a compress_buffering option to cowboy_compress_h Also changes the behavior to disable buffering by default, so that the default works in all cases, including server-sent events. --- src/cowboy_compress_h.erl | 20 +++++++++++----- src/cowboy_http.erl | 1 + src/cowboy_http2.erl | 1 + test/compress_SUITE.erl | 55 ++++++++++++++++++++++++++++++++++++++++++++ test/handlers/compress_h.erl | 8 +++++++ 5 files changed, 79 insertions(+), 6 deletions(-) diff --git a/src/cowboy_compress_h.erl b/src/cowboy_compress_h.erl index ed81db6..bd3df42 100644 --- a/src/cowboy_compress_h.erl +++ b/src/cowboy_compress_h.erl @@ -25,7 +25,8 @@ next :: any(), threshold :: non_neg_integer() | undefined, compress = undefined :: undefined | gzip, - deflate = undefined :: undefined | zlib:zstream() + deflate = undefined :: undefined | zlib:zstream(), + deflate_flush = sync :: none | sync }). -spec init(cowboy_stream:streamid(), cowboy_req:req(), cowboy:opts()) @@ -33,8 +34,14 @@ init(StreamID, Req, Opts) -> State0 = check_req(Req), CompressThreshold = maps:get(compress_threshold, Opts, 300), + DeflateFlush = case maps:get(compress_buffering, Opts, false) of + false -> sync; + true -> none + end, {Commands0, Next} = cowboy_stream:init(StreamID, Req, Opts), - fold(Commands0, State0#state{next=Next, threshold=CompressThreshold}). + fold(Commands0, State0#state{next=Next, + threshold=CompressThreshold, + deflate_flush=DeflateFlush}). -spec data(cowboy_stream:streamid(), cowboy_stream:fin(), cowboy_req:resp_body(), State) -> {cowboy_stream:commands(), State} when State::#state{}. @@ -176,9 +183,10 @@ gzip_headers({headers, Status, Headers0}, State) -> %% includes a checksum at the end of the stream. We have %% to read the file in memory, making this not suitable for %% large files. -gzip_data({data, nofin, Sendfile={sendfile, _, _, _}}, State=#state{deflate=Z}) -> +gzip_data({data, nofin, Sendfile={sendfile, _, _, _}}, + State=#state{deflate=Z, deflate_flush=Flush}) -> {ok, Data0} = read_file(Sendfile), - Data = zlib:deflate(Z, Data0), + Data = zlib:deflate(Z, Data0, Flush), {{data, nofin, Data}, State}; gzip_data({data, fin, Sendfile={sendfile, _, _, _}}, State=#state{deflate=Z}) -> {ok, Data0} = read_file(Sendfile), @@ -186,8 +194,8 @@ gzip_data({data, fin, Sendfile={sendfile, _, _, _}}, State=#state{deflate=Z}) -> zlib:deflateEnd(Z), zlib:close(Z), {{data, fin, Data}, State#state{deflate=undefined}}; -gzip_data({data, nofin, Data0}, State=#state{deflate=Z}) -> - Data = zlib:deflate(Z, Data0), +gzip_data({data, nofin, Data0}, State=#state{deflate=Z, deflate_flush=Flush}) -> + Data = zlib:deflate(Z, Data0, Flush), {{data, nofin, Data}, State}; gzip_data({data, fin, Data0}, State=#state{deflate=Z}) -> Data = zlib:deflate(Z, Data0, finish), diff --git a/src/cowboy_http.erl b/src/cowboy_http.erl index 56a4af8..9a10790 100644 --- a/src/cowboy_http.erl +++ b/src/cowboy_http.erl @@ -25,6 +25,7 @@ -export([system_code_change/4]). -type opts() :: #{ + compress_buffering => boolean(), compress_threshold => non_neg_integer(), connection_type => worker | supervisor, env => cowboy_middleware:env(), diff --git a/src/cowboy_http2.erl b/src/cowboy_http2.erl index fc1a3a0..3397241 100644 --- a/src/cowboy_http2.erl +++ b/src/cowboy_http2.erl @@ -27,6 +27,7 @@ -export([system_code_change/4]). -type opts() :: #{ + compress_buffering => boolean(), compress_threshold => non_neg_integer(), connection_type => worker | supervisor, enable_connect_protocol => boolean(), diff --git a/test/compress_SUITE.erl b/test/compress_SUITE.erl index 74be4f9..052d82d 100644 --- a/test/compress_SUITE.erl +++ b/test/compress_SUITE.erl @@ -18,6 +18,7 @@ -import(ct_helper, [config/2]). -import(ct_helper, [doc/1]). +-import(ct_helper, [name/0]). -import(cowboy_test, [gun_open/1]). %% ct. @@ -144,3 +145,57 @@ gzip_stream_reply_content_encoding(Config) -> {_, <<"compress">>} = lists:keyfind(<<"content-encoding">>, 1, Headers), 100000 = iolist_size(Body), ok. + +opts_compress_buffering_false(Config0) -> + doc("Confirm that the compress_buffering option can be set to false, " + "which is the default."), + Fun = case config(ref, Config0) of + https_compress -> init_https; + h2_compress -> init_http2; + _ -> init_http + end, + Config = cowboy_test:Fun(name(), #{ + env => #{dispatch => init_dispatch(Config0)}, + stream_handlers => [cowboy_compress_h, cowboy_stream_h], + compress_buffering => false + }, Config0), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/stream_reply/delayed", + [{<<"accept-encoding">>, <<"gzip">>}]), + {response, nofin, 200, Headers} = gun:await(ConnPid, Ref), + {_, <<"gzip">>} = lists:keyfind(<<"content-encoding">>, 1, Headers), + Z = zlib:open(), + zlib:inflateInit(Z, 31), + {data, nofin, Data1} = gun:await(ConnPid, Ref, 100), + <<"data: Hello!\r\n\r\n">> = iolist_to_binary(zlib:inflate(Z, Data1)), + timer:sleep(1000), + {data, nofin, Data2} = gun:await(ConnPid, Ref, 100), + <<"data: World!\r\n\r\n">> = iolist_to_binary(zlib:inflate(Z, Data2)), + gun:close(ConnPid), + cowboy:stop_listener(name()). + +opts_compress_buffering_true(Config0) -> + doc("Confirm that the compress_buffering option can be set to true, " + "and that the data received is buffered."), + Fun = case config(ref, Config0) of + https_compress -> init_https; + h2_compress -> init_http2; + _ -> init_http + end, + Config = cowboy_test:Fun(name(), #{ + env => #{dispatch => init_dispatch(Config0)}, + stream_handlers => [cowboy_compress_h, cowboy_stream_h], + compress_buffering => true + }, Config0), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/stream_reply/delayed", + [{<<"accept-encoding">>, <<"gzip">>}]), + {response, nofin, 200, Headers} = gun:await(ConnPid, Ref), + {_, <<"gzip">>} = lists:keyfind(<<"content-encoding">>, 1, Headers), + Z = zlib:open(), + zlib:inflateInit(Z, 31), + %% The data gets buffered because it is too small. + {data, nofin, Data1} = gun:await(ConnPid, Ref, 100), + <<>> = iolist_to_binary(zlib:inflate(Z, Data1)), + gun:close(ConnPid), + cowboy:stop_listener(name()). diff --git a/test/handlers/compress_h.erl b/test/handlers/compress_h.erl index 1509d82..ffea05f 100644 --- a/test/handlers/compress_h.erl +++ b/test/handlers/compress_h.erl @@ -50,6 +50,14 @@ init(Req0, State=stream_reply) -> cowboy_req:stream_body({sendfile, 0, Size, AppFile}, nofin, Req1), cowboy_req:stream_body(Data, nofin, Req1), cowboy_req:stream_body({sendfile, 0, Size, AppFile}, fin, Req1), + Req1; + <<"delayed">> -> + Req1 = cowboy_req:stream_reply(200, Req0), + cowboy_req:stream_body(<<"data: Hello!\r\n\r\n">>, nofin, Req1), + timer:sleep(1000), + cowboy_req:stream_body(<<"data: World!\r\n\r\n">>, nofin, Req1), + timer:sleep(1000), + cowboy_req:stream_body(<<"data: Closing!\r\n\r\n">>, fin, Req1), Req1 end, {ok, Req, State}. -- cgit v1.2.3