From 8d6d78575f64055be2d0992d8ccf802d9efa1faa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Sun, 18 Nov 2018 13:21:36 +0100 Subject: Add the chunked option for HTTP/1.1 It allows disabling the chunked transfer-encoding. It can also be disabled on a per-request basis, although it will be ignored for responses that are not streamed. --- doc/src/guide/migrating_from_2.5.asciidoc | 18 ++++--- doc/src/manual/cowboy_http.asciidoc | 10 +++- src/cowboy_http.erl | 24 ++++++++-- test/handlers/set_options_h.erl | 13 ++++++ test/http_SUITE.erl | 78 ++++++++++++++++++++++++++++++- 5 files changed, 129 insertions(+), 14 deletions(-) diff --git a/doc/src/guide/migrating_from_2.5.asciidoc b/doc/src/guide/migrating_from_2.5.asciidoc index 203775d..e3e1b80 100644 --- a/doc/src/guide/migrating_from_2.5.asciidoc +++ b/doc/src/guide/migrating_from_2.5.asciidoc @@ -24,23 +24,26 @@ experimental. data in order to compress them. This is the case for gzip compression. -* Add an `http10_keepalive` option to allow disabling +* Add the `chunked` option to allow disabling chunked + transfer-encoding for HTTP/1.1 connections. + +* Add the `http10_keepalive` option to allow disabling keep-alive for HTTP/1.0 connections. -* Add an `idle_timeout` option for HTTP/2. +* Add the `idle_timeout` option for HTTP/2. -* Add a `sendfile` option to both HTTP/1.1 and HTTP/2. +* Add the `sendfile` option to both HTTP/1.1 and HTTP/2. It allows disabling the sendfile syscall entirely for all connections. It is recommended to disable sendfile when using VirtualBox shared folders. * Add the `rate_limited/2` callback to REST handlers. -* Add a `deflate_opts` option to Websocket handlers that +* Add the `deflate_opts` option to Websocket handlers that allows configuring deflate options for the permessage-deflate extension. -* Add a `charset` option to `cowboy_static`. +* Add the `charset` option to `cowboy_static`. * Add support for the SameSite cookie attribute. @@ -81,8 +84,9 @@ experimental. handlers and Websocket handlers. This can be used to update options on a per-request basis. Allow overriding the `idle_timeout` option for both - HTTP/1.1 and Websocket, and the `cowboy_compress_h` - options for HTTP/1.1 and HTTP/2. + HTTP/1.1 and Websocket, the `cowboy_compress_h` + options for HTTP/1.1 and HTTP/2 and the `chunked` + option for HTTP/1.1. === Bugs fixed diff --git a/doc/src/manual/cowboy_http.asciidoc b/doc/src/manual/cowboy_http.asciidoc index 9ad1b27..3c33ab7 100644 --- a/doc/src/manual/cowboy_http.asciidoc +++ b/doc/src/manual/cowboy_http.asciidoc @@ -17,6 +17,7 @@ as a Ranch protocol. [source,erlang] ---- opts() :: #{ + chunked => boolean(), connection_type => worker | supervisor, env => cowboy_middleware:env(), http10_keepalive => boolean(), @@ -51,6 +52,13 @@ Ranch functions `ranch:get_protocol_options/1` and The default value is given next to the option name: +chunked (true):: + +Whether chunked transfer-encoding is enabled for HTTP/1.1 connections. +Note that a response streamed to the client without the chunked +transfer-encoding and without a content-length header will result +in the connection being closed at the end of the response body. + connection_type (supervisor):: Whether the connection process also acts as a supervisor. @@ -140,7 +148,7 @@ Ordered list of stream handlers that will handle all stream events. == Changelog -* *2.6*: The `http10_keepalive`, `proxy_header` and `sendfile` options were added. +* *2.6*: The `chunked`, `http10_keepalive`, `proxy_header` and `sendfile` options were added. * *2.5*: The `linger_timeout` option was added. * *2.2*: The `max_skip_body_length` option was added. * *2.0*: The `timeout` option was renamed `request_timeout`. diff --git a/src/cowboy_http.erl b/src/cowboy_http.erl index 17c19ed..15a08db 100644 --- a/src/cowboy_http.erl +++ b/src/cowboy_http.erl @@ -25,6 +25,7 @@ -export([system_code_change/4]). -type opts() :: #{ + chunked => boolean(), compress_buffering => boolean(), compress_threshold => non_neg_integer(), connection_type => worker | supervisor, @@ -963,21 +964,28 @@ commands(State0=#state{socket=Socket, transport=Transport, out_state=wait, strea end, commands(State, StreamID, Tail); %% Send response headers and initiate chunked encoding or streaming. -commands(State0=#state{socket=Socket, transport=Transport, streams=Streams0, out_state=OutState}, +commands(State0=#state{socket=Socket, transport=Transport, + opts=Opts, overriden_opts=Override, streams=Streams0, out_state=OutState}, StreamID, [{headers, StatusCode, Headers0}|Tail]) -> %% @todo Same as above (about the last stream in the list). Stream = #stream{version=Version} = lists:keyfind(StreamID, #stream.id, Streams0), Status = cow_http:status_to_integer(StatusCode), ContentLength = maps:get(<<"content-length">>, Headers0, undefined), + %% Chunked transfer-encoding can be disabled on a per-request basis. + Chunked = case Override of + #{chunked := Chunked0} -> Chunked0; + _ -> maps:get(chunked, Opts, true) + end, {State1, Headers1} = case {Status, ContentLength, Version} of {204, _, 'HTTP/1.1'} -> {State0#state{out_state=done}, Headers0}; {304, _, 'HTTP/1.1'} -> {State0#state{out_state=done}, Headers0}; - {_, undefined, 'HTTP/1.1'} -> + {_, undefined, 'HTTP/1.1'} when Chunked -> {State0#state{out_state=chunked}, Headers0#{<<"transfer-encoding">> => <<"chunked">>}}; - %% Close the connection after streaming without content-length to HTTP/1.0 client. - {_, undefined, 'HTTP/1.0'} -> + %% Close the connection after streaming without content-length + %% to all HTTP/1.0 clients and to HTTP/1.1 clients when chunked is disabled. + {_, undefined, _} -> {State0#state{out_state=streaming, last_streamid=StreamID}, Headers0}; %% Stream the response body without chunked transfer-encoding. _ -> @@ -1099,12 +1107,18 @@ commands(State0=#state{ref=Ref, parent=Parent, socket=Socket, transport=Transpor %% Set options dynamically. commands(State0=#state{overriden_opts=Opts}, StreamID, [{set_options, SetOpts}|Tail]) -> - State = case SetOpts of + State1 = case SetOpts of #{idle_timeout := IdleTimeout} -> set_timeout(State0#state{overriden_opts=Opts#{idle_timeout => IdleTimeout}}); _ -> State0 end, + State = case SetOpts of + #{chunked := Chunked} -> + State1#state{overriden_opts=Opts#{chunked => Chunked}}; + _ -> + State1 + end, commands(State, StreamID, Tail); %% Stream shutdown. commands(State, StreamID, [stop|Tail]) -> diff --git a/test/handlers/set_options_h.erl b/test/handlers/set_options_h.erl index a26bb98..1cefe92 100644 --- a/test/handlers/set_options_h.erl +++ b/test/handlers/set_options_h.erl @@ -8,6 +8,19 @@ init(Req, State) -> set_options(cowboy_req:binding(key, Req), Req, State). +set_options(<<"chunked_false">>, Req0, State) -> + %% @todo This should be replaced by a cowboy_req:cast/cowboy_stream:cast. + #{pid := Pid, streamid := StreamID} = Req0, + Pid ! {{Pid, StreamID}, {set_options, #{chunked => false}}}, + Req = cowboy_req:stream_reply(200, Req0), + cowboy_req:stream_body(<<0:8000000>>, fin, Req), + {ok, Req, State}; +set_options(<<"chunked_false_ignored">>, Req0, State) -> + %% @todo This should be replaced by a cowboy_req:cast/cowboy_stream:cast. + #{pid := Pid, streamid := StreamID} = Req0, + Pid ! {{Pid, StreamID}, {set_options, #{chunked => false}}}, + Req = cowboy_req:reply(200, #{}, <<"Hello world!">>, Req0), + {ok, Req, State}; set_options(<<"idle_timeout_short">>, Req0, State) -> %% @todo This should be replaced by a cowboy_req:cast/cowboy_stream:cast. #{pid := Pid, streamid := StreamID} = Req0, diff --git a/test/http_SUITE.erl b/test/http_SUITE.erl index f330d58..1d6c3fc 100644 --- a/test/http_SUITE.erl +++ b/test/http_SUITE.erl @@ -24,6 +24,8 @@ -import(cowboy_test, [raw_open/1]). -import(cowboy_test, [raw_send/2]). -import(cowboy_test, [raw_recv_head/1]). +-import(cowboy_test, [raw_recv/3]). +-import(cowboy_test, [raw_expect_recv/2]). all() -> [{group, clear}]. @@ -33,12 +35,39 @@ init_routes(_) -> [ {"localhost", [ {"/", hello_h, []}, {"/echo/:key", echo_h, []}, + {"/resp/:key[/:arg]", resp_h, []}, {"/set_options/:key", set_options_h, []} ]} ]. +chunked_false(Config) -> + doc("Confirm the option chunked => false disables chunked " + "transfer-encoding for HTTP/1.1 connections."), + {ok, _} = cowboy:start_clear(name(), [{port, 0}], #{ + env => #{dispatch => cowboy_router:compile(init_routes(Config))}, + chunked => false + }), + Port = ranch:get_port(name()), + Request = "GET /resp/stream_reply2/200 HTTP/1.1\r\nhost: localhost\r\n\r\n", + Client = raw_open([{type, tcp}, {port, Port}, {opts, []}|Config]), + ok = raw_send(Client, Request), + Rest = case catch raw_recv_head(Client) of + {'EXIT', _} -> error(closed); + Data -> + %% Cowboy always advertises itself as HTTP/1.1. + {'HTTP/1.1', 200, _, Rest0} = cow_http:parse_status_line(Data), + {Headers, Rest1} = cow_http:parse_headers(Rest0), + false = lists:keyfind(<<"content-length">>, 1, Headers), + false = lists:keyfind(<<"transfer-encoding">>, 1, Headers), + Rest1 + end, + Bits = 8000000 - bit_size(Rest), + raw_expect_recv(Client, <<0:Bits>>), + {error, closed} = raw_recv(Client, 1, 1000), + ok. + http10_keepalive_false(Config) -> - doc("Confirm the option {http10_keepalive, false} disables keep-alive " + doc("Confirm the option http10_keepalive => false disables keep-alive " "completely for HTTP/1.0 connections."), {ok, _} = cowboy:start_clear(name(), [{port, 0}], #{ env => #{dispatch => cowboy_router:compile(init_routes(Config))}, @@ -101,6 +130,53 @@ request_timeout_infinity(Config) -> ok end. +set_options_chunked_false(Config) -> + doc("Confirm the option chunked can be dynamically set to disable " + "chunked transfer-encoding. This results in the closing of the " + "connection after the current request."), + {ok, _} = cowboy:start_clear(name(), [{port, 0}], #{ + env => #{dispatch => cowboy_router:compile(init_routes(Config))}, + chunked => true + }), + Port = ranch:get_port(name()), + Request = "GET /set_options/chunked_false HTTP/1.1\r\nhost: localhost\r\n\r\n", + Client = raw_open([{type, tcp}, {port, Port}, {opts, []}|Config]), + ok = raw_send(Client, Request), + _ = case catch raw_recv_head(Client) of + {'EXIT', _} -> error(closed); + Data -> + %% Cowboy always advertises itself as HTTP/1.1. + {'HTTP/1.1', 200, _, Rest} = cow_http:parse_status_line(Data), + {Headers, <<>>} = cow_http:parse_headers(Rest), + false = lists:keyfind(<<"content-length">>, 1, Headers), + false = lists:keyfind(<<"transfer-encoding">>, 1, Headers) + end, + raw_expect_recv(Client, <<0:8000000>>), + {error, closed} = raw_recv(Client, 1, 1000), + ok. + +set_options_chunked_false_ignored(Config) -> + doc("Confirm the option chunked can be dynamically set to disable " + "chunked transfer-encoding, and that it is ignored if the " + "response is not streamed."), + {ok, _} = cowboy:start_clear(name(), [{port, 0}], #{ + env => #{dispatch => cowboy_router:compile(init_routes(Config))}, + chunked => true + }), + Port = ranch:get_port(name()), + ConnPid = gun_open([{type, tcp}, {protocol, http}, {port, Port}|Config]), + %% We do a first request setting the option but not + %% using chunked transfer-encoding in the response. + StreamRef1 = gun:get(ConnPid, "/set_options/chunked_false_ignored"), + {response, nofin, 200, _} = gun:await(ConnPid, StreamRef1), + {ok, <<"Hello world!">>} = gun:await_body(ConnPid, StreamRef1), + %% We then do a second request to confirm that chunked + %% is not disabled for that second request. + StreamRef2 = gun:get(ConnPid, "/resp/stream_reply2/200"), + {response, nofin, 200, Headers} = gun:await(ConnPid, StreamRef2), + {_, <<"chunked">>} = lists:keyfind(<<"transfer-encoding">>, 1, Headers), + ok. + set_options_idle_timeout(Config) -> doc("Confirm that the idle_timeout option can be dynamically " "set to change how long Cowboy will wait before it closes the connection."), -- cgit v1.2.3