From 01f57ad65d7c75fb455f48e354bb3a328c472ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 7 Jan 2013 22:42:16 +0100 Subject: Add optional automatic response body compression This behavior can be enabled with the `compress` protocol option. See the `compress_response` example for more details. All tests are now ran with and without compression for both HTTP and HTTPS. --- Makefile | 2 +- examples/README.md | 3 + examples/compress_response/README.md | 62 ++++++++++++++++++++ examples/compress_response/rebar.config | 4 ++ .../src/compress_response.app.src | 15 +++++ .../compress_response/src/compress_response.erl | 14 +++++ .../src/compress_response_app.erl | 26 +++++++++ .../src/compress_response_sup.erl | 23 ++++++++ examples/compress_response/src/toppage_handler.erl | 31 ++++++++++ examples/compress_response/start.sh | 3 + src/cowboy_protocol.erl | 14 +++-- src/cowboy_req.erl | 68 ++++++++++++++++++---- test/http_SUITE.erl | 52 +++++++++++++++-- 13 files changed, 294 insertions(+), 23 deletions(-) create mode 100644 examples/compress_response/README.md create mode 100644 examples/compress_response/rebar.config create mode 100644 examples/compress_response/src/compress_response.app.src create mode 100644 examples/compress_response/src/compress_response.erl create mode 100644 examples/compress_response/src/compress_response_app.erl create mode 100644 examples/compress_response/src/compress_response_sup.erl create mode 100644 examples/compress_response/src/toppage_handler.erl create mode 100755 examples/compress_response/start.sh diff --git a/Makefile b/Makefile index d1441fc..33ea27a 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ autobahn: build-plt: app @dialyzer --build_plt --output_plt .$(PROJECT).plt \ - --apps kernel stdlib sasl inets crypto public_key ssl deps/ranch + --apps erts kernel stdlib sasl inets crypto public_key ssl deps/ranch dialyze: @dialyzer --src src --plt .$(PROJECT).plt --no_native \ diff --git a/examples/README.md b/examples/README.md index c0e1f41..d50ebc9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -4,6 +4,9 @@ Cowboy Examples * [chunked_hello_world](./examples/chunked_hello_world): demonstrates chunked data transfer with two one-second delays + * [compress_response](./examples/compress_response) + send a response body compressed if the client supports it + * [cookie](./examples/cookie): set cookies from server and client side diff --git a/examples/compress_response/README.md b/examples/compress_response/README.md new file mode 100644 index 0000000..8afbe65 --- /dev/null +++ b/examples/compress_response/README.md @@ -0,0 +1,62 @@ +Cowboy Compress Response +======================== + +To compile this example you need rebar in your PATH. + +Type the following command: +``` +$ rebar get-deps compile +``` + +You can then start the Erlang node with the following command: +``` +./start.sh +``` + +Then point your browser to the indicated URL. + +Example +------- + +``` bash +$ curl -i http://localhost:8080 +HTTP/1.1 200 OK +connection: keep-alive +server: Cowboy +date: Mon, 07 Jan 2013 18:42:29 GMT +content-length: 909 + +A cowboy is an animal herder who tends cattle on ranches in North America, +traditionally on horseback, and often performs a multitude of other ranch- +related tasks. The historic American cowboy of the late 19th century arose +from the vaquero traditions of northern Mexico and became a figure of special +significance and legend. A subtype, called a wrangler, specifically tends the +horses used to work cattle. In addition to ranch work, some cowboys work for +or participate in rodeos. Cowgirls, first defined as such in the late 19th +century, had a less-well documented historical role, but in the modern world +have established the ability to work at virtually identical tasks and obtained +considerable respect for their achievements. There are also cattle handlers +in many other parts of the world, particularly South America and Australia, +who perform work similar to the cowboy in their respective nations. + +$ curl -i --compressed http://localhost:8080 +HTTP/1.1 200 OK +connection: keep-alive +server: Cowboy +date: Mon, 07 Jan 2013 18:42:30 GMT +content-encoding: gzip +content-length: 510 + +A cowboy is an animal herder who tends cattle on ranches in North America, +traditionally on horseback, and often performs a multitude of other ranch- +related tasks. The historic American cowboy of the late 19th century arose +from the vaquero traditions of northern Mexico and became a figure of special +significance and legend. A subtype, called a wrangler, specifically tends the +horses used to work cattle. In addition to ranch work, some cowboys work for +or participate in rodeos. Cowgirls, first defined as such in the late 19th +century, had a less-well documented historical role, but in the modern world +have established the ability to work at virtually identical tasks and obtained +considerable respect for their achievements. There are also cattle handlers +in many other parts of the world, particularly South America and Australia, +who perform work similar to the cowboy in their respective nations. +``` diff --git a/examples/compress_response/rebar.config b/examples/compress_response/rebar.config new file mode 100644 index 0000000..6ad3062 --- /dev/null +++ b/examples/compress_response/rebar.config @@ -0,0 +1,4 @@ +{deps, [ + {cowboy, ".*", + {git, "git://github.com/extend/cowboy.git", "master"}} +]}. diff --git a/examples/compress_response/src/compress_response.app.src b/examples/compress_response/src/compress_response.app.src new file mode 100644 index 0000000..3512084 --- /dev/null +++ b/examples/compress_response/src/compress_response.app.src @@ -0,0 +1,15 @@ +%% Feel free to use, reuse and abuse the code in this file. + +{application, compress_response, [ + {description, "Cowboy Compress Response example."}, + {vsn, "1"}, + {modules, []}, + {registered, []}, + {applications, [ + kernel, + stdlib, + cowboy + ]}, + {mod, {compress_response_app, []}}, + {env, []} +]}. diff --git a/examples/compress_response/src/compress_response.erl b/examples/compress_response/src/compress_response.erl new file mode 100644 index 0000000..ac2636c --- /dev/null +++ b/examples/compress_response/src/compress_response.erl @@ -0,0 +1,14 @@ +%% Feel free to use, reuse and abuse the code in this file. + +-module(compress_response). + +%% API. +-export([start/0]). + +%% API. + +start() -> + ok = application:start(crypto), + ok = application:start(ranch), + ok = application:start(cowboy), + ok = application:start(compress_response). diff --git a/examples/compress_response/src/compress_response_app.erl b/examples/compress_response/src/compress_response_app.erl new file mode 100644 index 0000000..b5f3054 --- /dev/null +++ b/examples/compress_response/src/compress_response_app.erl @@ -0,0 +1,26 @@ +%% Feel free to use, reuse and abuse the code in this file. + +%% @private +-module(compress_response_app). +-behaviour(application). + +%% API. +-export([start/2]). +-export([stop/1]). + +%% API. + +start(_Type, _Args) -> + Dispatch = [ + {'_', [ + {[], toppage_handler, []} + ]} + ], + {ok, _} = cowboy:start_http(http, 100, [{port, 8080}], [ + {compress, true}, + {env, [{dispatch, Dispatch}]} + ]), + compress_response_sup:start_link(). + +stop(_State) -> + ok. diff --git a/examples/compress_response/src/compress_response_sup.erl b/examples/compress_response/src/compress_response_sup.erl new file mode 100644 index 0000000..d1bc312 --- /dev/null +++ b/examples/compress_response/src/compress_response_sup.erl @@ -0,0 +1,23 @@ +%% Feel free to use, reuse and abuse the code in this file. + +%% @private +-module(compress_response_sup). +-behaviour(supervisor). + +%% API. +-export([start_link/0]). + +%% supervisor. +-export([init/1]). + +%% API. + +-spec start_link() -> {ok, pid()}. +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%% supervisor. + +init([]) -> + Procs = [], + {ok, {{one_for_one, 10, 10}, Procs}}. diff --git a/examples/compress_response/src/toppage_handler.erl b/examples/compress_response/src/toppage_handler.erl new file mode 100644 index 0000000..68d0e09 --- /dev/null +++ b/examples/compress_response/src/toppage_handler.erl @@ -0,0 +1,31 @@ +%% Feel free to use, reuse and abuse the code in this file. + +%% @doc Compress response handler. +-module(toppage_handler). + +-export([init/3]). +-export([handle/2]). +-export([terminate/2]). + +init(_Transport, Req, []) -> + {ok, Req, undefined}. + +handle(Req, State) -> + BigBody = +<<"A cowboy is an animal herder who tends cattle on ranches in North America, +traditionally on horseback, and often performs a multitude of other ranch- +related tasks. The historic American cowboy of the late 19th century arose +from the vaquero traditions of northern Mexico and became a figure of special +significance and legend. A subtype, called a wrangler, specifically tends the +horses used to work cattle. In addition to ranch work, some cowboys work for +or participate in rodeos. Cowgirls, first defined as such in the late 19th +century, had a less-well documented historical role, but in the modern world +have established the ability to work at virtually identical tasks and obtained +considerable respect for their achievements. There are also cattle handlers +in many other parts of the world, particularly South America and Australia, +who perform work similar to the cowboy in their respective nations.\n">>, + {ok, Req2} = cowboy_req:reply(200, [], BigBody, Req), + {ok, Req2, State}. + +terminate(_Req, _State) -> + ok. diff --git a/examples/compress_response/start.sh b/examples/compress_response/start.sh new file mode 100755 index 0000000..2e79031 --- /dev/null +++ b/examples/compress_response/start.sh @@ -0,0 +1,3 @@ +#!/bin/sh +erl -pa ebin deps/*/ebin -s compress_response \ + -eval "io:format(\"Point your browser at http://localhost:8080~n\")." diff --git a/src/cowboy_protocol.erl b/src/cowboy_protocol.erl index 48c0b00..0e9982b 100644 --- a/src/cowboy_protocol.erl +++ b/src/cowboy_protocol.erl @@ -17,6 +17,8 @@ %% %% The available options are: %%
+%%
compress
Whether to automatically compress the response +%% body when the conditions are met. Disabled by default.
%%
env
The environment passed and optionally modified %% by middlewares.
%%
max_empty_lines
Max number of empty lines before a request. @@ -64,6 +66,7 @@ socket :: inet:socket(), transport :: module(), middlewares :: [module()], + compress :: boolean(), env :: cowboy_middleware:env(), onrequest :: undefined | onrequest_fun(), onresponse = undefined :: undefined | onresponse_fun(), @@ -99,6 +102,7 @@ get_value(Key, Opts, Default) -> %% @private -spec init(pid(), inet:socket(), module(), any()) -> ok. init(ListenerPid, Socket, Transport, Opts) -> + Compress = get_value(compress, Opts, false), MaxEmptyLines = get_value(max_empty_lines, Opts, 5), MaxHeaderNameLength = get_value(max_header_name_length, Opts, 64), MaxHeaderValueLength = get_value(max_header_value_length, Opts, 4096), @@ -112,7 +116,7 @@ init(ListenerPid, Socket, Transport, Opts) -> Timeout = get_value(timeout, Opts, 5000), ok = ranch:accept_ack(ListenerPid), wait_request(<<>>, #state{socket=Socket, transport=Transport, - middlewares=Middlewares, env=Env, + middlewares=Middlewares, compress=Compress, env=Env, max_empty_lines=MaxEmptyLines, max_keepalive=MaxKeepalive, max_request_line_length=MaxRequestLineLength, max_header_name_length=MaxHeaderNameLength, @@ -457,11 +461,11 @@ parse_host(<< C, Rest/bits >>, Acc) -> request(Buffer, State=#state{socket=Socket, transport=Transport, req_keepalive=ReqKeepalive, max_keepalive=MaxKeepalive, - onresponse=OnResponse}, + compress=Compress, onresponse=OnResponse}, Method, Path, Query, Fragment, Version, Headers, Host, Port) -> Req = cowboy_req:new(Socket, Transport, Method, Path, Query, Fragment, Version, Headers, Host, Port, Buffer, ReqKeepalive < MaxKeepalive, - OnResponse), + Compress, OnResponse), onrequest(Req, State). %% Call the global onrequest callback. The callback can send a reply, @@ -546,13 +550,13 @@ error_terminate(Code, Req, State) -> %% Only send an error reply if there is no resp_sent message. -spec error_terminate(cowboy_http:status(), #state{}) -> ok. error_terminate(Code, State=#state{socket=Socket, transport=Transport, - onresponse=OnResponse}) -> + compress=Compress, onresponse=OnResponse}) -> receive {cowboy_req, resp_sent} -> ok after 0 -> _ = cowboy_req:reply(Code, cowboy_req:new(Socket, Transport, <<"GET">>, <<>>, <<>>, <<>>, {1, 1}, [], <<>>, undefined, - <<>>, false, OnResponse)), + <<>>, false, Compress, OnResponse)), ok end, terminate(State). diff --git a/src/cowboy_req.erl b/src/cowboy_req.erl index dab9410..973cc65 100644 --- a/src/cowboy_req.erl +++ b/src/cowboy_req.erl @@ -42,7 +42,7 @@ -module(cowboy_req). %% Request API. --export([new/13]). +-export([new/14]). -export([method/1]). -export([version/1]). -export([peer/1]). @@ -156,6 +156,7 @@ buffer = <<>> :: binary(), %% Response. + resp_compress = false :: boolean(), resp_state = waiting :: locked | waiting | chunks | done, resp_headers = [] :: cowboy_http:headers(), resp_body = <<>> :: iodata() | resp_body_fun() @@ -179,16 +180,16 @@ %% in an optimized way and add the parsed value to p_headers' cache. -spec new(inet:socket(), module(), binary(), binary(), binary(), binary(), cowboy_http:version(), cowboy_http:headers(), binary(), - inet:port_number() | undefined, binary(), boolean(), + inet:port_number() | undefined, binary(), boolean(), boolean(), undefined | cowboy_protocol:onresponse_fun()) -> req(). new(Socket, Transport, Method, Path, Query, Fragment, Version, Headers, Host, Port, Buffer, CanKeepalive, - OnResponse) -> + Compress, OnResponse) -> Req = #http_req{socket=Socket, transport=Transport, pid=self(), method=Method, path=Path, qs=Query, fragment=Fragment, version=Version, headers=Headers, host=Host, port=Port, buffer=Buffer, - onresponse=OnResponse}, + resp_compress=Compress, onresponse=OnResponse}, case CanKeepalive and (Version =:= {1, 1}) of false -> Req#http_req{connection=close}; @@ -892,7 +893,8 @@ reply(Status, Headers, Req=#http_req{resp_body=Body}) -> reply(Status, Headers, Body, Req=#http_req{ socket=Socket, transport=Transport, version=Version, connection=Connection, - method=Method, resp_state=waiting, resp_headers=RespHeaders}) -> + method=Method, resp_compress=Compress, + resp_state=waiting, resp_headers=RespHeaders}) -> RespConn = response_connection(Headers, Connection), HTTP11Headers = case Version of {1, 1} -> [{<<"connection">>, atom_to_connection(Connection)}]; @@ -922,18 +924,60 @@ reply(Status, Headers, Body, Req=#http_req{ BodyFun(Socket, Transport); true -> ok end; + _ when Compress -> + Req2 = reply_may_compress(Status, Headers, Body, Req, + RespHeaders, HTTP11Headers, Method); _ -> - {_, Req2} = response(Status, Headers, RespHeaders, [ - {<<"content-length">>, integer_to_list(iolist_size(Body))}, - {<<"date">>, cowboy_clock:rfc1123()}, - {<<"server">>, <<"Cowboy">>} - |HTTP11Headers], - case Method of <<"HEAD">> -> <<>>; _ -> Body end, - Req) + Req2 = reply_no_compress(Status, Headers, Body, Req, + RespHeaders, HTTP11Headers, Method, iolist_size(Body)) end, {ok, Req2#http_req{connection=RespConn, resp_state=done, resp_headers=[], resp_body= <<>>}}. +reply_may_compress(Status, Headers, Body, Req, + RespHeaders, HTTP11Headers, Method) -> + BodySize = iolist_size(Body), + {ok, Encodings, Req2} + = cowboy_req:parse_header(<<"accept-encoding">>, Req), + CanGzip = (BodySize > 300) + andalso (false =:= lists:keyfind(<<"content-encoding">>, + 1, Headers)) + andalso (false =:= lists:keyfind(<<"content-encoding">>, + 1, RespHeaders)) + andalso (false =:= lists:keyfind(<<"transfer-encoding">>, + 1, Headers)) + andalso (false =:= lists:keyfind(<<"transfer-encoding">>, + 1, RespHeaders)) + andalso (Encodings =/= undefined) + andalso (false =/= lists:keyfind(<<"gzip">>, 1, Encodings)), + case CanGzip of + true -> + GzBody = zlib:gzip(Body), + {_, Req3} = response(Status, Headers, RespHeaders, [ + {<<"content-length">>, integer_to_list(byte_size(GzBody))}, + {<<"content-encoding">>, <<"gzip">>}, + {<<"date">>, cowboy_clock:rfc1123()}, + {<<"server">>, <<"Cowboy">>} + |HTTP11Headers], + case Method of <<"HEAD">> -> <<>>; _ -> GzBody end, + Req2), + Req3; + false -> + reply_no_compress(Status, Headers, Body, Req, + RespHeaders, HTTP11Headers, Method, BodySize) + end. + +reply_no_compress(Status, Headers, Body, Req, + RespHeaders, HTTP11Headers, Method, BodySize) -> + {_, Req2} = response(Status, Headers, RespHeaders, [ + {<<"content-length">>, integer_to_list(BodySize)}, + {<<"date">>, cowboy_clock:rfc1123()}, + {<<"server">>, <<"Cowboy">>} + |HTTP11Headers], + case Method of <<"HEAD">> -> <<>>; _ -> Body end, + Req), + Req2. + %% @equiv chunked_reply(Status, [], Req) -spec chunked_reply(cowboy_http:status(), Req) -> {ok, Req} when Req::req(). chunked_reply(Status, Req) -> diff --git a/test/http_SUITE.erl b/test/http_SUITE.erl index 607178f..cd4e2dc 100644 --- a/test/http_SUITE.erl +++ b/test/http_SUITE.erl @@ -78,7 +78,14 @@ %% ct. all() -> - [{group, http}, {group, https}, {group, onrequest}, {group, onresponse}]. + [ + {group, http}, + {group, https}, + {group, http_compress}, + {group, https_compress}, + {group, onrequest}, + {group, onresponse} + ]. groups() -> Tests = [ @@ -130,6 +137,8 @@ groups() -> [ {http, [], Tests}, {https, [], Tests}, + {http_compress, [], Tests}, + {https_compress, [], Tests}, {onrequest, [], [ onrequest, onrequest_reply @@ -185,9 +194,42 @@ init_per_group(https, Config) -> {ok, Client} = cowboy_client:init(Opts), [{scheme, <<"https">>}, {port, Port}, {opts, Opts}, {transport, Transport}, {client, Client}|Config1]; -init_per_group(onrequest, Config) -> +init_per_group(http_compress, Config) -> Port = 33082, Transport = ranch_tcp, + Config1 = init_static_dir(Config), + {ok, _} = cowboy:start_http(http_compress, 100, [{port, Port}], [ + {compress, true}, + {env, [{dispatch, init_dispatch(Config1)}]}, + {max_keepalive, 50}, + {timeout, 500} + ]), + {ok, Client} = cowboy_client:init([]), + [{scheme, <<"http">>}, {port, Port}, {opts, []}, + {transport, Transport}, {client, Client}|Config1]; +init_per_group(https_compress, Config) -> + Port = 33083, + Transport = ranch_ssl, + Opts = [ + {certfile, ?config(data_dir, Config) ++ "cert.pem"}, + {keyfile, ?config(data_dir, Config) ++ "key.pem"}, + {password, "cowboy"} + ], + Config1 = init_static_dir(Config), + application:start(public_key), + application:start(ssl), + {ok, _} = cowboy:start_https(https_compress, 100, Opts ++ [{port, Port}], [ + {compress, true}, + {env, [{dispatch, init_dispatch(Config1)}]}, + {max_keepalive, 50}, + {timeout, 500} + ]), + {ok, Client} = cowboy_client:init(Opts), + [{scheme, <<"https">>}, {port, Port}, {opts, Opts}, + {transport, Transport}, {client, Client}|Config1]; +init_per_group(onrequest, Config) -> + Port = 33084, + Transport = ranch_tcp, {ok, _} = cowboy:start_http(onrequest, 100, [{port, Port}], [ {env, [{dispatch, init_dispatch(Config)}]}, {max_keepalive, 50}, @@ -198,7 +240,7 @@ init_per_group(onrequest, Config) -> [{scheme, <<"http">>}, {port, Port}, {opts, []}, {transport, Transport}, {client, Client}|Config]; init_per_group(onresponse, Config) -> - Port = 33083, + Port = 33085, Transport = ranch_tcp, {ok, _} = cowboy:start_http(onresponse, 100, [{port, Port}], [ {env, [{dispatch, init_dispatch(Config)}]}, @@ -210,13 +252,13 @@ init_per_group(onresponse, Config) -> [{scheme, <<"http">>}, {port, Port}, {opts, []}, {transport, Transport}, {client, Client}|Config]. -end_per_group(https, Config) -> +end_per_group(Group, Config) when Group =:= https; Group =:= https_compress -> cowboy:stop_listener(https), application:stop(ssl), application:stop(public_key), end_static_dir(Config), ok; -end_per_group(http, Config) -> +end_per_group(Group, Config) when Group =:= http; Group =:= http_compress -> cowboy:stop_listener(http), end_static_dir(Config); end_per_group(Name, _) -> -- cgit v1.2.3