From d7b7580b3913c17b404319cc4c153748d5e59194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Fri, 9 Nov 2018 17:42:37 +0100 Subject: Add sendfile support to cowboy_req:stream_body It is now possible to stream one or more sendfile tuples. A simple example of what can now be done would be for example to build a tar file on the fly using the sendfile syscall for sending the files, or to support Range requests with more than one range with the sendfile syscall. When using cowboy_compress_h unfortunately we have to read the file in order to send it. More options will be added at a later time to make sure users don't read too much into memory. This is a new feature however so existing code is not affected. Also rework cowboy_http's data sending to be flatter. --- test/compress_SUITE.erl | 18 ++++++++++++++++++ test/handlers/compress_h.erl | 25 ++++++++++++++++++++++++- test/handlers/resp_h.erl | 17 +++++++++++++++++ test/req_SUITE.erl | 33 ++++++++++++++++++++++++++++----- 4 files changed, 87 insertions(+), 6 deletions(-) (limited to 'test') diff --git a/test/compress_SUITE.erl b/test/compress_SUITE.erl index 36f8865..06b3950 100644 --- a/test/compress_SUITE.erl +++ b/test/compress_SUITE.erl @@ -121,6 +121,24 @@ gzip_stream_reply(Config) -> _ = zlib:gunzip(GzBody), ok. +gzip_stream_reply_sendfile(Config) -> + doc("Stream reply using sendfile for some chunks; get a gzipped response."), + {200, Headers, GzBody} = do_get("/stream_reply/sendfile", + [{<<"accept-encoding">>, <<"gzip">>}], Config), + {_, <<"gzip">>} = lists:keyfind(<<"content-encoding">>, 1, Headers), + file:write_file("/tmp/test.gz", GzBody), + _ = zlib:gunzip(GzBody), + ok. + +gzip_stream_reply_sendfile_fin(Config) -> + doc("Stream reply using sendfile for some chunks; get a gzipped response."), + {200, Headers, GzBody} = do_get("/stream_reply/sendfile_fin", + [{<<"accept-encoding">>, <<"gzip">>}], Config), + {_, <<"gzip">>} = lists:keyfind(<<"content-encoding">>, 1, Headers), + file:write_file("/tmp/test.gz", GzBody), + _ = zlib:gunzip(GzBody), + ok. + gzip_stream_reply_content_encoding(Config) -> doc("Stream reply with content-encoding header; get an uncompressed response."), {200, Headers, Body} = do_get("/stream_reply/content-encoding", diff --git a/test/handlers/compress_h.erl b/test/handlers/compress_h.erl index 4b4dbe7..1509d82 100644 --- a/test/handlers/compress_h.erl +++ b/test/handlers/compress_h.erl @@ -27,7 +27,30 @@ init(Req0, State=stream_reply) -> <<"large">> -> stream_reply(#{}, Req0); <<"content-encoding">> -> - stream_reply(#{<<"content-encoding">> => <<"compress">>}, Req0) + stream_reply(#{<<"content-encoding">> => <<"compress">>}, Req0); + <<"sendfile">> -> + Data = lists:duplicate(10000, $a), + AppFile = code:where_is_file("cowboy.app"), + Size = filelib:file_size(AppFile), + Req1 = cowboy_req:stream_reply(200, Req0), + %% We send a few files interspersed into other data. + cowboy_req:stream_body(Data, nofin, Req1), + cowboy_req:stream_body({sendfile, 0, Size, AppFile}, nofin, Req1), + cowboy_req:stream_body(Data, nofin, Req1), + cowboy_req:stream_body({sendfile, 0, Size, AppFile}, nofin, Req1), + cowboy_req:stream_body(Data, fin, Req1), + Req1; + <<"sendfile_fin">> -> + Data = lists:duplicate(10000, $a), + AppFile = code:where_is_file("cowboy.app"), + Size = filelib:file_size(AppFile), + Req1 = cowboy_req:stream_reply(200, Req0), + %% We send a few files interspersed into other data. + cowboy_req:stream_body(Data, nofin, Req1), + cowboy_req:stream_body({sendfile, 0, Size, AppFile}, nofin, Req1), + cowboy_req:stream_body(Data, nofin, Req1), + cowboy_req:stream_body({sendfile, 0, Size, AppFile}, fin, Req1), + Req1 end, {ok, Req, State}. diff --git a/test/handlers/resp_h.erl b/test/handlers/resp_h.erl index bfcb95c..7a80e75 100644 --- a/test/handlers/resp_h.erl +++ b/test/handlers/resp_h.erl @@ -219,6 +219,23 @@ do(<<"stream_body">>, Req0, Opts) -> Req = cowboy_req:stream_reply(200, Req0), cowboy_req:stream_body(<<"Hello world!">>, nofin, Req), {ok, Req, Opts}; + <<"sendfile">> -> + AppFile = code:where_is_file("cowboy.app"), + AppSize = filelib:file_size(AppFile), + Req = cowboy_req:stream_reply(200, Req0), + cowboy_req:stream_body(<<"Hello ">>, nofin, Req), + cowboy_req:stream_body({sendfile, 0, AppSize, AppFile}, nofin, Req), + cowboy_req:stream_body(<<" interspersed ">>, nofin, Req), + cowboy_req:stream_body({sendfile, 0, AppSize, AppFile}, nofin, Req), + cowboy_req:stream_body(<<" world!">>, fin, Req), + {ok, Req, Opts}; + <<"sendfile_fin">> -> + AppFile = code:where_is_file("cowboy.app"), + AppSize = filelib:file_size(AppFile), + Req = cowboy_req:stream_reply(200, Req0), + cowboy_req:stream_body(<<"Hello! ">>, nofin, Req), + cowboy_req:stream_body({sendfile, 0, AppSize, AppFile}, fin, Req), + {ok, Req, Opts}; _ -> %% Call stream_body without initiating streaming. cowboy_req:stream_body(<<0:800000>>, fin, Req0), diff --git a/test/req_SUITE.erl b/test/req_SUITE.erl index 6ca4521..9520699 100644 --- a/test/req_SUITE.erl +++ b/test/req_SUITE.erl @@ -885,21 +885,44 @@ stream_reply3(Config) -> {500, _, _} = do_get("/resp/stream_reply3/error", Config), ok. -stream_body_multiple(Config) -> - doc("Streamed body via multiple calls."), - {200, _, <<"Hello world!">>} = do_get("/resp/stream_body/multiple", Config), - ok. - stream_body_fin0(Config) -> doc("Streamed body with last chunk of size 0."), {200, _, <<"Hello world!">>} = do_get("/resp/stream_body/fin0", Config), ok. +stream_body_multiple(Config) -> + doc("Streamed body via multiple calls."), + {200, _, <<"Hello world!">>} = do_get("/resp/stream_body/multiple", Config), + ok. + stream_body_nofin(Config) -> doc("Unfinished streamed body."), {200, _, <<"Hello world!">>} = do_get("/resp/stream_body/nofin", Config), ok. +stream_body_sendfile(Config) -> + doc("Streamed body via multiple calls, including sendfile calls."), + {ok, AppFile} = file:read_file(code:where_is_file("cowboy.app")), + ExpectedBody = iolist_to_binary([ + <<"Hello ">>, + AppFile, + <<" interspersed ">>, + AppFile, + <<" world!">> + ]), + {200, _, ExpectedBody} = do_get("/resp/stream_body/sendfile", Config), + ok. + +stream_body_sendfile_fin(Config) -> + doc("Streamed body via multiple calls, including a sendfile final call."), + {ok, AppFile} = file:read_file(code:where_is_file("cowboy.app")), + ExpectedBody = iolist_to_binary([ + <<"Hello! ">>, + AppFile + ]), + {200, _, ExpectedBody} = do_get("/resp/stream_body/sendfile_fin", Config), + ok. + stream_body_content_length_multiple(Config) -> doc("Streamed body via multiple calls."), {200, _, <<"Hello world!">>} = do_get("/resp/stream_body_content_length/multiple", Config), -- cgit v1.2.3