From 29043aa7b4d11e377bc76d453f592ea5a6df1f43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Wed, 7 Nov 2018 18:55:06 +0100 Subject: Add support for range requests (RFC7233) in cowboy_rest This is currently undocumented but is planned to be documented in the next version. --- src/cowboy_req.erl | 3 + src/cowboy_rest.erl | 168 +++++++++++++- test/handlers/if_range_h.erl | 46 ++++ test/handlers/provide_range_callback_h.erl | 66 ++++++ test/handlers/range_satisfiable_h.erl | 39 ++++ test/handlers/ranges_provided_h.erl | 30 +++ test/handlers/stop_handler_h.erl | 21 +- test/handlers/switch_handler_h.erl | 21 +- test/rest_handler_SUITE.erl | 360 ++++++++++++++++++++++++++++- 9 files changed, 745 insertions(+), 9 deletions(-) create mode 100644 test/handlers/if_range_h.erl create mode 100644 test/handlers/provide_range_callback_h.erl create mode 100644 test/handlers/range_satisfiable_h.erl create mode 100644 test/handlers/ranges_provided_h.erl diff --git a/src/cowboy_req.erl b/src/cowboy_req.erl index 54f2613..46bc62a 100644 --- a/src/cowboy_req.erl +++ b/src/cowboy_req.erl @@ -158,6 +158,8 @@ media_type => {binary(), binary(), [{binary(), binary()}]}, language => binary() | undefined, charset => binary() | undefined, + range => {binary(), binary() + | [{non_neg_integer(), non_neg_integer() | infinity} | neg_integer()]}, websocket_version => 7 | 8 | 13 }. -export_type([req/0]). @@ -429,6 +431,7 @@ parse_header_fun(<<"expect">>) -> fun cow_http_hd:parse_expect/1; parse_header_fun(<<"if-match">>) -> fun cow_http_hd:parse_if_match/1; parse_header_fun(<<"if-modified-since">>) -> fun cow_http_hd:parse_if_modified_since/1; parse_header_fun(<<"if-none-match">>) -> fun cow_http_hd:parse_if_none_match/1; +parse_header_fun(<<"if-range">>) -> fun cow_http_hd:parse_if_range/1; parse_header_fun(<<"if-unmodified-since">>) -> fun cow_http_hd:parse_if_unmodified_since/1; parse_header_fun(<<"range">>) -> fun cow_http_hd:parse_range/1; parse_header_fun(<<"sec-websocket-extensions">>) -> fun cow_http_hd:parse_sec_websocket_extensions/1; diff --git a/src/cowboy_rest.erl b/src/cowboy_rest.erl index 5cfa007..8a574cd 100644 --- a/src/cowboy_rest.erl +++ b/src/cowboy_rest.erl @@ -180,6 +180,20 @@ when Req::cowboy_req:req(), State::any(). -optional_callbacks([previously_existed/2]). +-callback range_satisfiable(Req, State) + -> {boolean() | {false, non_neg_integer() | iodata()}, Req, State} + | {stop, Req, State} + | {switch_handler(), Req, State} + when Req::cowboy_req:req(), State::any(). +-optional_callbacks([range_satisfiable/2]). + +-callback ranges_provided(Req, State) + -> {[{binary(), atom()}], Req, State} + | {stop, Req, State} + | {switch_handler(), Req, State} + when Req::cowboy_req:req(), State::any(). +-optional_callbacks([ranges_provided/2]). + -callback rate_limited(Req, State) -> {{true, non_neg_integer() | calendar:datetime()} | false, Req, State} | {stop, Req, State} @@ -255,6 +269,9 @@ charsets_p = undefined :: undefined | [binary()], charset_a :: undefined | binary(), + %% Range units. + ranges_a = [] :: [{binary(), atom()}], + %% Whether the resource exists. exists = false :: boolean(), @@ -733,11 +750,28 @@ set_content_type_build_params([{Attr, Value}|Tail], Acc) -> %% @todo Don't forget to set the Content-Encoding header when we reply a body %% and the found encoding is something other than identity. encodings_provided(Req, State) -> - variances(Req, State). + ranges_provided(Req, State). not_acceptable(Req, State) -> respond(Req, State, 406). +ranges_provided(Req, State) -> + case call(Req, State, ranges_provided) of + no_call -> + variances(Req, State); + {stop, Req2, State2} -> + terminate(Req2, State2); + {Switch, Req2, State2} when element(1, Switch) =:= switch_handler -> + switch_handler(Switch, Req2, State2); + {[], Req2, State2} -> + Req3 = cowboy_req:set_resp_header(<<"accept-ranges">>, <<"none">>, Req2), + variances(Req3, State2#state{ranges_a=[]}); + {RP, Req2, State2} -> + <<", ", AcceptRanges/binary>> = <<<<", ", R/binary>> || {R, _} <- RP>>, + Req3 = cowboy_req:set_resp_header(<<"accept-ranges">>, AcceptRanges, Req2), + variances(Req3, State2#state{ranges_a=RP}) + end. + %% variances/2 should return a list of headers that will be added %% to the Vary response header. The Accept, Accept-Language, %% Accept-Charset and Accept-Encoding headers do not need to be @@ -1124,11 +1158,141 @@ set_resp_body_last_modified(Req, State) -> set_resp_body_expires(Req, State) -> try set_resp_expires(Req, State) of {Req2, State2} -> - set_resp_body(Req2, State2) + if_range(Req2, State2) catch Class:Reason -> error_terminate(Req, State, Class, Reason) end. +%% When both the if-range and range headers are set, we perform +%% a strong comparison. If it fails, we send a full response. +if_range(Req=#{headers := #{<<"if-range">> := _, <<"range">> := _}}, + State=#state{etag=Etag}) -> + try cowboy_req:parse_header(<<"if-range">>, Req) of + %% Strong etag comparison is an exact match with the generate_etag result. + Etag={strong, _} -> + range(Req, State); + %% We cannot do a strong date comparison because we have + %% no way of knowing whether the representation changed + %% twice during the second covered by the presented + %% validator. (RFC7232 2.2.2) + _ -> + set_resp_body(Req, State) + catch _:_ -> + set_resp_body(Req, State) + end; +if_range(Req, State) -> + range(Req, State). + +range(Req, State=#state{ranges_a=[]}) -> + set_resp_body(Req, State); +range(Req, State) -> + try cowboy_req:parse_header(<<"range">>, Req) of + undefined -> + set_resp_body(Req, State); + %% @todo Maybe change parse_header to return <<"bytes">> in 3.0. + {bytes, BytesRange} -> + choose_range(Req, State, {<<"bytes">>, BytesRange}); + Range -> + choose_range(Req, State, Range) + catch _:_ -> + %% We send a 416 response back when we can't parse the + %% range header at all. I'm not sure this is the right + %% way to go but at least this can help clients identify + %% what went wrong when their range requests never work. + range_not_satisfiable(Req, State, undefined) + end. + +choose_range(Req, State=#state{ranges_a=RangesAccepted}, Range={RangeUnit, _}) -> + case lists:keyfind(RangeUnit, 1, RangesAccepted) of + {_, Callback} -> + %% We pass the selected range onward in the Req. + range_satisfiable(Req#{range => Range}, State, Callback); + false -> + set_resp_body(Req, State) + end. + +range_satisfiable(Req, State, Callback) -> + case call(Req, State, range_satisfiable) of + no_call -> + set_ranged_body(Req, State, Callback); + {stop, Req2, State2} -> + terminate(Req2, State2); + {Switch, Req2, State2} when element(1, Switch) =:= switch_handler -> + switch_handler(Switch, Req2, State2); + {true, Req2, State2} -> + set_ranged_body(Req2, State2, Callback); + {false, Req2, State2} -> + range_not_satisfiable(Req2, State2, undefined); + {{false, Int}, Req2, State2} when is_integer(Int) -> + range_not_satisfiable(Req2, State2, [<<"*/">>, integer_to_binary(Int)]); + {{false, Iodata}, Req2, State2} when is_binary(Iodata); is_list(Iodata) -> + range_not_satisfiable(Req2, State2, Iodata) + end. + +%% We send the content-range header when we can on error. +range_not_satisfiable(Req, State, undefined) -> + respond(Req, State, 416); +range_not_satisfiable(Req0=#{range := {RangeUnit, _}}, State, RangeData) -> + Req = cowboy_req:set_resp_header(<<"content-range">>, + [RangeUnit, $\s, RangeData], Req0), + respond(Req, State, 416). + +set_ranged_body(Req, State=#state{handler=Handler}, Callback) -> + try case call(Req, State, Callback) of + {stop, Req2, State2} -> + terminate(Req2, State2); + {Switch, Req2, State2} when element(1, Switch) =:= switch_handler -> + switch_handler(Switch, Req2, State2); + %% When we receive a single range, we send it directly. + {[OneRange], Req2, State2} -> + {ContentRange, Body} = prepare_range(Req2, OneRange), + Req3 = cowboy_req:set_resp_header(<<"content-range">>, ContentRange, Req2), + Req4 = cowboy_req:set_resp_body(Body, Req3), + respond(Req4, State2, 206); + %% When we receive multiple ranges we have to send them as multipart/byteranges. + %% This also applies to non-bytes units. (RFC7233 A) If users don't want to use + %% this for non-bytes units they can always return a single range with a binary + %% content-range information. + {Ranges, Req2, State2} when length(Ranges) > 1 -> + set_multipart_ranged_body(Req2, State2, Ranges) + end catch Class:{case_clause, no_call} -> + error_terminate(Req, State, Class, {error, {missing_callback, {Handler, Callback, 2}}, + 'A callback specified in ranges_accepted/2 is not exported.'}) + end. + +set_multipart_ranged_body(Req, State, [FirstRange|MoreRanges]) -> + Boundary = cow_multipart:boundary(), + ContentType = cowboy_req:resp_header(<<"content-type">>, Req), + {FirstContentRange, FirstPartBody} = prepare_range(Req, FirstRange), + FirstPartHead = cow_multipart:first_part(Boundary, [ + {<<"content-type">>, ContentType}, + {<<"content-range">>, FirstContentRange} + ]), + MoreParts = [begin + {NextContentRange, NextPartBody} = prepare_range(Req, NextRange), + NextPartHead = cow_multipart:part(Boundary, [ + {<<"content-type">>, ContentType}, + {<<"content-range">>, NextContentRange} + ]), + [NextPartHead, NextPartBody] + end || NextRange <- MoreRanges], + Body = [FirstPartHead, FirstPartBody, MoreParts, cow_multipart:close(Boundary)], + Req2 = cowboy_req:set_resp_header(<<"content-type">>, + [<<"multipart/byteranges; boundary=">>, Boundary], Req), + Req3 = cowboy_req:set_resp_body(Body, Req2), + respond(Req3, State, 206). + +prepare_range(#{range := {RangeUnit, _}}, {{From, To, Total0}, Body}) -> + Total = case Total0 of + '*' -> <<"*">>; + _ -> integer_to_binary(Total0) + end, + ContentRange = [RangeUnit, $\s, integer_to_binary(From), + $-, integer_to_binary(To), $/, Total], + {ContentRange, Body}; +prepare_range(#{range := {RangeUnit, _}}, {RangeData, Body}) -> + {[RangeUnit, $\s, RangeData], Body}. + %% Set the response headers and call the callback found using %% content_types_provided/2 to obtain the request body and add %% it to the response. diff --git a/test/handlers/if_range_h.erl b/test/handlers/if_range_h.erl new file mode 100644 index 0000000..868edbf --- /dev/null +++ b/test/handlers/if_range_h.erl @@ -0,0 +1,46 @@ +%% This module defines the ranges_provided callback +%% and a generate_etag callback that returns something +%% different depending on query string. It also defines +%% a last_modified callback that must be ignored when a +%% date is provided in if_range. + +-module(if_range_h). + +-export([init/2]). +-export([content_types_provided/2]). +-export([ranges_provided/2]). +-export([generate_etag/2]). +-export([last_modified/2]). +-export([get_text_plain/2]). +-export([get_text_plain_bytes/2]). + +init(Req, State) -> + {cowboy_rest, Req, State}. + +content_types_provided(Req, State) -> + {[{{<<"text">>, <<"plain">>, []}, get_text_plain}], Req, State}. + +%% Simulate the callback being missing. +ranges_provided(#{qs := <<"missing-ranges_provided">>}, _) -> + no_call; +ranges_provided(Req=#{qs := <<"empty-ranges_provided">>}, State) -> + {[], Req, State}; +ranges_provided(Req, State) -> + {[{<<"bytes">>, get_text_plain_bytes}], Req, State}. + +generate_etag(Req=#{qs := <<"weak-etag">>}, State) -> + {{weak, <<"weak-no-match">>}, Req, State}; +generate_etag(Req, State) -> + {{strong, <<"strong-and-match">>}, Req, State}. + +last_modified(Req, State) -> + {{{2222, 2, 22}, {11, 11, 11}}, Req, State}. + +get_text_plain(Req, State) -> + {<<"This is REST!">>, Req, State}. + +get_text_plain_bytes(Req, State) -> + %% We send everything in one part, since we are not testing + %% this callback specifically. + Body = <<"This is ranged REST!">>, + {[{{0, byte_size(Body) - 1, byte_size(Body)}, Body}], Req, State}. diff --git a/test/handlers/provide_range_callback_h.erl b/test/handlers/provide_range_callback_h.erl new file mode 100644 index 0000000..f14a544 --- /dev/null +++ b/test/handlers/provide_range_callback_h.erl @@ -0,0 +1,66 @@ +%% This module defines the range_satisfiable callback +%% and return something different depending on query string. + +-module(provide_range_callback_h). + +-export([init/2]). +-export([content_types_provided/2]). +-export([ranges_provided/2]). +-export([expires/2]). +-export([generate_etag/2]). +-export([last_modified/2]). +-export([get_text_plain/2]). +-export([get_text_plain_bytes/2]). + +init(Req, State) -> + {cowboy_rest, Req, State}. + +content_types_provided(Req, State) -> + {[ + {{<<"text">>, <<"plain">>, []}, get_text_plain}, + %% This one only exists so we generate a vary header. + {{<<"text">>, <<"html">>, []}, get_text_html} + ], Req, State}. + +ranges_provided(Req, State) -> + {[{<<"bytes">>, get_text_plain_bytes}], Req, State}. + +generate_etag(Req=#{qs := <<"weak-etag">>}, State) -> + {{weak, <<"weak-no-match">>}, Req, State}; +generate_etag(Req, State) -> + {{strong, <<"strong-and-match">>}, Req, State}. + +last_modified(Req, State) -> + {{{2222, 2, 22}, {11, 11, 11}}, Req, State}. + +expires(Req, State) -> + {{{3333, 3, 3}, {11, 11, 11}}, Req, State}. + +get_text_plain(Req, State) -> + {<<"This is REST!">>, Req, State}. + +%% Simulate the callback being missing, otherwise expect true/false. +get_text_plain_bytes(#{qs := <<"missing">>}, _) -> + ct_helper_error_h:ignore(cowboy_rest, set_ranged_body, 3), + no_call; +get_text_plain_bytes(Req=#{range := {_, [{From=0, infinity}]}}, State) -> + %% We send everything in one part. + Body = <<"This is ranged REST!">>, + Total = byte_size(Body), + {[{{From, Total - 1, Total}, Body}], Req, State}; +get_text_plain_bytes(Req=#{range := {_, Range}}, State) -> + %% We check the range header we get and send everything hardcoded. + [ + {0, 3}, + {5, 6}, + {8, 13}, + {15, infinity} + ] = Range, + Body = <<"This is ranged REST!">>, + Total = byte_size(Body), + {[ + {{0, 3, Total}, <<"This">>}, + {{5, 6, Total}, <<"is">>}, + {{8, 13, Total}, <<"ranged">>}, + {{15, 19, Total}, <<"REST!">>} + ], Req, State}. diff --git a/test/handlers/range_satisfiable_h.erl b/test/handlers/range_satisfiable_h.erl new file mode 100644 index 0000000..357c30e --- /dev/null +++ b/test/handlers/range_satisfiable_h.erl @@ -0,0 +1,39 @@ +%% This module defines the range_satisfiable callback +%% and return something different depending on query string. + +-module(range_satisfiable_h). + +-export([init/2]). +-export([content_types_provided/2]). +-export([ranges_provided/2]). +-export([range_satisfiable/2]). +-export([get_text_plain/2]). +-export([get_text_plain_bytes/2]). + +init(Req, State) -> + {cowboy_rest, Req, State}. + +content_types_provided(Req, State) -> + {[{{<<"text">>, <<"plain">>, []}, get_text_plain}], Req, State}. + +ranges_provided(Req, State) -> + {[{<<"bytes">>, get_text_plain_bytes}], Req, State}. + +%% Simulate the callback being missing, otherwise expect true/false. +range_satisfiable(#{qs := <<"missing">>}, _) -> + no_call; +range_satisfiable(Req=#{qs := <<"false-int">>}, State) -> + {{false, 123}, Req, State}; +range_satisfiable(Req=#{qs := <<"false-bin">>}, State) -> + {{false, <<"*/456">>}, Req, State}; +range_satisfiable(Req=#{qs := Qs}, State) -> + {Qs =:= <<"true">>, Req, State}. + +get_text_plain(Req, State) -> + {<<"This is REST!">>, Req, State}. + +get_text_plain_bytes(Req, State) -> + %% We send everything in one part, since we are not testing + %% this callback specifically. + Body = <<"This is ranged REST!">>, + {[{{0, byte_size(Body) - 1, byte_size(Body)}, Body}], Req, State}. diff --git a/test/handlers/ranges_provided_h.erl b/test/handlers/ranges_provided_h.erl new file mode 100644 index 0000000..8e2b050 --- /dev/null +++ b/test/handlers/ranges_provided_h.erl @@ -0,0 +1,30 @@ +%% This module defines the ranges_provided callback +%% and return something different depending on query string. + +-module(ranges_provided_h). + +-export([init/2]). +-export([content_types_provided/2]). +-export([ranges_provided/2]). +-export([get_text_plain/2]). + +init(Req, State) -> + {cowboy_rest, Req, State}. + +content_types_provided(Req, State) -> + {[{{<<"text">>, <<"plain">>, []}, get_text_plain}], Req, State}. + +ranges_provided(Req=#{qs := <<"list">>}, State) -> + {[ + {<<"bytes">>, get_text_plain_bytes}, + {<<"pages">>, get_text_plain_pages}, + {<<"chapters">>, get_text_plain_chapters} + ], Req, State}; +ranges_provided(Req=#{qs := <<"none">>}, State) -> + {[], Req, State}; +%% Simulate the callback being missing in other cases. +ranges_provided(_, _) -> + no_call. + +get_text_plain(Req, State) -> + {<<"This is REST!">>, Req, State}. diff --git a/test/handlers/stop_handler_h.erl b/test/handlers/stop_handler_h.erl index 22abda7..423c3f9 100644 --- a/test/handlers/stop_handler_h.erl +++ b/test/handlers/stop_handler_h.erl @@ -23,6 +23,8 @@ -export([multiple_choices/2]). -export([options/2]). -export([previously_existed/2]). +-export([range_satisfiable/2]). +-export([ranges_provided/2]). -export([rate_limited/2]). -export([resource_exists/2]). -export([service_available/2]). @@ -32,6 +34,7 @@ -export([accept/2]). -export([provide/2]). +-export([provide_range/2]). init(Req, State) -> {cowboy_rest, Req, State}. @@ -90,6 +93,12 @@ options(Req, State) -> previously_existed(Req, State) -> maybe_stop_handler(Req, State, ?FUNCTION_NAME). +range_satisfiable(Req, State) -> + maybe_stop_handler(Req, State, ?FUNCTION_NAME). + +ranges_provided(Req, State) -> + maybe_stop_handler(Req, State, ?FUNCTION_NAME). + rate_limited(Req, State) -> maybe_stop_handler(Req, State, ?FUNCTION_NAME). @@ -114,6 +123,9 @@ accept(Req, State) -> provide(Req, State) -> maybe_stop_handler(Req, State, ?FUNCTION_NAME). +provide_range(Req, State) -> + maybe_stop_handler(Req, State, ?FUNCTION_NAME). + maybe_stop_handler(Req=#{qs := Qs}, State, StateName) -> case atom_to_binary(StateName, latin1) of Qs -> do_stop_handler(Req, State); @@ -128,6 +140,11 @@ do_default(Req, State, content_types_accepted) -> {[{<<"text/plain">>, accept}], Req, State}; do_default(Req, State, content_types_provided) -> {[{<<"text/plain">>, provide}], Req, State}; +%% We need to accept ranges to reach these callbacks. +do_default(Req=#{qs := <<"range_satisfiable">>}, State, ranges_provided) -> + {[{<<"bytes">>, provide_range}], Req, State}; +do_default(Req=#{qs := <<"provide_range">>}, State, ranges_provided) -> + {[{<<"bytes">>, provide_range}], Req, State}; %% We need resource_exists to return false to reach these callbacks. do_default(Req=#{qs := <<"allow_missing_post">>}, State, resource_exists) -> {false, Req, State}; @@ -145,11 +162,13 @@ do_default(Req=#{qs := <<"moved_temporarily">>}, State, previously_existed) -> %% We need the DELETE to suceed to reach this callback. do_default(Req=#{qs := <<"delete_completed">>}, State, delete_resource) -> {true, Req, State}; -%% We should never reach these two callbacks. +%% We should never reach these callbacks. do_default(Req, State, accept) -> {false, Req, State}; do_default(Req, State, provide) -> {<<"This is REST!">>, Req, State}; +do_default(Req, State, provide_range) -> + {<<"This is ranged REST!">>, Req, State}; %% Simulate the callback being missing in any other cases. do_default(_, _, _) -> no_call. diff --git a/test/handlers/switch_handler_h.erl b/test/handlers/switch_handler_h.erl index cf656ae..79a84e4 100644 --- a/test/handlers/switch_handler_h.erl +++ b/test/handlers/switch_handler_h.erl @@ -22,6 +22,8 @@ -export([multiple_choices/2]). -export([options/2]). -export([previously_existed/2]). +-export([range_satisfiable/2]). +-export([ranges_provided/2]). -export([rate_limited/2]). -export([resource_exists/2]). -export([service_available/2]). @@ -31,6 +33,7 @@ -export([accept/2]). -export([provide/2]). +-export([provide_range/2]). -export([info/3]). @@ -91,6 +94,12 @@ options(Req, State) -> previously_existed(Req, State) -> maybe_switch_handler(Req, State, ?FUNCTION_NAME). +range_satisfiable(Req, State) -> + maybe_switch_handler(Req, State, ?FUNCTION_NAME). + +ranges_provided(Req, State) -> + maybe_switch_handler(Req, State, ?FUNCTION_NAME). + rate_limited(Req, State) -> maybe_switch_handler(Req, State, ?FUNCTION_NAME). @@ -115,6 +124,9 @@ accept(Req, State) -> provide(Req, State) -> maybe_switch_handler(Req, State, ?FUNCTION_NAME). +provide_range(Req, State) -> + maybe_switch_handler(Req, State, ?FUNCTION_NAME). + maybe_switch_handler(Req=#{qs := Qs}, State, StateName) -> case atom_to_binary(StateName, latin1) of Qs -> do_switch_handler(Req, State); @@ -129,6 +141,11 @@ do_default(Req, State, content_types_accepted) -> {[{<<"text/plain">>, accept}], Req, State}; do_default(Req, State, content_types_provided) -> {[{<<"text/plain">>, provide}], Req, State}; +%% We need to accept ranges to reach these callbacks. +do_default(Req=#{qs := <<"range_satisfiable">>}, State, ranges_provided) -> + {[{<<"bytes">>, provide_range}], Req, State}; +do_default(Req=#{qs := <<"provide_range">>}, State, ranges_provided) -> + {[{<<"bytes">>, provide_range}], Req, State}; %% We need resource_exists to return false to reach these callbacks. do_default(Req=#{qs := <<"allow_missing_post">>}, State, resource_exists) -> {false, Req, State}; @@ -146,11 +163,13 @@ do_default(Req=#{qs := <<"moved_temporarily">>}, State, previously_existed) -> %% We need the DELETE to suceed to reach this callback. do_default(Req=#{qs := <<"delete_completed">>}, State, delete_resource) -> {true, Req, State}; -%% We should never reach these two callbacks. +%% We should never reach these callbacks. do_default(Req, State, accept) -> {false, Req, State}; do_default(Req, State, provide) -> {<<"This is REST!">>, Req, State}; +do_default(Req, State, provide_range) -> + {<<"This is ranged REST!">>, Req, State}; %% Simulate the callback being missing in any other cases. do_default(_, _, _) -> no_call. diff --git a/test/rest_handler_SUITE.erl b/test/rest_handler_SUITE.erl index fac4bbd..6116830 100644 --- a/test/rest_handler_SUITE.erl +++ b/test/rest_handler_SUITE.erl @@ -47,7 +47,11 @@ init_dispatch(_) -> charset_in_content_types_provided_implicit_h, []}, {"/charset_in_content_types_provided_implicit_no_callback", charset_in_content_types_provided_implicit_no_callback_h, []}, + {"/if_range", if_range_h, []}, {"/provide_callback_missing", provide_callback_missing_h, []}, + {"/provide_range_callback", provide_range_callback_h, []}, + {"/range_satisfiable", range_satisfiable_h, []}, + {"/ranges_provided", ranges_provided_h, []}, {"/rate_limited", rate_limited_h, []}, {"/stop_handler", stop_handler_h, []}, {"/switch_handler", switch_handler_h, run}, @@ -282,6 +286,110 @@ charsets_provided_empty_noheader(Config) -> {response, _, 406, _} = gun:await(ConnPid, Ref), ok. +if_range_etag_equal(Config) -> + doc("When the if-range header matches, a 206 partial content " + "response is expected for an otherwise valid range request. (RFC7233 3.2)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/if_range", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>}, + {<<"if-range">>, <<"\"strong-and-match\"">>} + ]), + {response, nofin, 206, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + {_, <<"bytes 0-19/20">>} = lists:keyfind(<<"content-range">>, 1, Headers), + ok. + +if_range_etag_not_equal(Config) -> + doc("When the if-range header does not match, the range header " + "must be ignored and a 200 OK response is expected for " + "an otherwise valid range request. (RFC7233 3.2)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/if_range", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>}, + {<<"if-range">>, <<"\"strong-but-no-match\"">>} + ]), + {response, nofin, 200, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + false = lists:keyfind(<<"content-range">>, 1, Headers), + ok. + +if_range_ignored_when_no_range_header(Config) -> + doc("When there is no range header the if-range header is ignored " + "and a 200 OK response is expected (RFC7233 3.2)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/if_range", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"if-range">>, <<"\"strong-and-match\"">>} + ]), + {response, nofin, 200, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + false = lists:keyfind(<<"content-range">>, 1, Headers), + ok. + +if_range_ignored_when_ranges_provided_missing(Config) -> + doc("When the resource does not support range requests " + "the range and if-range headers must be ignored" + "and a 200 OK response is expected (RFC7233 3.2)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/if_range?missing-ranges_provided", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>}, + {<<"if-range">>, <<"\"strong-and-match\"">>} + ]), + {response, nofin, 200, Headers} = gun:await(ConnPid, Ref), + false = lists:keyfind(<<"accept-ranges">>, 1, Headers), + false = lists:keyfind(<<"content-range">>, 1, Headers), + ok. + +if_range_ignored_when_ranges_provided_empty(Config) -> + doc("When the resource does not support range requests " + "the range and if-range headers must be ignored" + "and a 200 OK response is expected (RFC7233 3.2)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/if_range?empty-ranges_provided", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>}, + {<<"if-range">>, <<"\"strong-and-match\"">>} + ]), + {response, nofin, 200, Headers} = gun:await(ConnPid, Ref), + {_, <<"none">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + false = lists:keyfind(<<"content-range">>, 1, Headers), + ok. + +if_range_weak_etag_not_equal(Config) -> + doc("The if-range header must not match weak etags; the range header " + "must be ignored and a 200 OK response is expected for " + "an otherwise valid range request. (RFC7233 3.2)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/if_range?weak-etag", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>}, + {<<"if-range">>, <<"W/\"weak-no-match\"">>} + ]), + {response, nofin, 200, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + false = lists:keyfind(<<"content-range">>, 1, Headers), + ok. + +if_range_date_not_equal(Config) -> + doc("The if-range header must not match weak dates. Cowboy " + "currently has no way of knowing whether a resource was " + "updated twice within the same second. The range header " + "must be ignored and a 200 OK response is expected for " + "an otherwise valid range request. (RFC7233 3.2)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/if_range", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>}, + {<<"if-range">>, <<"Fri, 22 Feb 2222 11:11:11 GMT">>} + ]), + {response, nofin, 200, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + false = lists:keyfind(<<"content-range">>, 1, Headers), + ok. + provide_callback_missing(Config) -> doc("A 500 response must be sent when the ProvideCallback can't be called."), ConnPid = gun_open(Config), @@ -289,9 +397,228 @@ provide_callback_missing(Config) -> {response, fin, 500, _} = gun:await(ConnPid, Ref), ok. +provide_range_callback(Config) -> + doc("A successful request for a single range results in a " + "206 partial content response with content-range set. (RFC7233 4.1, RFC7233 4.2)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/provide_range_callback", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>} + ]), + {response, nofin, 206, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + {_, <<"bytes 0-19/20">>} = lists:keyfind(<<"content-range">>, 1, Headers), + {_, <<"text/plain">>} = lists:keyfind(<<"content-type">>, 1, Headers), + {ok, <<"This is ranged REST!">>} = gun:await_body(ConnPid, Ref), + ok. + +provide_range_callback_multipart(Config) -> + doc("A successful request for multiple ranges results in a " + "206 partial content response using the multipart/byteranges " + "content-type and the content-range not being set. The real " + "content-type and content-range of the parts can be found in " + "the multipart headers. (RFC7233 4.1, RFC7233 A)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/provide_range_callback", [ + {<<"accept-encoding">>, <<"gzip">>}, + %% This range selects everything except the space characters. + {<<"range">>, <<"bytes=0-3, 5-6, 8-13, 15-">>} + ]), + {response, nofin, 206, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + false = lists:keyfind(<<"content-range">>, 1, Headers), + {_, <<"multipart/byteranges; boundary=", Boundary/bits>>} + = lists:keyfind(<<"content-type">>, 1, Headers), + {ok, Body0} = gun:await_body(ConnPid, Ref), + Body = do_decode(Headers, Body0), + do_provide_range_callback_multipart_body(Body, Boundary, [], <<>>). + +do_provide_range_callback_multipart_body(Rest, Boundary, ContentRangesAcc, BodyAcc) -> + case cow_multipart:parse_headers(Rest, Boundary) of + {ok, Headers, Rest1} -> + {_, ContentRange0} = lists:keyfind(<<"content-range">>, 1, Headers), + ContentRange = cow_http_hd:parse_content_range(ContentRange0), + {_, <<"text/plain">>} = lists:keyfind(<<"content-type">>, 1, Headers), + case cow_multipart:parse_body(Rest1, Boundary) of + {done, Body} -> + do_provide_range_callback_multipart_body(<<>>, Boundary, + [ContentRange|ContentRangesAcc], + <>); + {done, Body, Rest2} -> + do_provide_range_callback_multipart_body(Rest2, Boundary, + [ContentRange|ContentRangesAcc], + <>) + end; + {done, <<>>} -> + [ + {bytes, 0, 3, 20}, + {bytes, 5, 6, 20}, + {bytes, 8, 13, 20}, + {bytes, 15, 19, 20} + ] = lists:reverse(ContentRangesAcc), + <<"ThisisrangedREST!">> = BodyAcc, + ok + end. + +provide_range_callback_metadata(Config) -> + doc("A successful request for a single range results in a " + "206 partial content response with the same headers that " + "a normal 200 OK response would, like vary or etag. (RFC7233 4.1)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/provide_range_callback", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>} + ]), + {response, nofin, 206, Headers} = gun:await(ConnPid, Ref), + {_, _} = lists:keyfind(<<"date">>, 1, Headers), + {_, _} = lists:keyfind(<<"etag">>, 1, Headers), + {_, _} = lists:keyfind(<<"expires">>, 1, Headers), + {_, _} = lists:keyfind(<<"last-modified">>, 1, Headers), + {_, _} = lists:keyfind(<<"vary">>, 1, Headers), + %% Also cache-control and content-location but we don't send those. + ok. + +provide_range_callback_missing(Config) -> + doc("A 500 response must be sent when the ProvideRangeCallback can't be called."), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/provide_range_callback?missing", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>} + ]), + {response, fin, 500, _} = gun:await(ConnPid, Ref), + ok. + +range_ignore_unknown_unit(Config) -> + doc("The range header must be ignored when the range unit " + "is not found in ranges_provided. (RFC7233 3.1)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/if_range", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"chapters=1-">>} + ]), + {response, nofin, 200, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + false = lists:keyfind(<<"content-range">>, 1, Headers), + ok. + +range_ignore_when_not_modified(Config) -> + doc("The range header must be ignored when a conditional " + "GET results in a 304 not modified response. (RFC7233 3.1)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/if_range", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>}, + {<<"if-none-match">>, <<"\"strong-and-match\"">>} + ]), + {response, fin, 304, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + false = lists:keyfind(<<"content-range">>, 1, Headers), + ok. + +range_satisfiable(Config) -> + doc("When the range_satisfiable callback returns true " + "a 206 partial content response is expected for " + "an otherwise valid range request. (RFC7233 4.1)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/range_satisfiable?true", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>} + ]), + {response, nofin, 206, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + {_, <<"bytes 0-19/20">>} = lists:keyfind(<<"content-range">>, 1, Headers), + ok. + +range_not_satisfiable(Config) -> + doc("When the range_satisfiable callback returns false " + "a 416 range not satisfiable response is expected for " + "an otherwise valid range request. (RFC7233 4.4)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/range_satisfiable?false", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>} + ]), + {response, fin, 416, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + false = lists:keyfind(<<"content-range">>, 1, Headers), + ok. + +range_not_satisfiable_int(Config) -> + doc("When the range_satisfiable callback returns false " + "a 416 range not satisfiable response is expected for " + "an otherwise valid range request. If an integer is " + "provided it is used to construct the content-range " + "header. (RFC7233 4.2, RFC7233 4.4)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/range_satisfiable?false-int", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>} + ]), + {response, fin, 416, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + {_, <<"bytes */123">>} = lists:keyfind(<<"content-range">>, 1, Headers), + ok. + +range_not_satisfiable_bin(Config) -> + doc("When the range_satisfiable callback returns false " + "a 416 range not satisfiable response is expected for " + "an otherwise valid range request. If a binary is " + "provided it is used to construct the content-range " + "header. (RFC7233 4.2, RFC7233 4.4)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/range_satisfiable?false-bin", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>} + ]), + {response, fin, 416, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + {_, <<"bytes */456">>} = lists:keyfind(<<"content-range">>, 1, Headers), + ok. + +range_satisfiable_missing(Config) -> + doc("When the range_satisfiable callback is missing " + "a 206 partial content response is expected for " + "an otherwise valid range request. (RFC7233 4.1)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/range_satisfiable?missing", [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>} + ]), + {response, _, 206, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + {_, <<"bytes ", _/bits>>} = lists:keyfind(<<"content-range">>, 1, Headers), + ok. + +ranges_provided_accept_ranges(Config) -> + doc("When the ranges_provided callback exists the accept-ranges header " + "is sent in the response. (RFC7233 2.3)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/ranges_provided?list", [{<<"accept-encoding">>, <<"gzip">>}]), + {response, _, 200, Headers} = gun:await(ConnPid, Ref), + {_, <<"bytes, pages, chapters">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + ok. + +ranges_provided_empty_accept_ranges_none(Config) -> + doc("When the ranges_provided callback exists but returns an empty list " + "the accept-ranges header is sent in the response with the value none. (RFC7233 2.3)"), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/ranges_provided?none", [{<<"accept-encoding">>, <<"gzip">>}]), + {response, _, 200, Headers} = gun:await(ConnPid, Ref), + {_, <<"none">>} = lists:keyfind(<<"accept-ranges">>, 1, Headers), + ok. + +ranges_provided_missing_no_accept_ranges(Config) -> + doc("When the ranges_provided callback does not exist " + "the accept-ranges header is not sent in the response."), + ConnPid = gun_open(Config), + Ref = gun:get(ConnPid, "/ranges_provided?missing", [{<<"accept-encoding">>, <<"gzip">>}]), + {response, _, 200, Headers} = gun:await(ConnPid, Ref), + false = lists:keyfind(<<"accept-ranges">>, 1, Headers), + ok. + rate_limited(Config) -> doc("A 429 response must be sent when the rate_limited callback returns true. " - "The retry-after header is specified as an integer."), + "The retry-after header is specified as an integer. (RFC6585 4, RFC7231 7.1.3)"), ConnPid = gun_open(Config), Ref = gun:get(ConnPid, "/rate_limited?true", [{<<"accept-encoding">>, <<"gzip">>}]), {response, fin, 429, Headers} = gun:await(ConnPid, Ref), @@ -300,7 +627,7 @@ rate_limited(Config) -> rate_limited_datetime(Config) -> doc("A 429 response must be sent when the rate_limited callback returns true. " - "The retry-after header is specified as a date/time tuple."), + "The retry-after header is specified as a date/time tuple. (RFC6585 4, RFC7231 7.1.3)"), ConnPid = gun_open(Config), Ref = gun:get(ConnPid, "/rate_limited?true-date", [{<<"accept-encoding">>, <<"gzip">>}]), {response, fin, 429, Headers} = gun:await(ConnPid, Ref), @@ -368,6 +695,12 @@ stop_handler_options(Config) -> stop_handler_previously_existed(Config) -> do_no_body_stop_handler(Config, get, ?FUNCTION_NAME). +stop_handler_range_satisfiable(Config) -> + do_no_body_stop_handler(Config, get, ?FUNCTION_NAME). + +stop_handler_ranges_provided(Config) -> + do_no_body_stop_handler(Config, get, ?FUNCTION_NAME). + stop_handler_rate_limited(Config) -> do_no_body_stop_handler(Config, get, ?FUNCTION_NAME). @@ -392,12 +725,17 @@ stop_handler_accept(Config) -> stop_handler_provide(Config) -> do_no_body_stop_handler(Config, get, ?FUNCTION_NAME). +stop_handler_provide_range(Config) -> + do_no_body_stop_handler(Config, get, ?FUNCTION_NAME). + do_no_body_stop_handler(Config, Method, StateName0) -> doc("Send a response manually and stop the REST handler."), ConnPid = gun_open(Config), "stop_handler_" ++ StateName = atom_to_list(StateName0), - Ref = gun:Method(ConnPid, "/stop_handler?" ++ StateName, - [{<<"accept-encoding">>, <<"gzip">>}]), + Ref = gun:Method(ConnPid, "/stop_handler?" ++ StateName, [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>} + ]), {response, fin, 248, _} = gun:await(ConnPid, Ref), ok. @@ -466,6 +804,12 @@ switch_handler_options(Config) -> switch_handler_previously_existed(Config) -> do_no_body_switch_handler(Config, get, ?FUNCTION_NAME). +switch_handler_range_satisfiable(Config) -> + do_no_body_switch_handler(Config, get, ?FUNCTION_NAME). + +switch_handler_ranges_provided(Config) -> + do_no_body_switch_handler(Config, get, ?FUNCTION_NAME). + switch_handler_rate_limited(Config) -> do_no_body_switch_handler(Config, get, ?FUNCTION_NAME). @@ -490,6 +834,9 @@ switch_handler_accept(Config) -> switch_handler_provide(Config) -> do_no_body_switch_handler(Config, get, ?FUNCTION_NAME). +switch_handler_provide_range(Config) -> + do_no_body_switch_handler(Config, get, ?FUNCTION_NAME). + do_no_body_switch_handler(Config, Method, StateName0) -> doc("Switch REST to loop handler for streaming the response body, " "with and without options."), @@ -499,7 +846,10 @@ do_no_body_switch_handler(Config, Method, StateName0) -> do_no_body_switch_handler1(Config, Method, Path) -> ConnPid = gun_open(Config), - Ref = gun:Method(ConnPid, Path, [{<<"accept-encoding">>, <<"gzip">>}]), + Ref = gun:Method(ConnPid, Path, [ + {<<"accept-encoding">>, <<"gzip">>}, + {<<"range">>, <<"bytes=0-">>} + ]), {response, nofin, 200, Headers} = gun:await(ConnPid, Ref), {ok, Body} = gun:await_body(ConnPid, Ref), <<"Hello\nstreamed\nworld!\n">> = do_decode(Headers, Body), -- cgit v1.2.3