From a1a254c4d001feb1b05c1f1620766de94c560823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Fri, 21 Mar 2014 10:55:42 +0100 Subject: Improve the HTTP/1.1 and HTTP/1.0 support --- src/gun.erl | 54 ++++++++++++----------- src/gun_http.erl | 128 ++++++++++++++++++++++++++++++++++++------------------- src/gun_spdy.erl | 8 ++-- src/gun_sup.erl | 5 +-- 4 files changed, 118 insertions(+), 77 deletions(-) (limited to 'src') diff --git a/src/gun.erl b/src/gun.erl index 38cb68a..8060776 100644 --- a/src/gun.erl +++ b/src/gun.erl @@ -75,7 +75,8 @@ | {text | binary | close | ping | pong, iodata()} | {close, ws_close_code(), iodata()}. --type opts() :: [{keepalive, pos_integer()} +-type opts() :: [{http, gun_http:opts()} + | {keepalive, pos_integer()} | {retry, non_neg_integer()} | {retry_timeout, pos_integer()} | {type, conn_type()}]. @@ -93,6 +94,7 @@ socket :: inet:socket() | ssl:sslsocket(), transport :: module(), protocol :: module(), + proto_opts :: gun_http:opts(), %% @todo Make a tuple with SPDY and WS too. protocol_state :: any() }). @@ -116,6 +118,8 @@ open(Host, Port, Opts) -> %% @private open_opts([]) -> ok; +open_opts([{http, O}|Opts]) when is_list(O) -> + open_opts(Opts); open_opts([{keepalive, K}|Opts]) when is_integer(K), K > 0 -> open_opts(Opts); open_opts([{retry, R}|Opts]) when is_integer(R), R >= 0 -> @@ -328,46 +332,44 @@ get_value(Key, Opts, Default) -> end. init(Parent, Owner, Host, Port, Opts) -> - try - ok = proc_lib:init_ack(Parent, {ok, self()}), - Keepalive = get_value(keepalive, Opts, 5000), - Retry = get_value(retry, Opts, 5), - RetryTimeout = get_value(retry_timeout, Opts, 5000), - Type = get_value(type, Opts, ssl), - connect(#state{parent=Parent, owner=Owner, host=Host, port=Port, - keepalive=Keepalive, type=Type, - retry=Retry, retry_timeout=RetryTimeout}, Retry) - catch Class:Reason -> - Owner ! {gun_error, self(), {{Class, Reason, erlang:get_stacktrace()}, - "An unexpected error occurred."}} - end. - -connect(State=#state{owner=Owner, host=Host, port=Port, type=ssl}, Retries) -> + ok = proc_lib:init_ack(Parent, {ok, self()}), + HTTPOpts = get_value(http, Opts, []), + Keepalive = get_value(keepalive, Opts, 5000), + Retry = get_value(retry, Opts, 5), + RetryTimeout = get_value(retry_timeout, Opts, 5000), + Type = get_value(type, Opts, ssl), + connect(#state{parent=Parent, owner=Owner, host=Host, port=Port, + keepalive=Keepalive, type=Type, retry=Retry, + proto_opts=HTTPOpts, retry_timeout=RetryTimeout}, Retry). + +connect(State=#state{owner=Owner, host=Host, port=Port, type=ssl, + proto_opts=HTTPOpts}, Retries) -> Transport = ranch_ssl, Opts = [binary, {active, false}, {client_preferred_next_protocols, - {client, [<<"spdy/3">>, <<"http/1.1">>], <<"spdy/3">>}}], + {client, [<<"spdy/3">>, <<"http/1.1">>], <<"http/1.1">>}}], case Transport:connect(Host, Port, Opts) of {ok, Socket} -> - Protocol = case ssl:negotiated_next_protocol(Socket) of - {ok, <<"spdy/3">>} -> gun_spdy; - _ -> gun_http + {Protocol, ProtoOpts} = case ssl:negotiated_next_protocol(Socket) of + {ok, <<"spdy/3">>} -> {gun_spdy, []}; + _ -> {gun_http, HTTPOpts} end, - ProtoState = Protocol:init(Owner, Socket, Transport), + ProtoState = Protocol:init(Owner, Socket, Transport, ProtoOpts), before_loop(State#state{socket=Socket, transport=Transport, protocol=Protocol, protocol_state=ProtoState}); {error, _} -> retry_loop(State, Retries - 1) end; -connect(State=#state{owner=Owner, host=Host, port=Port, type=Type}, Retries) -> +connect(State=#state{owner=Owner, host=Host, port=Port, type=Type, + proto_opts=HTTPOpts}, Retries) -> Transport = ranch_tcp, Opts = [binary, {active, false}], case Transport:connect(Host, Port, Opts) of {ok, Socket} -> - Protocol = case Type of - tcp_spdy -> gun_spdy; - tcp -> gun_http + {Protocol, ProtoOpts} = case Type of + tcp_spdy -> {gun_spdy, []}; + tcp -> {gun_http, HTTPOpts} end, - ProtoState = Protocol:init(Owner, Socket, Transport), + ProtoState = Protocol:init(Owner, Socket, Transport, ProtoOpts), before_loop(State#state{socket=Socket, transport=Transport, protocol=Protocol, protocol_state=ProtoState}); {error, _} -> diff --git a/src/gun_http.erl b/src/gun_http.erl index 26dd246..e05bcf8 100644 --- a/src/gun_http.erl +++ b/src/gun_http.erl @@ -14,7 +14,7 @@ -module(gun_http). --export([init/3]). +-export([init/4]). -export([handle/2]). -export([close/1]). -export([keepalive/1]). @@ -23,12 +23,16 @@ -export([data/4]). -export([cancel/2]). +-type opts() :: [{version, cow_http:version()}]. +-export_type([opts/0]). + -type io() :: head | {body, non_neg_integer()} | body_close | body_chunked. -record(http_state, { owner :: pid(), socket :: inet:socket() | ssl:sslsocket(), transport :: module(), + version = 'HTTP/1.1' :: cow_http:version(), connection = keepalive :: keepalive | close, buffer = <<>> :: binary(), streams = [] :: [{reference(), boolean()}], %% ref + whether stream is alive @@ -37,8 +41,11 @@ out = head :: io() }). -init(Owner, Socket, Transport) -> - #http_state{owner=Owner, socket=Socket, transport=Transport}. +init(Owner, Socket, Transport, []) -> + #http_state{owner=Owner, socket=Socket, transport=Transport}; +init(Owner, Socket, Transport, [{version, Version}]) -> + #http_state{owner=Owner, socket=Socket, transport=Transport, + version=Version}. %% Wait for the full response headers before trying to parse them. handle(Data, State=#http_state{in=head, buffer=Buffer}) -> @@ -48,11 +55,8 @@ handle(Data, State=#http_state{in=head, buffer=Buffer}) -> {_, _} -> handle_head(Data2, State#http_state{buffer= <<>>}) end; %% Everything sent to the socket until it closes is part of the response body. -handle(Data, State=#http_state{in=body_close, - owner=Owner, streams=[{StreamRef, true}|_]}) -> - Owner ! {gun_data, self(), StreamRef, nofin, Data}, - State; -handle(_, State=#http_state{in=body_close, streams=[{_StreamRef, false}|_]}) -> +handle(Data, State=#http_state{in=body_close}) -> + send_data_if_alive(Data, State, nofin), State; handle(Data, State=#http_state{in=body_chunked, in_state=InState, buffer=Buffer, connection=Conn}) -> @@ -95,7 +99,7 @@ handle(Data, State=#http_state{in={body, Length}, connection=Conn}) -> %% More data coming. DataSize < Length -> send_data_if_alive(Data, State, nofin), - State; + State#http_state{in={body, Length - DataSize}}; %% Stream finished, no rest. DataSize =:= Length -> send_data_if_alive(Data, State, fin), @@ -113,27 +117,39 @@ handle(Data, State=#http_state{in={body, Length}, connection=Conn}) -> end end. -handle_head(Data, State=#http_state{owner=Owner, connection=Conn, - streams=[{StreamRef, IsAlive}|_]}) -> - {Version, Status, StatusStr, Rest} = cow_http:parse_status_line(Data), +handle_head(Data, State=#http_state{owner=Owner, version=ClientVersion, + connection=Conn, streams=[{StreamRef, IsAlive}|_]}) -> + {Version, Status, _, Rest} = cow_http:parse_status_line(Data), {Headers, Rest2} = cow_http:parse_headers(Rest), + In = io_from_headers(Version, Headers), + IsFin = case In of head -> fin; _ -> nofin end, case IsAlive of false -> ok; true -> - Owner ! {gun, response, self(), StreamRef, - Status, StatusStr, Headers}, + Owner ! {gun_response, self(), StreamRef, + IsFin, Status, Headers}, ok end, Conn2 = if Conn =:= close -> close; Version =:= 'HTTP/1.0' -> close; + ClientVersion =:= 'HTTP/1.0' -> close; true -> conn_from_headers(Headers) end, - In = io_from_headers(Version, Headers), %% We always reset in_state even if not chunked. - handle(Rest2, State#http_state{in=In, in_state={0, 0}, connection=Conn2}). + if + IsFin =:= fin, Conn2 =:= close -> + close; + IsFin =:= fin -> + handle(Rest2, end_stream(State#http_state{in=In, + in_state={0, 0}, connection=Conn2})); + true -> + handle(Rest2, State#http_state{in=In, in_state={0, 0}, connection=Conn2}) + end. +send_data_if_alive(<<>>, _, nofin) -> + ok; send_data_if_alive(Data, #http_state{owner=Owner, streams=[{StreamRef, true}|_]}, IsFin) -> Owner ! {gun_data, self(), StreamRef, IsFin, Data}, @@ -161,28 +177,37 @@ keepalive(State=#http_state{socket=Socket, transport=Transport}) -> Transport:send(Socket, <<"\r\n">>), State. -request(State=#http_state{socket=Socket, transport=Transport, out=head}, - StreamRef, Method, Host, Path, Headers) -> - Conn = conn_from_headers(Headers), - Out = io_from_headers('HTTP/1.1', Headers), - Transport:send(Socket, - cow_http:request(Method, Path, 'HTTP/1.1', - [{<<"host">>, Host}|Headers]) - ), +request(State=#http_state{socket=Socket, transport=Transport, version=Version, + out=head}, StreamRef, Method, Host, Path, Headers) -> + Headers2 = case Version of + 'HTTP/1.0' -> lists:keydelete(<<"transfer-encoding">>, 1, Headers); + 'HTTP/1.1' -> Headers + end, + Headers3 = case lists:keymember(<<"host">>, 1, Headers) of + false -> [{<<"host">>, Host}|Headers2]; + true -> Headers2 + end, + %% We use Headers2 because this is the smallest list. + Conn = conn_from_headers(Headers2), + Out = io_from_headers(Version, Headers2), + Transport:send(Socket, cow_http:request(Method, Path, Version, Headers3)), new_stream(State#http_state{connection=Conn, out=Out}, StreamRef). -request(State=#http_state{socket=Socket, transport=Transport, out=head}, - StreamRef, Method, Host, Path, Headers, Body) -> +request(State=#http_state{socket=Socket, transport=Transport, version=Version, + out=head}, StreamRef, Method, Host, Path, Headers, Body) -> Headers2 = lists:keydelete(<<"content-length">>, 1, lists:keydelete(<<"transfer-encoding">>, 1, Headers)), + Headers3 = case lists:keymember(<<"host">>, 1, Headers) of + false -> [{<<"host">>, Host}|Headers2]; + true -> Headers2 + end, + %% We use Headers2 because this is the smallest list. Conn = conn_from_headers(Headers2), - Transport:send(Socket, - cow_http:request(Method, Path, 'HTTP/1.1', [ - {<<"host">>, Host}, + Transport:send(Socket, [ + cow_http:request(Method, Path, Version, [ {<<"content-length">>, integer_to_list(iolist_size(Body))} - |Headers2]), - Body - ), + |Headers3]), + Body]), new_stream(State#http_state{connection=Conn}, StreamRef). %% We are expecting a new stream. @@ -192,19 +217,24 @@ data(State=#http_state{out=head}, _, _, _) -> data(State=#http_state{streams=[]}, _, _, _) -> error_stream_not_found(State); %% We can only send data on the last created stream. -data(State=#http_state{socket=Socket, transport=Transport, out=Out, - streams=Streams}, StreamRef, IsFin, Data) -> +data(State=#http_state{socket=Socket, transport=Transport, version=Version, + out=Out, streams=Streams}, StreamRef, IsFin, Data) -> case lists:last(Streams) of {StreamRef, true} -> DataSize = byte_size(Data), case Out of - body_chunked when IsFin -> - Transport:send(Socket, [ - integer_to_list(DataSize), <<"\r\n">>, - Data, <<"\r\n0\r\n\r\n">> - ]), + body_chunked when Version =:= 'HTTP/1.1', IsFin =:= fin -> + case Data of + <<>> -> + Transport:send(Socket, <<"0\r\n\r\n">>); + _ -> + Transport:send(Socket, [ + integer_to_list(DataSize), <<"\r\n">>, + Data, <<"\r\n0\r\n\r\n">> + ]) + end, State#http_state{out=head}; - body_chunked -> + body_chunked when Version =:= 'HTTP/1.1' -> Transport:send(Socket, [ integer_to_list(DataSize), <<"\r\n">>, Data, <<"\r\n">> @@ -218,7 +248,10 @@ data(State=#http_state{socket=Socket, transport=Transport, out=Out, State#http_state{out=head}; Length2 > 0, not IsFin -> State#http_state{out={body, Length2}} - end + end; + body_chunked -> %% HTTP/1.0 + Transport:send(Socket, Data), + State end; {_, _} -> error_stream_not_found(State) @@ -250,7 +283,11 @@ conn_from_headers(Headers) -> false -> keepalive; {_, ConnHd} -> - cow_http_hd:parse_connection(ConnHd) + ConnList = cow_http_hd:parse_connection(ConnHd), + case lists:member(<<"keep-alive">>, ConnList) of + true -> keepalive; + false -> close + end end. io_from_headers(Version, Headers) -> @@ -266,9 +303,10 @@ io_from_headers(Version, Headers) -> false -> head; {_, TE} -> - %% We only support chunked transfer-encoding. - [<<"chunked">>] = cow_http_hd:parse_transfer_encoding(TE), - body_chunked + case cow_http_hd:parse_transfer_encoding(TE) of + [<<"chunked">>] -> body_chunked; + [<<"identity">>] -> body_close + end end end. diff --git a/src/gun_spdy.erl b/src/gun_spdy.erl index d8070d1..f787831 100644 --- a/src/gun_spdy.erl +++ b/src/gun_spdy.erl @@ -14,7 +14,7 @@ -module(gun_spdy). --export([init/3]). +-export([init/4]). -export([handle/2]). -export([close/1]). -export([keepalive/1]). @@ -43,7 +43,7 @@ ping_id = 1 :: non_neg_integer() }). -init(Owner, Socket, Transport) -> +init(Owner, Socket, Transport, []) -> #spdy_state{owner=Owner, socket=Socket, transport=Transport, zdef=cow_spdy:deflate_init(), zinf=cow_spdy:inflate_init()}. @@ -183,6 +183,7 @@ keepalive(State=#spdy_state{socket=Socket, transport=Transport, Transport:send(Socket, cow_spdy:ping(PingID)), State#spdy_state{ping_id=PingID + 2}. +%% @todo Allow overriding the host when doing requests. request(State=#spdy_state{socket=Socket, transport=Transport, zdef=Zdef, stream_id=StreamID}, StreamRef, Method, Host, Path, Headers) -> Out = false =/= lists:keyfind(<<"content-type">>, 1, Headers), @@ -192,7 +193,7 @@ request(State=#spdy_state{socket=Socket, transport=Transport, zdef=Zdef, new_stream(StreamID, StreamRef, true, Out, <<"HTTP/1.1">>, State#spdy_state{stream_id=StreamID + 2}). -%% @todo Handle Body > 16MB. +%% @todo Handle Body > 16MB. (split it out into many frames) request(State=#spdy_state{socket=Socket, transport=Transport, zdef=Zdef, stream_id=StreamID}, StreamRef, Method, Host, Path, Headers, Body) -> Transport:send(Socket, [ @@ -242,6 +243,7 @@ error_stream_not_found(State=#spdy_state{owner=Owner}) -> State. %% Streams. +%% @todo probably change order of args and have state first? new_stream(StreamID, StreamRef, In, Out, Version, State=#spdy_state{streams=Streams}) -> diff --git a/src/gun_sup.erl b/src/gun_sup.erl index b7a9c82..b765dd9 100644 --- a/src/gun_sup.erl +++ b/src/gun_sup.erl @@ -33,7 +33,6 @@ start_link() -> %% supervisor. init([]) -> - Procs = [ - {gun, {gun, start_link, []}, - transient, 5000, worker, [gun]}], + Procs = [{gun, {gun, start_link, []}, + temporary, 5000, worker, [gun]}], {ok, {{simple_one_for_one, 10, 10}, Procs}}. -- cgit v1.2.3