diff options
Diffstat (limited to 'lib/inets/src/http_client')
| -rw-r--r-- | lib/inets/src/http_client/httpc.erl | 82 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_handler.erl | 274 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_handler_sup.erl | 8 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_manager.erl | 509 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_request.erl | 104 |
5 files changed, 342 insertions, 635 deletions
diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index 04fae13b20..b70b16f57f 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -126,7 +126,10 @@ request(Url, Profile) -> %% Header = {Field, Value} %% Field = string() %% Value = string() -%% Body = string() | binary() - HTLM-code +%% Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} | +%% {chunkify, fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code +%% SendFunResult = eof | {ok, iolist(), NewSendAcc} +%% SendAcc = NewSendAcc = term() %% %% Description: Sends a HTTP-request. The function can be both %% syncronus and asynchronous in the later case the function will @@ -426,26 +429,44 @@ service_info(Pid) -> handle_request(Method, Url, {Scheme, UserInfo, Host, Port, Path, Query}, - Headers, ContentType, Body, + Headers0, ContentType, Body0, HTTPOptions0, Options0, Profile) -> - Started = http_util:timestamp(), - NewHeaders = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers], + Started = http_util:timestamp(), + NewHeaders0 = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers0], try begin + ?hcrt("begin processing", [{started, Started}, + {new_headers, NewHeaders0}]), + + {NewHeaders, Body} = + case Body0 of + {chunkify, ProcessBody, Acc} + when is_function(ProcessBody, 1) -> + NewHeaders1 = ensure_chunked_encoding(NewHeaders0), + Body1 = {mk_chunkify_fun(ProcessBody), Acc}, + {NewHeaders1, Body1}; + {ProcessBody, _} + when is_function(ProcessBody, 1) -> + {NewHeaders0, Body0}; + _ when is_list(Body0) orelse is_binary(Body0) -> + {NewHeaders0, Body0}; + _ -> + throw({error, {bad_body, Body0}}) + end, + HTTPOptions = http_options(HTTPOptions0), Options = request_options(Options0), Sync = proplists:get_value(sync, Options), Stream = proplists:get_value(stream, Options), Host2 = header_host(Scheme, Host, Port), HeadersRecord = header_record(NewHeaders, Host2, HTTPOptions), - Receiver = proplists:get_value(receiver, Options), - SocketOpts = proplists:get_value(socket_opts, Options), - UrlEncodeBool = HTTPOptions#http_options.url_encode, - MaybeEscPath = url_encode(Path, UrlEncodeBool), - MaybeEscQuery = url_encode(Query, UrlEncodeBool), - AbsUri = url_encode(Url, UrlEncodeBool), + Receiver = proplists:get_value(receiver, Options), + SocketOpts = proplists:get_value(socket_opts, Options), + MaybeEscPath = maybe_encode_uri(HTTPOptions, Path), + MaybeEscQuery = maybe_encode_uri(HTTPOptions, Query), + AbsUri = maybe_encode_uri(HTTPOptions, Url), Request = #request{from = Receiver, scheme = Scheme, @@ -458,38 +479,71 @@ handle_request(Method, Url, settings = HTTPOptions, abs_uri = AbsUri, userinfo = UserInfo, - stream = Stream, - headers_as_is = headers_as_is(Headers, Options), + stream = Stream, + headers_as_is = headers_as_is(Headers0, Options), socket_opts = SocketOpts, started = Started}, + case httpc_manager:request(Request, profile_name(Profile)) of {ok, RequestId} -> handle_answer(RequestId, Sync, Options); {error, Reason} -> + ?hcrd("request failed", [{reason, Reason}]), {error, Reason} end end catch error:{noproc, _} -> + ?hcrv("noproc", [{profile, Profile}]), {error, {not_started, Profile}}; throw:Error -> + ?hcrv("throw", [{error, Error}]), Error end. -url_encode(URI, true) -> +ensure_chunked_encoding(Hdrs) -> + Key = "transfer-encoding", + lists:keystore(Key, 1, Hdrs, {Key, "chunked"}). + +maybe_encode_uri(#http_options{url_encode = true}, URI) -> http_uri:encode(URI); -url_encode(URI, false) -> +maybe_encode_uri(_, URI) -> URI. +mk_chunkify_fun(ProcessBody) -> + fun(eof_body) -> + eof; + (Acc) -> + case ProcessBody(Acc) of + eof -> + {ok, <<"0\r\n\r\n">>, eof_body}; + {ok, Data, NewAcc} -> + {ok, mk_chunk_bin(Data), NewAcc} + end + end. + +mk_chunk_bin(Data) -> + Bin = iolist_to_binary(Data), + iolist_to_binary([hex_size(Bin), "\r\n", Bin, "\r\n"]). + +hex_size(Bin) -> + hd(io_lib:format("~.16B", [size(Bin)])). + + handle_answer(RequestId, false, _) -> {ok, RequestId}; handle_answer(RequestId, true, Options) -> receive {http, {RequestId, saved_to_file}} -> + ?hcrt("received saved-to-file", [{request_id, RequestId}]), {ok, saved_to_file}; {http, {RequestId, {_,_,_} = Result}} -> + ?hcrt("received answer", [{request_id, RequestId}, + {result, Result}]), return_answer(Options, Result); {http, {RequestId, {error, Reason}}} -> + ?hcrt("received error", [{request_id, RequestId}, + {reason, Reason}]), {error, Reason} end. diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index cb6f3e2841..1f0e012e7e 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2010. All Rights Reserved. +%% Copyright Ericsson AB 2002-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,10 +29,10 @@ %%-------------------------------------------------------------------- %% Internal Application API -export([ - start_link/2, - connect_and_send/2, + start_link/4, + %% connect_and_send/2, send/2, - cancel/2, + cancel/3, stream/3, stream_next/1, info/1 @@ -51,7 +51,7 @@ -record(state, { request, % #request{} - session, % #tcp_session{} + session, % #session{} status_line, % {Version, StatusCode, ReasonPharse} headers, % #http_response_h{} body, % binary() @@ -94,13 +94,9 @@ %%-------------------------------------------------------------------- %%-------------------------------------------------------------------- -start_link(Options, ProfileName) -> - Args = [Options, ProfileName], - gen_server:start_link(?MODULE, Args, []). - -connect_and_send(Request, HandlerPid) -> - call({connect_and_send, Request}, HandlerPid). - +start_link(Parent, Request, Options, ProfileName) -> + {ok, proc_lib:start_link(?MODULE, init, [[Parent, Request, Options, + ProfileName]])}. %%-------------------------------------------------------------------- %% Function: send(Request, Pid) -> ok @@ -122,8 +118,8 @@ send(Request, Pid) -> %% Description: Cancels a request. Intended to be called by the httpc %% manager process. %%-------------------------------------------------------------------- -cancel(RequestId, Pid) -> - cast({cancel, RequestId}, Pid). +cancel(RequestId, Pid, From) -> + cast({cancel, RequestId, From}, Pid). %%-------------------------------------------------------------------- @@ -229,16 +225,27 @@ stream(BodyPart, Request,_) -> % only 200 and 206 responses can be streamed %% but we do not want that so errors will be handled by the process %% sending an init_error message to itself. %%-------------------------------------------------------------------- -init([Options, ProfileName]) -> - ?hcrv("init - starting", [{options, Options}, {profile, ProfileName}]), +init([Parent, Request, Options, ProfileName]) -> process_flag(trap_exit, true), - handle_verbose(Options#options.verbose), - State = #state{status = undefined, - options = Options, - profile_name = ProfileName}, - ?hcrd("init - started", []), - {ok, State}. + %% Do not let initial tcp-connection block the manager-process + proc_lib:init_ack(Parent, self()), + handle_verbose(Options#options.verbose), + Address = handle_proxy(Request#request.address, Options#options.proxy), + {ok, State} = + case {Address /= Request#request.address, Request#request.scheme} of + {true, https} -> + Error = https_through_proxy_is_not_currently_supported, + self() ! {init_error, + Error, httpc_response:error(Request, Error)}, + {ok, #state{request = Request, options = Options, + status = ssl_tunnel}}; + {_, _} -> + connect_and_send_first_request(Address, Request, + #state{options = Options, + profile_name = ProfileName}) + end, + gen_server:enter_loop(?MODULE, [], State). %%-------------------------------------------------------------------- %% Function: handle_call(Request, From, State) -> {reply, Reply, State} | @@ -249,41 +256,6 @@ init([Options, ProfileName]) -> %% {stop, Reason, State} (terminate/2 is called) %% Description: Handling call messages %%-------------------------------------------------------------------- - - -%% This is the first request, the reason the proc was started -handle_call({connect_and_send, #request{address = Address0, - scheme = Scheme} = Request}, - _From, - #state{options = #options{proxy = Proxy}, - status = undefined, - session = undefined} = State) -> - ?hcrv("connect and send", [{address0, Address0}, {proxy, Proxy}]), - Address = handle_proxy(Address0, Proxy), - if - ((Address =/= Address0) andalso (Scheme =:= https)) -> - %% This is what we should do if and when ssl supports - %% "socket upgrading" - %%send_ssl_tunnel_request(Address, Request, - %% #state{options = Options, - %% status = ssl_tunnel}); - Reason = {failed_connecting, - https_through_proxy_is_not_currently_supported}, - %% Send a reply to the original caller - ErrorResponse = httpc_response:error(Request, Reason), - httpc_response:send(Request#request.from, ErrorResponse), - %% Reply to the manager - ErrorReply = {error, Reason}, - {stop, normal, ErrorReply, State}; - true -> - case connect_and_send_first_request(Address, Request, State) of - {ok, NewState} -> - {reply, ok, NewState}; - {stop, Error, NewState} -> - {stop, normal, Error, NewState} - end - end; - handle_call(#request{address = Addr} = Request, _, #state{status = Status, session = #session{type = pipeline} = Session, @@ -445,25 +417,27 @@ handle_call(info, _, State) -> %% handle_keep_alive_queue/2 on the other hand will just skip the %% request as if it was never issued as in this case the request will %% not have been sent. -handle_cast({cancel, RequestId}, +handle_cast({cancel, RequestId, From}, #state{request = #request{id = RequestId} = Request, profile_name = ProfileName, canceled = Canceled} = State) -> ?hcrv("cancel current request", [{request_id, RequestId}, {profile, ProfileName}, {canceled, Canceled}]), - httpc_manager:request_canceled(RequestId, ProfileName), + httpc_manager:request_canceled(RequestId, ProfileName, From), ?hcrv("canceled", []), {stop, normal, State#state{canceled = [RequestId | Canceled], request = Request#request{from = answer_sent}}}; -handle_cast({cancel, RequestId}, +handle_cast({cancel, RequestId, From}, #state{profile_name = ProfileName, + request = #request{id = CurrId}, canceled = Canceled} = State) -> - ?hcrv("cancel", [{request_id, RequestId}, + ?hcrv("cancel", [{request_id, RequestId}, + {curr_req_id, CurrId}, {profile, ProfileName}, {canceled, Canceled}]), - httpc_manager:request_canceled(RequestId, ProfileName), + httpc_manager:request_canceled(RequestId, ProfileName, From), ?hcrv("canceled", []), {noreply, State#state{canceled = [RequestId | Canceled]}}; @@ -872,62 +846,55 @@ connect(SocketType, ToAddress, Opts3 = [IpFamily | Opts2], http_transport:connect(SocketType, ToAddress, Opts3, Timeout) end. - -connect_and_send_first_request(Address, - #request{settings = Settings, - headers = Headers, - address = OrigAddress, - scheme = Scheme} = Request, - #state{options = Options} = State) -> - - ?hcrd("connect", - [{address, Address}, {request, Request}, {options, Options}]), +connect_and_send_first_request(Address, Request, #state{options = Options} = State) -> SocketType = socket_type(Request), - ConnTimeout = Settings#http_options.connect_timeout, + ConnTimeout = (Request#request.settings)#http_options.connect_timeout, + ?hcri("connect", + [{address, Address}, {request, Request}, {options, Options}]), case connect(SocketType, Address, Options, ConnTimeout) of {ok, Socket} -> - Session = #session{id = {OrigAddress, self()}, - scheme = Scheme, - socket = Socket, - socket_type = SocketType}, - ?hcrd("connected - now send first request", [{socket, Socket}]), + ClientClose = + httpc_request:is_client_closing( + Request#request.headers), + SessionType = httpc_manager:session_type(Options), + SocketType = socket_type(Request), + Session = #session{id = {Request#request.address, self()}, + scheme = Request#request.scheme, + socket = Socket, + socket_type = SocketType, + client_close = ClientClose, + type = SessionType}, + ?hcri("connected - now send first request", [{socket, Socket}]), + case httpc_request:send(Address, Session, Request) of ok -> - ?hcrd("first request sent", []), - ClientClose = - httpc_request:is_client_closing(Headers), - SessionType = httpc_manager:session_type(Options), - Session2 = - Session#session{client_close = ClientClose, - type = SessionType}, - TmpState = - State#state{request = Request, - session = Session2, - mfa = init_mfa(Request, State), - status_line = init_status_line(Request), - headers = undefined, - body = undefined, - status = new}, - ?hcrt("activate socket", []), - activate_once(Session), + ?hcri("first request sent", []), + TmpState = State#state{request = Request, + session = Session, + mfa = init_mfa(Request, State), + status_line = + init_status_line(Request), + headers = undefined, + body = undefined, + status = new}, + http_transport:setopts(SocketType, + Socket, [{active, once}]), NewState = activate_request_timeout(TmpState), {ok, NewState}; - - {error, Reason} = Error -> - ?hcrv("failed sending request", [{reason, Reason}]), - {stop, Error, - State#state{session = {send_failed, Reason}, - request = Request}} + {error, Reason} -> + self() ! {init_error, error_sending, + httpc_response:error(Request, Reason)}, + {ok, State#state{request = Request, + session = + #session{socket = Socket}}} end; - - {error, Reason} = Error -> - ?hcri("connect failed", [{reason, Reason}]), - {stop, Error, State#state{session = {connect_failed, Reason}, - request = Request}} + {error, Reason} -> + self() ! {init_error, error_connecting, + httpc_response:error(Request, Reason)}, + {ok, State#state{request = Request}} end. - handler_info(#state{request = Request, session = Session, status_line = _StatusLine, @@ -1167,12 +1134,12 @@ handle_response(#state{request = Request, {ok, Msg, Data} -> ?hcrd("handle response - ok", []), end_stream(StatusLine, Request), - NewState = answer_request(Request, Msg, State), + NewState = maybe_send_answer(Request, Msg, State), handle_queue(NewState, Data); {stop, Msg} -> ?hcrd("handle response - stop", [{msg, Msg}]), end_stream(StatusLine, Request), - NewState = answer_request(Request, Msg, State), + NewState = maybe_send_answer(Request, Msg, State), {stop, normal, NewState} end. @@ -1242,7 +1209,8 @@ handle_pipeline(#state{status = pipeline, %% See comment for handle_cast({cancel, RequestId}) {stop, normal, State#state{request = - NextRequest#request{from = answer_sent}}}; + NextRequest#request{from = answer_sent}, + pipeline = Pipeline}}; false -> ?hcrv("next request", [{request, NextRequest}]), NewSession = @@ -1443,6 +1411,7 @@ answer_request(#request{id = RequestId, from = From} = Request, Msg, Timer = {RequestId, TimerRef}, cancel_timer(TimerRef, {timeout, Request#request.id}), httpc_manager:request_done(RequestId, ProfileName), + State#state{request = Request#request{from = answer_sent}, timers = Timers#timers{request_timers = @@ -1662,71 +1631,28 @@ handle_verbose(_) -> ok. -%%% Normaly I do not comment out code, I throw it away. But this might -%%% actually be used one day if ssl is improved. -%% send_ssl_tunnel_request(Address, Request = #request{address = {Host, Port}}, -%% State) -> -%% %% A ssl tunnel request is a special http request that looks like -%% %% CONNECT host:port HTTP/1.1 -%% SslTunnelRequest = #request{method = connect, scheme = http, -%% headers = -%% #http_request_h{ -%% host = Host, -%% address = Address, -%% path = Host ++ ":", -%% pquery = integer_to_list(Port), -%% other = [{ "Proxy-Connection", "keep-alive"}]}, -%% Ipv6 = (State#state.options)#options.ipv6, -%% SocketType = socket_type(SslTunnelRequest), -%% case http_transport:connect(SocketType, -%% SslTunnelRequest#request.address, Ipv6) of -%% {ok, Socket} -> -%% case httpc_request:send(Address, SslTunnelRequest, Socket) of -%% ok -> -%% Session = #tcp_session{id = -%% {SslTunnelRequest#request.address, -%% self()}, -%% scheme = -%% SslTunnelRequest#request.scheme, -%% socket = Socket}, -%% NewState = State#state{mfa = -%% {httpc_response, parse, -%% [State#state.max_header_size]}, -%% request = Request, -%% session = Session}, -%% http_transport:setopts(socket_type( -%% SslTunnelRequest#request.scheme), -%% Socket, -%% [{active, once}]), -%% {ok, NewState}; -%% {error, Reason} -> -%% self() ! {init_error, error_sending, -%% httpc_response:error(Request, Reason)}, -%% {ok, State#state{request = Request, -%% session = #tcp_session{socket = -%% Socket}}} -%% end; -%% {error, Reason} -> -%% self() ! {init_error, error_connecting, -%% httpc_response:error(Request, Reason)}, -%% {ok, State#state{request = Request}} -%% end. - -%% d(F) -> -%% d(F, []). - -%% d(F, A) -> -%% d(get(dbg), F, A). - -%% d(true, F, A) -> -%% io:format(user, "~w:~w:" ++ F ++ "~n", [self(), ?MODULE | A]); -%% d(_, _, _) -> -%% ok. - +send_raw(#session{socket = Socket, socket_type = SocketType}, + {ProcessBody, Acc}) when is_function(ProcessBody, 1) -> + ?hcrt("send raw", [{acc, Acc}]), + send_raw(SocketType, Socket, ProcessBody, Acc); send_raw(#session{socket = Socket, socket_type = SocketType}, Body) -> http_transport:send(SocketType, Socket, Body). +send_raw(SocketType, Socket, ProcessBody, Acc) -> + case ProcessBody(Acc) of + eof -> + ok; + {ok, Data, NewAcc} -> + DataBin = iolist_to_binary(Data), + ?hcrd("send", [{data, DataBin}]), + case http_transport:send(SocketType, Socket, DataBin) of + ok -> + send_raw(SocketType, Socket, ProcessBody, NewAcc); + Error -> + Error + end + end. call(Msg, Pid) -> @@ -1738,11 +1664,5 @@ call(Msg, Pid, Timeout) -> cast(Msg, Pid) -> gen_server:cast(Pid, Msg). - -%% to(To, Start) when is_integer(Start) andalso (Start >= 0) -> -%% http_util:timeout(To, Start); -%% to(To, _Start) -> -%% http_util:timeout(To, t()). - t() -> http_util:timestamp(). diff --git a/lib/inets/src/http_client/httpc_handler_sup.erl b/lib/inets/src/http_client/httpc_handler_sup.erl index 2a69fd15d0..f7a0b014b3 100644 --- a/lib/inets/src/http_client/httpc_handler_sup.erl +++ b/lib/inets/src/http_client/httpc_handler_sup.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2010. All Rights Reserved. +%% Copyright Ericsson AB 2007-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -23,7 +23,7 @@ %% API -export([start_link/0]). --export([start_child/2]). +-export([start_child/1]). %% Supervisor callback -export([init/1]). @@ -34,11 +34,9 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). -start_child(Options, Profile) -> - Args = [Options, Profile], +start_child(Args) -> supervisor:start_child(?MODULE, Args). - %%%========================================================================= %%% Supervisor callback %%%========================================================================= diff --git a/lib/inets/src/http_client/httpc_manager.erl b/lib/inets/src/http_client/httpc_manager.erl index 591cb78c29..7f66b477eb 100644 --- a/lib/inets/src/http_client/httpc_manager.erl +++ b/lib/inets/src/http_client/httpc_manager.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2010. All Rights Reserved. +%% Copyright Ericsson AB 2002-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,7 +29,7 @@ start_link/3, request/2, cancel_request/2, - request_canceled/2, + request_canceled/3, request_done/2, retry_request/2, redirect_request/2, @@ -66,6 +66,7 @@ state % State of the handler: initiating | started | operational | canceled }). +-define(DELAY, 500). %%==================================================================== %% Internal Application API @@ -158,7 +159,8 @@ cancel_request(RequestId, ProfileName) -> %% be called by the httpc handler process. %%-------------------------------------------------------------------- -request_canceled(RequestId, ProfileName) -> +request_canceled(RequestId, ProfileName, From) -> + gen_server:reply(From, ok), cast(ProfileName, {request_canceled, RequestId}). @@ -355,44 +357,32 @@ do_init(ProfileName, CookiesDir) -> %% {stop, Reason, State} (terminate/2 is called) %% Description: Handling call messages %%-------------------------------------------------------------------- -handle_call({request, Request}, _From, State) -> - ?hcrv("request", [{request, Request}]), +handle_call({request, Request}, _, State) -> + ?hcri("request", [{request, Request}]), case (catch handle_request(Request, State)) of - {ok, ReqId, NewState} -> - {reply, {ok, ReqId}, NewState}; - + {reply, Msg, NewState} -> + {reply, Msg, NewState}; Error -> - NewError = {error, {failed_process_request, Error}}, - {reply, NewError, State} + {stop, Error, httpc_response:error(Request, Error), State} end; - -handle_call({cancel_request, RequestId}, From, - #state{handler_db = HandlerDb} = State) -> - ?hcrv("cancel_request", [{request_id, RequestId}]), + +handle_call({cancel_request, RequestId}, From, State) -> + ?hcri("cancel_request", [{request_id, RequestId}]), case ets:lookup(State#state.handler_db, RequestId) of [] -> - ?hcrd("nothing to cancel", []), - Reply = ok, %% Nothing to cancel - {reply, Reply, State}; - - [#handler_info{handler = Pid}] when is_pid(Pid) -> - ?hcrd("found operational handler for this request", - [{handler, Pid}]), - httpc_handler:cancel(RequestId, Pid), - {noreply, State#state{cancel = - [{RequestId, Pid, From} | - State#state.cancel]}}; - - [#handler_info{starter = Pid, state = HandlerState}] - when is_pid(Pid) -> - ?hcri("found *initiating* handler for this request", - [{starter, Pid}, {state, HandlerState}]), - ets:update_element(HandlerDb, RequestId, - {#handler_info.state, canceled}), + %% The request has allready compleated make sure + %% it is deliverd to the client process queue so + %% it can be thrown away by httpc:cancel_request + %% This delay is hopfully a temporary workaround. + %% Note that it will not not delay the manager, + %% only the client that called httpc:cancel_request + timer:apply_after(?DELAY, gen_server, reply, [From, ok]), + {noreply, State}; + [{_, Pid, _}] -> + httpc_handler:cancel(RequestId, Pid, From), {noreply, State#state{cancel = - [{RequestId, Pid, From} | + [{RequestId, Pid, From} | State#state.cancel]}} - end; handle_call(reset_cookies, _, #state{cookie_db = CookieDb} = State) -> @@ -437,43 +427,16 @@ handle_call(Req, From, #state{profile_name = ProfileName} = State) -> %%-------------------------------------------------------------------- handle_cast({retry_or_redirect_request, {Time, Request}}, #state{profile_name = ProfileName} = State) -> - ?hcrv("retry or redirect request", [{time, Time}, {request, Request}]), - case timer:apply_after(Time, ?MODULE, retry_request, - [Request, ProfileName]) of - {ok, _} -> - {noreply, State}; - {error, Reason} -> - error_report(ProfileName, - "failed scheduling retry/redirect request" - "~n Time: ~p" - "~n Request: ~p" - "~n Reason: ~p", [Time, Request, Reason]), - {noreply, State} - end; + {ok, _} = timer:apply_after(Time, ?MODULE, retry_request, [Request, ProfileName]), + {noreply, State}; -handle_cast({retry_or_redirect_request, Request}, - #state{profile_name = Profile, - handler_db = HandlerDb} = State) -> - ?hcrv("retry or redirect request", [{request, Request}]), +handle_cast({retry_or_redirect_request, Request}, State) -> case (catch handle_request(Request, State)) of - {ok, _, NewState} -> + {reply, {ok, _}, NewState} -> {noreply, NewState}; - Error -> - ReqId = Request#request.id, - error_report(Profile, - "failed to retry or redirect request ~p" - "~n Error: ~p", [ReqId, Error]), - case ets:lookup(HandlerDb, ReqId) of - [#handler_info{from = From}] -> - Error2 = httpc_response:error(Request, Error), - httpc_response:send(From, Error2), - ok; - - _ -> - ok - end, - {noreply, State} + httpc_response:error(Request, Error), + {stop, Error, State} end; handle_cast({request_canceled, RequestId}, State) -> @@ -482,7 +445,6 @@ handle_cast({request_canceled, RequestId}, State) -> case lists:keysearch(RequestId, 1, State#state.cancel) of {value, Entry = {RequestId, _, From}} -> ?hcrt("found in cancel", [{from, From}]), - gen_server:reply(From, ok), {noreply, State#state{cancel = lists:delete(Entry, State#state.cancel)}}; Else -> @@ -539,8 +501,6 @@ handle_cast(Msg, #state{profile_name = ProfileName} = State) -> "recived unknown message" "~n Msg: ~p", [Msg]), {noreply, State}. - - %%-------------------------------------------------------------------- %% Function: handle_info(Info, State) -> {noreply, State} | @@ -548,39 +508,17 @@ handle_cast(Msg, #state{profile_name = ProfileName} = State) -> %% {stop, Reason, State} (terminate/2 is called) %% Description: Handling all non call/cast messages %%--------------------------------------------------------- - -handle_info({started, StarterPid, ReqId, HandlerPid}, State) -> - handle_started(StarterPid, ReqId, HandlerPid, State), - {noreply, State}; - -handle_info({connect_and_send, StarterPid, ReqId, HandlerPid, Res}, State) -> - handle_connect_and_send(StarterPid, ReqId, HandlerPid, Res, State), - {noreply, State}; - -handle_info({failed_starting_handler, StarterPid, ReqId, Res}, State) -> - handle_failed_starting_handler(StarterPid, ReqId, Res, State), - {noreply, State}; - -handle_info({'EXIT', Pid, Reason}, #state{handler_db = HandlerDb} = State) -> - maybe_handle_terminating_starter(Pid, Reason, HandlerDb), +handle_info({'EXIT', _, _}, State) -> + %% Handled in DOWN {noreply, State}; - handle_info({'DOWN', _, _, Pid, _}, State) -> - - %% - %% Normally this should have been cleaned up already - %% (when receiving {request_done, PequestId}), but - %% just in case there is a glitch, cleanup anyway. - %% - - Pattern = #handler_info{handler = Pid, _ = '_'}, - ets:match_delete(State#state.handler_db, Pattern), + ets:match_delete(State#state.handler_db, {'_', Pid, '_'}), %% If there where any canceled request, handled by the %% the process that now has terminated, the %% cancelation can be viewed as sucessfull! - NewCanceledList = - lists:foldl(fun({_, HandlerPid, From} = Entry, Acc) -> + NewCanceldList = + lists:foldl(fun(Entry = {_, HandlerPid, From}, Acc) -> case HandlerPid of Pid -> gen_server:reply(From, ok), @@ -589,15 +527,13 @@ handle_info({'DOWN', _, _, Pid, _}, State) -> Acc end end, State#state.cancel, State#state.cancel), - {noreply, State#state{cancel = NewCanceledList}}; - -handle_info(Info, #state{profile_name = ProfileName} = State) -> - error_report(ProfileName, - "received unknown info" - "~n Info: ~p", [Info]), + {noreply, State#state{cancel = NewCanceldList}}; +handle_info(Info, State) -> + Report = io_lib:format("Unknown message in " + "httpc_manager:handle_info ~p~n", [Info]), + error_logger:error_report(Report), {noreply, State}. - %%-------------------------------------------------------------------- %% Function: terminate(Reason, State) -> _ (ignored by gen_server) %% Description: Shutdown the httpc_handler @@ -655,224 +591,79 @@ get_handler_info(Tab) -> {Pid, State} <- Handlers2], Handlers3. - -%% -%% The request handler process is started asynchronously by a -%% "starter process". When the handler has sucessfully been started, -%% this message (started) is sent. -%% - -handle_started(StarterPid, ReqId, HandlerPid, - #state{profile_name = Profile, - handler_db = HandlerDb}) -> - case ets:lookup(HandlerDb, ReqId) of - [#handler_info{state = initiating} = HandlerInfo] -> - ?hcri("received started ack for initiating handler", []), - %% As a last resort, make sure we know when it exits, - %% in case it forgets to notify us. - %% We dont need to know the ref id? - erlang:monitor(process, HandlerPid), - HandlerInfo2 = HandlerInfo#handler_info{handler = HandlerPid, - state = started}, - ets:insert(HandlerDb, HandlerInfo2), - ok; - - [#handler_info{state = State}] -> - error_report(Profile, - "unexpected (started) message for handler (~p) in state " - "~p regarding request ~p - ignoring", [HandlerPid, State, ReqId]), - ?hcri("received unexpected started message", [{state, State}]), - ok; - - [] -> - error_report(Profile, - "unknown handler ~p (~p) started for request ~w - canceling", - [HandlerPid, StarterPid, ReqId]), - httpc_handler:cancel(ReqId, HandlerPid) - end. - - -%% -%% The request handler process is started asynchronously by a -%% "starter process". When that process terminates it sends -%% one of two messages. These ara handled by the two functions -%% below. -%% - -handle_connect_and_send(_StarterPid, ReqId, HandlerPid, Result, - #state{profile_name = Profile, - handler_db = HandlerDb}) -> - case ets:lookup(HandlerDb, ReqId) of - [#handler_info{state = started} = HandlerInfo] when Result =:= ok -> - ?hcri("received connect-and-send ack for started handler", []), - HandlerInfo2 = HandlerInfo#handler_info{starter = undefined, - handler = HandlerPid, - state = operational}, - ets:insert(HandlerDb, HandlerInfo2), - ok; - - [#handler_info{state = canceled} = HandlerInfo] when Result =:= ok -> - ?hcri("received connect-and-send ack for canceled handler", []), - httpc_handler:cancel(ReqId, HandlerPid), - HandlerInfo2 = HandlerInfo#handler_info{starter = undefined, - handler = HandlerPid}, - ets:insert(HandlerDb, HandlerInfo2), - ok; - - [#handler_info{state = State}] when Result =/= ok -> - error_report(Profile, - "handler (~p, ~w) failed to connect and/or " - "send request ~p" - "~n Result: ~p", - [HandlerPid, State, ReqId, Result]), - ?hcri("received connect-and-send error", - [{result, Result}, {state, State}]), - %% We don't need to send a response to the original caller - %% because the handler already sent one in its terminate - %% function. - ets:delete(HandlerDb, ReqId), - ok; - - [] -> - ?hcri("handler successfully started " - "for unknown request => canceling", - [{profile, Profile}, - {handler, HandlerPid}, - {request, ReqId}]), - httpc_handler:cancel(ReqId, HandlerPid) - end. - - -handle_failed_starting_handler(_StarterPid, ReqId, Error, - #state{profile_name = Profile, - handler_db = HandlerDb}) -> - case ets:lookup(HandlerDb, ReqId) of - [#handler_info{state = canceled}] -> - error_report(Profile, - "failed starting handler for request ~p" - "~n Error: ~p", [ReqId, Error]), - request_canceled(Profile, ReqId), % Fake signal from handler - ets:delete(HandlerDb, ReqId), - ok; - - [#handler_info{from = From}] -> - error_report(Profile, - "failed starting handler for request ~p" - "~n Error: ~p", [ReqId, Error]), - Reason2 = - case Error of - {error, Reason} -> - {failed_connecting, Reason}; - _ -> - {failed_connecting, Error} - end, - DummyReq = #request{id = ReqId}, - httpc_response:send(From, httpc_response:error(DummyReq, Reason2)), - ets:delete(HandlerDb, ReqId), - ok; - - [] -> - error_report(Profile, - "failed starting handler for unknown request ~p" - "~n Error: ~p", [ReqId, Error]), - ok - end. - - -maybe_handle_terminating_starter(MeybeStarterPid, Reason, HandlerDb) -> - Pattern = #handler_info{starter = MeybeStarterPid, _ = '_'}, - case ets:match_object(HandlerDb, Pattern) of - [#handler_info{id = ReqId, from = From, state = initiating}] -> - %% The starter process crashed before it could start the - %% the handler process, therefor we need to answer the - %% original caller. - ?hcri("starter process crashed bfore starting handler", - [{starter, MeybeStarterPid}, {reason, Reason}]), - Reason2 = - case Reason of - {error, Error} -> - {failed_connecting, Error}; - _ -> - {failed_connecting, Reason} - end, - DummyReq = #request{id = ReqId}, - httpc_response:send(From, httpc_response:error(DummyReq, Reason2)), - ets:delete(HandlerDb, ReqId), - ok; - - [#handler_info{state = State} = HandlerInfo] -> - %% The starter process crashed after the handler was started. - %% The handler will answer to the original caller. - ?hcri("starter process crashed after starting handler", - [{starter, MeybeStarterPid}, {reason, Reason}, {state, State}]), - HandlerInfo2 = HandlerInfo#handler_info{starter = undefined}, - ets:insert(HandlerDb, HandlerInfo2), - ok; - - _ -> - ok - end. - - -%% ----- -%% Act as an HTTP/0.9 client that does not know anything -%% about persistent connections handle_request(#request{settings = - #http_options{version = "HTTP/0.9"}} = Request0, + #http_options{version = "HTTP/0.9"}} = Request, State) -> - Request1 = handle_cookies(generate_request_id(Request0), State), - Hdrs0 = Request1#request.headers, - Hdrs1 = Hdrs0#http_request_h{connection = undefined}, - Request2 = Request1#request{headers = Hdrs1}, - create_handler_starter(Request2, State), - {ok, Request2#request.id, State}; - -%% ----- -%% Act as an HTTP/1.0 client that does not -%% use persistent connections + %% Act as an HTTP/0.9 client that does not know anything + %% about persistent connections + + NewRequest = handle_cookies(generate_request_id(Request), State), + NewHeaders = + (NewRequest#request.headers)#http_request_h{connection + = undefined}, + start_handler(NewRequest#request{headers = NewHeaders}, State), + {reply, {ok, NewRequest#request.id}, State}; + handle_request(#request{settings = - #http_options{version = "HTTP/1.0"}} = Request0, + #http_options{version = "HTTP/1.0"}} = Request, State) -> - Request1 = handle_cookies(generate_request_id(Request0), State), - Hdrs0 = Request1#request.headers, - Hdrs1 = Hdrs0#http_request_h{connection = "close"}, - Request2 = Request1#request{headers = Hdrs1}, - create_handler_starter(Request2, State), - {ok, Request2#request.id, State}; - - -%% ----- -handle_request(#request{method = Method, - address = Address, - scheme = Scheme} = Request0, - #state{options = Opts} = State) -> - Request1 = handle_cookies(generate_request_id(Request0), State), - SessionType = session_type(Opts), - case select_session(Method, Address, Scheme, SessionType, State) of + %% Act as an HTTP/1.0 client that does not + %% use persistent connections + + NewRequest = handle_cookies(generate_request_id(Request), State), + NewHeaders = + (NewRequest#request.headers)#http_request_h{connection + = "close"}, + start_handler(NewRequest#request{headers = NewHeaders}, State), + {reply, {ok, NewRequest#request.id}, State}; + +handle_request(Request, State = #state{options = Options}) -> + + NewRequest = handle_cookies(generate_request_id(Request), State), + SessionType = session_type(Options), + case select_session(Request#request.method, + Request#request.address, + Request#request.scheme, SessionType, State) of {ok, HandlerPid} -> - pipeline_or_keep_alive(Request1, HandlerPid, State); + pipeline_or_keep_alive(NewRequest, HandlerPid, State); no_connection -> - create_handler_starter(Request1, State); - {no_session, OpenSessions} - when OpenSessions < Opts#options.max_sessions -> - create_handler_starter(Request1, State); + start_handler(NewRequest, State); + {no_session, OpenSessions} when OpenSessions + < Options#options.max_sessions -> + start_handler(NewRequest, State); {no_session, _} -> %% Do not start any more persistent connections %% towards this server. - Hdrs0 = Request1#request.headers, - Hdrs1 = Hdrs0#http_request_h{connection = "close"}, - Request2 = Request1#request{headers = Hdrs1}, - create_handler_starter(Request2, State) + NewHeaders = + (NewRequest#request.headers)#http_request_h{connection + = "close"}, + start_handler(NewRequest#request{headers = NewHeaders}, State) end, - {ok, Request1#request.id, State}. + {reply, {ok, NewRequest#request.id}, State}. + + +start_handler(Request, State) -> + {ok, Pid} = + case is_inets_manager() of + true -> + httpc_handler_sup:start_child([whereis(httpc_handler_sup), + Request, State#state.options, + State#state.profile_name]); + false -> + httpc_handler:start_link(self(), Request, State#state.options, + State#state.profile_name) + end, + ets:insert(State#state.handler_db, {Request#request.id, + Pid, Request#request.from}), + erlang:monitor(process, Pid). select_session(Method, HostPort, Scheme, SessionType, #state{options = #options{max_pipeline_length = MaxPipe, max_keep_alive_length = MaxKeepAlive}, session_db = SessionDb}) -> - ?hcrd("select session", [{session_type, SessionType}, - {max_pipeline_length, MaxPipe}, + ?hcrd("select session", [{session_type, SessionType}, + {max_pipeline_length, MaxPipe}, {max_keep_alive_length, MaxKeepAlive}]), case httpc_request:is_idempotent(Method) orelse (SessionType =:= keep_alive) of @@ -918,92 +709,17 @@ select_session(Candidates, Max) -> ?hcrd("select session - found one", [{handler, HandlerPid}]), {ok, HandlerPid} end. - -pipeline_or_keep_alive(#request{id = Id} = Request, HandlerPid, State) -> - ?hcrd("pipeline of keep-alive", [{id, Id}, {handler, HandlerPid}]), + +pipeline_or_keep_alive(Request, HandlerPid, State) -> case (catch httpc_handler:send(Request, HandlerPid)) of ok -> - ?hcrd("pipeline or keep-alive - successfully sent", []), - Entry = #handler_info{id = Id, - handler = HandlerPid, - state = operational}, - ets:insert(State#state.handler_db, Entry); - - _ -> %% timeout pipelining failed - ?hcrd("pipeline or keep-alive - failed sending -> " - "start a new handler", []), - create_handler_starter(Request, State) + ets:insert(State#state.handler_db, {Request#request.id, + HandlerPid, + Request#request.from}); + _ -> %timeout pipelining failed + start_handler(Request, State) end. - -create_handler_starter(#request{socket_opts = SocketOpts} = Request, - #state{options = Options} = State) - when is_list(SocketOpts) -> - %% The user provided us with (override) socket options - ?hcrt("create handler starter", [{socket_opts, SocketOpts}, {options, Options}]), - Options2 = Options#options{socket_opts = SocketOpts}, - create_handler_starter(Request#request{socket_opts = undefined}, - State#state{options = Options2}); - -create_handler_starter(#request{id = Id, - from = From} = Request, - #state{profile_name = ProfileName, - options = Options, - handler_db = HandlerDb} = _State) -> - ?hcrv("create handler starter", [{id, Id}, {profile, ProfileName}]), - IsInetsManager = is_inets_manager(), - ManagerPid = self(), - StarterFun = - fun() -> - ?hcrd("handler starter - start", - [{id, Id}, - {profile, ProfileName}, - {inets_manager, IsInetsManager}]), - Result1 = - case IsInetsManager of - true -> - httpc_handler_sup:start_child(Options, - ProfileName); - false -> - httpc_handler:start_link(Options, - ProfileName) - end, - ?hcrd("handler starter - maybe connect and send", - [{id, Id}, {profile, ProfileName}, {result, Result1}]), - case Result1 of - {ok, HandlerPid} -> - StartedMessage = - {started, self(), Id, HandlerPid}, - ManagerPid ! StartedMessage, - Result2 = httpc_handler:connect_and_send(Request, - HandlerPid), - ?hcrd("handler starter - connected and sent", - [{id, Id}, {profile, ProfileName}, - {handler, HandlerPid}, {result, Result2}]), - ConnAndSendMessage = - {connect_and_send, - self(), Id, HandlerPid, Result2}, - ManagerPid ! ConnAndSendMessage; - {error, Reason} -> - StartFailureMessage = - {failed_starting_handler, self(), Id, Reason}, - ManagerPid ! StartFailureMessage; - _ -> - StartFailureMessage = - {failed_starting_handler, self(), Id, Result1}, - ManagerPid ! StartFailureMessage - end - end, - Starter = erlang:spawn_link(StarterFun), - ?hcrd("create handler starter - started", [{id, Id}, {starter, Starter}]), - Entry = #handler_info{id = Id, - starter = Starter, - from = From, - state = initiating}, - ets:insert(HandlerDb, Entry), - ok. - - is_inets_manager() -> case get('$ancestors') of [httpc_profile_sup | _] -> @@ -1045,8 +761,6 @@ do_store_cookies([Cookie | Cookies], #state{cookie_db = CookieDb} = State) -> ok = httpc_cookie:insert(CookieDb, Cookie), do_store_cookies(Cookies, State). - - session_db_name(ProfileName) -> make_db_name(ProfileName, "__session_db"). @@ -1074,7 +788,6 @@ cast(ProfileName, Msg) -> gen_server:cast(ProfileName, Msg). - get_proxy(Opts, #options{proxy = Default}) -> proplists:get_value(proxy, Opts, Default). @@ -1133,20 +846,6 @@ handle_verbose(trace) -> handle_verbose(_) -> ok. - error_report(Profile, F, A) -> Report = io_lib:format("HTTPC-MANAGER<~p> " ++ F ++ "~n", [Profile | A]), error_logger:error_report(Report). - - -%% d(F) -> -%% d(F, []). - -%% d(F, A) -> -%% d(get(dbg), F, A). - -%% d(true, F, A) -> -%% io:format(user, "~w:~w:" ++ F ++ "~n", [self(), ?MODULE | A]); -%% d(_, _, _) -> -%% ok. - diff --git a/lib/inets/src/http_client/httpc_request.erl b/lib/inets/src/http_client/httpc_request.erl index d4df97ad40..879053f0f2 100644 --- a/lib/inets/src/http_client/httpc_request.erl +++ b/lib/inets/src/http_client/httpc_request.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2010. All Rights Reserved. +%% Copyright Ericsson AB 2004-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -79,36 +79,62 @@ send(SendAddr, Socket, SocketType, {settings, HttpOptions}, {userinfo, UserInfo}]), - TmpHeaders = handle_user_info(UserInfo, Headers), + TmpHdrs = handle_user_info(UserInfo, Headers), - {TmpHeaders2, Body} = - post_data(Method, TmpHeaders, Content, HeadersAsIs), + {TmpHdrs2, Body} = post_data(Method, TmpHdrs, Content, HeadersAsIs), - {NewHeaders, Uri} = case Address of - SendAddr -> - {TmpHeaders2, Path ++ Query}; - _Proxy -> - TmpHeaders3 = - handle_proxy(HttpOptions, TmpHeaders2), - {TmpHeaders3, AbsUri} - end, - - FinalHeaders = case NewHeaders of - HeaderList when is_list(HeaderList) -> - http_headers(HeaderList, []); - _ -> - http_request:http_headers(NewHeaders) - end, + {NewHeaders, Uri} = + case Address of + SendAddr -> + {TmpHdrs2, Path ++ Query}; + _Proxy -> + TmpHdrs3 = handle_proxy(HttpOptions, TmpHdrs2), + {TmpHdrs3, AbsUri} + end, + + FinalHeaders = + case NewHeaders of + HeaderList when is_list(HeaderList) -> + http_headers(HeaderList, []); + _ -> + http_request:http_headers(NewHeaders) + end, Version = HttpOptions#http_options.version, - Message = [method(Method), " ", Uri, " ", - version(Version), ?CRLF, - headers(FinalHeaders, Version), ?CRLF, Body], + do_send_body(SocketType, Socket, Method, Uri, Version, FinalHeaders, Body). + + +do_send_body(SocketType, Socket, Method, Uri, Version, Headers, + {ProcessBody, Acc}) when is_function(ProcessBody, 1) -> + ?hcrt("send", [{acc, Acc}]), + case do_send_body(SocketType, Socket, Method, Uri, Version, Headers, []) of + ok -> + do_send_body(SocketType, Socket, ProcessBody, Acc); + Error -> + Error + end; +do_send_body(SocketType, Socket, Method, Uri, Version, Headers, Body) -> + ?hcrt("create message", [{body, Body}]), + Message = [method(Method), " ", Uri, " ", + version(Version), ?CRLF, + headers(Headers, Version), ?CRLF, Body], ?hcrd("send", [{message, Message}]), - - http_transport:send(SocketType, Socket, lists:append(Message)). + http_transport:send(SocketType, Socket, Message). + +do_send_body(SocketType, Socket, ProcessBody, Acc) -> + case ProcessBody(Acc) of + eof -> + ok; + {ok, Data, NewAcc} -> + case http_transport:send(SocketType, Socket, Data) of + ok -> + do_send_body(SocketType, Socket, ProcessBody, NewAcc); + Error -> + Error + end + end. %%------------------------------------------------------------------------- @@ -161,7 +187,6 @@ is_client_closing(Headers) -> %%%======================================================================== post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) when (Method =:= post) orelse (Method =:= put) -> - ContentLength = body_length(Body), NewBody = case Headers#http_request_h.expect of "100-continue" -> ""; @@ -170,14 +195,22 @@ post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) end, NewHeaders = case HeadersAsIs of - [] -> - Headers#http_request_h{'content-type' = - ContentType, - 'content-length' = - ContentLength}; - _ -> - HeadersAsIs - end, + [] -> + Headers#http_request_h{ + 'content-type' = ContentType, + 'content-length' = case body_length(Body) of + undefined -> + % on upload streaming the caller must give a + % value to the Content-Length header + % (or use chunked Transfer-Encoding) + Headers#http_request_h.'content-length'; + Len when is_list(Len) -> + Len + end + }; + _ -> + HeadersAsIs + end, {NewHeaders, NewBody}; @@ -190,7 +223,10 @@ body_length(Body) when is_binary(Body) -> integer_to_list(size(Body)); body_length(Body) when is_list(Body) -> - integer_to_list(length(Body)). + integer_to_list(length(Body)); + +body_length({DataFun, _Acc}) when is_function(DataFun, 1) -> + undefined. method(Method) -> http_util:to_upper(atom_to_list(Method)). |