diff options
Diffstat (limited to 'lib/inets/src/http_client')
| -rw-r--r-- | lib/inets/src/http_client/httpc.erl | 58 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_handler.erl | 109 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_internal.hrl | 6 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_manager.erl | 46 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_response.erl | 44 |
5 files changed, 207 insertions, 56 deletions
diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index 418b6247b0..dd493d7554 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% Copyright Ericsson AB 2009-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -171,6 +171,7 @@ request(Method, HTTPOptions, Options, Profile) when (Method =:= options) orelse (Method =:= get) orelse + (Method =:= put) orelse (Method =:= head) orelse (Method =:= delete) orelse (Method =:= trace) andalso @@ -528,6 +529,7 @@ handle_request(Method, Url, HeadersRecord = header_record(NewHeaders, Host2, HTTPOptions), Receiver = proplists:get_value(receiver, Options), SocketOpts = proplists:get_value(socket_opts, Options), + UnixSocket = proplists:get_value(unix_socket, Options), BracketedHost = proplists:get_value(ipv6_host_with_brackets, Options), MaybeEscPath = maybe_encode_uri(HTTPOptions, Path), @@ -549,6 +551,7 @@ handle_request(Method, Url, headers_as_is = headers_as_is(Headers0, Options), socket_opts = SocketOpts, started = Started, + unix_socket = UnixSocket, ipv6_host_with_brackets = BracketedHost}, case httpc_manager:request(Request, profile_name(Profile)) of @@ -798,7 +801,7 @@ request_options_defaults() -> error end, - VerifyBrackets = VerifyBoolean, + VerifyBrackets = VerifyBoolean, [ {sync, true, VerifySync}, @@ -869,11 +872,36 @@ request_options_sanity_check(Opts) -> end, ok. -validate_options(Options) -> - (catch validate_options(Options, [])). - -validate_options([], ValidateOptions) -> - {ok, lists:reverse(ValidateOptions)}; +validate_ipfamily_unix_socket(Options0) -> + IpFamily = proplists:get_value(ipfamily, Options0, inet), + UnixSocket = proplists:get_value(unix_socket, Options0, undefined), + Options1 = proplists:delete(ipfamily, Options0), + Options2 = proplists:delete(ipfamily, Options1), + validate_ipfamily_unix_socket(IpFamily, UnixSocket, Options2, + [{ipfamily, IpFamily}, {unix_socket, UnixSocket}]). +%% +validate_ipfamily_unix_socket(local, undefined, _Options, _Acc) -> + bad_option(unix_socket, undefined); +validate_ipfamily_unix_socket(IpFamily, UnixSocket, _Options, _Acc) + when IpFamily =/= local, UnixSocket =/= undefined -> + bad_option(ipfamily, IpFamily); +validate_ipfamily_unix_socket(IpFamily, UnixSocket, Options, Acc) -> + validate_ipfamily(IpFamily), + validate_unix_socket(UnixSocket), + {Options, Acc}. + + +validate_options(Options0) -> + try + {Options, Acc} = validate_ipfamily_unix_socket(Options0), + validate_options(Options, Acc) + catch + error:Reason -> + {error, Reason} + end. +%% +validate_options([], ValidOptions) -> + {ok, lists:reverse(ValidOptions)}; validate_options([{proxy, Proxy} = Opt| Tail], Acc) -> validate_proxy(Proxy), @@ -933,6 +961,10 @@ validate_options([{verbose, Value} = Opt| Tail], Acc) -> validate_verbose(Value), validate_options(Tail, [Opt | Acc]); +validate_options([{unix_socket, Value} = Opt| Tail], Acc) -> + validate_unix_socket(Value), + validate_options(Tail, [Opt | Acc]); + validate_options([{_, _} = Opt| _], _Acc) -> {error, {not_an_option, Opt}}. @@ -1001,7 +1033,8 @@ validate_ipv6(BadValue) -> bad_option(ipv6, BadValue). validate_ipfamily(Value) - when (Value =:= inet) orelse (Value =:= inet6) orelse (Value =:= inet6fb4) -> + when (Value =:= inet) orelse (Value =:= inet6) orelse + (Value =:= inet6fb4) orelse (Value =:= local) -> Value; validate_ipfamily(BadValue) -> bad_option(ipfamily, BadValue). @@ -1031,6 +1064,15 @@ validate_verbose(Value) validate_verbose(BadValue) -> bad_option(verbose, BadValue). +validate_unix_socket(Value) + when (Value =:= undefined) -> + Value; +validate_unix_socket(Value) + when is_list(Value) andalso length(Value) > 0 -> + Value; +validate_unix_socket(BadValue) -> + bad_option(unix_socket, BadValue). + bad_option(Option, BadValue) -> throw({error, {bad_option, Option, BadValue}}). diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index c99200777b..26e4f4e699 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2016. All Rights Reserved. +%% Copyright Ericsson AB 2002-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -109,7 +109,7 @@ start_link(Parent, Request, Options, ProfileName) -> %% to be called by the httpc manager process. %%-------------------------------------------------------------------- send(Request, Pid) -> - call(Request, Pid, 5000). + call(Request, Pid). %%-------------------------------------------------------------------- @@ -711,13 +711,17 @@ do_handle_info({'EXIT', _, _}, State = #state{request = undefined}) -> %% can retry requests in the pipeline. do_handle_info({'EXIT', _, _}, State) -> {noreply, State#state{status = close}}. - call(Msg, Pid) -> - call(Msg, Pid, infinity). - -call(Msg, Pid, Timeout) -> - gen_server:call(Pid, Msg, Timeout). + try gen_server:call(Pid, Msg, infinity) + catch + exit:{noproc, _} -> + {error, closed}; + exit:{normal, _} -> + {error, closed}; + exit:{{shutdown, _},_} -> + {error, closed} + end. cast(Msg, Pid) -> gen_server:cast(Pid, Msg). @@ -736,7 +740,7 @@ maybe_send_answer(Request, Answer, State) -> answer_request(Request, Answer, State). deliver_answer(#request{from = From} = Request) - when is_pid(From) -> + when From =/= answer_sent -> Response = httpc_response:error(Request, socket_closed_remotely), httpc_response:send(From, Response); deliver_answer(_Request) -> @@ -750,6 +754,7 @@ connect(SocketType, ToAddress, #options{ipfamily = IpFamily, ip = FromAddress, port = FromPort, + unix_socket = UnixSocket, socket_opts = Opts0}, Timeout) -> Opts1 = case FromPort of @@ -785,6 +790,16 @@ connect(SocketType, ToAddress, OK -> OK end; + local -> + Opts3 = [IpFamily | Opts2], + SocketAddr = {local, UnixSocket}, + case http_transport:connect(SocketType, {SocketAddr, 0}, Opts3, Timeout) of + {error, Reason} -> + {error, {failed_connect, [{to_address, SocketAddr}, + {IpFamily, Opts3, Reason}]}}; + Else -> + Else + end; _ -> Opts3 = [IpFamily | Opts2], case http_transport:connect(SocketType, ToAddress, Opts3, Timeout) of @@ -796,9 +811,23 @@ connect(SocketType, ToAddress, end end. -connect_and_send_first_request(Address, Request, #state{options = Options} = State) -> +handle_unix_socket_options(#request{unix_socket = UnixSocket}, Options) + when UnixSocket =:= undefined -> + Options; + +handle_unix_socket_options(#request{unix_socket = UnixSocket}, + Options = #options{ipfamily = IpFamily}) -> + case IpFamily of + local -> + Options#options{unix_socket = UnixSocket}; + Else -> + error({badarg, [{ipfamily, Else}, {unix_socket, UnixSocket}]}) + end. + +connect_and_send_first_request(Address, Request, #state{options = Options0} = State) -> SocketType = socket_type(Request), ConnTimeout = (Request#request.settings)#http_options.connect_timeout, + Options = handle_unix_socket_options(Request, Options0), case connect(SocketType, Address, Options, ConnTimeout) of {ok, Socket} -> ClientClose = @@ -837,9 +866,10 @@ connect_and_send_first_request(Address, Request, #state{options = Options} = Sta {ok, State#state{request = Request}} end. -connect_and_send_upgrade_request(Address, Request, #state{options = Options} = State) -> +connect_and_send_upgrade_request(Address, Request, #state{options = Options0} = State) -> ConnTimeout = (Request#request.settings)#http_options.connect_timeout, SocketType = ip_comm, + Options = handle_unix_socket_options(Request, Options0), case connect(SocketType, Address, Options, ConnTimeout) of {ok, Socket} -> SessionType = httpc_manager:session_type(Options), @@ -1175,17 +1205,20 @@ handle_empty_queue(Session, ProfileName, TimeOut, State) -> %% If a pipline | keep_alive session has been idle for some time is not %% closed by the server, the client may want to close it. NewState = activate_queue_timeout(TimeOut, State), - update_session(ProfileName, Session, #session.queue_length, 0), - %% Note mfa will be initialized when a new request - %% arrives. - {noreply, - NewState#state{request = undefined, - mfa = undefined, - status_line = undefined, - headers = undefined, - body = undefined - } - }. + case update_session(ProfileName, Session, #session.queue_length, 0) of + {stop, Reason} -> + {stop, {shutdown, Reason}, State}; + _ -> + %% Note mfa will be initialized when a new request + %% arrives. + {noreply, + NewState#state{request = undefined, + mfa = undefined, + status_line = undefined, + headers = undefined, + body = undefined + }} + end. receive_response(Request, Session, Data, State) -> NewState = init_wait_for_response_state(Request, State), @@ -1224,7 +1257,7 @@ close_socket(#session{socket = Socket, socket_type = SocketType}) -> http_transport:close(SocketType, Socket). activate_request_timeout( - #state{request = #request{timer = undefined} = Request} = State) -> + #state{request = #request{timer = OldRef} = Request} = State) -> Timeout = (Request#request.settings)#http_options.timeout, case Timeout of infinity -> @@ -1232,17 +1265,21 @@ activate_request_timeout( _ -> ReqId = Request#request.id, Msg = {timeout, ReqId}, + case OldRef of + undefined -> + ok; + _ -> + %% Timer is already running! This is the case for a redirect or retry + %% We need to restart the timer because the handler pid has changed + cancel_timer(OldRef, Msg) + end, Ref = erlang:send_after(Timeout, self(), Msg), Request2 = Request#request{timer = Ref}, ReqTimers = [{Request#request.id, Ref} | (State#state.timers)#timers.request_timers], Timers = #timers{request_timers = ReqTimers}, State#state{request = Request2, timers = Timers} - end; - -%% Timer is already running! This is the case for a redirect or retry -activate_request_timeout(State) -> - State. + end. activate_queue_timeout(infinity, State) -> State; @@ -1673,7 +1710,7 @@ update_session(ProfileName, #session{id = SessionId} = Session, Pos, Value) -> Session2 = erlang:setelement(Pos, Session, Value), insert_session(Session2, ProfileName); error:badarg -> - exit(normal); %% Manager has been shutdown + {stop, normal}; T:E -> %% Unexpected this must be an error! Stacktrace = erlang:get_stacktrace(), @@ -1693,14 +1730,14 @@ update_session(ProfileName, #session{id = SessionId} = Session, Pos, Value) -> Session, (catch httpc_manager:lookup_session(SessionId, ProfileName)), T, E]), - exit({failed_updating_session, - [{profile, ProfileName}, - {session_id, SessionId}, - {pos, Pos}, - {value, Value}, - {etype, T}, - {error, E}, - {stacktrace, Stacktrace}]}) + {stop, {failed_updating_session, + [{profile, ProfileName}, + {session_id, SessionId}, + {pos, Pos}, + {value, Value}, + {etype, T}, + {error, E}, + {stacktrace, Stacktrace}]}} end. diff --git a/lib/inets/src/http_client/httpc_internal.hrl b/lib/inets/src/http_client/httpc_internal.hrl index 5f8c70f28d..c5fe439722 100644 --- a/lib/inets/src/http_client/httpc_internal.hrl +++ b/lib/inets/src/http_client/httpc_internal.hrl @@ -83,10 +83,11 @@ max_sessions = ?HTTP_MAX_TCP_SESSIONS, cookies = disabled, % enabled | disabled | verify verbose = false, % boolean(), - ipfamily = inet, % inet | inet6 | inet6fb4 + ipfamily = inet, % inet | inet6 | inet6fb4 | local ip = default, % specify local interface port = default, % specify local port - socket_opts = [] % other socket options + socket_opts = [], % other socket options + unix_socket = undefined % Local unix socket } ). -type options() :: #options{}. @@ -115,6 +116,7 @@ % request timer :: undefined | reference(), socket_opts, % undefined | [socket_option()] + unix_socket, % undefined | string() ipv6_host_with_brackets % boolean() } ). diff --git a/lib/inets/src/http_client/httpc_manager.erl b/lib/inets/src/http_client/httpc_manager.erl index a63864493f..c3404dbb37 100644 --- a/lib/inets/src/http_client/httpc_manager.erl +++ b/lib/inets/src/http_client/httpc_manager.erl @@ -553,7 +553,8 @@ handle_cast({set_options, Options}, State = #state{options = OldOptions}) -> ip = get_ip(Options, OldOptions), port = get_port(Options, OldOptions), verbose = get_verbose(Options, OldOptions), - socket_opts = get_socket_opts(Options, OldOptions) + socket_opts = get_socket_opts(Options, OldOptions), + unix_socket = get_unix_socket_opts(Options, OldOptions) }, case {OldOptions#options.verbose, NewOptions#options.verbose} of {Same, Same} -> @@ -749,8 +750,26 @@ handle_request(#request{settings = start_handler(NewRequest#request{headers = NewHeaders}, State), {reply, {ok, NewRequest#request.id}, State}; -handle_request(Request, State = #state{options = Options}) -> +%% Simple socket options handling (ERL-441). +%% +%% TODO: Refactor httpc to enable sending socket options in requests +%% using persistent connections. This workaround opens a new +%% connection for each request with non-empty socket_opts. +handle_request(Request0 = #request{socket_opts = SocketOpts}, + State0 = #state{options = Options0}) + when is_list(SocketOpts) andalso length(SocketOpts) > 0 -> + Request = handle_cookies(generate_request_id(Request0), State0), + Options = convert_options(SocketOpts, Options0), + State = State0#state{options = Options}, + Headers = + (Request#request.headers)#http_request_h{connection + = "close"}, + %% Reset socket_opts to avoid setopts failure. + start_handler(Request#request{headers = Headers, socket_opts = []}, State), + %% Do not change the state + {reply, {ok, Request#request.id}, State0}; +handle_request(Request, State = #state{options = Options}) -> NewRequest = handle_cookies(generate_request_id(Request), State), SessionType = session_type(Options), case select_session(Request#request.method, @@ -774,6 +793,18 @@ handle_request(Request, State = #state{options = Options}) -> {reply, {ok, NewRequest#request.id}, State}. +%% Convert Request options to State options +convert_options([], Options) -> + Options; +convert_options([{ipfamily, Value}|T], Options) -> + convert_options(T, Options#options{ipfamily = Value}); +convert_options([{ip, Value}|T], Options) -> + convert_options(T, Options#options{ip = Value}); +convert_options([{port, Value}|T], Options) -> + convert_options(T, Options#options{port = Value}); +convert_options([Option|T], Options = #options{socket_opts = SocketOpts}) -> + convert_options(T, Options#options{socket_opts = SocketOpts ++ [Option]}). + start_handler(#request{id = Id, from = From} = Request, #state{profile_name = ProfileName, @@ -849,11 +880,11 @@ pipeline_or_keep_alive(#request{id = Id, from = From} = Request, HandlerPid, #state{handler_db = HandlerDb} = State) -> - case (catch httpc_handler:send(Request, HandlerPid)) of + case httpc_handler:send(Request, HandlerPid) of ok -> HandlerInfo = {Id, HandlerPid, From}, ets:insert(HandlerDb, HandlerInfo); - _ -> % timeout pipelining failed + {error, closed} -> % timeout pipelining failed start_handler(Request, State) end. @@ -963,7 +994,10 @@ get_option(ip, #options{ip = IP}) -> get_option(port, #options{port = Port}) -> Port; get_option(socket_opts, #options{socket_opts = SocketOpts}) -> - SocketOpts. + SocketOpts; +get_option(unix_socket, #options{unix_socket = UnixSocket}) -> + UnixSocket. + get_proxy(Opts, #options{proxy = Default}) -> proplists:get_value(proxy, Opts, Default). @@ -1016,6 +1050,8 @@ get_verbose(Opts, #options{verbose = Default}) -> get_socket_opts(Opts, #options{socket_opts = Default}) -> proplists:get_value(socket_opts, Opts, Default). +get_unix_socket_opts(Opts, #options{unix_socket = Default}) -> + proplists:get_value(unix_socket, Opts, Default). handle_verbose(debug) -> dbg:p(self(), [call]), diff --git a/lib/inets/src/http_client/httpc_response.erl b/lib/inets/src/http_client/httpc_response.erl index 5fbceb8ad0..91638f5d2e 100644 --- a/lib/inets/src/http_client/httpc_response.erl +++ b/lib/inets/src/http_client/httpc_response.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -269,7 +269,7 @@ parse_headers(<<?LF,?LF,Body/binary>>, Header, Headers, MaxHeaderSize, Result, Relaxed); parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, Header, Headers, - MaxHeaderSize, Result, _) -> + MaxHeaderSize, Result, Relaxed) -> HTTPHeaders = [lists:reverse(Header) | Headers], Length = lists:foldl(fun(H, Acc) -> length(H) + Acc end, 0, HTTPHeaders), @@ -277,8 +277,42 @@ parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, Header, Headers, true -> ResponseHeaderRcord = http_response:headers(HTTPHeaders, #http_response_h{}), - {ok, list_to_tuple( - lists:reverse([Body, ResponseHeaderRcord | Result]))}; + + %% RFC7230, Section 3.3.3 + %% If a message is received with both a Transfer-Encoding and a + %% Content-Length header field, the Transfer-Encoding overrides the + %% Content-Length. Such a message might indicate an attempt to + %% perform request smuggling (Section 9.5) or response splitting + %% (Section 9.4) and ought to be handled as an error. A sender MUST + %% remove the received Content-Length field prior to forwarding such + %% a message downstream. + case ResponseHeaderRcord#http_response_h.'transfer-encoding' of + undefined -> + {ok, list_to_tuple( + lists:reverse([Body, ResponseHeaderRcord | Result]))}; + Value -> + TransferEncoding = string:lowercase(Value), + ContentLength = ResponseHeaderRcord#http_response_h.'content-length', + if + %% Respond without error but remove Content-Length field in relaxed mode + (Relaxed =:= true) + andalso (TransferEncoding =:= "chunked") + andalso (ContentLength =/= "-1") -> + ResponseHeaderRcordFixed = + ResponseHeaderRcord#http_response_h{'content-length' = "-1"}, + {ok, list_to_tuple( + lists:reverse([Body, ResponseHeaderRcordFixed | Result]))}; + %% Respond with error in default (not relaxed) mode + (Relaxed =:= false) + andalso (TransferEncoding =:= "chunked") + andalso (ContentLength =/= "-1") -> + throw({error, {headers_conflict, {'content-length', + 'transfer-encoding'}}}); + true -> + {ok, list_to_tuple( + lists:reverse([Body, ResponseHeaderRcord | Result]))} + end + end; false -> throw({error, {header_too_long, MaxHeaderSize, MaxHeaderSize-Length}}) @@ -435,7 +469,7 @@ format_response({StatusLine, Headers, Body}) -> Length = list_to_integer(Headers#http_response_h.'content-length'), {NewBody, Data} = case Length of - -1 -> % When no lenght indicator is provided + -1 -> % When no length indicator is provided {Body, <<>>}; Length when (Length =< size(Body)) -> <<BodyThisReq:Length/binary, Next/binary>> = Body, |