diff options
Diffstat (limited to 'lib/inets/src/http_client')
| -rw-r--r-- | lib/inets/src/http_client/Makefile | 102 | ||||
| -rw-r--r-- | lib/inets/src/http_client/http.erl | 801 | ||||
| -rw-r--r-- | lib/inets/src/http_client/http_cookie.erl | 391 | ||||
| -rw-r--r-- | lib/inets/src/http_client/http_uri.erl | 116 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_handler.erl | 1499 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_handler_sup.erl | 66 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_internal.hrl | 136 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_manager.erl | 634 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_profile_sup.erl | 107 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_request.erl | 209 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_response.erl | 431 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_sup.erl | 75 |
12 files changed, 4567 insertions, 0 deletions
diff --git a/lib/inets/src/http_client/Makefile b/lib/inets/src/http_client/Makefile new file mode 100644 index 0000000000..23170f439f --- /dev/null +++ b/lib/inets/src/http_client/Makefile @@ -0,0 +1,102 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2005-2009. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# +# + +include $(ERL_TOP)/make/target.mk +EBIN = ../../ebin +include $(ERL_TOP)/make/$(TARGET)/otp.mk + +# ---------------------------------------------------- +# Application version +# ---------------------------------------------------- +include ../../vsn.mk + +VSN = $(INETS_VSN) + +# ---------------------------------------------------- +# Release directory specification +# ---------------------------------------------------- +RELSYSDIR = $(RELEASE_PATH)/lib/inets-$(VSN) + +# ---------------------------------------------------- +# Target Specs +# ---------------------------------------------------- +MODULES = \ + http \ + http_cookie \ + httpc_handler \ + httpc_manager \ + httpc_sup \ + httpc_handler_sup \ + httpc_profile_sup \ + httpc_response \ + httpc_request \ + http_uri \ + +HRL_FILES = httpc_internal.hrl + +ERL_FILES = $(MODULES:%=%.erl) + +TARGET_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) + +# ---------------------------------------------------- +# INETS FLAGS +# ---------------------------------------------------- +INETS_FLAGS = -D'SERVER_SOFTWARE="inets/$(VSN)"' \ + +# ---------------------------------------------------- +# FLAGS +# ---------------------------------------------------- +INETS_ERL_FLAGS += -I ../http_lib -I ../inets_app -pa ../../ebin + +ERL_COMPILE_FLAGS += $(INETS_ERL_FLAGS)\ + $(INETS_FLAGS) \ + +'{parse_transform,sys_pre_attributes}' \ + +'{attribute,insert,app_vsn,$(APP_VSN)}' +# ---------------------------------------------------- +# Targets +# ---------------------------------------------------- + +debug opt: $(TARGET_FILES) + +clean: + rm -f $(TARGET_FILES) + rm -f core + +docs: + +# Release Target +# ---------------------------------------------------- +include $(ERL_TOP)/make/otp_release_targets.mk + +release_spec: opt + $(INSTALL_DIR) $(RELSYSDIR)/src + $(INSTALL_DATA) $(HRL_FILES) $(ERL_FILES) $(RELSYSDIR)/src + $(INSTALL_DIR) $(RELSYSDIR)/ebin + $(INSTALL_DATA) $(TARGET_FILES) $(RELSYSDIR)/ebin + +release_docs_spec: + +info: + @echo "INETS_DEBUG = $(INETS_DEBUG)" + @echo "INETS_FLAGS = $(INETS_FLAGS)" + @echo "ERL_COMPILE_FLAGS = $(ERL_COMPILE_FLAGS)" + + + diff --git a/lib/inets/src/http_client/http.erl b/lib/inets/src/http_client/http.erl new file mode 100644 index 0000000000..ce5d7723f0 --- /dev/null +++ b/lib/inets/src/http_client/http.erl @@ -0,0 +1,801 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2002-2009. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% + +%% Description: +%%% This version of the HTTP/1.1 client supports: +%%% - RFC 2616 HTTP 1.1 client part +%%% - RFC 2818 HTTP Over TLS + +-module(http). +-behaviour(inets_service). + +%% API +-export([request/1, request/2, request/4, request/5, + cancel_request/1, cancel_request/2, + set_option/2, set_option/3, + set_options/1, set_options/2, + verify_cookies/2, verify_cookies/3, cookie_header/1, + cookie_header/2, stream_next/1, + default_profile/0]). + +%% Behavior callbacks +-export([start_standalone/1, start_service/1, + stop_service/1, services/0, service_info/1]). + +-include("http_internal.hrl"). +-include("httpc_internal.hrl"). + +-define(DEFAULT_PROFILE, default). + + +%%%========================================================================= +%%% API +%%%========================================================================= + +%%-------------------------------------------------------------------------- +%% request(Url [, Profile]) -> +%% {ok, {StatusLine, Headers, Body}} | {error,Reason} +%% +%% Url - string() +%% Description: Calls request/4 with default values. +%%-------------------------------------------------------------------------- + +request(Url) -> + request(Url, default_profile()). + +request(Url, Profile) -> + request(get, {Url, []}, [], [], Profile). + + +%%-------------------------------------------------------------------------- +%% request(Method, Request, HTTPOptions, Options [, Profile]) -> +%% {ok, {StatusLine, Headers, Body}} | {ok, {Status, Body}} | +%% {ok, RequestId} | {error,Reason} | {ok, {saved_as, FilePath} +%% +%% Method - atom() = head | get | put | post | trace | options| delete +%% Request - {Url, Headers} | {Url, Headers, ContentType, Body} +%% Url - string() +%% HTTPOptions - [HttpOption] +%% HTTPOption - {timeout, Time} | {connect_timeout, Time} | +%% {ssl, SSLOptions} | {proxy_auth, {User, Password}} +%% Ssloptions = [SSLOption] +%% SSLOption = {verify, code()} | {depth, depth()} | {certfile, path()} | +%% {keyfile, path()} | {password, string()} | {cacertfile, path()} | +%% {ciphers, string()} +%% Options - [Option] +%% Option - {sync, Boolean} | {body_format, BodyFormat} | +%% {full_result, Boolean} | {stream, To} | +%% {headers_as_is, Boolean} +%% StatusLine = {HTTPVersion, StatusCode, ReasonPhrase}</v> +%% HTTPVersion = string() +%% StatusCode = integer() +%% ReasonPhrase = string() +%% Headers = [Header] +%% Header = {Field, Value} +%% Field = string() +%% Value = string() +%% Body = string() | binary() - HTLM-code +%% +%% Description: Sends a HTTP-request. The function can be both +%% syncronus and asynchronous in the later case the function will +%% return {ok, RequestId} and later on a message will be sent to the +%% calling process on the format {http, {RequestId, {StatusLine, +%% Headers, Body}}} or {http, {RequestId, {error, Reason}}} +%%-------------------------------------------------------------------------- + +request(Method, Request, HttpOptions, Options) -> + request(Method, Request, HttpOptions, Options, default_profile()). + +request(Method, {Url, Headers}, HTTPOptions, Options, Profile) + when (Method =:= options) orelse + (Method =:= get) orelse + (Method =:= head) orelse + (Method =:= delete) orelse + (Method =:= trace) -> + case http_uri:parse(Url) of + {error, Reason} -> + {error, Reason}; + ParsedUrl -> + handle_request(Method, Url, ParsedUrl, Headers, [], [], + HTTPOptions, Options, Profile) + end; + +request(Method, {Url,Headers,ContentType,Body}, HTTPOptions, Options, Profile) + when (Method =:= post) orelse (Method =:= put) -> + case http_uri:parse(Url) of + {error, Reason} -> + {error, Reason}; + ParsedUrl -> + handle_request(Method, Url, + ParsedUrl, Headers, ContentType, Body, + HTTPOptions, Options, Profile) + end. + +%%-------------------------------------------------------------------------- +%% request(RequestId) -> ok +%% RequestId - As returned by request/4 +%% +%% Description: Cancels a HTTP-request. +%%------------------------------------------------------------------------- +cancel_request(RequestId) -> + cancel_request(RequestId, default_profile()). + +cancel_request(RequestId, Profile) -> + ok = httpc_manager:cancel_request(RequestId, profile_name(Profile)), + receive + %% If the request was allready fullfilled throw away the + %% answer as the request has been canceled. + {http, {RequestId, _}} -> + ok + after 0 -> + ok + end. + + +set_option(Key, Value) -> + set_option(Key, Value, default_profile()). + +set_option(Key, Value, Profile) -> + set_options([{Key, Value}], Profile). + + +%%-------------------------------------------------------------------------- +%% set_options(Options [, Profile]) -> ok | {error, Reason} +%% Options - [Option] +%% Profile - atom() +%% Option - {proxy, {Proxy, NoProxy}} | {max_sessions, MaxSessions} | +%% {max_pipeline_length, MaxPipeline} | +%% {pipeline_timeout, PipelineTimeout} | {cookies, CookieMode} | +%% {ipfamily, IpFamily} +%% Proxy - {Host, Port} +%% NoProxy - [Domain | HostName | IPAddress] +%% MaxSessions, MaxPipeline, PipelineTimeout = integer() +%% CookieMode - enabled | disabled | verify +%% IpFamily - inet | inet6 | inet6fb4 +%% Description: Informs the httpc_manager of the new settings. +%%------------------------------------------------------------------------- +set_options(Options) -> + set_options(Options, default_profile()). +set_options(Options, Profile) -> + case validate_options(Options) of + {ok, Opts} -> + try httpc_manager:set_options(Opts, profile_name(Profile)) of + Result -> + Result + catch + exit:{noproc, _} -> + {error, inets_not_started} + end; + {error, Reason} -> + {error, Reason} + end. + + +%%-------------------------------------------------------------------------- +%% verify_cookies(SetCookieHeaders, Url [, Profile]) -> ok | {error, reason} +%% +%% +%% Description: Store the cookies from <SetCookieHeaders> +%% in the cookie database +%% for the profile <Profile>. This function shall be used when the option +%% cookie is set to verify. +%%------------------------------------------------------------------------- +verify_cookies(SetCookieHeaders, Url) -> + verify_cookies(SetCookieHeaders, Url, default_profile()). + +verify_cookies(SetCookieHeaders, Url, Profile) -> + {_, _, Host, Port, Path, _} = http_uri:parse(Url), + ProfileName = profile_name(Profile), + Cookies = http_cookie:cookies(SetCookieHeaders, Path, Host), + try httpc_manager:store_cookies(Cookies, {Host, Port}, ProfileName) of + _ -> + ok + catch + exit:{noproc, _} -> + {error, {not_started, Profile}} + end. + +%%-------------------------------------------------------------------------- +%% cookie_header(Url [, Profile]) -> Header | {error, Reason} +%% +%% Description: Returns the cookie header that would be sent when making +%% a request to <Url>. +%%------------------------------------------------------------------------- +cookie_header(Url) -> + cookie_header(Url, default_profile()). + +cookie_header(Url, Profile) -> + try httpc_manager:cookies(Url, profile_name(Profile)) of + Header -> + Header + catch + exit:{noproc, _} -> + {error, {not_started, Profile}} + end. + + +stream_next(Pid) -> + httpc_handler:stream_next(Pid). + +%%%======================================================================== +%%% Behavior callbacks +%%%======================================================================== +start_standalone(PropList) -> + case proplists:get_value(profile, PropList) of + undefined -> + {error, no_profile}; + Profile -> + Dir = + proplists:get_value(data_dir, PropList, only_session_cookies), + httpc_manager:start_link({Profile, Dir}, stand_alone) + end. + +start_service(Config) -> + httpc_profile_sup:start_child(Config). + +stop_service(Profile) when is_atom(Profile) -> + httpc_profile_sup:stop_child(Profile); +stop_service(Pid) when is_pid(Pid) -> + case service_info(Pid) of + {ok, [{profile, Profile}]} -> + stop_service(Profile); + Error -> + Error + end. + +services() -> + [{httpc, Pid} || {_, Pid, _, _} <- + supervisor:which_children(httpc_profile_sup)]. +service_info(Pid) -> + try [{ChildName, ChildPid} || + {ChildName, ChildPid, _, _} <- + supervisor:which_children(httpc_profile_sup)] of + Children -> + child_name2info(child_name(Pid, Children)) + catch + exit:{noproc, _} -> + {error, service_not_available} + end. + + +%%%======================================================================== +%%% Internal functions +%%%======================================================================== +handle_request(Method, Url, + {Scheme, UserInfo, Host, Port, Path, Query}, + Headers, ContentType, Body, + HTTPOptions0, Options, Profile) -> + + HTTPOptions = http_options(HTTPOptions0), + Sync = proplists:get_value(sync, Options, true), + NewHeaders = lists:map(fun({Key, Val}) -> + {http_util:to_lower(Key), Val} end, + Headers), + Stream = proplists:get_value(stream, Options, none), + case {Sync, Stream} of + {true, self} -> + {error, streaming_error}; + _ -> + RecordHeaders = header_record(NewHeaders, + #http_request_h{}, + Host, + HTTPOptions#http_options.version), + Request = #request{from = self(), + scheme = Scheme, + address = {Host,Port}, + path = Path, + pquery = Query, + method = Method, + headers = RecordHeaders, + content = {ContentType,Body}, + settings = HTTPOptions, + abs_uri = Url, + userinfo = UserInfo, + stream = Stream, + headers_as_is = headers_as_is(Headers, Options)}, + try httpc_manager:request(Request, profile_name(Profile)) of + {ok, RequestId} -> + handle_answer(RequestId, Sync, Options); + {error, Reason} -> + {error, Reason} + catch + error:{noproc, _} -> + {error, {not_started, Profile}} + end + end. + + +handle_answer(RequestId, false, _) -> + {ok, RequestId}; +handle_answer(RequestId, true, Options) -> + receive + {http, {RequestId, saved_to_file}} -> + {ok, saved_to_file}; + {http, {RequestId, Result = {_,_,_}}} -> + return_answer(Options, Result); + {http, {RequestId, {error, Reason}}} -> + {error, Reason} + end. + +return_answer(Options, {{"HTTP/0.9",_,_}, _, BinBody}) -> + Body = format_body(BinBody, Options), + {ok, Body}; + +return_answer(Options, {StatusLine, Headers, BinBody}) -> + + Body = format_body(BinBody, Options), + + case proplists:get_value(full_result, Options, true) of + true -> + {ok, {StatusLine, Headers, Body}}; + false -> + {_, Status, _} = StatusLine, + {ok, {Status, Body}} + end. + +format_body(BinBody, Options) -> + case proplists:get_value(body_format, Options, string) of + string -> + binary_to_list(BinBody); + _ -> + BinBody + end. + +%% This options is a workaround for http servers that do not follow the +%% http standard and have case sensative header parsing. Should only be +%% used if there is no other way to communicate with the server or for +%% testing purpose. +headers_as_is(Headers, Options) -> + case proplists:get_value(headers_as_is, Options, false) of + false -> + []; + true -> + Headers + end. + + +http_options(HttpOptions) -> + HttpOptionsDefault = http_options_default(), + http_options(HttpOptionsDefault, HttpOptions, #http_options{}). + +http_options([], [], Acc) -> + Acc; +http_options([], HttpOptions, Acc) -> + Fun = fun(BadOption) -> + Report = io_lib:format("Invalid option ~p ignored ~n", + [BadOption]), + error_logger:info_report(Report) + end, + lists:foreach(Fun, HttpOptions), + Acc; +http_options([{Tag, Default, Idx, Post} | Defaults], HttpOptions, Acc) -> + case lists:keysearch(Tag, 1, HttpOptions) of + {value, {Tag, Val0}} -> + case Post(Val0) of + {ok, Val} -> + Acc2 = setelement(Idx, Acc, Val), + HttpOptions2 = lists:keydelete(Tag, 1, HttpOptions), + http_options(Defaults, HttpOptions2, Acc2); + error -> + Report = io_lib:format("Invalid option ~p:~p ignored ~n", + [Tag, Val0]), + error_logger:info_report(Report), + HttpOptions2 = lists:keydelete(Tag, 1, HttpOptions), + http_options(Defaults, HttpOptions2, Acc) + end; + false -> + DefaultVal = + case Default of + {value, Val} -> + Val; + {field, DefaultIdx} -> + element(DefaultIdx, Acc) + end, + Acc2 = setelement(Idx, Acc, DefaultVal), + http_options(Defaults, HttpOptions, Acc2) + end. + +http_options_default() -> + VersionPost = + fun(Value) when is_atom(Value) -> + {ok, http_util:to_upper(atom_to_list(Value))}; + (Value) when is_list(Value) -> + {ok, http_util:to_upper(Value)}; + (_) -> + error + end, + TimeoutPost = fun(Value) when is_integer(Value) andalso (Value >= 0) -> + {ok, Value}; + (infinity = Value) -> + {ok, Value}; + (_) -> + error + end, + AutoRedirectPost = fun(Value) when (Value =:= true) orelse + (Value =:= false) -> + {ok, Value}; + (_) -> + error + end, + SslPost = fun(Value) when is_list(Value) -> + {ok, Value}; + (_) -> + error + end, + ProxyAuthPost = fun({User, Passwd} = Value) when is_list(User) andalso + is_list(Passwd) -> + {ok, Value}; + (_) -> + error + end, + RelaxedPost = fun(Value) when (Value =:= true) orelse + (Value =:= false) -> + {ok, Value}; + (_) -> + error + end, + ConnTimeoutPost = + fun(Value) when is_integer(Value) andalso (Value >= 0) -> + {ok, Value}; + (infinity = Value) -> + {ok, Value}; + (_) -> + error + end, + [ + {version, {value, "HTTP/1.1"}, #http_options.version, VersionPost}, + {timeout, {value, ?HTTP_REQUEST_TIMEOUT}, #http_options.timeout, TimeoutPost}, + {autoredirect, {value, true}, #http_options.autoredirect, AutoRedirectPost}, + {ssl, {value, []}, #http_options.ssl, SslPost}, + {proxy_auth, {value, undefined}, #http_options.proxy_auth, ProxyAuthPost}, + {relaxed, {value, false}, #http_options.relaxed, RelaxedPost}, + %% this field has to be *after* the timeout field (as that field is used for the default value) + {connect_timeout, {field, #http_options.timeout}, #http_options.connect_timeout, ConnTimeoutPost} + ]. + +validate_options(Options) -> + (catch validate_options(Options, [])). + +validate_options([], ValidateOptions) -> + {ok, lists:reverse(ValidateOptions)}; + +validate_options([{proxy, Proxy} = Opt| Tail], Acc) -> + validate_proxy(Proxy), + validate_options(Tail, [Opt | Acc]); + +validate_options([{max_sessions, Value} = Opt| Tail], Acc) -> + validate_max_sessions(Value), + validate_options(Tail, [Opt | Acc]); + +validate_options([{keep_alive_timeout, Value} = Opt| Tail], Acc) -> + validate_keep_alive_timeout(Value), + validate_options(Tail, [Opt | Acc]); + +validate_options([{max_keep_alive_length, Value} = Opt| Tail], Acc) -> + validate_max_keep_alive_length(Value), + validate_options(Tail, [Opt | Acc]); + +validate_options([{pipeline_timeout, Value} = Opt| Tail], Acc) -> + validate_pipeline_timeout(Value), + validate_options(Tail, [Opt | Acc]); + +validate_options([{max_pipeline_length, Value} = Opt| Tail], Acc) -> + validate_max_pipeline_length(Value), + validate_options(Tail, [Opt | Acc]); + +validate_options([{cookies, Value} = Opt| Tail], Acc) -> + validate_cookies(Value), + validate_options(Tail, [Opt | Acc]); + +validate_options([{ipfamily, Value} = Opt| Tail], Acc) -> + validate_ipfamily(Value), + validate_options(Tail, [Opt | Acc]); + +%% For backward compatibillity +validate_options([{ipv6, Value}| Tail], Acc) -> + NewValue = validate_ipv6(Value), + Opt = {ipfamily, NewValue}, + validate_options(Tail, [Opt | Acc]); + +validate_options([{ip, Value} = Opt| Tail], Acc) -> + validate_ip(Value), + validate_options(Tail, [Opt | Acc]); + +validate_options([{port, Value} = Opt| Tail], Acc) -> + validate_port(Value), + validate_options(Tail, [Opt | Acc]); + +validate_options([{verbose, Value} = Opt| Tail], Acc) -> + validate_verbose(Value), + validate_options(Tail, [Opt | Acc]); + +validate_options([{_, _} = Opt| _], _Acc) -> + {error, {not_an_option, Opt}}. + + +validate_proxy({{ProxyHost, ProxyPort}, NoProxy} = Proxy) + when is_list(ProxyHost) andalso + is_integer(ProxyPort) andalso + is_list(NoProxy) -> + Proxy; +validate_proxy(BadProxy) -> + bad_option(proxy, BadProxy). + +validate_max_sessions(Value) when is_integer(Value) andalso (Value >= 0) -> + Value; +validate_max_sessions(BadValue) -> + bad_option(max_sessions, BadValue). + +validate_keep_alive_timeout(Value) when is_integer(Value) andalso (Value >= 0) -> + Value; +validate_keep_alive_timeout(infinity = Value) -> + Value; +validate_keep_alive_timeout(BadValue) -> + bad_option(keep_alive_timeout, BadValue). + +validate_max_keep_alive_length(Value) when is_integer(Value) andalso (Value >= 0) -> + Value; +validate_max_keep_alive_length(BadValue) -> + bad_option(max_keep_alive_length, BadValue). + +validate_pipeline_timeout(Value) when is_integer(Value) -> + Value; +validate_pipeline_timeout(infinity = Value) -> + Value; +validate_pipeline_timeout(BadValue) -> + bad_option(pipeline_timeout, BadValue). + +validate_max_pipeline_length(Value) when is_integer(Value) -> + Value; +validate_max_pipeline_length(BadValue) -> + bad_option(max_pipeline_length, BadValue). + +validate_cookies(Value) + when ((Value =:= enabled) orelse + (Value =:= disabled) orelse + (Value =:= verify)) -> + Value; +validate_cookies(BadValue) -> + bad_option(cookies, BadValue). + +validate_ipv6(Value) when (Value =:= enabled) orelse (Value =:= disabled) -> + case Value of + enabled -> + inet6fb4; + disabled -> + inet + end; +validate_ipv6(BadValue) -> + bad_option(ipv6, BadValue). + +validate_ipfamily(Value) + when (Value =:= inet) orelse (Value =:= inet6) orelse (Value =:= inet6fb4) -> + Value; +validate_ipfamily(BadValue) -> + bad_option(ipfamily, BadValue). + +validate_ip(Value) + when is_tuple(Value) andalso ((size(Value) =:= 4) orelse (size(Value) =:= 8)) -> + Value; +validate_ip(BadValue) -> + bad_option(ip, BadValue). + +validate_port(Value) when is_integer(Value) -> + Value; +validate_port(BadValue) -> + bad_option(port, BadValue). + +validate_verbose(Value) + when ((Value =:= false) orelse + (Value =:= verbose) orelse + (Value =:= debug) orelse + (Value =:= trace)) -> + ok; +validate_verbose(BadValue) -> + bad_option(verbose, BadValue). + +bad_option(Option, BadValue) -> + throw({error, {bad_option, Option, BadValue}}). + + + +header_record([], RequestHeaders, Host, Version) -> + validate_headers(RequestHeaders, Host, Version); +header_record([{"cache-control", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{'cache-control' = Val}, + Host, Version); +header_record([{"connection", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{connection = Val}, Host, + Version); +header_record([{"date", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{date = Val}, Host, + Version); +header_record([{"pragma", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{pragma = Val}, Host, + Version); +header_record([{"trailer", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{trailer = Val}, Host, + Version); +header_record([{"transfer-encoding", Val} | Rest], RequestHeaders, Host, + Version) -> + header_record(Rest, + RequestHeaders#http_request_h{'transfer-encoding' = Val}, + Host, Version); +header_record([{"upgrade", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{upgrade = Val}, Host, + Version); +header_record([{"via", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{via = Val}, Host, + Version); +header_record([{"warning", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{warning = Val}, Host, + Version); +header_record([{"accept", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{accept = Val}, Host, + Version); +header_record([{"accept-charset", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{'accept-charset' = Val}, + Host, Version); +header_record([{"accept-encoding", Val} | Rest], RequestHeaders, Host, + Version) -> + header_record(Rest, RequestHeaders#http_request_h{'accept-encoding' = Val}, + Host, Version); +header_record([{"accept-language", Val} | Rest], RequestHeaders, Host, + Version) -> + header_record(Rest, RequestHeaders#http_request_h{'accept-language' = Val}, + Host, Version); +header_record([{"authorization", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{authorization = Val}, + Host, Version); +header_record([{"expect", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{expect = Val}, Host, + Version); +header_record([{"from", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{from = Val}, Host, + Version); +header_record([{"host", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{host = Val}, Host, + Version); +header_record([{"if-match", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{'if-match' = Val}, + Host, Version); +header_record([{"if-modified-since", Val} | Rest], RequestHeaders, Host, + Version) -> + header_record(Rest, + RequestHeaders#http_request_h{'if-modified-since' = Val}, + Host, Version); +header_record([{"if-none-match", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{'if-none-match' = Val}, + Host, Version); +header_record([{"if-range", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{'if-range' = Val}, + Host, Version); + +header_record([{"if-unmodified-since", Val} | Rest], RequestHeaders, Host, + Version) -> + header_record(Rest, RequestHeaders#http_request_h{'if-unmodified-since' + = Val}, Host, Version); +header_record([{"max-forwards", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{'max-forwards' = Val}, + Host, Version); +header_record([{"proxy-authorization", Val} | Rest], RequestHeaders, Host, + Version) -> + header_record(Rest, RequestHeaders#http_request_h{'proxy-authorization' + = Val}, Host, Version); +header_record([{"range", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{range = Val}, Host, + Version); +header_record([{"referer", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{referer = Val}, Host, + Version); +header_record([{"te", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{te = Val}, Host, + Version); +header_record([{"user-agent", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{'user-agent' = Val}, + Host, Version); +header_record([{"allow", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{allow = Val}, Host, + Version); +header_record([{"content-encoding", Val} | Rest], RequestHeaders, Host, + Version) -> + header_record(Rest, + RequestHeaders#http_request_h{'content-encoding' = Val}, + Host, Version); +header_record([{"content-language", Val} | Rest], RequestHeaders, + Host, Version) -> + header_record(Rest, + RequestHeaders#http_request_h{'content-language' = Val}, + Host, Version); +header_record([{"content-length", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{'content-length' = Val}, + Host, Version); +header_record([{"content-location", Val} | Rest], RequestHeaders, + Host, Version) -> + header_record(Rest, + RequestHeaders#http_request_h{'content-location' = Val}, + Host, Version); +header_record([{"content-md5", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{'content-md5' = Val}, + Host, Version); +header_record([{"content-range", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{'content-range' = Val}, + Host, Version); +header_record([{"content-type", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{'content-type' = Val}, + Host, Version); +header_record([{"expires", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{expires = Val}, Host, + Version); +header_record([{"last-modified", Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{'last-modified' = Val}, + Host, Version); +header_record([{Key, Val} | Rest], RequestHeaders, Host, Version) -> + header_record(Rest, RequestHeaders#http_request_h{ + other = [{Key, Val} | + RequestHeaders#http_request_h.other]}, + Host, Version). + +validate_headers(RequestHeaders = #http_request_h{te = undefined}, Host, + "HTTP/1.1" = Version) -> + validate_headers(RequestHeaders#http_request_h{te = ""}, Host, + "HTTP/1.1" = Version); +validate_headers(RequestHeaders = #http_request_h{host = undefined}, + Host, "HTTP/1.1" = Version) -> + validate_headers(RequestHeaders#http_request_h{host = Host}, Host, Version); +validate_headers(RequestHeaders, _, _) -> + RequestHeaders. + + +default_profile() -> + ?DEFAULT_PROFILE. + +profile_name(?DEFAULT_PROFILE) -> + httpc_manager; +profile_name(Pid) when is_pid(Pid) -> + Pid; +profile_name(Profile) -> + list_to_atom("httpc_manager_" ++ atom_to_list(Profile)). + +child_name2info(undefined) -> + {error, no_such_service}; +child_name2info(httpc_manager) -> + {ok, [{profile, default}]}; +child_name2info({http, Profile}) -> + {ok, [{profile, Profile}]}. + +child_name(_, []) -> + undefined; +child_name(Pid, [{Name, Pid} | _]) -> + Name; +child_name(Pid, [_ | Children]) -> + child_name(Pid, Children). + +%% d(F) -> +%% d(F, []). + +%% d(F, A) -> +%% d(get(dbg), F, A). + +%% d(true, F, A) -> +%% io:format(user, "~w:~w:" ++ F ++ "~n", [self(), ?MODULE | A]); +%% d(_, _, _) -> +%% ok. + diff --git a/lib/inets/src/http_client/http_cookie.erl b/lib/inets/src/http_client/http_cookie.erl new file mode 100644 index 0000000000..e091070f72 --- /dev/null +++ b/lib/inets/src/http_client/http_cookie.erl @@ -0,0 +1,391 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2009. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% Description: Cookie handling according to RFC 2109 + +-module(http_cookie). + +-include("httpc_internal.hrl"). + +-export([header/4, cookies/3, open_cookie_db/1, close_cookie_db/1, insert/2]). + +%%%========================================================================= +%%% API +%%%========================================================================= +header(Scheme, {Host, _}, Path, CookieDb) -> + case lookup_cookies(Host, Path, CookieDb) of + [] -> + {"cookie", ""}; + Cookies -> + {"cookie", cookies_to_string(Scheme, Cookies)} + end. + +cookies(Headers, RequestPath, RequestHost) -> + Cookies = parse_set_cookies(Headers, {RequestPath, RequestHost}), + accept_cookies(Cookies, RequestPath, RequestHost). + +open_cookie_db({{_, only_session_cookies}, SessionDbName}) -> + EtsDb = ets:new(SessionDbName, [protected, bag, + {keypos, #http_cookie.domain}]), + {undefined, EtsDb}; + +open_cookie_db({{DbName, Dbdir}, SessionDbName}) -> + File = filename:join(Dbdir, atom_to_list(DbName)), + {ok, DetsDb} = dets:open_file(DbName, [{keypos, #http_cookie.domain}, + {type, bag}, + {file, File}, + {ram_file, true}]), + EtsDb = ets:new(SessionDbName, [protected, bag, + {keypos, #http_cookie.domain}]), + {DetsDb, EtsDb}. + +close_cookie_db({undefined, EtsDb}) -> + ets:delete(EtsDb); + +close_cookie_db({DetsDb, EtsDb}) -> + dets:close(DetsDb), + ets:delete(EtsDb). + +%% If no persistent cookie database is defined we +%% treat all cookies as if they where session cookies. +insert(Cookie = #http_cookie{max_age = Int}, + Dbs = {undefined, _}) when is_integer(Int) -> + insert(Cookie#http_cookie{max_age = session}, Dbs); + +insert(Cookie = #http_cookie{domain = Key, name = Name, + path = Path, max_age = session}, + Db = {_, CookieDb}) -> + case ets:match_object(CookieDb, #http_cookie{domain = Key, + name = Name, + path = Path, + _ = '_'}) of + [] -> + ets:insert(CookieDb, Cookie); + [NewCookie] -> + delete(NewCookie, Db), + ets:insert(CookieDb, Cookie) + end, + ok; +insert(#http_cookie{domain = Key, name = Name, + path = Path, max_age = 0}, + Db = {CookieDb, _}) -> + case dets:match_object(CookieDb, #http_cookie{domain = Key, + name = Name, + path = Path, + _ = '_'}) of + [] -> + ok; + [NewCookie] -> + delete(NewCookie, Db) + end, + ok; +insert(Cookie = #http_cookie{domain = Key, name = Name, path = Path}, + Db = {CookieDb, _}) -> + case dets:match_object(CookieDb, #http_cookie{domain = Key, + name = Name, + path = Path, + _ = '_'}) of + [] -> + dets:insert(CookieDb, Cookie); + [NewCookie] -> + delete(NewCookie, Db), + dets:insert(CookieDb, Cookie) + end, + ok. + +%%%======================================================================== +%%% Internal functions +%%%======================================================================== +lookup_cookies(Key, {undefined, Ets}) -> + ets:match_object(Ets, #http_cookie{domain = Key, + _ = '_'}); +lookup_cookies(Key, {Dets,Ets}) -> + SessionCookies = ets:match_object(Ets, #http_cookie{domain = Key, + _ = '_'}), + Cookies = dets:match_object(Dets, #http_cookie{domain = Key, + _ = '_'}), + Cookies ++ SessionCookies. + +delete(Cookie = #http_cookie{max_age = session}, {_, CookieDb}) -> + ets:delete_object(CookieDb, Cookie); +delete(Cookie, {CookieDb, _}) -> + dets:delete_object(CookieDb, Cookie). + +lookup_cookies(Host, Path, Db) -> + Cookies = + case http_util:is_hostname(Host) of + true -> + HostCookies = lookup_cookies(Host, Db), + [_| DomainParts] = string:tokens(Host, "."), + lookup_domain_cookies(DomainParts, Db, HostCookies); + false -> % IP-adress + lookup_cookies(Host, Db) + end, + ValidCookies = valid_cookies(Cookies, [], Db), + lists:filter(fun(Cookie) -> + lists:prefix(Cookie#http_cookie.path, Path) + end, ValidCookies). + +%% For instance if Host=localhost +lookup_domain_cookies([], _, AccCookies) -> + lists:flatten(AccCookies); +%% Top domains can not have cookies +lookup_domain_cookies([_], _, AccCookies) -> + lists:flatten(AccCookies); +lookup_domain_cookies([Next | DomainParts], CookieDb, AccCookies) -> + Domain = merge_domain_parts(DomainParts, [Next ++ "."]), + lookup_domain_cookies(DomainParts, CookieDb, + [lookup_cookies(Domain, CookieDb) + | AccCookies]). + +merge_domain_parts([Part], Merged) -> + lists:flatten(["." | lists:reverse([Part | Merged])]); +merge_domain_parts([Part| Rest], Merged) -> + merge_domain_parts(Rest, [".", Part | Merged]). + +cookies_to_string(Scheme, Cookies = [Cookie | _]) -> + Version = "$Version=" ++ Cookie#http_cookie.version ++ "; ", + cookies_to_string(Scheme, path_sort(Cookies), [Version]). + +cookies_to_string(_, [], CookieStrs) -> + case length(CookieStrs) of + 1 -> + ""; + _ -> + lists:flatten(lists:reverse(CookieStrs)) + end; + +cookies_to_string(https, [Cookie = #http_cookie{secure = true}| Cookies], + CookieStrs) -> + Str = case Cookies of + [] -> + cookie_to_string(Cookie); + _ -> + cookie_to_string(Cookie) ++ "; " + end, + cookies_to_string(https, Cookies, [Str | CookieStrs]); + +cookies_to_string(Scheme, [#http_cookie{secure = true}| Cookies], + CookieStrs) -> + cookies_to_string(Scheme, Cookies, CookieStrs); + +cookies_to_string(Scheme, [Cookie | Cookies], CookieStrs) -> + Str = case Cookies of + [] -> + cookie_to_string(Cookie); + _ -> + cookie_to_string(Cookie) ++ "; " + end, + cookies_to_string(Scheme, Cookies, [Str | CookieStrs]). + +cookie_to_string(Cookie = #http_cookie{name = Name, value = Value}) -> + Str = Name ++ "=" ++ Value, + add_domain(add_path(Str, Cookie), Cookie). + +add_path(Str, #http_cookie{path_default = true}) -> + Str; +add_path(Str, #http_cookie{path = Path}) -> + Str ++ "; $Path=" ++ Path. + +add_domain(Str, #http_cookie{domain_default = true}) -> + Str; +add_domain(Str, #http_cookie{domain = Domain}) -> + Str ++ "; $Domain=" ++ Domain. + +parse_set_cookies(OtherHeaders, DefaultPathDomain) -> + SetCookieHeaders = lists:foldl(fun({"set-cookie", Value}, Acc) -> + [string:tokens(Value, ",")| Acc]; + (_, Acc) -> + Acc + end, [], OtherHeaders), + + lists:flatten(lists:map(fun(CookieHeader) -> + NewHeader = + fix_netscape_cookie(CookieHeader, + []), + parse_set_cookie(NewHeader, [], + DefaultPathDomain) end, + SetCookieHeaders)). + +parse_set_cookie([], AccCookies, _) -> + AccCookies; +parse_set_cookie([CookieHeader | CookieHeaders], AccCookies, + Defaults = {DefaultPath, DefaultDomain}) -> + [CookieStr | Attributes] = case string:tokens(CookieHeader, ";") of + [CStr] -> + [CStr, ""]; + [CStr | Attr] -> + [CStr, Attr] + end, + Pos = string:chr(CookieStr, $=), + Name = string:substr(CookieStr, 1, Pos - 1), + Value = string:substr(CookieStr, Pos + 1), + Cookie = #http_cookie{name = string:strip(Name), + value = string:strip(Value)}, + NewAttributes = parse_set_cookie_attributes(Attributes), + TmpCookie = cookie_attributes(NewAttributes, Cookie), + %% Add runtime defult values if necessary + NewCookie = domain_default(path_default(TmpCookie, DefaultPath), + DefaultDomain), + parse_set_cookie(CookieHeaders, [NewCookie | AccCookies], Defaults). + +parse_set_cookie_attributes([]) -> + []; +parse_set_cookie_attributes([Attributes]) -> + lists:map(fun(Attr) -> + [AttrName, AttrValue] = + case string:tokens(Attr, "=") of + %% All attributes have the form + %% Name=Value except "secure"! + [Name] -> + [Name, ""]; + [Name, Value] -> + [Name, Value]; + %% Anything not expected will be + %% disregarded + _ -> + ["Dummy",""] + end, + {http_util:to_lower(string:strip(AttrName)), + string:strip(AttrValue)} + end, Attributes). + +cookie_attributes([], Cookie) -> + Cookie; +cookie_attributes([{"comment", Value}| Attributes], Cookie) -> + cookie_attributes(Attributes, + Cookie#http_cookie{comment = Value}); +cookie_attributes([{"domain", Value}| Attributes], Cookie) -> + cookie_attributes(Attributes, + Cookie#http_cookie{domain = Value}); +cookie_attributes([{"max-age", Value}| Attributes], Cookie) -> + ExpireTime = cookie_expires(list_to_integer(Value)), + cookie_attributes(Attributes, + Cookie#http_cookie{max_age = ExpireTime}); +%% Backwards compatibility with netscape cookies +cookie_attributes([{"expires", Value}| Attributes], Cookie) -> + Time = http_util:convert_netscapecookie_date(Value), + ExpireTime = calendar:datetime_to_gregorian_seconds(Time), + cookie_attributes(Attributes, + Cookie#http_cookie{max_age = ExpireTime}); +cookie_attributes([{"path", Value}| Attributes], Cookie) -> + cookie_attributes(Attributes, + Cookie#http_cookie{path = Value}); +cookie_attributes([{"secure", _}| Attributes], Cookie) -> + cookie_attributes(Attributes, + Cookie#http_cookie{secure = true}); +cookie_attributes([{"version", Value}| Attributes], Cookie) -> + cookie_attributes(Attributes, + Cookie#http_cookie{version = Value}); +%% Disregard unknown attributes. +cookie_attributes([_| Attributes], Cookie) -> + cookie_attributes(Attributes, Cookie). + +domain_default(Cookie = #http_cookie{domain = undefined}, + DefaultDomain) -> + Cookie#http_cookie{domain = DefaultDomain, domain_default = true}; +domain_default(Cookie, _) -> + Cookie. + +path_default(Cookie = #http_cookie{path = undefined}, + DefaultPath) -> + Cookie#http_cookie{path = skip_right_most_slash(DefaultPath), + path_default = true}; +path_default(Cookie, _) -> + Cookie. + +%% Note: if the path is only / that / will be keept +skip_right_most_slash("/") -> + "/"; +skip_right_most_slash(Str) -> + string:strip(Str, right, $/). + +accept_cookies(Cookies, RequestPath, RequestHost) -> + lists:filter(fun(Cookie) -> + accept_cookie(Cookie, RequestPath, RequestHost) + end, Cookies). + +accept_cookie(Cookie, RequestPath, RequestHost) -> + accept_path(Cookie, RequestPath) and accept_domain(Cookie, RequestHost). + +accept_path(#http_cookie{path = Path}, RequestPath) -> + lists:prefix(Path, RequestPath). + +accept_domain(#http_cookie{domain = RequestHost}, RequestHost) -> + true; + +accept_domain(#http_cookie{domain = Domain}, RequestHost) -> + HostCheck = case http_util:is_hostname(RequestHost) of + true -> + (lists:suffix(Domain, RequestHost) andalso + (not + lists:member($., + string:substr(RequestHost, 1, + (length(RequestHost) - + length(Domain)))))); + false -> + false + end, + HostCheck andalso (hd(Domain) == $.) + andalso (length(string:tokens(Domain, ".")) > 1). + +cookie_expires(0) -> + 0; +cookie_expires(DeltaSec) -> + NowSec = calendar:datetime_to_gregorian_seconds({date(), time()}), + NowSec + DeltaSec. + +is_cookie_expired(#http_cookie{max_age = session}) -> + false; +is_cookie_expired(#http_cookie{max_age = ExpireTime}) -> + NowSec = calendar:datetime_to_gregorian_seconds({date(), time()}), + ExpireTime - NowSec =< 0. + +valid_cookies([], Valid, _) -> + Valid; + +valid_cookies([Cookie | Cookies], Valid, Db) -> + case is_cookie_expired(Cookie) of + true -> + delete(Cookie, Db), + valid_cookies(Cookies, Valid, Db); + false -> + valid_cookies(Cookies, [Cookie | Valid], Db) + end. + +path_sort(Cookies)-> + lists:reverse(lists:keysort(#http_cookie.path, Cookies)). + + +%% Informally, the Set-Cookie response header comprises the token +%% Set-Cookie:, followed by a comma-separated list of one or more +%% cookies. Netscape cookies expires attribute may also have a +%% , in this case the header list will have been incorrectly split +%% in parse_set_cookies/2 this functions fixs that problem. +fix_netscape_cookie([Cookie1, Cookie2 | Rest], Acc) -> + case inets_regexp:match(Cookie1, "expires=") of + {_, _, _} -> + fix_netscape_cookie(Rest, [Cookie1 ++ Cookie2 | Acc]); + nomatch -> + fix_netscape_cookie([Cookie2 |Rest], [Cookie1| Acc]) + end; +fix_netscape_cookie([Cookie | Rest], Acc) -> + fix_netscape_cookie(Rest, [Cookie | Acc]); + +fix_netscape_cookie([], Acc) -> + Acc. diff --git a/lib/inets/src/http_client/http_uri.erl b/lib/inets/src/http_client/http_uri.erl new file mode 100644 index 0000000000..615a0d8ec4 --- /dev/null +++ b/lib/inets/src/http_client/http_uri.erl @@ -0,0 +1,116 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2006-2009. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% + +-module(http_uri). + +-export([parse/1]). + +%%%========================================================================= +%%% API +%%%========================================================================= +parse(AbsURI) -> + case parse_scheme(AbsURI) of + {error, Reason} -> + {error, Reason}; + {Scheme, Rest} -> + case (catch parse_uri_rest(Scheme, Rest)) of + {UserInfo, Host, Port, Path, Query} -> + {Scheme, UserInfo, Host, Port, Path, Query}; + _ -> + {error, {malformed_url, AbsURI}} + end + end. + +%%%======================================================================== +%%% Internal functions +%%%======================================================================== +parse_scheme(AbsURI) -> + case split_uri(AbsURI, ":", {error, no_scheme}, 1, 1) of + {error, no_scheme} -> + {error, no_scheme}; + {StrScheme, Rest} -> + case list_to_atom(http_util:to_lower(StrScheme)) of + Scheme when Scheme == http; Scheme == https -> + {Scheme, Rest}; + Scheme -> + {error, {not_supported_scheme, Scheme}} + end + end. + +parse_uri_rest(Scheme, "//" ++ URIPart) -> + + {Authority, PathQuery} = + case split_uri(URIPart, "/", URIPart, 1, 0) of + Split = {_, _} -> + Split; + URIPart -> + case split_uri(URIPart, "\\?", URIPart, 1, 0) of + Split = {_, _} -> + Split; + URIPart -> + {URIPart,""} + end + end, + + {UserInfo, HostPort} = split_uri(Authority, "@", {"", Authority}, 1, 1), + {Host, Port} = parse_host_port(Scheme, HostPort), + {Path, Query} = parse_path_query(PathQuery), + {UserInfo, Host, Port, Path, Query}. + + +parse_path_query(PathQuery) -> + {Path, Query} = split_uri(PathQuery, "\\?", {PathQuery, ""}, 1, 0), + {path(Path), Query}. + + +parse_host_port(Scheme,"[" ++ HostPort) -> %ipv6 + DefaultPort = default_port(Scheme), + {Host, ColonPort} = split_uri(HostPort, "\\]", {HostPort, ""}, 1, 1), + {_, Port} = split_uri(ColonPort, ":", {"", DefaultPort}, 0, 1), + {Host, int_port(Port)}; + +parse_host_port(Scheme, HostPort) -> + DefaultPort = default_port(Scheme), + {Host, Port} = split_uri(HostPort, ":", {HostPort, DefaultPort}, 1, 1), + {Host, int_port(Port)}. + +split_uri(UriPart, SplitChar, NoMatchResult, SkipLeft, SkipRight) -> + case inets_regexp:first_match(UriPart, SplitChar) of + {match, Match, _} -> + {string:substr(UriPart, 1, Match - SkipLeft), + string:substr(UriPart, Match + SkipRight, length(UriPart))}; + nomatch -> + NoMatchResult + end. + +default_port(http) -> + 80; +default_port(https) -> + 443. + +int_port(Port) when is_integer(Port) -> + Port; +int_port(Port) when is_list(Port) -> + list_to_integer(Port). + +path("") -> + "/"; +path(Path) -> + Path. diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl new file mode 100644 index 0000000000..7b737c2f86 --- /dev/null +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -0,0 +1,1499 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2002-2009. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% + +-module(httpc_handler). + +-behaviour(gen_server). + +-include("httpc_internal.hrl"). +-include("http_internal.hrl"). + + +%%-------------------------------------------------------------------- +%% Internal Application API +-export([start_link/3, send/2, cancel/2, stream/3, stream_next/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-record(timers, + { + request_timers = [], % [ref()] + queue_timer % ref() + }). + +-record(state, + { + request, % #request{} + session, % #tcp_session{} + status_line, % {Version, StatusCode, ReasonPharse} + headers, % #http_response_h{} + body, % binary() + mfa, % {Moduel, Function, Args} + pipeline = queue:new(), % queue() + keep_alive = queue:new(), % queue() + status = new, % new | pipeline | keep_alive | close | ssl_tunnel + canceled = [], % [RequestId] + max_header_size = nolimit, % nolimit | integer() + max_body_size = nolimit, % nolimit | integer() + options, % #options{} + timers = #timers{}, % #timers{} + profile_name, % atom() - id of httpc_manager process. + once % send | undefined + }). + + +%%==================================================================== +%% External functions +%%==================================================================== +%%-------------------------------------------------------------------- +%% Function: start_link(Request, Options, ProfileName) -> {ok, Pid} +%% +%% Request = #request{} +%% Options = #options{} +%% ProfileName = atom() - id of httpc manager process +%% +%% Description: Starts a http-request handler process. Intended to be +%% called by the httpc profile supervisor or the http manager process +%% if the client is started stand alone form inets. +%% +%% Note: Uses proc_lib and gen_server:enter_loop so that waiting +%% for gen_tcp:connect to timeout in init/1 will not +%% block the httpc manager process in odd cases such as trying to call +%% a server that does not exist. (See OTP-6735) The only API function +%% sending messages to the handler process that can be called before +%% init has compleated is cancel and that is not a problem! (Send and +%% stream will not be called before the first request has been sent and +%% the reply or part of it has arrived.) +%%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +start_link(Request, Options, ProfileName) -> + {ok, proc_lib:spawn_link(?MODULE, init, [[Request, Options, + ProfileName]])}. + + +%%-------------------------------------------------------------------- +%% Function: send(Request, Pid) -> ok +%% Request = #request{} +%% Pid = pid() - the pid of the http-request handler process. +%% +%% Description: Uses this handlers session to send a request. Intended +%% to be called by the httpc manager process. +%%-------------------------------------------------------------------- +send(Request, Pid) -> + call(Request, Pid, 5000). + + +%%-------------------------------------------------------------------- +%% Function: cancel(RequestId, Pid) -> ok +%% RequestId = ref() +%% Pid = pid() - the pid of the http-request handler process. +%% +%% Description: Cancels a request. Intended to be called by the httpc +%% manager process. +%%-------------------------------------------------------------------- +cancel(RequestId, Pid) -> + cast({cancel, RequestId}, Pid). + + +%%-------------------------------------------------------------------- +%% Function: stream_next(Pid) -> ok +%% Pid = pid() - the pid of the http-request handler process. +%% +%% Description: Works as inets:setopts(active, once) but for +%% body chunks sent to the user. +%%-------------------------------------------------------------------- +stream_next(Pid) -> + cast(stream_next, Pid). + + +%%-------------------------------------------------------------------- +%% Function: stream(BodyPart, Request, Code) -> _ +%% BodyPart = binary() +%% Request = #request{} +%% Code = integer() +%% +%% Description: Stream the HTTP body to the caller process (client) +%% or to a file. Note that the data that has been stream +%% does not have to be saved. (We do not want to use up +%% memory in vain.) +%%-------------------------------------------------------------------- +%% Request should not be streamed +stream(BodyPart, Request = #request{stream = none}, _) -> + ?hcrt("stream - none", [{body_part, BodyPart}]), + {BodyPart, Request}; + +%% Stream to caller +stream(BodyPart, Request = #request{stream = Self}, Code) + when ((Code =:= 200) orelse (Code =:= 206)) andalso + ((Self =:= self) orelse (Self =:= {self, once})) -> + ?hcrt("stream - self", [{stream, Self}, {code, Code}, {body_part, BodyPart}]), + httpc_response:send(Request#request.from, + {Request#request.id, stream, BodyPart}), + {<<>>, Request}; + +stream(BodyPart, Request = #request{stream = Self}, 404) + when (Self =:= self) orelse (Self =:= {self, once}) -> + ?hcrt("stream - self with 404", [{stream, Self}, {body_part, BodyPart}]), + httpc_response:send(Request#request.from, + {Request#request.id, stream, BodyPart}), + {<<>>, Request}; + +%% Stream to file +%% This has been moved to start_stream/3 +%% We keep this for backward compatibillity... +stream(BodyPart, Request = #request{stream = Filename}, Code) + when ((Code =:= 200) orelse (Code =:= 206)) andalso is_list(Filename) -> + ?hcrt("stream - filename", [{stream, Filename}, {code, Code}, {body_part, BodyPart}]), + case file:open(Filename, [write, raw, append, delayed_write]) of + {ok, Fd} -> + ?hcrt("stream - file open ok", [{fd, Fd}]), + stream(BodyPart, Request#request{stream = Fd}, 200); + {error, Reason} -> + exit({stream_to_file_failed, Reason}) + end; + +%% Stream to file +stream(BodyPart, Request = #request{stream = Fd}, Code) + when ((Code =:= 200) orelse (Code =:= 206)) -> + ?hcrt("stream to file", [{stream, Fd}, {code, Code}, {body_part, BodyPart}]), + case file:write(Fd, BodyPart) of + ok -> + {<<>>, Request}; + {error, Reason} -> + exit({stream_to_file_failed, Reason}) + end; + +stream(BodyPart, Request,_) -> % only 200 and 206 responses can be streamed + ?hcrt("stream - ignore", [{request, Request}, {body_part, BodyPart}]), + {BodyPart, Request}. + + +%%==================================================================== +%% Server functions +%%==================================================================== + +%%-------------------------------------------------------------------- +%% Function: init([Request, Options, ProfileName]) -> {ok, State} | +%% {ok, State, Timeout} | ignore |{stop, Reason} +%% +%% Request = #request{} +%% Options = #options{} +%% ProfileName = atom() - id of httpc manager process +%% +%% Description: Initiates the httpc_handler process +%% +%% Note: The init function may not fail, that will kill the +%% httpc_manager process. We could make the httpc_manager more comlex +%% but we do not want that so errors will be handled by the process +%% sending an init_error message to itself. +%%-------------------------------------------------------------------- +init([Request, Options, ProfileName]) -> + process_flag(trap_exit, true), + + handle_verbose(Options#options.verbose), + Address = handle_proxy(Request#request.address, Options#options.proxy), + {ok, State} = + case {Address /= Request#request.address, Request#request.scheme} of + {true, https} -> + Error = https_through_proxy_is_not_currently_supported, + self() ! {init_error, + Error, httpc_response:error(Request, Error)}, + {ok, #state{request = Request, options = Options, + status = ssl_tunnel}}; + %% This is what we should do if and when ssl supports + %% "socket upgrading" + %%send_ssl_tunnel_request(Address, Request, + %% #state{options = Options, + %% status = ssl_tunnel}); + {_, _} -> + send_first_request(Address, Request, + #state{options = Options, + profile_name = ProfileName}) + end, + gen_server:enter_loop(?MODULE, [], State). + +%%-------------------------------------------------------------------- +%% Function: handle_call(Request, From, State) -> {reply, Reply, State} | +%% {reply, Reply, State, Timeout} | +%% {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, Reply, State} | (terminate/2 is called) +%% {stop, Reason, State} (terminate/2 is called) +%% Description: Handling call messages +%%-------------------------------------------------------------------- +handle_call(Request, _, State = #state{session = Session = + #tcp_session{socket = Socket, + type = pipeline}, + timers = Timers, + options = Options, + profile_name = ProfileName}) -> + Address = handle_proxy(Request#request.address, Options#options.proxy), + + case httpc_request:send(Address, Request, Socket) of + ok -> + %% Activate the request time out for the new request + NewState = activate_request_timeout(State#state{request = + Request}), + + ClientClose = httpc_request:is_client_closing( + Request#request.headers), + case State#state.request of + #request{} -> %% Old request no yet finished + %% Make sure to use the new value of timers in state + NewTimers = NewState#state.timers, + NewPipeline = queue:in(Request, State#state.pipeline), + NewSession = + Session#tcp_session{queue_length = + %% Queue + current + queue:len(NewPipeline) + 1, + client_close = ClientClose}, + httpc_manager:insert_session(NewSession, ProfileName), + {reply, ok, State#state{pipeline = NewPipeline, + session = NewSession, + timers = NewTimers}}; + undefined -> + %% Note: tcp-message reciving has already been + %% activated by handle_pipeline/2. + cancel_timer(Timers#timers.queue_timer, + timeout_queue), + NewSession = + Session#tcp_session{queue_length = 1, + client_close = ClientClose}, + httpc_manager:insert_session(NewSession, ProfileName), + Relaxed = + (Request#request.settings)#http_options.relaxed, + {reply, ok, + NewState#state{request = Request, + session = NewSession, + mfa = {httpc_response, parse, + [State#state.max_header_size, + Relaxed]}, + timers = + Timers#timers{queue_timer = + undefined}}} + end; + {error, Reason} -> + {reply, {pipeline_failed, Reason}, State} + end; + +handle_call(Request, _, #state{session = Session = + #tcp_session{type = keep_alive, + socket = Socket}, + timers = Timers, + options = Options, + profile_name = ProfileName} = State) -> + + ClientClose = httpc_request:is_client_closing(Request#request.headers), + + Address = handle_proxy(Request#request.address, + Options#options.proxy), + case httpc_request:send(Address, Request, Socket) of + ok -> + NewState = + activate_request_timeout(State#state{request = + Request}), + + case State#state.request of + #request{} -> %% Old request not yet finished + %% Make sure to use the new value of timers in state + NewTimers = NewState#state.timers, + NewKeepAlive = queue:in(Request, State#state.keep_alive), + NewSession = + Session#tcp_session{queue_length = + %% Queue + current + queue:len(NewKeepAlive) + 1, + client_close = ClientClose}, + httpc_manager:insert_session(NewSession, ProfileName), + {reply, ok, State#state{keep_alive = NewKeepAlive, + session = NewSession, + timers = NewTimers}}; + undefined -> + %% Note: tcp-message reciving has already been + %% activated by handle_pipeline/2. + cancel_timer(Timers#timers.queue_timer, + timeout_queue), + NewSession = + Session#tcp_session{queue_length = 1, + client_close = ClientClose}, + httpc_manager:insert_session(NewSession, ProfileName), + Relaxed = + (Request#request.settings)#http_options.relaxed, + {reply, ok, + NewState#state{request = Request, + session = NewSession, + mfa = {httpc_response, parse, + [State#state.max_header_size, + Relaxed]}}} + end; + {error, Reason} -> + {reply, {request_failed, Reason}, State} + end. + +%%-------------------------------------------------------------------- +%% Function: handle_cast(Msg, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} (terminate/2 is called) +%% Description: Handling cast messages +%%-------------------------------------------------------------------- + +%% When the request in process has been canceled the handler process is +%% stopped and the pipelined requests will be reissued or remaining +%% requests will be sent on a new connection. This is is +%% based on the assumption that it is proably cheaper to reissue the +%% requests than to wait for a potentiall large response that we then +%% only throw away. This of course is not always true maybe we could +%% do something smarter here?! If the request canceled is not +%% the one handled right now the same effect will take place in +%% handle_pipeline/2 when the canceled request is on turn, +%% handle_keep_alive_queue/2 on the other hand will just skip the +%% request as if it was never issued as in this case the request will +%% not have been sent. +handle_cast({cancel, RequestId}, State = #state{request = Request = + #request{id = RequestId}, + profile_name = ProfileName}) -> + httpc_manager:request_canceled(RequestId, ProfileName), + {stop, normal, + State#state{canceled = [RequestId | State#state.canceled], + request = Request#request{from = answer_sent}}}; +handle_cast({cancel, RequestId}, State = #state{profile_name = ProfileName}) -> + httpc_manager:request_canceled(RequestId, ProfileName), + {noreply, State#state{canceled = [RequestId | State#state.canceled]}}; +handle_cast(stream_next, #state{session = Session} = State) -> + http_transport:setopts(socket_type(Session#tcp_session.scheme), + Session#tcp_session.socket, [{active, once}]), + {noreply, State#state{once = once}}. + + +%%-------------------------------------------------------------------- +%% Function: handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} (terminate/2 is called) +%% Description: Handling all non call/cast messages +%%-------------------------------------------------------------------- +handle_info({Proto, _Socket, Data}, + #state{mfa = {Module, Function, Args} = MFA, + request = #request{method = Method, + stream = Stream} = Request, + session = Session, + status_line = StatusLine} = State) + when (Proto =:= tcp) orelse + (Proto =:= ssl) orelse + (Proto =:= httpc_handler) -> + + ?hcri("received data", [{proto, Proto}, {data, Data}, {mfa, MFA}, {method, Method}, {stream, Stream}, {session, Session}, {status_line, StatusLine}]), + + FinalResult = + try Module:Function([Data | Args]) of + {ok, Result} -> + ?hcrd("data processed - ok", [{result, Result}]), + handle_http_msg(Result, State); + {_, whole_body, _} when Method =:= head -> + ?hcrd("data processed - whole body", []), + handle_response(State#state{body = <<>>}); + {Module, whole_body, [Body, Length]} -> + ?hcrd("data processed - whole body", [{module, Module}, {body, Body}, {length, Length}]), + {_, Code, _} = StatusLine, + {NewBody, NewRequest} = stream(Body, Request, Code), + %% When we stream we will not keep the already + %% streamed data, that would be a waste of memory. + NewLength = case Stream of + none -> + Length; + _ -> + Length - size(Body) + end, + + NewState = next_body_chunk(State), + + {noreply, NewState#state{mfa = {Module, whole_body, + [NewBody, NewLength]}, + request = NewRequest}}; + NewMFA -> + ?hcrd("data processed", [{new_mfa, NewMFA}]), + http_transport:setopts(socket_type(Session#tcp_session.scheme), + Session#tcp_session.socket, + [{active, once}]), + {noreply, State#state{mfa = NewMFA}} + catch + exit:_ -> + ClientErrMsg = httpc_response:error(Request, + {could_not_parse_as_http, + Data}), + NewState = answer_request(Request, ClientErrMsg, State), + {stop, normal, NewState}; + error:_ -> + ClientErrMsg = httpc_response:error(Request, + {could_not_parse_as_http, + Data}), + NewState = answer_request(Request, ClientErrMsg, State), + {stop, normal, NewState} + + end, + ?hcri("data processed", [{result, FinalResult}]), + FinalResult; + + +handle_info({Proto, Socket, Data}, + #state{mfa = MFA, + request = Request, + session = Session, + status = Status, + status_line = StatusLine, + profile_name = Profile} = State) + when (Proto =:= tcp) orelse + (Proto =:= ssl) orelse + (Proto =:= httpc_handler) -> + + error_logger:warning_msg("Received unexpected ~p data on ~p" + "~n Data: ~p" + "~n MFA: ~p" + "~n Request: ~p" + "~n Session: ~p" + "~n Status: ~p" + "~n StatusLine: ~p" + "~n Profile: ~p" + "~n", + [Proto, Socket, Data, MFA, + Request, Session, Status, StatusLine, Profile]), + {noreply, State}; + + +%% The Server may close the connection to indicate that the +%% whole body is now sent instead of sending an length +%% indicator. +handle_info({tcp_closed, _}, State = #state{mfa = {_, whole_body, Args}}) -> + handle_response(State#state{body = hd(Args)}); +handle_info({ssl_closed, _}, State = #state{mfa = {_, whole_body, Args}}) -> + handle_response(State#state{body = hd(Args)}); + +%%% Server closes idle pipeline +handle_info({tcp_closed, _}, State = #state{request = undefined}) -> + {stop, normal, State}; +handle_info({ssl_closed, _}, State = #state{request = undefined}) -> + {stop, normal, State}; + +%%% Error cases +handle_info({tcp_closed, _}, #state{session = Session0} = State) -> + Socket = Session0#tcp_session.socket, + Session = Session0#tcp_session{socket = {remote_close, Socket}}, + %% {stop, session_remotly_closed, State}; + {stop, normal, State#state{session = Session}}; +handle_info({ssl_closed, _}, #state{session = Session0} = State) -> + Socket = Session0#tcp_session.socket, + Session = Session0#tcp_session{socket = {remote_close, Socket}}, + %% {stop, session_remotly_closed, State}; + {stop, normal, State#state{session = Session}}; +handle_info({tcp_error, _, _} = Reason, State) -> + {stop, Reason, State}; +handle_info({ssl_error, _, _} = Reason, State) -> + {stop, Reason, State}; + +%% Timeouts +%% Internally, to a request handling process, a request timeout is +%% seen as a canceled request. +handle_info({timeout, RequestId}, + #state{request = #request{id = RequestId} = Request, + canceled = Canceled} = State) -> + httpc_response:send(Request#request.from, + httpc_response:error(Request,timeout)), + {stop, normal, + State#state{request = Request#request{from = answer_sent}, + canceled = [RequestId | Canceled]}}; + +handle_info({timeout, RequestId}, #state{canceled = Canceled} = State) -> + Filter = + fun(#request{id = Id, from = From} = Request) when Id =:= RequestId -> + %% Notify the owner + Response = httpc_response:error(Request, timeout), + httpc_response:send(From, Response), + [Request#request{from = answer_sent}]; + (_) -> + true + end, + case State#state.status of + pipeline -> + Pipeline = queue:filter(Filter, State#state.pipeline), + {noreply, State#state{canceled = [RequestId | Canceled], + pipeline = Pipeline}}; + keep_alive -> + KeepAlive = queue:filter(Filter, State#state.keep_alive), + {noreply, State#state{canceled = [RequestId | Canceled], + keep_alive = KeepAlive}} + end; + +handle_info(timeout_queue, State = #state{request = undefined}) -> + {stop, normal, State}; + +%% Timing was such as the pipeline_timout was not canceled! +handle_info(timeout_queue, #state{timers = Timers} = State) -> + {noreply, State#state{timers = + Timers#timers{queue_timer = undefined}}}; + +%% Setting up the connection to the server somehow failed. +handle_info({init_error, _, ClientErrMsg}, + State = #state{request = Request}) -> + NewState = answer_request(Request, ClientErrMsg, State), + {stop, normal, NewState}; + + +%%% httpc_manager process dies. +handle_info({'EXIT', _, _}, State = #state{request = undefined}) -> + {stop, normal, State}; +%%Try to finish the current request anyway, +%% there is a fairly high probability that it can be done successfully. +%% Then close the connection, hopefully a new manager is started that +%% can retry requests in the pipeline. +handle_info({'EXIT', _, _}, State) -> + {noreply, State#state{status = close}}. + + +%%-------------------------------------------------------------------- +%% Function: terminate(Reason, State) -> _ (ignored by gen_server) +%% Description: Shutdown the httpc_handler +%%-------------------------------------------------------------------- + +%% Init error there is no socket to be closed. +terminate(normal, #state{session = undefined}) -> + ok; + +%% Init error sending, no session information has been setup but +%% there is a socket that needs closing. +terminate(normal, #state{request = Request, + session = #tcp_session{id = undefined, + socket = Socket}}) -> + http_transport:close(socket_type(Request), Socket); + +%% Socket closed remotely +terminate(normal, + #state{session = #tcp_session{socket = {remote_close, Socket}, + id = Id}, + profile_name = ProfileName, + request = Request, + timers = Timers, + pipeline = Pipeline}) -> + %% Clobber session + (catch httpc_manager:delete_session(Id, ProfileName)), + + %% Cancel timers + #timers{request_timers = ReqTmrs, queue_timer = QTmr} = Timers, + cancel_timer(QTmr, timeout_queue), + lists:foreach(fun({_, Timer}) -> cancel_timer(Timer, timeout) end, + ReqTmrs), + + %% Maybe deliver answers to requests + deliver_answers([Request | queue:to_list(Pipeline)]), + + %% And, just in case, close our side (**really** overkill) + http_transport:close(socket_type(Request), Socket); + +terminate(_, State = #state{session = Session, + request = undefined, + profile_name = ProfileName, + timers = Timers, + pipeline = Pipeline, + keep_alive = KeepAlive}) -> + catch httpc_manager:delete_session(Session#tcp_session.id, + ProfileName), + + maybe_retry_queue(Pipeline, State), + maybe_retry_queue(KeepAlive, State), + + cancel_timer(Timers#timers.queue_timer, timeout_queue), + Socket = Session#tcp_session.socket, + http_transport:close(socket_type(Session#tcp_session.scheme), Socket); + +terminate(Reason, State = #state{request = Request}) -> + NewState = maybe_send_answer(Request, + httpc_response:error(Request, Reason), + State), + terminate(Reason, NewState#state{request = undefined}). + +maybe_retry_queue(Q, State) -> + case queue:is_empty(Q) of + false -> + retry_pipeline(queue:to_list(Q), State); + true -> + ok + end. + +maybe_send_answer(#request{from = answer_sent}, _Reason, State) -> + State; +maybe_send_answer(Request, Answer, State) -> + answer_request(Request, Answer, State). + +deliver_answers([]) -> + ok; +deliver_answers([#request{from = From} = Request | Requests]) + when is_pid(From) -> + Response = httpc_response:error(Request, socket_closed_remotely), + httpc_response:send(From, Response), + deliver_answers(Requests); +deliver_answers([_|Requests]) -> + deliver_answers(Requests). + + +%%-------------------------------------------------------------------- +%% Func: code_change(_OldVsn, State, Extra) -> {ok, NewState} +%% Purpose: Convert process state when code is changed +%%-------------------------------------------------------------------- +code_change(_, #state{request = Request, pipeline = Queue} = State, + [{from, '5.0.1'}, {to, '5.0.2'}]) -> + Settings = new_http_options(Request#request.settings), + NewRequest = Request#request{settings = Settings}, + NewQueue = new_queue(Queue, fun new_http_options/1), + {ok, State#state{request = NewRequest, pipeline = NewQueue}}; + +code_change(_, #state{request = Request, pipeline = Queue} = State, + [{from, '5.0.2'}, {to, '5.0.1'}]) -> + Settings = old_http_options(Request#request.settings), + NewRequest = Request#request{settings = Settings}, + NewQueue = new_queue(Queue, fun old_http_options/1), + {ok, State#state{request = NewRequest, pipeline = NewQueue}}; + +code_change(_, State, _) -> + {ok, State}. + +new_http_options({http_options, TimeOut, AutoRedirect, SslOpts, + Auth, Relaxed}) -> + {http_options, "HTTP/1.1", TimeOut, AutoRedirect, SslOpts, + Auth, Relaxed}. + +old_http_options({http_options, _, TimeOut, AutoRedirect, + SslOpts, Auth, Relaxed}) -> + {http_options, TimeOut, AutoRedirect, SslOpts, Auth, Relaxed}. + +new_queue(Queue, Fun) -> + List = queue:to_list(Queue), + NewList = + lists:map(fun(Request) -> + Settings = + Fun(Request#request.settings), + Request#request{settings = Settings} + end, List), + queue:from_list(NewList). + +%%-------------------------------------------------------------------- +%%% Internal functions +%%-------------------------------------------------------------------- + +connect(SocketType, ToAddress, #options{ipfamily = IpFamily, + ip = FromAddress, + port = FromPort}, Timeout) -> + Opts1 = + case FromPort of + default -> + []; + _ -> + [{port, FromPort}] + end, + Opts2 = + case FromAddress of + default -> + Opts1; + _ -> + [{ip, FromAddress} | Opts1] + end, + case IpFamily of + inet6fb4 -> + Opts3 = [inet6 | Opts2], + case http_transport:connect(SocketType, ToAddress, Opts3, Timeout) of + {error, Reason} when ((Reason =:= nxdomain) orelse + (Reason =:= eafnosupport)) -> + Opts4 = [inet | Opts2], + http_transport:connect(SocketType, ToAddress, Opts4, Timeout); + Other -> + Other + end; + _ -> + Opts3 = [IpFamily | Opts2], + http_transport:connect(SocketType, ToAddress, Opts3, Timeout) + end. + + +send_first_request(Address, Request, #state{options = Options} = State) -> + SocketType = socket_type(Request), + ConnTimeout = (Request#request.settings)#http_options.connect_timeout, + ?hcri("connect", + [{address, Address}, {request, Request}, {options, Options}]), + case connect(SocketType, Address, Options, ConnTimeout) of + {ok, Socket} -> + ?hcri("connected - now send first request", [{socket, Socket}]), + case httpc_request:send(Address, Request, Socket) of + ok -> + ?hcri("first request sent", []), + ClientClose = + httpc_request:is_client_closing( + Request#request.headers), + SessionType = httpc_manager:session_type(Options), + Session = + #tcp_session{id = {Request#request.address, self()}, + scheme = Request#request.scheme, + socket = Socket, + client_close = ClientClose, + type = SessionType}, + TmpState = State#state{request = Request, + session = Session, + mfa = init_mfa(Request, State), + status_line = + init_status_line(Request), + headers = undefined, + body = undefined, + status = new}, + http_transport:setopts(SocketType, + Socket, [{active, once}]), + NewState = activate_request_timeout(TmpState), + {ok, NewState}; + + {error, Reason} -> + %% Commented out in wait of ssl support to avoid + %% dialyzer warning + %%case State#state.status of + %% new -> % Called from init/1 + self() ! {init_error, error_sending, + httpc_response:error(Request, Reason)}, + {ok, State#state{request = Request, + session = + #tcp_session{socket = Socket}}} + %%ssl_tunnel -> % Not called from init/1 + %% NewState = + %% answer_request(Request, + %%httpc_response:error(Request, + %%Reason), + %% State), + %% {stop, normal, NewState} + %% end + end; + + {error, Reason} -> + %% Commented out in wait of ssl support to avoid + %% dialyzer warning + %% case State#state.status of + %% new -> % Called from init/1 + self() ! {init_error, error_connecting, + httpc_response:error(Request, Reason)}, + {ok, State#state{request = Request}} + %% ssl_tunnel -> % Not called from init/1 + %% NewState = + %% answer_request(Request, + %% httpc_response:error(Request, + %% Reason), + %% State), + %% {stop, normal, NewState} + %%end + end. + +handle_http_msg({Version, StatusCode, ReasonPharse, Headers, Body}, + State = #state{request = Request}) -> + ?hcrt("handle_http_msg", [{body, Body}]), + case Headers#http_response_h.'content-type' of + "multipart/byteranges" ++ _Param -> + exit(not_yet_implemented); + _ -> + StatusLine = {Version, StatusCode, ReasonPharse}, + {ok, NewRequest} = start_stream(StatusLine, Headers, Request), + handle_http_body(Body, + State#state{request = NewRequest, + status_line = StatusLine, + headers = Headers}) + end; +handle_http_msg({ChunkedHeaders, Body}, + State = #state{headers = Headers}) -> + ?hcrt("handle_http_msg", [{chunked_headers, ChunkedHeaders}, {body, Body}]), + NewHeaders = http_chunk:handle_headers(Headers, ChunkedHeaders), + handle_response(State#state{headers = NewHeaders, body = Body}); +handle_http_msg(Body, State = #state{status_line = {_,Code, _}}) -> + ?hcrt("handle_http_msg", [{body, Body}, {code, Code}]), + {NewBody, NewRequest}= stream(Body, State#state.request, Code), + handle_response(State#state{body = NewBody, request = NewRequest}). + +handle_http_body(<<>>, State = #state{status_line = {_,304, _}}) -> + ?hcrt("handle_http_body - 304", []), + handle_response(State#state{body = <<>>}); + +handle_http_body(<<>>, State = #state{status_line = {_,204, _}}) -> + ?hcrt("handle_http_body - 204", []), + handle_response(State#state{body = <<>>}); + +handle_http_body(<<>>, State = #state{request = #request{method = head}}) -> + ?hcrt("handle_http_body - head", []), + handle_response(State#state{body = <<>>}); + +handle_http_body(Body, State = #state{headers = Headers, + max_body_size = MaxBodySize, + status_line = {_,Code, _}, + request = Request}) -> + ?hcrt("handle_http_body", [{body, Body}, {max_body_size, MaxBodySize}, {code, Code}]), + TransferEnc = Headers#http_response_h.'transfer-encoding', + case case_insensitive_header(TransferEnc) of + "chunked" -> + ?hcrt("handle_http_body - chunked", []), + case http_chunk:decode(Body, State#state.max_body_size, + State#state.max_header_size, + {Code, Request}) of + {Module, Function, Args} -> + ?hcrt("handle_http_body - new mfa", [{module, Module}, {function, Function}, {args, Args}]), + NewState = next_body_chunk(State), + {noreply, NewState#state{mfa = + {Module, Function, Args}}}; + {ok, {ChunkedHeaders, NewBody}} -> + ?hcrt("handle_http_body - nyew body", [{chunked_headers, ChunkedHeaders}, {new_body, NewBody}]), + NewHeaders = http_chunk:handle_headers(Headers, + ChunkedHeaders), + handle_response(State#state{headers = NewHeaders, + body = NewBody}) + end; + Encoding when is_list(Encoding) -> + ?hcrt("handle_http_body - encoding", [{encoding, Encoding}]), + NewState = answer_request(Request, + httpc_response:error(Request, + unknown_encoding), + State), + {stop, normal, NewState}; + _ -> + ?hcrt("handle_http_body - other", []), + Length = + list_to_integer(Headers#http_response_h.'content-length'), + case ((Length =< MaxBodySize) or (MaxBodySize == nolimit)) of + true -> + case httpc_response:whole_body(Body, Length) of + {ok, Body} -> + {NewBody, NewRequest}= stream(Body, Request, Code), + handle_response(State#state{body = NewBody, + request = NewRequest}); + MFA -> + NewState = next_body_chunk(State), + {noreply, NewState#state{mfa = MFA}} + end; + false -> + NewState = + answer_request(Request, + httpc_response:error(Request, + body_too_big), + State), + {stop, normal, NewState} + end + end. + +%%% Normaly I do not comment out code, I throw it away. But this might +%%% actually be used on day if ssl is improved. +%% handle_response(State = #state{status = ssl_tunnel, +%% request = Request, +%% options = Options, +%% session = #tcp_session{socket = Socket, +%% scheme = Scheme}, +%% status_line = {_, 200, _}}) -> +%% %%% Insert code for upgrading the socket if and when ssl supports this. +%% Address = handle_proxy(Request#request.address, Options#options.proxy), +%% send_first_request(Address, Request, State); +%% handle_response(State = #state{status = ssl_tunnel, +%% request = Request}) -> +%% NewState = answer_request(Request, +%% httpc_response:error(Request, +%% ssl_proxy_tunnel_failed), +%% State), +%% {stop, normal, NewState}; + +handle_response(State = #state{status = new}) -> + handle_response(try_to_enable_pipeline_or_keep_alive(State)); + +handle_response(State = + #state{request = Request, + status = Status, + session = Session, + status_line = StatusLine, + headers = Headers, + body = Body, + options = Options, + profile_name = ProfileName}) when Status =/= new -> + ?hcrt("handle response", [{status, Status}, {session, Session}, {status_line, StatusLine}, {profile_name, ProfileName}]), + handle_cookies(Headers, Request, Options, ProfileName), + case httpc_response:result({StatusLine, Headers, Body}, Request) of + %% 100-continue + continue -> + %% Send request body + {_, RequestBody} = Request#request.content, + http_transport:send(socket_type(Session#tcp_session.scheme), + Session#tcp_session.socket, + RequestBody), + %% Wait for next response + http_transport:setopts(socket_type(Session#tcp_session.scheme), + Session#tcp_session.socket, + [{active, once}]), + Relaxed = (Request#request.settings)#http_options.relaxed, + {noreply, + State#state{mfa = {httpc_response, parse, + [State#state.max_header_size, + Relaxed]}, + status_line = undefined, + headers = undefined, + body = undefined + }}; + %% Ignore unexpected 100-continue response and receive the + %% actual response that the server will send right away. + {ignore, Data} -> + Relaxed = (Request#request.settings)#http_options.relaxed, + NewState = State#state{mfa = + {httpc_response, parse, + [State#state.max_header_size, + Relaxed]}, + status_line = undefined, + headers = undefined, + body = undefined}, + handle_info({httpc_handler, dummy, Data}, NewState); + %% On a redirect or retry the current request becomes + %% obsolete and the manager will create a new request + %% with the same id as the current. + {redirect, NewRequest, Data} -> + ?hcrt("handle response - redirect", [{new_request, NewRequest}, {data, Data}]), + ok = httpc_manager:redirect_request(NewRequest, ProfileName), + handle_queue(State#state{request = undefined}, Data); + {retry, TimeNewRequest, Data} -> + ?hcrt("handle response - retry", [{time_new_request, TimeNewRequest}, {data, Data}]), + ok = httpc_manager:retry_request(TimeNewRequest, ProfileName), + handle_queue(State#state{request = undefined}, Data); + {ok, Msg, Data} -> + ?hcrt("handle response - result ok", [{msg, Msg}, {data, Data}]), + end_stream(StatusLine, Request), + NewState = answer_request(Request, Msg, State), + handle_queue(NewState, Data); + {stop, Msg} -> + ?hcrt("handle response - result stop", [{msg, Msg}]), + end_stream(StatusLine, Request), + NewState = answer_request(Request, Msg, State), + {stop, normal, NewState} + end. + +handle_cookies(_,_, #options{cookies = disabled}, _) -> + ok; +%% User wants to verify the cookies before they are stored, +%% so the user will have to call a store command. +handle_cookies(_,_, #options{cookies = verify}, _) -> + ok; +handle_cookies(Headers, Request, #options{cookies = enabled}, ProfileName) -> + {Host, _ } = Request#request.address, + Cookies = http_cookie:cookies(Headers#http_response_h.other, + Request#request.path, Host), + httpc_manager:store_cookies(Cookies, Request#request.address, + ProfileName). + +%% This request could not be pipelined or used as sequential keept alive +%% queue +handle_queue(State = #state{status = close}, _) -> + {stop, normal, State}; + +handle_queue(State = #state{status = keep_alive}, Data) -> + handle_keep_alive_queue(State, Data); + +handle_queue(State = #state{status = pipeline}, Data) -> + handle_pipeline(State, Data). + +handle_pipeline(State = + #state{status = pipeline, session = Session, + profile_name = ProfileName, + options = #options{pipeline_timeout = TimeOut}}, + Data) -> + case queue:out(State#state.pipeline) of + {empty, _} -> + %% The server may choose too teminate an idle pipeline + %% in this case we want to receive the close message + %% at once and not when trying to pipeline the next + %% request. + http_transport:setopts(socket_type(Session#tcp_session.scheme), + Session#tcp_session.socket, + [{active, once}]), + %% If a pipeline that has been idle for some time is not + %% closed by the server, the client may want to close it. + NewState = activate_queue_timeout(TimeOut, State), + NewSession = Session#tcp_session{queue_length = 0}, + httpc_manager:insert_session(NewSession, ProfileName), + %% Note mfa will be initilized when a new request + %% arrives. + {noreply, + NewState#state{request = undefined, + mfa = undefined, + status_line = undefined, + headers = undefined, + body = undefined + } + }; + {{value, NextRequest}, Pipeline} -> + case lists:member(NextRequest#request.id, + State#state.canceled) of + true -> + %% See comment for handle_cast({cancel, RequestId}) + {stop, normal, + State#state{request = + NextRequest#request{from = answer_sent}}}; + false -> + NewSession = + Session#tcp_session{queue_length = + %% Queue + current + queue:len(Pipeline) + 1}, + httpc_manager:insert_session(NewSession, ProfileName), + Relaxed = + (NextRequest#request.settings)#http_options.relaxed, + NewState = + State#state{pipeline = Pipeline, + request = NextRequest, + mfa = {httpc_response, parse, + [State#state.max_header_size, + Relaxed]}, + status_line = undefined, + headers = undefined, + body = undefined}, + case Data of + <<>> -> + http_transport:setopts( + socket_type(Session#tcp_session.scheme), + Session#tcp_session.socket, + [{active, once}]), + {noreply, NewState}; + _ -> + %% If we already received some bytes of + %% the next response + handle_info({httpc_handler, dummy, Data}, + NewState) + end + end + end. + +handle_keep_alive_queue(State = #state{status = keep_alive, + session = Session, + profile_name = ProfileName, + options = #options{keep_alive_timeout + = TimeOut} + }, + Data) -> + case queue:out(State#state.keep_alive) of + {empty, _} -> + %% The server may choose too terminate an idle keep_alive session + %% in this case we want to receive the close message + %% at once and not when trying to send the next + %% request. + http_transport:setopts(socket_type(Session#tcp_session.scheme), + Session#tcp_session.socket, + [{active, once}]), + %% If a keep_alive session has been idle for some time is not + %% closed by the server, the client may want to close it. + NewState = activate_queue_timeout(TimeOut, State), + NewSession = Session#tcp_session{queue_length = 0}, + httpc_manager:insert_session(NewSession, ProfileName), + %% Note mfa will be initilized when a new request + %% arrives. + {noreply, + NewState#state{request = undefined, + mfa = undefined, + status_line = undefined, + headers = undefined, + body = undefined + } + }; + {{value, NextRequest}, KeepAlive} -> + case lists:member(NextRequest#request.id, + State#state.canceled) of + true -> + handle_keep_alive_queue(State#state{keep_alive = + KeepAlive}, Data); + false -> + Relaxed = + (NextRequest#request.settings)#http_options.relaxed, + NewState = + State#state{request = NextRequest, + keep_alive = KeepAlive, + mfa = {httpc_response, parse, + [State#state.max_header_size, + Relaxed]}, + status_line = undefined, + headers = undefined, + body = undefined}, + case Data of + <<>> -> + http_transport:setopts( + socket_type(Session#tcp_session.scheme), + Session#tcp_session.socket, [{active, once}]), + {noreply, NewState}; + _ -> + %% If we already received some bytes of + %% the next response + handle_info({httpc_handler, dummy, Data}, + NewState) + end + end + end. + +call(Msg, Pid, Timeout) -> + gen_server:call(Pid, Msg, Timeout). + +cast(Msg, Pid) -> + gen_server:cast(Pid, Msg). + +case_insensitive_header(Str) when is_list(Str) -> + http_util:to_lower(Str); +%% Might be undefined if server does not send such a header +case_insensitive_header(Str) -> + Str. + +activate_request_timeout(State = #state{request = Request}) -> + Time = (Request#request.settings)#http_options.timeout, + case Time of + infinity -> + State; + _ -> + Ref = erlang:send_after(Time, self(), + {timeout, Request#request.id}), + State#state + {timers = + #timers{request_timers = + [{Request#request.id, Ref}| + (State#state.timers)#timers.request_timers]}} + end. + +activate_queue_timeout(infinity, State) -> + State; +activate_queue_timeout(Time, State) -> + Ref = erlang:send_after(Time, self(), timeout_queue), + State#state{timers = #timers{queue_timer = Ref}}. + + +is_pipeline_enabled_client(#tcp_session{type = pipeline}) -> + true; +is_pipeline_enabled_client(_) -> + false. + +is_keep_alive_enabled_server("HTTP/1." ++ N, _) when (hd(N) >= $1) -> + true; +is_keep_alive_enabled_server("HTTP/1.0", + #http_response_h{connection = "keep-alive"}) -> + true; +is_keep_alive_enabled_server(_,_) -> + false. + +is_keep_alive_connection(Headers, Session) -> + (not ((Session#tcp_session.client_close) or + httpc_response:is_server_closing(Headers))). + +try_to_enable_pipeline_or_keep_alive(State = + #state{session = Session, + request = #request{method = Method}, + status_line = {Version, _, _}, + headers = Headers, + profile_name = ProfileName}) -> + case (is_keep_alive_enabled_server(Version, Headers) andalso + is_keep_alive_connection(Headers, Session)) of + true -> + case (is_pipeline_enabled_client(Session) andalso + httpc_request:is_idempotent(Method)) of + true -> + httpc_manager:insert_session(Session, ProfileName), + State#state{status = pipeline}; + false -> + httpc_manager:insert_session(Session, ProfileName), + %% Make sure type is keep_alive in session + %% as it in this case might be pipeline + State#state{status = keep_alive, + session = + Session#tcp_session{type = keep_alive}} + end; + false -> + State#state{status = close} + end. + +answer_request(Request, Msg, #state{timers = Timers} = State) -> + httpc_response:send(Request#request.from, Msg), + RequestTimers = Timers#timers.request_timers, + TimerRef = + proplists:get_value(Request#request.id, RequestTimers, undefined), + Timer = {Request#request.id, TimerRef}, + cancel_timer(TimerRef, {timeout, Request#request.id}), + State#state{request = Request#request{from = answer_sent}, + timers = + Timers#timers{request_timers = + lists:delete(Timer, RequestTimers)}}. +cancel_timer(undefined, _) -> + ok; +cancel_timer(Timer, TimeoutMsg) -> + erlang:cancel_timer(Timer), + receive + TimeoutMsg -> + ok + after 0 -> + ok + end. + +retry_pipeline([], _) -> + ok; + +%% Skip requests when the answer has already been sent +retry_pipeline([#request{from = answer_sent}|PipeLine], State) -> + retry_pipeline(PipeLine, State); + +retry_pipeline([Request | PipeLine], + #state{timers = Timers, + profile_name = ProfileName} = State) -> + NewState = + case (catch httpc_manager:retry_request(Request, ProfileName)) of + ok -> + RequestTimers = Timers#timers.request_timers, + TimerRef = + proplists:get_value(Request#request.id, RequestTimers, + undefined), + cancel_timer(TimerRef, {timeout, Request#request.id}), + State#state{timers = Timers#timers{request_timers = + lists:delete({Request#request.id, + TimerRef}, + RequestTimers)}}; + Error -> + answer_request(Request#request.from, + httpc_response:error(Request, Error), State) + end, + retry_pipeline(PipeLine, NewState). + +%%% Check to see if the given {Host,Port} tuple is in the NoProxyList +%%% Returns an eventually updated {Host,Port} tuple, with the proxy address +handle_proxy(HostPort = {Host, _Port}, {Proxy, NoProxy}) -> + case Proxy of + undefined -> + HostPort; + Proxy -> + case is_no_proxy_dest(Host, NoProxy) of + true -> + HostPort; + false -> + Proxy + end + end. + +is_no_proxy_dest(_, []) -> + false; +is_no_proxy_dest(Host, [ "*." ++ NoProxyDomain | NoProxyDests]) -> + + case is_no_proxy_dest_domain(Host, NoProxyDomain) of + true -> + true; + false -> + is_no_proxy_dest(Host, NoProxyDests) + end; + +is_no_proxy_dest(Host, [NoProxyDest | NoProxyDests]) -> + IsNoProxyDest = case http_util:is_hostname(NoProxyDest) of + true -> + fun is_no_proxy_host_name/2; + false -> + fun is_no_proxy_dest_address/2 + end, + + case IsNoProxyDest(Host, NoProxyDest) of + true -> + true; + false -> + is_no_proxy_dest(Host, NoProxyDests) + end. + +is_no_proxy_host_name(Host, Host) -> + true; +is_no_proxy_host_name(_,_) -> + false. + +is_no_proxy_dest_domain(Dest, DomainPart) -> + lists:suffix(DomainPart, Dest). + +is_no_proxy_dest_address(Dest, Dest) -> + true; +is_no_proxy_dest_address(Dest, AddressPart) -> + lists:prefix(AddressPart, Dest). + +init_mfa(#request{settings = Settings}, State) -> + case Settings#http_options.version of + "HTTP/0.9" -> + {httpc_response, whole_body, [<<>>, -1]}; + _ -> + Relaxed = Settings#http_options.relaxed, + {httpc_response, parse, [State#state.max_header_size, Relaxed]} + end. + +init_status_line(#request{settings = Settings}) -> + case Settings#http_options.version of + "HTTP/0.9" -> + {"HTTP/0.9", 200, "OK"}; + _ -> + undefined + end. + +socket_type(#request{scheme = http}) -> + ip_comm; +socket_type(#request{scheme = https, settings = Settings}) -> + {ssl, Settings#http_options.ssl}; +socket_type(http) -> + ip_comm; +socket_type(https) -> + {ssl, []}. %% Dummy value ok for ex setops that does not use this value + +start_stream({_Version, _Code, _ReasonPhrase}, _Headers, #request{stream = none} = Request) -> + ?hcrt("start stream - none", []), + {ok, Request}; +start_stream({_Version, Code, _ReasonPhrase}, Headers, #request{stream = self} = Request) + when (Code =:= 200) orelse (Code =:= 206) -> + ?hcrt("start stream - self", [{code, Code}]), + Msg = httpc_response:stream_start(Headers, Request, ignore), + httpc_response:send(Request#request.from, Msg), + {ok, Request}; +start_stream({_Version, Code, _ReasonPhrase}, Headers, + #request{stream = {self, once}} = Request) + when (Code =:= 200) orelse (Code =:= 206) -> + ?hcrt("start stream - self:once", [{code, Code}]), + Msg = httpc_response:stream_start(Headers, Request, self()), + httpc_response:send(Request#request.from, Msg), + {ok, Request}; +start_stream({_Version, Code, _ReasonPhrase}, _Headers, #request{stream = Filename} = Request) + when ((Code =:= 200) orelse (Code =:= 206)) andalso is_list(Filename) -> + ?hcrt("start stream", [{code, Code}, {filename, Filename}]), + case file:open(Filename, [write, raw, append, delayed_write]) of + {ok, Fd} -> + ?hcri("start stream - file open ok", [{fd, Fd}]), + {ok, Request#request{stream = Fd}}; + {error, Reason} -> + exit({stream_to_file_failed, Reason}) + end; +start_stream(_StatusLine, _Headers, Request) -> + ?hcrt("start stream - no op", []), + {ok, Request}. + + +%% Note the end stream message is handled by httpc_response and will +%% be sent by answer_request +end_stream(_, #request{stream = none}) -> + ?hcrt("end stream - none", []), + ok; +end_stream(_, #request{stream = self}) -> + ?hcrt("end stream - self", []), + ok; +end_stream(_, #request{stream = {self, once}}) -> + ?hcrt("end stream - self:once", []), + ok; +end_stream({_,200,_}, #request{stream = Fd}) -> + ?hcrt("end stream - 200", [{stream, Fd}]), + case file:close(Fd) of + ok -> + ok; + {error, enospc} -> % Could be due to delayed_write + file:close(Fd) + end; +end_stream({_,206,_}, #request{stream = Fd}) -> + ?hcrt("end stream - 206", [{stream, Fd}]), + case file:close(Fd) of + ok -> + ok; + {error, enospc} -> % Could be due to delayed_write + file:close(Fd) + end; +end_stream(SL, R) -> + ?hcrt("end stream", [{status_line, SL}, {request, R}]), + ok. + + +next_body_chunk(#state{request = #request{stream = {self, once}}, + once = once, session = Session} = State) -> + http_transport:setopts(socket_type(Session#tcp_session.scheme), + Session#tcp_session.socket, + [{active, once}]), + State#state{once = inactive}; +next_body_chunk(#state{request = #request{stream = {self, once}}, + once = inactive} = State) -> + State; %% Wait for user to call stream_next +next_body_chunk(#state{session = Session} = State) -> + http_transport:setopts(socket_type(Session#tcp_session.scheme), + Session#tcp_session.socket, + [{active, once}]), + State. + +handle_verbose(verbose) -> + dbg:p(self(), [r]); +handle_verbose(debug) -> + dbg:p(self(), [call]), + dbg:tp(?MODULE, [{'_', [], [{return_trace}]}]); +handle_verbose(trace) -> + dbg:p(self(), [call]), + dbg:tpl(?MODULE, [{'_', [], [{return_trace}]}]); +handle_verbose(_) -> + ok. + +%%% Normaly I do not comment out code, I throw it away. But this might +%%% actually be used one day if ssl is improved. +%% send_ssl_tunnel_request(Address, Request = #request{address = {Host, Port}}, +%% State) -> +%% %% A ssl tunnel request is a special http request that looks like +%% %% CONNECT host:port HTTP/1.1 +%% SslTunnelRequest = #request{method = connect, scheme = http, +%% headers = +%% #http_request_h{ +%% host = Host, +%% address = Address, +%% path = Host ++ ":", +%% pquery = integer_to_list(Port), +%% other = [{ "Proxy-Connection", "keep-alive"}]}, +%% Ipv6 = (State#state.options)#options.ipv6, +%% SocketType = socket_type(SslTunnelRequest), +%% case http_transport:connect(SocketType, +%% SslTunnelRequest#request.address, Ipv6) of +%% {ok, Socket} -> +%% case httpc_request:send(Address, SslTunnelRequest, Socket) of +%% ok -> +%% Session = #tcp_session{id = +%% {SslTunnelRequest#request.address, +%% self()}, +%% scheme = +%% SslTunnelRequest#request.scheme, +%% socket = Socket}, +%% NewState = State#state{mfa = +%% {httpc_response, parse, +%% [State#state.max_header_size]}, +%% request = Request, +%% session = Session}, +%% http_transport:setopts(socket_type( +%% SslTunnelRequest#request.scheme), +%% Socket, +%% [{active, once}]), +%% {ok, NewState}; +%% {error, Reason} -> +%% self() ! {init_error, error_sending, +%% httpc_response:error(Request, Reason)}, +%% {ok, State#state{request = Request, +%% session = #tcp_session{socket = +%% Socket}}} +%% end; +%% {error, Reason} -> +%% self() ! {init_error, error_connecting, +%% httpc_response:error(Request, Reason)}, +%% {ok, State#state{request = Request}} +%% end. + +%% d(F) -> +%% d(F, []). + +%% d(F, A) -> +%% d(get(dbg), F, A). + +%% d(true, F, A) -> +%% io:format(user, "~w:~w:" ++ F ++ "~n", [self(), ?MODULE | A]); +%% d(_, _, _) -> +%% ok. + diff --git a/lib/inets/src/http_client/httpc_handler_sup.erl b/lib/inets/src/http_client/httpc_handler_sup.erl new file mode 100644 index 0000000000..d9edaa0599 --- /dev/null +++ b/lib/inets/src/http_client/httpc_handler_sup.erl @@ -0,0 +1,66 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2007-2009. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% +-module(httpc_handler_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/0]). +-export([start_child/1]). + +%% Supervisor callback +-export([init/1]). + +%%%========================================================================= +%%% API +%%%========================================================================= +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +start_child(Args) -> + supervisor:start_child(?MODULE, Args). + +%%%========================================================================= +%%% Supervisor callback +%%%========================================================================= +init(Args) -> + RestartStrategy = simple_one_for_one, + MaxR = 0, + MaxT = 3600, + + Name = undefined, % As simple_one_for_one is used. + StartFunc = {httpc_handler, start_link, Args}, + Restart = temporary, % E.g. should not be restarted + Shutdown = 4000, + Modules = [httpc_handler], + Type = worker, + + ChildSpec = {Name, StartFunc, Restart, Shutdown, Type, Modules}, + {ok, {{RestartStrategy, MaxR, MaxT}, [ChildSpec]}}. + + + + + + + + + + diff --git a/lib/inets/src/http_client/httpc_internal.hrl b/lib/inets/src/http_client/httpc_internal.hrl new file mode 100644 index 0000000000..ec709b9860 --- /dev/null +++ b/lib/inets/src/http_client/httpc_internal.hrl @@ -0,0 +1,136 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2005-2009. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% + +-include("inets_internal.hrl"). +-define(SERVICE, httpc). +-define(hcri(Label, Data), ?report_important(Label, ?SERVICE, Data)). +-define(hcrv(Label, Data), ?report_verbose(Label, ?SERVICE, Data)). +-define(hcrd(Label, Data), ?report_debug(Label, ?SERVICE, Data)). +-define(hcrt(Label, Data), ?report_trace(Label, ?SERVICE, Data)). + +-define(HTTP_REQUEST_TIMEOUT, infinity). +-define(HTTP_REQUEST_CTIMEOUT, ?HTTP_REQUEST_TIMEOUT). +-define(HTTP_PIPELINE_TIMEOUT, 0). +-define(HTTP_PIPELINE_LENGTH, 2). +-define(HTTP_MAX_TCP_SESSIONS, 2). +-define(HTTP_MAX_REDIRECTS, 4). +-define(HTTP_KEEP_ALIVE_TIMEOUT, 120000). +-define(HTTP_KEEP_ALIVE_LENGTH, 5). + +%%% HTTP Client per request settings +-record(http_options, + { + %% string() - "HTTP/1.1" | "HTTP/1.0" | "HTTP/0.9" + version, + + %% integer() | infinity - ms before a request times out + timeout = ?HTTP_REQUEST_TIMEOUT, + + %% bool() - true if auto redirect on 30x response + autoredirect = true, + + %% Ssl socket options + ssl = [], + + %% {User, Password} = {string(), string()} + proxy_auth, + + %% bool() - true if not strictly std compliant + relaxed = false, + + %% integer() - ms before a connect times out + connect_timeout = ?HTTP_REQUEST_CTIMEOUT + } + ). + +%%% HTTP Client per profile setting. +-record(options, + { + proxy = {undefined, []}, % {{ProxyHost, ProxyPort}, [NoProxy]}, + %% 0 means persistent connections are used without pipelining + pipeline_timeout = ?HTTP_PIPELINE_TIMEOUT, + max_pipeline_length = ?HTTP_PIPELINE_LENGTH, + max_keep_alive_length = ?HTTP_KEEP_ALIVE_LENGTH, + keep_alive_timeout = ?HTTP_KEEP_ALIVE_TIMEOUT, % Used when pipeline_timeout = 0 + max_sessions = ?HTTP_MAX_TCP_SESSIONS, + cookies = disabled, % enabled | disabled | verify + verbose = false, + ipfamily = inet, % inet | inet6 | inet6fb4 + ip = default, % specify local interface + port = default % specify local port + } + ). + +%%% All data associated to a specific HTTP request +-record(request, + { + id, % ref() - Request Id + from, % pid() - Caller + redircount = 0,% Number of redirects made for this request + scheme, % http | https + address, % ({Host,Port}) Destination Host and Port + path, % string() - Path of parsed URL + pquery, % string() - Rest of parsed URL + method, % atom() - HTTP request Method + headers, % #http_request_h{} + content, % {ContentType, Body} - Current HTTP request + settings, % #http_options{} - User defined settings + abs_uri, % string() ex: "http://www.erlang.org" + userinfo, % string() - optinal "<userinfo>@<host>:<port>" + stream, % Boolean() - stream async reply? + headers_as_is % Boolean() - workaround for servers that does + %% not honor the http standard, can also be used for testing purposes. + } + ). + +-record(tcp_session, + { + id, % {{Host, Port}, HandlerPid} + client_close, % true | false + scheme, % http (HTTP/TCP) | https (HTTP/SSL/TCP) + socket, % Open socket, used by connection + queue_length = 1, % Current length of pipeline or keep alive queue + type % pipeline | keep_alive (wait for response before sending new request) + }). + +-record(http_cookie, + { + domain, + domain_default = false, + name, + value, + comment, + max_age = session, + path, + path_default = false, + secure = false, + version = "0" + }). + + +%% -record(parsed_uri, +%% { +%% scheme, % http | https +%% uinfo, % string() +%% host, % string() +%% port, % integer() +%% path, % string() +%% q % query: string() +%% }). diff --git a/lib/inets/src/http_client/httpc_manager.erl b/lib/inets/src/http_client/httpc_manager.erl new file mode 100644 index 0000000000..63b00c7dce --- /dev/null +++ b/lib/inets/src/http_client/httpc_manager.erl @@ -0,0 +1,634 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2002-2009. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% +-module(httpc_manager). + +-behaviour(gen_server). + +-include("httpc_internal.hrl"). +-include("http_internal.hrl"). + +%% Internal Application API +-export([start_link/1, start_link/2, request/2, cancel_request/2, + request_canceled/2, retry_request/2, redirect_request/2, + insert_session/2, delete_session/2, set_options/2, store_cookies/3, + cookies/2, session_type/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-record(state, { + cancel = [], % [{RequestId, HandlerPid, ClientPid}] + handler_db, % ets() - Entry: {Requestid, HandlerPid, ClientPid} + cookie_db, % {ets(), dets()} - {session_cookie_db, cookie_db} + session_db, % ets() - Entry: #tcp_session{} + profile_name, % atom() + options = #options{} + }). + + +%%==================================================================== +%% Internal Application API +%%==================================================================== +%%-------------------------------------------------------------------- +%% Function: start_link({ProfileName, CookieDir}) -> {ok, Pid} +%% +%% ProfileName - httpc_manager_<Profile> +%% CookieDir - directory() +%% +%% Description: Starts the http request manger process. (Started by +%% the intes supervisor.) +%%-------------------------------------------------------------------- +start_link({default, CookieDir}) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, + [?MODULE, {http_default_cookie_db, CookieDir}], + []); +start_link({Profile, CookieDir}) -> + ProfileName = list_to_atom("httpc_manager_" ++ atom_to_list(Profile)), + gen_server:start_link({local, ProfileName}, ?MODULE, + [ProfileName, + {http_default_cookie_db, CookieDir}], []). +start_link({Profile, CookieDir}, stand_alone) -> + ProfileName = list_to_atom("stand_alone_" ++ atom_to_list(Profile)), + gen_server:start_link(?MODULE, [ProfileName, + {http_default_cookie_db, CookieDir}], + []). +%%-------------------------------------------------------------------- +%% Function: request(Request, ProfileName) -> +%% {ok, Requestid} | {error, Reason} +%% Request = #request{} +%% ProfileName = atom() +%% +%% Description: Sends a request to the httpc manager process. +%%-------------------------------------------------------------------- +request(Request, ProfileName) -> + call(ProfileName, {request, Request}, infinity). + +%%-------------------------------------------------------------------- +%% Function: retry_request(Request, ProfileName) -> _ +%% Request = #request{} +%% ProfileName = atom() +%% +%% Description: Resends a request to the httpc manager process, intended +%% to be called by the httpc handler process if it has to terminate with +%% a non empty pipeline. +%%-------------------------------------------------------------------- +retry_request(Request, ProfileName) -> + cast(ProfileName, {retry_or_redirect_request, Request}). + +%%-------------------------------------------------------------------- +%% Function: redirect_request(Request, ProfileName) -> _ +%% Request = #request{} +%% ProfileName = atom() +%% +%% Description: Sends an atoumatic redirect request to the httpc +%% manager process, intended to be called by the httpc handler process +%% when the automatic redirect option is set. +%%-------------------------------------------------------------------- +redirect_request(Request, ProfileName) -> + cast(ProfileName, {retry_or_redirect_request, Request}). + +%%-------------------------------------------------------------------- +%% Function: cancel_request(RequestId, ProfileName) -> ok +%% RequestId - ref() +%% ProfileName = atom() +%% +%% Description: Cancels the request with <RequestId>. +%%-------------------------------------------------------------------- +cancel_request(RequestId, ProfileName) -> + call(ProfileName, {cancel_request, RequestId}, infinity). + +%%-------------------------------------------------------------------- +%% Function: request_canceled(RequestId, ProfileName) -> ok +%% RequestId - ref() +%% ProfileName = atom() +%% +%% Description: Confirms that a request has been canceld. Intended to +%% be called by the httpc handler process. +%%-------------------------------------------------------------------- +request_canceled(RequestId, ProfileName) -> + cast(ProfileName, {request_canceled, RequestId}). + +%%-------------------------------------------------------------------- +%% Function: insert_session(Session, ProfileName) -> _ +%% Session - #tcp_session{} +%% ProfileName - atom() +%% +%% Description: Inserts session information into the httpc manager +%% table <ProfileName>_session_db. Intended to be called by +%% the httpc request handler process. +%%-------------------------------------------------------------------- +insert_session(Session, ProfileName) -> + Db = list_to_atom(atom_to_list(ProfileName) ++ "_session_db"), + ets:insert(Db, Session). + +%%-------------------------------------------------------------------- +%% Function: delete_session(SessionId, ProfileName) -> _ +%% SessionId - {{Host, Port}, HandlerPid} +%% ProfileName - atom() +%% +%% Description: Deletes session information from the httpc manager +%% table httpc_manager_session_db_<Profile>. Intended to be called by +%% the httpc request handler process. +%%-------------------------------------------------------------------- +delete_session(SessionId, ProfileName) -> + Db = list_to_atom(atom_to_list(ProfileName) ++ "_session_db"), + ets:delete(Db, SessionId). + +%%-------------------------------------------------------------------- +%% Function: set_options(Options, ProfileName) -> ok +%% +%% Options = [Option] +%% Option = {proxy, {Proxy, [NoProxy]}} +%% | {max_pipeline_length, integer()} | +%% {max_sessions, integer()} | {pipeline_timeout, integer()} +%% Proxy = {Host, Port} +%% NoProxy - [Domain | HostName | IPAddress] +%% Max - integer() +%% ProfileName = atom() +%% +%% Description: Sets the options to be used by the client. +%%-------------------------------------------------------------------- +set_options(Options, ProfileName) -> + cast(ProfileName, {set_options, Options}). + +%%-------------------------------------------------------------------- +%% Function: store_cookies(Cookies, Address, ProfileName) -> ok +%% +%% Cookies = [Cookie] +%% Cookie = #http_cookie{} +%% ProfileName = atom() +%% +%% Description: Stores cookies from the server. +%%-------------------------------------------------------------------- +store_cookies([], _, _) -> + ok; +store_cookies(Cookies, Address, ProfileName) -> + cast(ProfileName, {store_cookies, {Cookies, Address}}). + +%%-------------------------------------------------------------------- +%% Function: cookies(Url, ProfileName) -> ok +%% +%% Url = string() +%% ProfileName = atom() +%% +%% Description: Retrieves the cookies that would be sent when +%% requesting <Url>. +%%-------------------------------------------------------------------- +cookies(Url, ProfileName) -> + call(ProfileName, {cookies, Url}, infinity). + +%%-------------------------------------------------------------------- +%% Function: session_type(Options) -> ok +%% +%% Options = #options{} +%% +%% Description: Determines if to use pipelined sessions or not. +%%-------------------------------------------------------------------- +session_type(#options{pipeline_timeout = 0}) -> + keep_alive; +session_type(_) -> + pipeline. + +%%==================================================================== +%% gen_server callback functions +%%==================================================================== + +%%-------------------------------------------------------------------- +%% Function: init([ProfileName, CookiesConf]) -> {ok, State} | +%% {ok, State, Timeout} | ignore |{stop, Reason} +%% Description: Initiates the httpc_manger process +%%-------------------------------------------------------------------- +init([ProfileName, CookiesConf | _]) -> + process_flag(trap_exit, true), + SessionDb = list_to_atom(atom_to_list(ProfileName) ++ "_session_db"), + ets:new(SessionDb, + [public, set, named_table, {keypos, #tcp_session.id}]), + ?hcri("starting", [{profile, ProfileName}]), + {ok, #state{handler_db = ets:new(handler_db, [protected, set]), + cookie_db = + http_cookie:open_cookie_db({CookiesConf, + http_session_cookie_db}), + session_db = SessionDb, + profile_name = ProfileName + }}. + +%%-------------------------------------------------------------------- +%% Function: handle_call(Request, From, State) -> {reply, Reply, State} | +%% {reply, Reply, State, Timeout} | +%% {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, Reply, State} | (terminate/2 is called) +%% {stop, Reason, State} (terminate/2 is called) +%% Description: Handling call messages +%%-------------------------------------------------------------------- +handle_call({request, Request}, _, State) -> + ?hcri("request", [{request, Request}]), + case (catch handle_request(Request, State)) of + {reply, Msg, NewState} -> + {reply, Msg, NewState}; + Error -> + {stop, Error, httpc_response:error(Request, Error), State} + end; + +handle_call({cancel_request, RequestId}, From, State) -> + ?hcri("cancel_request", [{request_id, RequestId}]), + case ets:lookup(State#state.handler_db, RequestId) of + [] -> + ok, %% Nothing to cancel + {reply, ok, State}; + [{_, Pid, _}] -> + httpc_handler:cancel(RequestId, Pid), + {noreply, State#state{cancel = + [{RequestId, Pid, From} | + State#state.cancel]}} + end; + +handle_call({cookies, Url}, _, State) -> + case http_uri:parse(Url) of + {Scheme, _, Host, Port, Path, _} -> + CookieHeaders = + http_cookie:header(Scheme, {Host, Port}, + Path, State#state.cookie_db), + {reply, CookieHeaders, State}; + Msg -> + {reply, Msg, State} + end; + +handle_call(Msg, From, State) -> + Report = io_lib:format("HTTPC_MANAGER recived unkown call: ~p" + "from: ~p~n", [Msg, From]), + error_logger:error_report(Report), + {reply, {error, 'API_violation'}, State}. + +%%-------------------------------------------------------------------- +%% Function: handle_cast(Msg, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} (terminate/2 is called) +%% Description: Handling cast messages +%%-------------------------------------------------------------------- +handle_cast({retry_or_redirect_request, {Time, Request}}, + #state{profile_name = ProfileName} = State) -> + {ok, _} = timer:apply_after(Time, ?MODULE, retry_request, [Request, ProfileName]), + {noreply, State}; + +handle_cast({retry_or_redirect_request, Request}, State) -> + case (catch handle_request(Request, State)) of + {reply, {ok, _}, NewState} -> + {noreply, NewState}; + Error -> + httpc_response:error(Request, Error), + {stop, Error, State} + end; + +handle_cast({request_canceled, RequestId}, State) -> + ets:delete(State#state.handler_db, RequestId), + case lists:keysearch(RequestId, 1, State#state.cancel) of + {value, Entry = {RequestId, _, From}} -> + gen_server:reply(From, ok), + {noreply, + State#state{cancel = lists:delete(Entry, State#state.cancel)}}; + _ -> + {noreply, State} + end; +handle_cast({set_options, Options}, State = #state{options = OldOptions}) -> + NewOptions = + #options{proxy = get_proxy(Options, OldOptions), + pipeline_timeout = get_pipeline_timeout(Options, OldOptions), + max_pipeline_length = get_max_pipeline_length(Options, OldOptions), + max_keep_alive_length = get_max_keep_alive_length(Options, OldOptions), + keep_alive_timeout = get_keep_alive_timeout(Options, OldOptions), + max_sessions = get_max_sessions(Options, OldOptions), + cookies = get_cookies(Options, OldOptions), + ipfamily = get_ipfamily(Options, OldOptions), + ip = get_ip(Options, OldOptions), + port = get_port(Options, OldOptions), + verbose = get_verbose(Options, OldOptions) + }, + case {OldOptions#options.verbose, NewOptions#options.verbose} of + {Same, Same} -> + ok; + {_, false} -> + dbg:stop(); + {false, Level} -> + dbg:tracer(), + handle_verbose(Level); + {_, Level} -> + dbg:stop(), + dbg:tracer(), + handle_verbose(Level) + end, + {noreply, State#state{options = NewOptions}}; + +handle_cast({store_cookies, _}, + State = #state{options = #options{cookies = disabled}}) -> + {noreply, State}; + +handle_cast({store_cookies, {Cookies, _}}, State) -> + ok = do_store_cookies(Cookies, State), + {noreply, State}; + +handle_cast(Msg, State) -> + Report = io_lib:format("HTTPC_MANAGER recived unkown cast: ~p", + [Msg]), + error_logger:error_report(Report), + {noreply, State}. + + + +%%-------------------------------------------------------------------- +%% Function: handle_info(Info, State) -> {noreply, State} | +%% {noreply, State, Timeout} | +%% {stop, Reason, State} (terminate/2 is called) +%% Description: Handling all non call/cast messages +%%--------------------------------------------------------- +handle_info({'EXIT', _, _}, State) -> + %% Handled in DOWN + {noreply, State}; +handle_info({'DOWN', _, _, Pid, _}, State) -> + ets:match_delete(State#state.handler_db, {'_', Pid, '_'}), + + %% If there where any canceled request, handled by the + %% the process that now has terminated, the + %% cancelation can be viewed as sucessfull! + NewCanceldList = + lists:foldl(fun(Entry = {_, HandlerPid, From}, Acc) -> + case HandlerPid of + Pid -> + gen_server:reply(From, ok), + lists:delete(Entry, Acc); + _ -> + Acc + end + end, State#state.cancel, State#state.cancel), + {noreply, State#state{cancel = NewCanceldList}}; +handle_info(Info, State) -> + Report = io_lib:format("Unknown message in " + "httpc_manager:handle_info ~p~n", [Info]), + error_logger:error_report(Report), + {noreply, State}. +%%-------------------------------------------------------------------- +%% Function: terminate(Reason, State) -> _ (ignored by gen_server) +%% Description: Shutdown the httpc_handler +%%-------------------------------------------------------------------- +terminate(_, State) -> + http_cookie:close_cookie_db(State#state.cookie_db), + ets:delete(State#state.session_db), + ets:delete(State#state.handler_db). + +%%-------------------------------------------------------------------- +%% Func: code_change(_OldVsn, State, Extra) -> {ok, NewState} +%% Purpose: Convert process state when code is changed +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +handle_request(#request{settings = + #http_options{version = "HTTP/0.9"}} = Request, + State) -> + %% Act as an HTTP/0.9 client that does not know anything + %% about persistent connections + + NewRequest = handle_cookies(generate_request_id(Request), State), + NewHeaders = + (NewRequest#request.headers)#http_request_h{connection + = undefined}, + start_handler(NewRequest#request{headers = NewHeaders}, State), + {reply, {ok, NewRequest#request.id}, State}; + +handle_request(#request{settings = + #http_options{version = "HTTP/1.0"}} = Request, + State) -> + %% Act as an HTTP/1.0 client that does not + %% use persistent connections + + NewRequest = handle_cookies(generate_request_id(Request), State), + NewHeaders = + (NewRequest#request.headers)#http_request_h{connection + = "close"}, + start_handler(NewRequest#request{headers = NewHeaders}, State), + {reply, {ok, NewRequest#request.id}, State}; + +handle_request(Request, State = #state{options = Options}) -> + + NewRequest = handle_cookies(generate_request_id(Request), State), + SessionType = session_type(Options), + case select_session(Request#request.method, + Request#request.address, + Request#request.scheme, SessionType, State) of + {ok, HandlerPid} -> + pipeline_or_keep_alive(NewRequest, HandlerPid, State); + no_connection -> + start_handler(NewRequest, State); + {no_session, OpenSessions} when OpenSessions + < Options#options.max_sessions -> + start_handler(NewRequest, State); + {no_session, _} -> + %% Do not start any more persistent connections + %% towards this server. + NewHeaders = + (NewRequest#request.headers)#http_request_h{connection + = "close"}, + start_handler(NewRequest#request{headers = NewHeaders}, State) + end, + {reply, {ok, NewRequest#request.id}, State}. + +select_session(Method, HostPort, Scheme, SessionTyp, + #state{options = #options{max_pipeline_length = + MaxPipe, + max_keep_alive_length = MaxKeepAlive}, + session_db = SessionDb}) -> + case httpc_request:is_idempotent(Method) or (SessionTyp == keep_alive) of + true -> + Candidates = ets:match(SessionDb, + {'_', {HostPort, '$1'}, + false, Scheme, '_', '$2', SessionTyp}), + select_session(Candidates, MaxKeepAlive, MaxPipe, SessionTyp); + false -> + no_connection + end. + +select_session(Candidates, Max, _, keep_alive) -> + select_session(Candidates, Max); +select_session(Candidates, _, Max, pipeline) -> + select_session(Candidates, Max). + +select_session(Candidates, Max) -> + case Candidates of + [] -> + no_connection; + _ -> + NewCandidates = + lists:foldl( + fun([Pid, Length], Acc) when Length =< Max -> + [{Pid, Length} | Acc]; + (_, Acc) -> + Acc + end, [], Candidates), + + case lists:keysort(2, NewCandidates) of + [] -> + {no_session, length(Candidates)}; + [{HandlerPid, _} | _] -> + {ok, HandlerPid} + end + end. + +pipeline_or_keep_alive(Request, HandlerPid, State) -> + case (catch httpc_handler:send(Request, HandlerPid)) of + ok -> + ets:insert(State#state.handler_db, {Request#request.id, + HandlerPid, + Request#request.from}); + _ -> %timeout pipelining failed + start_handler(Request, State) + end. + +start_handler(Request, State) -> + {ok, Pid} = + case is_inets_manager() of + true -> + httpc_handler_sup:start_child([Request, State#state.options, + State#state.profile_name]); + false -> + httpc_handler:start_link(Request, State#state.options, + State#state.profile_name) + end, + ets:insert(State#state.handler_db, {Request#request.id, + Pid, Request#request.from}), + erlang:monitor(process, Pid). + +is_inets_manager() -> + case get('$ancestors') of + [httpc_profile_sup | _] -> + true; + _ -> + false + end. + +generate_request_id(Request) -> + case Request#request.id of + undefined -> + RequestId = make_ref(), + Request#request{id = RequestId}; + _ -> + %% This is an automatic redirect or a retryed pipelined + %% request keep the old id. + Request + end. + +handle_cookies(Request, #state{options = #options{cookies = disabled}}) -> + Request; +handle_cookies(Request = #request{scheme = Scheme, address = Address, + path = Path, headers = + Headers = #http_request_h{other = Other}}, + #state{cookie_db = Db}) -> + case http_cookie:header(Scheme, Address, Path, Db) of + {"cookie", ""} -> + Request; + CookieHeader -> + NewHeaders = + Headers#http_request_h{other = [CookieHeader | Other]}, + Request#request{headers = NewHeaders} + end. + +do_store_cookies([], _) -> + ok; +do_store_cookies([Cookie | Cookies], State) -> + ok = http_cookie:insert(Cookie, State#state.cookie_db), + do_store_cookies(Cookies, State). + +call(ProfileName, Msg, Timeout) -> + gen_server:call(ProfileName, Msg, Timeout). + +cast(ProfileName, Msg) -> + gen_server:cast(ProfileName, Msg). + + + +get_proxy(Opts, #options{proxy = Default}) -> + proplists:get_value(proxy, Opts, Default). + +get_pipeline_timeout(Opts, #options{pipeline_timeout = Default}) -> + proplists:get_value(pipeline_timeout, Opts, Default). + +get_max_pipeline_length(Opts, #options{max_pipeline_length = Default}) -> + proplists:get_value(max_pipeline_length, Opts, Default). + +get_max_keep_alive_length(Opts, #options{max_keep_alive_length = Default}) -> + proplists:get_value(max_keep_alive_length, Opts, Default). + +get_keep_alive_timeout(Opts, #options{keep_alive_timeout = Default}) -> + proplists:get_value(keep_alive_timeout, Opts, Default). + +get_max_sessions(Opts, #options{max_sessions = Default}) -> + proplists:get_value(max_sessions, Opts, Default). + +get_cookies(Opts, #options{cookies = Default}) -> + proplists:get_value(cookies, Opts, Default). + +get_ipfamily(Opts, #options{ipfamily = IpFamily}) -> + case lists:keysearch(ipfamily, 1, Opts) of + false -> + case proplists:get_value(ipv6, Opts) of + enabled -> + inet6fb4; + disabled -> + inet; + _ -> + IpFamily + end; + {value, {_, Value}} -> + Value + end. + +get_ip(Opts, #options{ip = Default}) -> + proplists:get_value(ip, Opts, Default). + +get_port(Opts, #options{port = Default}) -> + proplists:get_value(port, Opts, Default). + +get_verbose(Opts, #options{verbose = Default}) -> + proplists:get_value(verbose, Opts, Default). + + +handle_verbose(debug) -> + dbg:p(self(), [call]), + dbg:tp(?MODULE, [{'_', [], [{return_trace}]}]); +handle_verbose(trace) -> + dbg:p(self(), [call]), + dbg:tpl(?MODULE, [{'_', [], [{return_trace}]}]); +handle_verbose(_) -> + ok. + +%% d(F) -> +%% d(F, []). + +%% d(F, A) -> +%% d(get(dbg), F, A). + +%% d(true, F, A) -> +%% io:format(user, "~w:~w:" ++ F ++ "~n", [self(), ?MODULE | A]); +%% d(_, _, _) -> +%% ok. + diff --git a/lib/inets/src/http_client/httpc_profile_sup.erl b/lib/inets/src/http_client/httpc_profile_sup.erl new file mode 100644 index 0000000000..2351083435 --- /dev/null +++ b/lib/inets/src/http_client/httpc_profile_sup.erl @@ -0,0 +1,107 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2007-2009. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% +-module(httpc_profile_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/1]). +-export([start_child/1, restart_child/1, stop_child/1]). + +%% Supervisor callback +-export([init/1]). + +%%%========================================================================= +%%% API +%%%========================================================================= +start_link(HttpcServices) -> + supervisor:start_link({local, ?MODULE}, ?MODULE, [HttpcServices]). + +start_child(PropList) -> + case proplists:get_value(profile, PropList) of + undefined -> + {error, no_profile}; + Profile -> + Dir = proplists:get_value(data_dir, PropList, only_session_cookies), + Spec = httpc_child_spec(Profile, Dir), + supervisor:start_child(?MODULE, Spec) + end. + +restart_child(Profile) -> + Name = id(Profile), + case supervisor:terminate_child(?MODULE, Name) of + ok -> + supervisor:restart_child(?MODULE, Name); + Error -> + Error + end. + +stop_child(Profile) -> + Name = id(Profile), + case supervisor:terminate_child(?MODULE, Name) of + ok -> + supervisor:delete_child(?MODULE, Name); + Error -> + Error + end. + +id(Profile) -> + DefaultProfile = http:default_profile(), + case Profile of + DefaultProfile -> + httpc_manager; + _ -> + {http, Profile} + end. + + +%%%========================================================================= +%%% Supervisor callback +%%%========================================================================= +init([]) -> + init([[]]); +init([HttpcServices]) -> + RestartStrategy = one_for_one, + MaxR = 10, + MaxT = 3600, + Children = child_spec(HttpcServices, []), + {ok, {{RestartStrategy, MaxR, MaxT}, Children}}. + +child_spec([], Acc) -> + Acc; +%% For backwards compatibility +child_spec([{httpc, {Profile, Dir}} | Rest], Acc) -> + Spec = httpc_child_spec(Profile, Dir), + child_spec(Rest, [Spec | Acc]); +child_spec([{httpc, PropList} | Rest], Acc) when is_list(PropList) -> + Profile = proplists:get_value(profile, PropList), + Dir = proplists:get_value(data_dir, PropList), + Spec = httpc_child_spec(Profile, Dir), + child_spec(Rest, [Spec | Acc]). + +httpc_child_spec(Profile, Dir) -> + Name = id(Profile), + StartFunc = {httpc_manager, start_link, [{Profile, Dir}]}, + Restart = permanent, + Shutdown = 4000, + Modules = [httpc_manager], + Type = worker, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + diff --git a/lib/inets/src/http_client/httpc_request.erl b/lib/inets/src/http_client/httpc_request.erl new file mode 100644 index 0000000000..3d66638d66 --- /dev/null +++ b/lib/inets/src/http_client/httpc_request.erl @@ -0,0 +1,209 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2009. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(httpc_request). + +-include("http_internal.hrl"). +-include("httpc_internal.hrl"). + +%%% Internal API +-export([send/3, is_idempotent/1, is_client_closing/1]). + +%%%========================================================================= +%%% Internal application API +%%%========================================================================= +%%------------------------------------------------------------------------- +%% send(MaybeProxy, Request) -> +%% MaybeProxy - {Host, Port} +%% Host = string() +%% Port = integer() +%% Request - #request{} +%% Socket - socket() +%% CookieSupport - enabled | disabled | verify +%% +%% Description: Composes and sends a HTTP-request. +%%------------------------------------------------------------------------- +send(SendAddr, #request{method = Method, scheme = Scheme, + path = Path, pquery = Query, headers = Headers, + content = Content, address = Address, + abs_uri = AbsUri, headers_as_is = HeadersAsIs, + settings = HttpOptions, + userinfo = UserInfo}, + Socket) -> + + TmpHeaders = handle_user_info(UserInfo, Headers), + + {TmpHeaders2, Body} = + post_data(Method, TmpHeaders, Content, HeadersAsIs), + + {NewHeaders, Uri} = case Address of + SendAddr -> + {TmpHeaders2, Path ++ Query}; + _Proxy -> + TmpHeaders3 = + handle_proxy(HttpOptions, TmpHeaders2), + {TmpHeaders3, AbsUri} + end, + + FinalHeaders = case NewHeaders of + HeaderList when is_list(HeaderList) -> + http_headers(HeaderList, []); + _ -> + http_request:http_headers(NewHeaders) + end, + Version = HttpOptions#http_options.version, + + Message = [method(Method), " ", Uri, " ", + version(Version), ?CRLF, headers(FinalHeaders, Version), ?CRLF, Body], + + http_transport:send(socket_type(Scheme), Socket, lists:append(Message)). + +%%------------------------------------------------------------------------- +%% is_idempotent(Method) -> +%% Method = atom() +%% +%% Description: Checks if Method is considered idempotent. +%%------------------------------------------------------------------------- + +%% In particular, the convention has been established that the GET and +%% HEAD methods SHOULD NOT have the significance of taking an action +%% other than retrieval. These methods ought to be considered "safe". +is_idempotent(head) -> + true; +is_idempotent(get) -> + true; +%% Methods can also have the property of "idempotence" in that (aside +%% from error or expiration issues) the side-effects of N > 0 +%% identical requests is the same as for a single request. +is_idempotent(put) -> + true; +is_idempotent(delete) -> + true; +%% Also, the methods OPTIONS and TRACE SHOULD NOT have side effects, +%% and so are inherently idempotent. +is_idempotent(trace) -> + true; +is_idempotent(options) -> + true; +is_idempotent(_) -> + false. + +%%------------------------------------------------------------------------- +%% is_client_closing(Headers) -> +%% Headers = #http_request_h{} +%% +%% Description: Checks if the client has supplied a "Connection: +%% close" header. +%%------------------------------------------------------------------------- +is_client_closing(Headers) -> + case Headers#http_request_h.connection of + "close" -> + true; + _ -> + false + end. + +%%%======================================================================== +%%% Internal functions +%%%======================================================================== +post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) + when Method == post; Method == put -> + ContentLength = body_length(Body), + NewBody = case Headers#http_request_h.expect of + "100-continue" -> + ""; + _ -> + Body + end, + + NewHeaders = case HeadersAsIs of + [] -> + Headers#http_request_h{'content-type' = + ContentType, + 'content-length' = + ContentLength}; + _ -> + HeadersAsIs + end, + + {NewHeaders, NewBody}; + +post_data(_, Headers, _, []) -> + {Headers, ""}; +post_data(_, _, _, HeadersAsIs = [_|_]) -> + {HeadersAsIs, ""}. + +body_length(Body) when is_binary(Body) -> + integer_to_list(size(Body)); + +body_length(Body) when is_list(Body) -> + integer_to_list(length(Body)). + +method(Method) -> + http_util:to_upper(atom_to_list(Method)). + +version("HTTP/0.9") -> + ""; +version(Version) -> + Version. + +headers(_, "HTTP/0.9") -> + ""; +%% HTTP 1.1 headers not present in HTTP 1.0 should be +%% consider as unknown extension headers that should be +%% ignored. +headers(Headers, _) -> + Headers. + +socket_type(http) -> + ip_comm; +socket_type(https) -> + {ssl, []}. + +http_headers([], Headers) -> + lists:flatten(Headers); +http_headers([{Key,Value} | Rest], Headers) -> + Header = Key ++ ": " ++ Value ++ ?CRLF, + http_headers(Rest, [Header | Headers]). + +handle_proxy(_, Headers) when is_list(Headers) -> + Headers; %% Headers as is option was specified +handle_proxy(HttpOptions, Headers) -> + case HttpOptions#http_options.proxy_auth of + undefined -> + Headers; + {User, Password} -> + UserPasswd = base64:encode_to_string(User ++ ":" ++ Password), + Headers#http_request_h{'proxy-authorization' = + "Basic " ++ UserPasswd} + end. + +handle_user_info([], Headers) -> + Headers; +handle_user_info(UserInfo, Headers) -> + case string:tokens(UserInfo, ":") of + [User, Passwd] -> + UserPasswd = base64:encode_to_string(User ++ ":" ++ Passwd), + Headers#http_request_h{authorization = "Basic " ++ UserPasswd}; + [User] -> + UserPasswd = base64:encode_to_string(User ++ ":"), + Headers#http_request_h{authorization = "Basic " ++ UserPasswd}; + _ -> + Headers + end. diff --git a/lib/inets/src/http_client/httpc_response.erl b/lib/inets/src/http_client/httpc_response.erl new file mode 100644 index 0000000000..e2ba66f730 --- /dev/null +++ b/lib/inets/src/http_client/httpc_response.erl @@ -0,0 +1,431 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2009. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(httpc_response). + +-include("http_internal.hrl"). +-include("httpc_internal.hrl"). + +%% API +-export([parse/1, result/2, send/2, error/2, is_server_closing/1, + stream_start/3]). + +%% Callback API - used for example if the header/body is received a +%% little at a time on a socket. +-export([parse_version/1, parse_status_code/1, parse_reason_phrase/1, + parse_headers/1, whole_body/1, whole_body/2]). + +%%%========================================================================= +%%% API +%%%========================================================================= + +parse([Bin, MaxHeaderSize, Relaxed]) -> + parse_version(Bin, [], MaxHeaderSize, [], Relaxed). + +whole_body([Bin, Body, Length]) -> + whole_body(<<Body/binary, Bin/binary>>, Length). + +%% Functions that may be returned during the decoding process +%% if the input data is incompleate. +parse_version([Bin, Version, MaxHeaderSize, Result, Relaxed]) -> + parse_version(Bin, Version, MaxHeaderSize, Result, Relaxed). + +parse_status_code([Bin, Code, MaxHeaderSize, Result, Relaxed]) -> + parse_status_code(Bin, Code, MaxHeaderSize, Result, Relaxed). + +parse_reason_phrase([Bin, Rest, Phrase, MaxHeaderSize, Result, Relaxed]) -> + parse_reason_phrase(<<Rest/binary, Bin/binary>>, Phrase, + MaxHeaderSize, Result, Relaxed). + +parse_headers([Bin, Rest,Header, Headers, MaxHeaderSize, Result, Relaxed]) -> + parse_headers(<<Rest/binary, Bin/binary>>, Header, Headers, + MaxHeaderSize, Result, Relaxed). + +whole_body(Body, Length) -> + case size(Body) of + N when (N < Length) andalso (N > 0) -> + {?MODULE, whole_body, [Body, Length]}; + %% OBS! The Server may close the connection to indicate that the + %% whole body is now sent instead of sending a lengh + %% indicator.In this case the lengh indicator will be + %% -1. + N when (N >= Length) andalso (Length >= 0) -> + %% Potential trailing garbage will be thrown away in + %% format_response/1 Some servers may send a 100-continue + %% response without the client requesting it through an + %% expect header in this case the trailing bytes may be + %% part of the real response message. + {ok, Body}; + _ -> %% Length == -1 + {?MODULE, whole_body, [Body, Length]} + end. + +%%------------------------------------------------------------------------- +%% result(Response, Request) -> +%% Response - {StatusLine, Headers, Body} +%% Request - #request{} +%% Session - #tcp_session{} +%% +%% Description: Checks the status code ... +%%------------------------------------------------------------------------- +result(Response = {{_, Code,_}, _, _}, + Request = #request{stream = Stream}) + when ((Code =:= 200) orelse (Code =:= 206)) andalso (Stream =/= none) -> + stream_end(Response, Request); + +result(Response = {{_,100,_}, _, _}, Request) -> + status_continue(Response, Request); + +%% In redirect loop +result(Response = {{_, Code, _}, _, _}, Request = + #request{redircount = Redirects, + settings = #http_options{autoredirect = true}}) + when ((Code div 100) =:= 3) andalso (Redirects > ?HTTP_MAX_REDIRECTS) -> + transparent(Response, Request); + +%% multiple choices +result(Response = {{_, 300, _}, _, _}, + Request = #request{settings = + #http_options{autoredirect = + true}}) -> + redirect(Response, Request); + +result(Response = {{_, Code, _}, _, _}, + Request = #request{settings = + #http_options{autoredirect = true}, + method = head}) when (Code =:= 301) orelse + (Code =:= 302) orelse + (Code =:= 303) orelse + (Code =:= 307) -> + redirect(Response, Request); +result(Response = {{_, Code, _}, _, _}, + Request = #request{settings = + #http_options{autoredirect = true}, + method = get}) when (Code =:= 301) orelse + (Code =:= 302) orelse + (Code =:= 303) orelse + (Code =:= 307) -> + redirect(Response, Request); + + +result(Response = {{_,503,_}, _, _}, Request) -> + status_service_unavailable(Response, Request); +result(Response = {{_,Code,_}, _, _}, Request) when (Code div 100) =:= 5 -> + status_server_error_50x(Response, Request); + +result(Response, Request) -> + transparent(Response, Request). + +send(To, Msg) -> + To ! {http, Msg}. + +%%%======================================================================== +%%% Internal functions +%%%======================================================================== +parse_version(<<>>, Version, MaxHeaderSize, Result, Relaxed) -> + {?MODULE, parse_version, [Version, MaxHeaderSize,Result, Relaxed]}; +parse_version(<<?SP, Rest/binary>>, Version, + MaxHeaderSize, Result, Relaxed) -> + case lists:reverse(Version) of + "HTTP/" ++ _ = Newversion -> + parse_status_code(Rest, [], MaxHeaderSize, + [Newversion | Result], Relaxed); + NewVersion -> + throw({error, {invalid_version, NewVersion}}) + end; + +parse_version(<<Octet, Rest/binary>>, Version, + MaxHeaderSize, Result, Relaxed) -> + parse_version(Rest, [Octet | Version], MaxHeaderSize,Result, Relaxed). + +parse_status_code(<<>>, StatusCodeStr, MaxHeaderSize, Result, Relaxed) -> + {?MODULE, parse_status_code, + [StatusCodeStr, MaxHeaderSize, Result, Relaxed]}; + +%% Some Apache servers has been known to leave out the reason phrase, +%% in relaxed mode we will allow this. +parse_status_code(<<?CR>> = Data, StatusCodeStr, + MaxHeaderSize, Result, true) -> + {?MODULE, parse_status_code, + [Data, StatusCodeStr, MaxHeaderSize, Result, true]}; +parse_status_code(<<?LF>>, StatusCodeStr, + MaxHeaderSize, Result, true) -> + %% If ?CR is is missing RFC2616 section-19.3 + parse_status_code(<<?CR, ?LF>>, StatusCodeStr, + MaxHeaderSize, Result, true); + +parse_status_code(<<?CR, ?LF, Rest/binary>>, StatusCodeStr, + MaxHeaderSize, Result, true) -> + parse_headers(Rest, [], [], MaxHeaderSize, + [" ", list_to_integer(lists:reverse( + string:strip(StatusCodeStr))) + | Result], true); + +parse_status_code(<<?SP, Rest/binary>>, StatusCodeStr, + MaxHeaderSize, Result, Relaxed) -> + parse_reason_phrase(Rest, [], MaxHeaderSize, + [list_to_integer(lists:reverse(StatusCodeStr)) | + Result], Relaxed); + +parse_status_code(<<Octet, Rest/binary>>, StatusCodeStr, + MaxHeaderSize,Result, Relaxed) -> + parse_status_code(Rest, [Octet | StatusCodeStr], MaxHeaderSize, Result, + Relaxed). + +parse_reason_phrase(<<>>, Phrase, MaxHeaderSize, Result, Relaxed) -> + {?MODULE, parse_reason_phrase, + [<<>>, Phrase, MaxHeaderSize, Result, Relaxed]}; + +parse_reason_phrase(<<?CR, ?LF, ?LF, Body/binary>>, Phrase, + MaxHeaderSize, Result, Relaxed) -> + %% If ?CR is is missing RFC2616 section-19.3 + parse_reason_phrase(<<?CR, ?LF, ?CR, ?LF, Body/binary>>, Phrase, + MaxHeaderSize, Result, Relaxed); + +parse_reason_phrase(<<?CR, ?LF, ?CR, ?LF, Body/binary>>, Phrase, + _, Result, _) -> + ResponseHeaderRcord = + http_response:headers([], #http_response_h{}), + {ok, list_to_tuple( + lists:reverse([Body, ResponseHeaderRcord | + [lists:reverse(Phrase) | Result]]))}; + +parse_reason_phrase(<<?CR, ?LF, ?CR>> = Data, Phrase, MaxHeaderSize, Result, + Relaxed) -> + {?MODULE, parse_reason_phrase, [Data, Phrase, MaxHeaderSize, Result], + Relaxed}; + +parse_reason_phrase(<<?CR, ?LF>> = Data, Phrase, MaxHeaderSize, Result, + Relaxed) -> + {?MODULE, parse_reason_phrase, [Data, Phrase, MaxHeaderSize, Result, + Relaxed]}; +parse_reason_phrase(<<?LF, Rest/binary>>, Phrase, + MaxHeaderSize, Result, Relaxed) -> + %% If ?CR is is missing RFC2616 section-19.3 + parse_reason_phrase(<<?CR, ?LF, Rest/binary>>, Phrase, + MaxHeaderSize, Result, Relaxed); +parse_reason_phrase(<<?CR, ?LF, Rest/binary>>, Phrase, + MaxHeaderSize, Result, Relaxed) -> + parse_headers(Rest, [], [], MaxHeaderSize, + [lists:reverse(Phrase) | Result], Relaxed); +parse_reason_phrase(<<?LF>>, Phrase, MaxHeaderSize, Result, Relaxed) -> + %% If ?CR is is missing RFC2616 section-19.3 + parse_reason_phrase(<<?CR, ?LF>>, Phrase, MaxHeaderSize, Result, + Relaxed); +parse_reason_phrase(<<?CR>> = Data, Phrase, MaxHeaderSize, Result, Relaxed) -> + {?MODULE, parse_reason_phrase, + [Data, Phrase, MaxHeaderSize, Result, Relaxed]}; +parse_reason_phrase(<<Octet, Rest/binary>>, Phrase, MaxHeaderSize, Result, + Relaxed) -> + parse_reason_phrase(Rest, [Octet | Phrase], MaxHeaderSize, + Result, Relaxed). + +parse_headers(<<>>, Header, Headers, MaxHeaderSize, Result, Relaxed) -> + {?MODULE, parse_headers, [<<>>, Header, Headers, MaxHeaderSize, Result, + Relaxed]}; + +parse_headers(<<?CR,?LF,?LF,Body/binary>>, Header, Headers, + MaxHeaderSize, Result, Relaxed) -> + %% If ?CR is is missing RFC2616 section-19.3 + parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, Header, Headers, + MaxHeaderSize, Result, Relaxed); + +parse_headers(<<?LF,?LF,Body/binary>>, Header, Headers, + MaxHeaderSize, Result, Relaxed) -> + %% If ?CR is is missing RFC2616 section-19.3 + parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, Header, Headers, + MaxHeaderSize, Result, Relaxed); + +parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, Header, Headers, + MaxHeaderSize, Result, _) -> + HTTPHeaders = [lists:reverse(Header) | Headers], + Length = lists:foldl(fun(H, Acc) -> length(H) + Acc end, + 0, HTTPHeaders), + case ((Length =< MaxHeaderSize) or (MaxHeaderSize == nolimit)) of + true -> + ResponseHeaderRcord = + http_response:headers(HTTPHeaders, #http_response_h{}), + {ok, list_to_tuple( + lists:reverse([Body, ResponseHeaderRcord | Result]))}; + false -> + throw({error, {header_too_long, MaxHeaderSize, + MaxHeaderSize-Length}}) + end; +parse_headers(<<?CR,?LF,?CR>> = Data, Header, Headers, + MaxHeaderSize, Result, Relaxed) -> + {?MODULE, parse_headers, [Data, Header, Headers, + MaxHeaderSize, Result, Relaxed]}; +parse_headers(<<?CR,?LF>> = Data, Header, Headers, + MaxHeaderSize, Result, Relaxed) -> + {?MODULE, parse_headers, [Data, Header, Headers, MaxHeaderSize, + Result, Relaxed]}; +parse_headers(<<?CR,?LF, Octet, Rest/binary>>, Header, Headers, + MaxHeaderSize, Result, Relaxed) -> + parse_headers(Rest, [Octet], + [lists:reverse(Header) | Headers], MaxHeaderSize, + Result, Relaxed); +parse_headers(<<?CR>> = Data, Header, Headers, + MaxHeaderSize, Result, Relaxed) -> + {?MODULE, parse_headers, [Data, Header, Headers, MaxHeaderSize, + Result, Relaxed]}; + +parse_headers(<<?LF>>, Header, Headers, + MaxHeaderSize, Result, Relaxed) -> + %% If ?CR is is missing RFC2616 section-19.3 + parse_headers(<<?CR, ?LF>>, Header, Headers, + MaxHeaderSize, Result, Relaxed); + +parse_headers(<<Octet, Rest/binary>>, Header, Headers, + MaxHeaderSize, Result, Relaxed) -> + parse_headers(Rest, [Octet | Header], Headers, MaxHeaderSize, + Result, Relaxed). + + +%% RFC2616, Section 10.1.1 +%% Note: +%% - Only act on the 100 status if the request included the +%% "Expect:100-continue" header, otherwise just ignore this response. +status_continue(_, #request{headers = + #http_request_h{expect = "100-continue"}}) -> + continue; + +status_continue({_,_, Data}, _) -> + %% The data in the body in this case is actually part of the real + %% response sent after the "fake" 100-continue. + {ignore, Data}. + +status_service_unavailable(Response = {_, Headers, _}, Request) -> + case Headers#http_response_h.'retry-after' of + undefined -> + status_server_error_50x(Response, Request); + Time when (length(Time) < 3) -> % Wait only 99 s or less + NewTime = list_to_integer(Time) * 100, % time in ms + {_, Data} = format_response(Response), + {retry, {NewTime, Request}, Data}; + _ -> + status_server_error_50x(Response, Request) + end. + +status_server_error_50x(Response, Request) -> + {Msg, _} = format_response(Response), + {stop, {Request#request.id, Msg}}. + + +redirect(Response = {StatusLine, Headers, Body}, Request) -> + {_, Data} = format_response(Response), + case Headers#http_response_h.location of + undefined -> + transparent(Response, Request); + RedirUrl -> + case http_uri:parse(RedirUrl) of + {error, no_scheme} when + (Request#request.settings)#http_options.relaxed -> + NewLocation = fix_relative_uri(Request, RedirUrl), + redirect({StatusLine, Headers#http_response_h{ + location = NewLocation}, + Body}, Request); + {error, Reason} -> + {ok, error(Request, Reason), Data}; + %% Automatic redirection + {Scheme, _, Host, Port, Path, Query} -> + NewHeaders = + (Request#request.headers)#http_request_h{host = + Host}, + NewRequest = + Request#request{redircount = + Request#request.redircount+1, + scheme = Scheme, + headers = NewHeaders, + address = {Host,Port}, + path = Path, + pquery = Query, + abs_uri = + atom_to_list(Scheme) ++ "://" ++ + Host ++ ":" ++ + integer_to_list(Port) ++ + Path ++ Query}, + {redirect, NewRequest, Data} + end + end. + +maybe_to_list(Port) when is_integer(Port) -> + integer_to_list(Port); +maybe_to_list(Port) when is_list(Port) -> + Port. + +%%% Guessing that we received a relative URI, fix it to become an absoluteURI +fix_relative_uri(Request, RedirUrl) -> + {Server, Port0} = Request#request.address, + Port = maybe_to_list(Port0), + Path = Request#request.path, + atom_to_list(Request#request.scheme) ++ "://" ++ Server ++ ":" ++ Port + ++ Path ++ RedirUrl. + +error(#request{id = Id}, Reason) -> + {Id, {error, Reason}}. + +transparent(Response, Request) -> + {Msg, Data} = format_response(Response), + {ok, {Request#request.id, Msg}, Data}. + +stream_start(Headers, Request, ignore) -> + {Request#request.id, stream_start, http_response:header_list(Headers)}; + +stream_start(Headers, Request, Pid) -> + {Request#request.id, stream_start, + http_response:header_list(Headers), Pid}. + +stream_end(Response, Request = #request{stream = Self}) + when (Self =:= self) orelse (Self =:= {self, once}) -> + {{_, Headers, _}, Data} = format_response(Response), + {ok, {Request#request.id, stream_end, Headers}, Data}; + +stream_end(Response, Request) -> + {_, Data} = format_response(Response), + {ok, {Request#request.id, saved_to_file}, Data}. + +is_server_closing(Headers) when is_record(Headers, http_response_h) -> + case Headers#http_response_h.connection of + "close" -> + true; + _ -> + false + end. + +format_response({{"HTTP/0.9", _, _} = StatusLine, _, Body}) -> + {{StatusLine, [], Body}, <<>>}; +format_response({StatusLine, Headers, Body = <<>>}) -> + {{StatusLine, http_response:header_list(Headers), Body}, <<>>}; + +format_response({StatusLine, Headers, Body}) -> + Length = list_to_integer(Headers#http_response_h.'content-length'), + {NewBody, Data} = + case Length of + 0 -> + {Body, <<>>}; + -1 -> % When no lenght indicator is provided + {Body, <<>>}; + Length when (Length =< size(Body)) -> + <<BodyThisReq:Length/binary, Next/binary>> = Body, + {BodyThisReq, Next}; + _ -> %% Connection prematurely ended. + {Body, <<>>} + end, + {{StatusLine, http_response:header_list(Headers), NewBody}, Data}. + diff --git a/lib/inets/src/http_client/httpc_sup.erl b/lib/inets/src/http_client/httpc_sup.erl new file mode 100644 index 0000000000..152a57d32d --- /dev/null +++ b/lib/inets/src/http_client/httpc_sup.erl @@ -0,0 +1,75 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2009. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% +%%---------------------------------------------------------------------- +%% Purpose: The top supervisor for the http client hangs under +%% inets_sup. +%%---------------------------------------------------------------------- + +-module(httpc_sup). + +-behaviour(supervisor). + +%% API +-export([start_link/1]). + +%% Supervisor callback +-export([init/1]). + +%%%========================================================================= +%%% API +%%%========================================================================= +start_link(HttpcServices) -> + supervisor:start_link({local, ?MODULE}, ?MODULE, [HttpcServices]). + +%%%========================================================================= +%%% Supervisor callback +%%%========================================================================= +init([HttpcServices]) -> + RestartStrategy = one_for_one, + MaxR = 10, + MaxT = 3600, + Children = child_specs(HttpcServices), + {ok, {{RestartStrategy, MaxR, MaxT}, Children}}. + +%%%========================================================================= +%%% Internal functions +%%%========================================================================= +child_specs(HttpcServices) -> + [httpc_profile_sup(HttpcServices), httpc_handler_sup()]. + +httpc_profile_sup(HttpcServices) -> + Name = httpc_profile_sup, + StartFunc = {httpc_profile_sup, start_link, [HttpcServices]}, + Restart = permanent, + Shutdown = infinity, + Modules = [httpc_profile_sup], + Type = supervisor, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + +httpc_handler_sup() -> + Name = httpc_handler_sup, + StartFunc = {httpc_handler_sup, start_link, []}, + Restart = permanent, + Shutdown = infinity, + Modules = [httpc_handler_sup], + Type = supervisor, + {Name, StartFunc, Restart, Shutdown, Type, Modules}. + + |