aboutsummaryrefslogtreecommitdiffstats
path: root/lib/inets/src/http_client
diff options
context:
space:
mode:
Diffstat (limited to 'lib/inets/src/http_client')
-rw-r--r--lib/inets/src/http_client/httpc.erl116
-rw-r--r--lib/inets/src/http_client/httpc_handler.erl18
-rw-r--r--lib/inets/src/http_client/httpc_manager.erl4
-rw-r--r--lib/inets/src/http_client/httpc_response.erl264
4 files changed, 275 insertions, 127 deletions
diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl
index bf2da82603..821eb7f02f 100644
--- a/lib/inets/src/http_client/httpc.erl
+++ b/lib/inets/src/http_client/httpc.erl
@@ -175,10 +175,10 @@ request(Method,
(Method =:= delete) orelse
(Method =:= trace) andalso
(is_atom(Profile) orelse is_pid(Profile)) ->
- case uri_parse(Url, Options) of
- {error, Reason} ->
+ case uri_string:parse(uri_string:normalize(Url)) of
+ {error, Reason, _} ->
{error, Reason};
- {ok, ParsedUrl} ->
+ ParsedUrl ->
case header_parse(Headers) of
{error, Reason} ->
{error, Reason};
@@ -189,10 +189,10 @@ request(Method,
end.
do_request(Method, {Url, Headers, ContentType, Body}, HTTPOptions, Options, Profile) ->
- case uri_parse(Url, Options) of
- {error, Reason} ->
+ case uri_string:parse(uri_string:normalize(Url)) of
+ {error, Reason, _} ->
{error, Reason};
- {ok, ParsedUrl} ->
+ ParsedUrl ->
handle_request(Method, Url,
ParsedUrl, Headers, ContentType, Body,
HTTPOptions, Options, Profile)
@@ -312,23 +312,28 @@ store_cookies(SetCookieHeaders, Url) ->
store_cookies(SetCookieHeaders, Url, Profile)
when is_atom(Profile) orelse is_pid(Profile) ->
- try
- begin
+ case uri_string:parse(uri_string:normalize(Url)) of
+ {error, Bad, _} ->
+ {error, {parse_failed, Bad}};
+ URI ->
+ Scheme = scheme_to_atom(maps:get(scheme, URI, '')),
+ Host = maps:get(host, URI, ""),
+ Port = maps:get(port, URI, default_port(Scheme)),
+ Path = uri_string:recompose(#{path => maps:get(path, URI, "")}),
%% Since the Address part is not actually used
%% by the manager when storing cookies, we dont
%% care about ipv6-host-with-brackets.
- {ok, {_, _, Host, Port, Path, _}} = uri_parse(Url),
Address = {Host, Port},
ProfileName = profile_name(Profile),
Cookies = httpc_cookie:cookies(SetCookieHeaders, Path, Host),
httpc_manager:store_cookies(Cookies, Address, ProfileName),
ok
- end
- catch
- error:{badmatch, Bad} ->
- {error, {parse_failed, Bad}}
end.
+default_port(http) ->
+ 80;
+default_port(https) ->
+ 443.
%%--------------------------------------------------------------------------
%% cookie_header(Url) -> Header | {error, Reason}
@@ -495,7 +500,7 @@ service_info(Pid) ->
%%% Internal functions
%%%========================================================================
handle_request(Method, Url,
- {Scheme, UserInfo, Host, Port, Path, Query},
+ URI,
Headers0, ContentType, Body0,
HTTPOptions0, Options0, Profile) ->
@@ -520,37 +525,40 @@ handle_request(Method, Url,
throw({error, {bad_body, Body0}})
end,
- HTTPOptions = http_options(HTTPOptions0),
- Options = request_options(Options0),
- Sync = proplists:get_value(sync, Options),
- Stream = proplists:get_value(stream, Options),
- Host2 = http_request:normalize_host(Scheme, Host, Port),
- HeadersRecord = header_record(NewHeaders, Host2, HTTPOptions),
- Receiver = proplists:get_value(receiver, Options),
- SocketOpts = proplists:get_value(socket_opts, Options),
- BracketedHost = proplists:get_value(ipv6_host_with_brackets,
- Options),
- MaybeEscPath = maybe_encode_uri(HTTPOptions, Path),
- MaybeEscQuery = maybe_encode_uri(HTTPOptions, Query),
- AbsUri = maybe_encode_uri(HTTPOptions, Url),
+ HTTPOptions = http_options(HTTPOptions0),
+ Options = request_options(Options0),
+ Sync = proplists:get_value(sync, Options),
+ Stream = proplists:get_value(stream, Options),
+ Receiver = proplists:get_value(receiver, Options),
+ SocketOpts = proplists:get_value(socket_opts, Options),
+ BracketedHost = proplists:get_value(ipv6_host_with_brackets,
+ Options),
+
+ Scheme = scheme_to_atom(maps:get(scheme, URI, '')),
+ Userinfo = maps:get(userinfo, URI, ""),
+ Host = http_util:maybe_add_brackets(maps:get(host, URI, ""), BracketedHost),
+ Port = maps:get(port, URI, default_port(Scheme)),
+ Host2 = http_request:normalize_host(Scheme, Host, Port),
+ Path = uri_string:recompose(#{path => maps:get(path, URI, "")}),
+ Query = add_question_mark(maps:get(query, URI, "")),
+ HeadersRecord = header_record(NewHeaders, Host2, HTTPOptions),
Request = #request{from = Receiver,
- scheme = Scheme,
- address = {host_address(Host, BracketedHost), Port},
- path = MaybeEscPath,
- pquery = MaybeEscQuery,
+ scheme = Scheme,
+ address = {Host, Port},
+ path = Path,
+ pquery = Query,
method = Method,
headers = HeadersRecord,
content = {ContentType, Body},
settings = HTTPOptions,
- abs_uri = AbsUri,
- userinfo = UserInfo,
+ abs_uri = Url,
+ userinfo = Userinfo,
stream = Stream,
headers_as_is = headers_as_is(Headers0, Options),
socket_opts = SocketOpts,
started = Started,
ipv6_host_with_brackets = BracketedHost},
-
case httpc_manager:request(Request, profile_name(Profile)) of
{ok, RequestId} ->
handle_answer(RequestId, Sync, Options);
@@ -565,14 +573,31 @@ handle_request(Method, Url,
Error
end.
+
+add_question_mark(<<>>) ->
+ <<>>;
+add_question_mark([]) ->
+ [];
+add_question_mark(Comp) when is_binary(Comp) ->
+ <<$?, Comp/binary>>;
+add_question_mark(Comp) when is_list(Comp) ->
+ [$?|Comp].
+
+
+scheme_to_atom("http") ->
+ http;
+scheme_to_atom("https") ->
+ https;
+scheme_to_atom('') ->
+ '';
+scheme_to_atom(Scheme) ->
+ throw({error, {bad_scheme, Scheme}}).
+
+
ensure_chunked_encoding(Hdrs) ->
Key = "transfer-encoding",
lists:keystore(Key, 1, Hdrs, {Key, "chunked"}).
-maybe_encode_uri(#http_options{url_encode = true}, URI) ->
- http_uri:encode(URI);
-maybe_encode_uri(_, URI) ->
- URI.
mk_chunkify_fun(ProcessBody) ->
fun(eof_body) ->
@@ -1190,17 +1215,6 @@ validate_headers(RequestHeaders, _, _) ->
%% These functions is just simple wrappers to parse specifically HTTP URIs
%%--------------------------------------------------------------------------
-scheme_defaults() ->
- [{http, 80}, {https, 443}].
-
-uri_parse(URI) ->
- http_uri:parse(URI, [{scheme_defaults, scheme_defaults()}]).
-
-uri_parse(URI, Opts) ->
- http_uri:parse(URI, [{scheme_defaults, scheme_defaults()} | Opts]).
-
-
-%%--------------------------------------------------------------------------
header_parse([]) ->
ok;
header_parse([{Field, Value}|T]) when is_list(Field), is_list(Value) ->
@@ -1221,10 +1235,6 @@ child_name(Pid, [{Name, Pid} | _]) ->
child_name(Pid, [_ | Children]) ->
child_name(Pid, Children).
-host_address(Host, false) ->
- Host;
-host_address(Host, true) ->
- string:strip(string:strip(Host, right, $]), left, $[).
check_body_gen({Fun, _}) when is_function(Fun) ->
ok;
diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl
index bd1d2e833a..1482f4f922 100644
--- a/lib/inets/src/http_client/httpc_handler.erl
+++ b/lib/inets/src/http_client/httpc_handler.erl
@@ -109,7 +109,7 @@ start_link(Parent, Request, Options, ProfileName) ->
%% to be called by the httpc manager process.
%%--------------------------------------------------------------------
send(Request, Pid) ->
- call(Request, Pid, 5000).
+ call(Request, Pid).
%%--------------------------------------------------------------------
@@ -712,12 +712,16 @@ do_handle_info({'EXIT', _, _}, State = #state{request = undefined}) ->
do_handle_info({'EXIT', _, _}, State) ->
{noreply, State#state{status = close}}.
-
call(Msg, Pid) ->
- call(Msg, Pid, infinity).
-
-call(Msg, Pid, Timeout) ->
- gen_server:call(Pid, Msg, Timeout).
+ try gen_server:call(Pid, Msg)
+ catch
+ exit:{noproc, _} ->
+ {error, closed};
+ exit:{normal, _} ->
+ {error, closed};
+ exit:{{shutdown, _},_} ->
+ {error, closed}
+ end.
cast(Msg, Pid) ->
gen_server:cast(Pid, Msg).
@@ -736,7 +740,7 @@ maybe_send_answer(Request, Answer, State) ->
answer_request(Request, Answer, State).
deliver_answer(#request{from = From} = Request)
- when is_pid(From) ->
+ when From =/= answer_sent ->
Response = httpc_response:error(Request, socket_closed_remotely),
httpc_response:send(From, Response);
deliver_answer(_Request) ->
diff --git a/lib/inets/src/http_client/httpc_manager.erl b/lib/inets/src/http_client/httpc_manager.erl
index a63864493f..ffdf1603b3 100644
--- a/lib/inets/src/http_client/httpc_manager.erl
+++ b/lib/inets/src/http_client/httpc_manager.erl
@@ -849,11 +849,11 @@ pipeline_or_keep_alive(#request{id = Id,
from = From} = Request,
HandlerPid,
#state{handler_db = HandlerDb} = State) ->
- case (catch httpc_handler:send(Request, HandlerPid)) of
+ case httpc_handler:send(Request, HandlerPid) of
ok ->
HandlerInfo = {Id, HandlerPid, From},
ets:insert(HandlerDb, HandlerInfo);
- _ -> % timeout pipelining failed
+ {error, closed} -> % timeout pipelining failed
start_handler(Request, State)
end.
diff --git a/lib/inets/src/http_client/httpc_response.erl b/lib/inets/src/http_client/httpc_response.erl
index b3b11b74ab..58ab9144df 100644
--- a/lib/inets/src/http_client/httpc_response.erl
+++ b/lib/inets/src/http_client/httpc_response.erl
@@ -190,7 +190,7 @@ parse_status_code(<<?CR, ?LF, Rest/binary>>, StatusCodeStr,
MaxHeaderSize, Result, true) ->
parse_headers(Rest, [], [], MaxHeaderSize,
[" ", list_to_integer(lists:reverse(
- string:strip(StatusCodeStr)))
+ string:trim(StatusCodeStr)))
| Result], true);
parse_status_code(<<?SP, Rest/binary>>, StatusCodeStr,
@@ -269,7 +269,7 @@ parse_headers(<<?LF,?LF,Body/binary>>, Header, Headers,
MaxHeaderSize, Result, Relaxed);
parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, Header, Headers,
- MaxHeaderSize, Result, _) ->
+ MaxHeaderSize, Result, Relaxed) ->
HTTPHeaders = [lists:reverse(Header) | Headers],
Length = lists:foldl(fun(H, Acc) -> length(H) + Acc end,
0, HTTPHeaders),
@@ -277,8 +277,42 @@ parse_headers(<<?CR,?LF,?CR,?LF,Body/binary>>, Header, Headers,
true ->
ResponseHeaderRcord =
http_response:headers(HTTPHeaders, #http_response_h{}),
- {ok, list_to_tuple(
- lists:reverse([Body, ResponseHeaderRcord | Result]))};
+
+ %% RFC7230, Section 3.3.3
+ %% If a message is received with both a Transfer-Encoding and a
+ %% Content-Length header field, the Transfer-Encoding overrides the
+ %% Content-Length. Such a message might indicate an attempt to
+ %% perform request smuggling (Section 9.5) or response splitting
+ %% (Section 9.4) and ought to be handled as an error. A sender MUST
+ %% remove the received Content-Length field prior to forwarding such
+ %% a message downstream.
+ case ResponseHeaderRcord#http_response_h.'transfer-encoding' of
+ undefined ->
+ {ok, list_to_tuple(
+ lists:reverse([Body, ResponseHeaderRcord | Result]))};
+ Value ->
+ TransferEncoding = string:lowercase(Value),
+ ContentLength = ResponseHeaderRcord#http_response_h.'content-length',
+ if
+ %% Respond without error but remove Content-Length field in relaxed mode
+ (Relaxed =:= true)
+ andalso (TransferEncoding =:= "chunked")
+ andalso (ContentLength =/= "-1") ->
+ ResponseHeaderRcordFixed =
+ ResponseHeaderRcord#http_response_h{'content-length' = "-1"},
+ {ok, list_to_tuple(
+ lists:reverse([Body, ResponseHeaderRcordFixed | Result]))};
+ %% Respond with error in default (not relaxed) mode
+ (Relaxed =:= false)
+ andalso (TransferEncoding =:= "chunked")
+ andalso (ContentLength =/= "-1") ->
+ throw({error, {headers_conflict, {'content-length',
+ 'transfer-encoding'}}});
+ true ->
+ {ok, list_to_tuple(
+ lists:reverse([Body, ResponseHeaderRcord | Result]))}
+ end
+ end;
false ->
throw({error, {header_too_long, MaxHeaderSize,
MaxHeaderSize-Length}})
@@ -343,58 +377,173 @@ status_server_error_50x(Response, Request) ->
{stop, {Request#request.id, Msg}}.
-redirect(Response = {StatusLine, Headers, Body}, Request) ->
+redirect(Response = {_, Headers, _}, Request) ->
{_, Data} = format_response(Response),
case Headers#http_response_h.location of
- undefined ->
- transparent(Response, Request);
- RedirUrl ->
- UrlParseOpts = [{ipv6_host_with_brackets,
- Request#request.ipv6_host_with_brackets}],
- case uri_parse(RedirUrl, UrlParseOpts) of
- {error, no_scheme} when
- (Request#request.settings)#http_options.relaxed ->
- NewLocation = fix_relative_uri(Request, RedirUrl),
- redirect({StatusLine, Headers#http_response_h{
- location = NewLocation},
- Body}, Request);
- {error, Reason} ->
- {ok, error(Request, Reason), Data};
- %% Automatic redirection
- {ok, {Scheme, _, Host, Port, Path, Query}} ->
- HostPort = http_request:normalize_host(Scheme, Host, Port),
- NewHeaders =
- (Request#request.headers)#http_request_h{host = HostPort},
- NewRequest =
- Request#request{redircount =
- Request#request.redircount+1,
- scheme = Scheme,
- headers = NewHeaders,
- address = {Host,Port},
- path = Path,
- pquery = Query,
- abs_uri =
- atom_to_list(Scheme) ++ "://" ++
- Host ++ ":" ++
- integer_to_list(Port) ++
- Path ++ Query},
- {redirect, NewRequest, Data}
- end
+ undefined ->
+ transparent(Response, Request);
+ RedirUrl ->
+ Brackets = Request#request.ipv6_host_with_brackets,
+ case uri_string:parse(RedirUrl) of
+ {error, Reason, _} ->
+ {ok, error(Request, Reason), Data};
+ %% Automatic redirection
+ URI ->
+ {Host, Port0} = Request#request.address,
+ Port = maybe_to_integer(Port0),
+ Path = Request#request.path,
+ Scheme = atom_to_list(Request#request.scheme),
+ Query = Request#request.pquery,
+ URIMap = resolve_uri(Scheme, Host, Port, Path, Query, URI),
+ TScheme = list_to_atom(maps:get(scheme, URIMap)),
+ THost = http_util:maybe_add_brackets(maps:get(host, URIMap), Brackets),
+ TPort = maps:get(port, URIMap),
+ TPath = maps:get(path, URIMap),
+ TQuery = maps:get(query, URIMap, ""),
+ NewURI = uri_string:normalize(
+ uri_string:recompose(URIMap)),
+ HostPort = http_request:normalize_host(TScheme, THost, TPort),
+ NewHeaders =
+ (Request#request.headers)#http_request_h{host = HostPort},
+ NewRequest =
+ Request#request{redircount =
+ Request#request.redircount+1,
+ scheme = TScheme,
+ headers = NewHeaders,
+ address = {THost,TPort},
+ path = TPath,
+ pquery = TQuery,
+ abs_uri = NewURI},
+ {redirect, NewRequest, Data}
+ end
+ end.
+
+
+%% RFC3986 - 5.2.2. Transform References
+resolve_uri(Scheme, Host, Port, Path, Query, URI) ->
+ resolve_uri(Scheme, Host, Port, Path, Query, URI, #{}).
+%%
+resolve_uri(Scheme, Host, Port, Path, Query, URI, Map0) ->
+ case maps:is_key(scheme, URI) of
+ true ->
+ Port = get_port(URI),
+ maybe_add_query(
+ Map0#{scheme => maps:get(scheme, URI),
+ host => maps:get(host, URI),
+ port => Port,
+ path => maps:get(path, URI)},
+ URI);
+ false ->
+ Map = Map0#{scheme => Scheme},
+ resolve_authority(Host, Port, Path, Query, URI, Map)
+ end.
+
+
+get_port(URI) ->
+ Scheme = maps:get(scheme, URI),
+ case maps:get(port, URI, undefined) of
+ undefined ->
+ get_default_port(Scheme);
+ Port ->
+ Port
+ end.
+
+
+get_default_port("http") ->
+ 80;
+get_default_port("https") ->
+ 443.
+
+
+resolve_authority(Host, Port, Path, Query, RelURI, Map) ->
+ case maps:is_key(host, RelURI) of
+ true ->
+ Port = get_port(RelURI),
+ maybe_add_query(
+ Map#{host => maps:get(host, RelURI),
+ port => Port,
+ path => maps:get(path, RelURI)},
+ RelURI);
+ false ->
+ Map1 = Map#{host => Host,
+ port => Port},
+ resolve_path(Path, Query, RelURI, Map1)
+ end.
+
+
+maybe_add_query(Map, RelURI) ->
+ case maps:is_key(query, RelURI) of
+ true ->
+ Map#{query => maps:get(query, RelURI)};
+ false ->
+ Map
+ end.
+
+
+resolve_path(Path, Query, RelURI, Map) ->
+ case maps:is_key(path, RelURI) of
+ true ->
+ Path1 = calculate_path(Path, maps:get(path, RelURI)),
+ maybe_add_query(
+ Map#{path => Path1},
+ RelURI);
+ false ->
+ Map1 = Map#{path => Path},
+ resolve_query(Query, RelURI, Map1)
+ end.
+
+
+calculate_path(BaseP, RelP) ->
+ case starts_with_slash(RelP) of
+ true ->
+ RelP;
+ false ->
+ merge_paths(BaseP, RelP)
end.
-maybe_to_list(Port) when is_integer(Port) ->
- integer_to_list(Port);
-maybe_to_list(Port) when is_list(Port) ->
+
+starts_with_slash([$/|_]) ->
+ true;
+starts_with_slash(<<$/,_/binary>>) ->
+ true;
+starts_with_slash(_) ->
+ false.
+
+
+%% RFC3986 - 5.2.3. Merge Paths
+merge_paths("", RelP) ->
+ [$/|RelP];
+merge_paths(BaseP, RelP) when is_list(BaseP) ->
+ do_merge_paths(lists:reverse(BaseP), RelP);
+merge_paths(BaseP, RelP) when is_binary(BaseP) ->
+ B = binary_to_list(BaseP),
+ R = binary_to_list(RelP),
+ Res = merge_paths(B, R),
+ list_to_binary(Res).
+
+
+do_merge_paths([$/|_] = L, RelP) ->
+ lists:reverse(L) ++ RelP;
+do_merge_paths([_|T], RelP) ->
+ do_merge_paths(T, RelP).
+
+
+resolve_query(Query, RelURI, Map) ->
+ case maps:is_key(query, RelURI) of
+ true ->
+ Map#{query => maps:get(query, RelURI)};
+ false ->
+ Map#{query => Query}
+ end.
+
+
+maybe_to_integer(Port) when is_list(Port) ->
+ {Port1, _} = string:to_integer(Port),
+ Port1;
+maybe_to_integer(Port) when is_integer(Port) ->
Port.
-%%% Guessing that we received a relative URI, fix it to become an absoluteURI
-fix_relative_uri(Request, RedirUrl) ->
- {Server, Port0} = Request#request.address,
- Port = maybe_to_list(Port0),
- Path = Request#request.path,
- atom_to_list(Request#request.scheme) ++ "://" ++ Server ++ ":" ++ Port
- ++ Path ++ RedirUrl.
-
+
error(#request{id = Id}, Reason) ->
{Id, {error, Reason}}.
@@ -444,18 +593,3 @@ format_response({StatusLine, Headers, Body}) ->
{Body, <<>>}
end,
{{StatusLine, http_response:header_list(Headers), NewBody}, Data}.
-
-%%--------------------------------------------------------------------------
-%% These functions is just simple wrappers to parse specifically HTTP URIs
-%%--------------------------------------------------------------------------
-
-scheme_defaults() ->
- [{http, 80}, {https, 443}].
-
-uri_parse(URI, Opts) ->
- http_uri:parse(URI, [{scheme_defaults, scheme_defaults()} | Opts]).
-
-
-%%--------------------------------------------------------------------------
-
-
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-14 17:56:11 +0000