aboutsummaryrefslogtreecommitdiffstats
path: root/lib/inets
diff options
context:
space:
mode:
Diffstat (limited to 'lib/inets')
-rw-r--r--lib/inets/doc/src/http_uri.xml13
-rw-r--r--lib/inets/src/http_client/httpc_handler.erl59
-rw-r--r--lib/inets/src/http_lib/http_response.erl30
-rw-r--r--lib/inets/src/http_lib/http_uri.erl31
-rw-r--r--lib/inets/test/httpc_SUITE.erl70
-rw-r--r--lib/inets/test/uri_SUITE.erl23
6 files changed, 180 insertions, 46 deletions
diff --git a/lib/inets/doc/src/http_uri.xml b/lib/inets/doc/src/http_uri.xml
index 47c40da96a..64e6c7a6cc 100644
--- a/lib/inets/doc/src/http_uri.xml
+++ b/lib/inets/doc/src/http_uri.xml
@@ -117,7 +117,8 @@
<v>Options = [Option]</v>
<v>Option = {ipv6_host_with_brackets, boolean()} |
{scheme_defaults, scheme_defaults()} |
- {fragment, boolean()}]</v>
+ {fragment, boolean()} |
+ {schema_validation_fun, fun()}]</v>
<v>Result = {Scheme, UserInfo, Host, Port, Path, Query} |
{Scheme, UserInfo, Host, Port, Path, Query, Fragment}</v>
<v>UserInfo = user_info()</v>
@@ -141,6 +142,16 @@
<p>If the fragment option is <c>true</c>, the URI fragment is returned as
part of the parsing result, otherwise it is ignored.</p>
+ <p>Scheme validation fun is to be defined as follows:
+
+ <code>
+fun(SchemeStr :: string()) ->
+ valid | {error, Reason :: term()}.
+ </code>
+
+ It is called before scheme string gets converted into scheme atom and
+ thus possible atom leak could be prevented</p>
+
<marker id="encode"></marker>
</desc>
</func>
diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl
index 6e6cc38c06..1044cffe6f 100644
--- a/lib/inets/src/http_client/httpc_handler.erl
+++ b/lib/inets/src/http_client/httpc_handler.erl
@@ -26,6 +26,7 @@
-include_lib("inets/src/http_lib/http_internal.hrl").
-include("httpc_internal.hrl").
+-define(IS_STREAMED(Code), ((Code =:= 200) orelse (Code =:= 206))).
%%--------------------------------------------------------------------
%% Internal Application API
@@ -163,22 +164,22 @@ info(Pid) ->
%% Request should not be streamed
stream(BodyPart, #request{stream = none} = Request, _) ->
?hcrt("stream - none", []),
- {BodyPart, Request};
+ {false, BodyPart, Request};
%% Stream to caller
stream(BodyPart, #request{stream = Self} = Request, Code)
- when ((Code =:= 200) orelse (Code =:= 206)) andalso
+ when ?IS_STREAMED(Code) andalso
((Self =:= self) orelse (Self =:= {self, once})) ->
?hcrt("stream - self", [{stream, Self}, {code, Code}]),
httpc_response:send(Request#request.from,
{Request#request.id, stream, BodyPart}),
- {<<>>, Request};
+ {true, <<>>, Request};
%% Stream to file
%% This has been moved to start_stream/3
%% We keep this for backward compatibillity...
stream(BodyPart, #request{stream = Filename} = Request, Code)
- when ((Code =:= 200) orelse (Code =:= 206)) andalso is_list(Filename) ->
+ when ?IS_STREAMED(Code) andalso is_list(Filename) ->
?hcrt("stream - filename", [{stream, Filename}, {code, Code}]),
case file:open(Filename, [write, raw, append, delayed_write]) of
{ok, Fd} ->
@@ -190,18 +191,18 @@ stream(BodyPart, #request{stream = Filename} = Request, Code)
%% Stream to file
stream(BodyPart, #request{stream = Fd} = Request, Code)
- when ((Code =:= 200) orelse (Code =:= 206)) ->
+ when ?IS_STREAMED(Code) ->
?hcrt("stream to file", [{stream, Fd}, {code, Code}]),
case file:write(Fd, BodyPart) of
ok ->
- {<<>>, Request};
+ {true, <<>>, Request};
{error, Reason} ->
exit({stream_to_file_failed, Reason})
end;
stream(BodyPart, Request,_) -> % only 200 and 206 responses can be streamed
?hcrt("stream - ignore", [{request, Request}]),
- {BodyPart, Request}.
+ {false, BodyPart, Request}.
%%====================================================================
@@ -474,18 +475,18 @@ handle_info({Proto, _Socket, Data},
{Module, whole_body, [Body, Length]} ->
?hcrd("data processed - whole body", [{length, Length}]),
{_, Code, _} = StatusLine,
- {NewBody, NewRequest} = stream(Body, Request, Code),
+ {Streamed, NewBody, NewRequest} = stream(Body, Request, Code),
%% When we stream we will not keep the already
%% streamed data, that would be a waste of memory.
NewLength =
- case Stream of
- none ->
+ case Streamed of
+ false ->
Length;
- _ ->
+ true ->
Length - size(Body)
end,
- NewState = next_body_chunk(State),
+ NewState = next_body_chunk(State, Code),
NewMFA = {Module, whole_body, [NewBody, NewLength]},
{noreply, NewState#state{mfa = NewMFA,
request = NewRequest}};
@@ -497,8 +498,8 @@ handle_info({Proto, _Socket, Data},
%% The response body is chunk-encoded. Steal decoded
%% chunks as much as possible to stream.
{_, Code, _} = StatusLine,
- {NewBody, NewRequest} = stream(BodySoFar, Request, Code),
- NewState = next_body_chunk(State),
+ {_, NewBody, NewRequest} = stream(BodySoFar, Request, Code),
+ NewState = next_body_chunk(State, Code),
NewMFA = {Module, decode_size,
[TotalChunk, HexList,
{MaxBodySize, NewBody, AccLength, MaxHeaderSize}]},
@@ -517,8 +518,8 @@ handle_info({Proto, _Socket, Data},
NewChunkSize = ChunkSize - ChunkSizeToSteal,
{_, Code, _} = StatusLine,
- {NewBody, NewRequest} = stream(StolenBody, Request, Code),
- NewState = next_body_chunk(State),
+ {_, NewBody, NewRequest} = stream(StolenBody, Request, Code),
+ NewState = next_body_chunk(State, Code),
NewMFA = {Module, decode_data,
[NewChunkSize, NewTotalChunk,
{MaxBodySize, NewBody, AccLength, MaxHeaderSize}]},
@@ -1071,13 +1072,13 @@ handle_http_msg({ChunkedHeaders, Body},
?hcrt("handle_http_msg",
[{chunked_headers, ChunkedHeaders}, {headers, Headers}]),
NewHeaders = http_chunk:handle_headers(Headers, ChunkedHeaders),
- {NewBody, NewRequest} = stream(Body, State#state.request, Code),
+ {_, NewBody, NewRequest} = stream(Body, State#state.request, Code),
handle_response(State#state{headers = NewHeaders,
body = NewBody,
request = NewRequest});
handle_http_msg(Body, #state{status_line = {_,Code, _}} = State) ->
?hcrt("handle_http_msg", [{code, Code}]),
- {NewBody, NewRequest} = stream(Body, State#state.request, Code),
+ {_, NewBody, NewRequest} = stream(Body, State#state.request, Code),
handle_response(State#state{body = NewBody, request = NewRequest}).
handle_http_body(_, #state{status = {ssl_tunnel, _},
@@ -1119,7 +1120,7 @@ handle_http_body(Body, #state{headers = Headers,
[{module, Module},
{function, Function},
{args, Args}]),
- NewState = next_body_chunk(State),
+ NewState = next_body_chunk(State, Code),
{noreply, NewState#state{mfa =
{Module, Function, Args}}};
{ok, {ChunkedHeaders, NewBody}} ->
@@ -1133,7 +1134,7 @@ handle_http_body(Body, #state{headers = Headers,
handle_response(State#state{headers = NewHeaders,
body = NewBody});
_ ->
- {NewBody2, _NewRequest} =
+ {_, NewBody2, _} =
stream(NewBody, Request, Code),
handle_response(State#state{headers = NewHeaders,
body = NewBody2})
@@ -1147,12 +1148,12 @@ handle_http_body(Body, #state{headers = Headers,
true ->
case httpc_response:whole_body(Body, Length) of
{ok, Body} ->
- {NewBody, NewRequest} =
+ {_, NewBody, NewRequest} =
stream(Body, Request, Code),
handle_response(State#state{body = NewBody,
request = NewRequest});
MFA ->
- NewState = next_body_chunk(State),
+ NewState = next_body_chunk(State, Code),
{noreply, NewState#state{mfa = MFA}}
end;
false ->
@@ -1646,21 +1647,21 @@ start_stream({_Version, _Code, _ReasonPhrase}, _Headers,
{ok, Request};
start_stream({_Version, Code, _ReasonPhrase}, Headers,
#request{stream = self} = Request)
- when (Code =:= 200) orelse (Code =:= 206) ->
+ when ?IS_STREAMED(Code) ->
?hcrt("start stream - self", [{code, Code}]),
Msg = httpc_response:stream_start(Headers, Request, ignore),
httpc_response:send(Request#request.from, Msg),
{ok, Request};
start_stream({_Version, Code, _ReasonPhrase}, Headers,
#request{stream = {self, once}} = Request)
- when (Code =:= 200) orelse (Code =:= 206) ->
+ when ?IS_STREAMED(Code) ->
?hcrt("start stream - self:once", [{code, Code}]),
Msg = httpc_response:stream_start(Headers, Request, self()),
httpc_response:send(Request#request.from, Msg),
{ok, Request};
start_stream({_Version, Code, _ReasonPhrase}, _Headers,
#request{stream = Filename} = Request)
- when ((Code =:= 200) orelse (Code =:= 206)) andalso is_list(Filename) ->
+ when ?IS_STREAMED(Code) andalso is_list(Filename) ->
?hcrt("start stream", [{code, Code}, {filename, Filename}]),
case file:open(Filename, [write, raw, append, delayed_write]) of
{ok, Fd} ->
@@ -1712,13 +1713,15 @@ end_stream(SL, R) ->
next_body_chunk(#state{request = #request{stream = {self, once}},
once = once,
- session = Session} = State) ->
+ session = Session} = State,
+ Code) when ?IS_STREAMED(Code) ->
activate_once(Session),
State#state{once = inactive};
next_body_chunk(#state{request = #request{stream = {self, once}},
- once = inactive} = State) ->
+ once = inactive} = State,
+ Code) when ?IS_STREAMED(Code) ->
State; %% Wait for user to call stream_next
-next_body_chunk(#state{session = Session} = State) ->
+next_body_chunk(#state{session = Session} = State, _) ->
activate_once(Session),
State.
diff --git a/lib/inets/src/http_lib/http_response.erl b/lib/inets/src/http_lib/http_response.erl
index 58b30c4e9e..d13670700c 100644
--- a/lib/inets/src/http_lib/http_response.erl
+++ b/lib/inets/src/http_lib/http_response.erl
@@ -31,16 +31,11 @@
%% Value - string()
%%
%% Description: Creates a http_response_h-record used internally to
-%% handle http-headers.
+%% handle http-headers, assumes reversed list of headers
+%% to unfold multiline headers with obs-folds
%%-------------------------------------------------------------------------
-headers([], Headers) ->
- Headers;
-
-headers([Header | Tail], Headers) ->
- {Key, [$: | Value]} =
- lists:splitwith(fun($:) -> false; (_) -> true end, Header),
- headers(Tail, headers(http_util:to_lower(string:strip(Key)),
- string:strip(Value), Headers)).
+headers(RevLines, Headers) ->
+ fill_headers(RevLines, [], Headers).
%%-------------------------------------------------------------------------
%% headers(#http_response_h{}) -> HeaderList
@@ -68,6 +63,23 @@ header_list(Headers) ->
%%%========================================================================
%%% Internal functions
%%%========================================================================
+fill_headers([], _, Headers) ->
+ Headers;
+fill_headers([[Ch|HeaderFold]|Tail], Folded, Headers)
+ when Ch == $\t; Ch == $\s ->
+ fill_headers(Tail, [HeaderFold|Folded], Headers);
+fill_headers([Header | Tail], Folded, Headers) ->
+ Unfolded = unfold([Header|Folded]),
+ {Key, [$: | Value]} =
+ lists:splitwith(fun($:) -> false; (_) -> true end, Unfolded),
+ fill_headers(Tail, [], headers(http_util:to_lower(string:strip(Key)),
+ string:strip(Value), Headers)).
+
+unfold([L]) ->
+ L;
+unfold(Folded) ->
+ string:join(Folded, " ").
+
headers("cache-control", Value, Headers) ->
Headers#http_response_h{'cache-control'= Value};
headers("connection", Value, Headers) ->
diff --git a/lib/inets/src/http_lib/http_uri.erl b/lib/inets/src/http_lib/http_uri.erl
index 79591eec29..6fe8c1776d 100644
--- a/lib/inets/src/http_lib/http_uri.erl
+++ b/lib/inets/src/http_lib/http_uri.erl
@@ -138,16 +138,33 @@ parse_scheme(AbsURI, Opts) ->
{error, no_scheme} ->
{error, no_scheme};
{SchemeStr, Rest} ->
- Scheme = list_to_atom(http_util:to_lower(SchemeStr)),
- SchemeDefaults = which_scheme_defaults(Opts),
- case lists:keysearch(Scheme, 1, SchemeDefaults) of
- {value, {Scheme, DefaultPort}} ->
- {Scheme, DefaultPort, Rest};
- false ->
- {Scheme, no_default_port, Rest}
+ case extract_scheme(SchemeStr, Opts) of
+ {error, Error} ->
+ {error, Error};
+ {ok, Scheme} ->
+ SchemeDefaults = which_scheme_defaults(Opts),
+ case lists:keysearch(Scheme, 1, SchemeDefaults) of
+ {value, {Scheme, DefaultPort}} ->
+ {Scheme, DefaultPort, Rest};
+ false ->
+ {Scheme, no_default_port, Rest}
+ end
end
end.
+extract_scheme(Str, Opts) ->
+ case lists:keysearch(scheme_validation_fun, 1, Opts) of
+ {value, {scheme_validation_fun, Fun}} when is_function(Fun) ->
+ case Fun(Str) of
+ valid ->
+ {ok, list_to_atom(http_util:to_lower(Str))};
+ {error, Error} ->
+ {error, Error}
+ end;
+ _ ->
+ {ok, list_to_atom(http_util:to_lower(Str))}
+ end.
+
parse_uri_rest(Scheme, DefaultPort, "//" ++ URIPart, Opts) ->
{Authority, PathQueryFragment} =
split_uri(URIPart, "[/?#]", {URIPart, ""}, 1, 0),
diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl
index 5b40d08859..989563cdbc 100644
--- a/lib/inets/test/httpc_SUITE.erl
+++ b/lib/inets/test/httpc_SUITE.erl
@@ -98,6 +98,8 @@ only_simulated() ->
stream_once,
stream_single_chunk,
stream_no_length,
+ not_streamed_once,
+ stream_large_not_200_or_206,
no_content_204,
tolerate_missing_CR,
userinfo,
@@ -105,6 +107,7 @@ only_simulated() ->
internal_server_error,
invalid_http,
headers_dummy,
+ headers_with_obs_fold,
empty_response_header,
remote_socket_close,
remote_socket_close_async,
@@ -407,6 +410,21 @@ stream_no_length(Config) when is_list(Config) ->
stream_test(Request1, {stream, self}),
Request2 = {url(group_name(Config), "/http_1_0_no_length_multiple.html", Config), []},
stream_test(Request2, {stream, self}).
+%%-------------------------------------------------------------------------
+stream_large_not_200_or_206() ->
+ [{doc, "Test the option stream for large responses with status codes "
+ "other than 200 or 206" }].
+stream_large_not_200_or_206(Config) when is_list(Config) ->
+ Request = {url(group_name(Config), "/large_404_response.html", Config), []},
+ {404, _} = not_streamed_test(Request, {stream, self}).
+%%-------------------------------------------------------------------------
+not_streamed_once() ->
+ [{doc, "Test not streamed responses with once streaming"}].
+not_streamed_once(Config) when is_list(Config) ->
+ Request0 = {url(group_name(Config), "/404.html", Config), []},
+ {404, _} = not_streamed_test(Request0, {stream, {self, once}}),
+ Request1 = {url(group_name(Config), "/404_chunked.html", Config), []},
+ {404, _} = not_streamed_test(Request1, {stream, {self, once}}).
%%-------------------------------------------------------------------------
@@ -893,6 +911,13 @@ headers_dummy(Config) when is_list(Config) ->
%%-------------------------------------------------------------------------
+headers_with_obs_fold(Config) when is_list(Config) ->
+ Request = {url(group_name(Config), "/obs_folded_headers.html", Config), []},
+ {ok, {{_,200,_}, Headers, [_|_]}} = httpc:request(get, Request, [], []),
+ "a b" = proplists:get_value("folded", Headers).
+
+%%-------------------------------------------------------------------------
+
invalid_headers(Config) ->
Request = {url(group_name(Config), "/dummy.html", Config), [{"cookie", undefined}]},
{error, _} = httpc:request(get, Request, [], []).
@@ -1109,6 +1134,19 @@ stream_test(Request, To) ->
Body = binary_to_list(StreamedBody).
+not_streamed_test(Request, To) ->
+ {ok, {{_,Code,_}, [_ | _], Body}} =
+ httpc:request(get, Request, [], [{body_format, binary}]),
+ {ok, RequestId} =
+ httpc:request(get, Request, [], [{body_format, binary}, {sync, false}, To]),
+
+ receive
+ {http, {RequestId, {{_, Code, _}, _Headers, Body}}} ->
+ {Code, binary_to_list(Body)};
+ {http, Msg} ->
+ ct:fail(Msg)
+ end.
+
url(http, End, Config) ->
Port = ?config(port, Config),
{ok,Host} = inet:gethostname(),
@@ -1640,6 +1678,11 @@ handle_uri(_,"/307.html",Port,_,Socket,_) ->
"Content-Length:" ++ integer_to_list(length(Body))
++ "\r\n\r\n" ++ Body;
+handle_uri(_,"/404.html",_,_,_,_) ->
+ "HTTP/1.1 404 not found\r\n" ++
+ "Content-Length:14\r\n\r\n" ++
+ "Page not found";
+
handle_uri(_,"/500.html",_,_,_,_) ->
"HTTP/1.1 500 Internal Server Error\r\n" ++
"Content-Length:47\r\n\r\n" ++
@@ -1713,6 +1756,13 @@ handle_uri(_,"/dummy_headers.html",_,_,Socket,_) ->
send(Socket, http_chunk:encode("obar</BODY></HTML>")),
http_chunk:encode_last();
+handle_uri(_,"/obs_folded_headers.html",_,_,_,_) ->
+ "HTTP/1.1 200 ok\r\n"
+ "Content-Length:5\r\n"
+ "Folded: a\r\n"
+ " b\r\n\r\n"
+ "Hello";
+
handle_uri(_,"/capital_transfer_encoding.html",_,_,Socket,_) ->
Head = "HTTP/1.1 200 ok\r\n" ++
"Transfer-Encoding:Chunked\r\n\r\n",
@@ -1768,6 +1818,15 @@ handle_uri(_,"/once_chunked.html",_,_,Socket,_) ->
http_chunk:encode("obar</BODY></HTML>")),
http_chunk:encode_last();
+handle_uri(_,"/404_chunked.html",_,_,Socket,_) ->
+ Head = "HTTP/1.1 404 not found\r\n" ++
+ "Transfer-Encoding:Chunked\r\n\r\n",
+ send(Socket, Head),
+ send(Socket, http_chunk:encode("<HTML><BODY>Not ")),
+ send(Socket,
+ http_chunk:encode("found</BODY></HTML>")),
+ http_chunk:encode_last();
+
handle_uri(_,"/single_chunk.html",_,_,Socket,_) ->
Chunk = "HTTP/1.1 200 ok\r\n" ++
"Transfer-Encoding:Chunked\r\n\r\n" ++
@@ -1792,6 +1851,17 @@ handle_uri(_,"/http_1_0_no_length_multiple.html",_,_,Socket,_) ->
send(Socket, string:copies("other multiple packets ", 200)),
close(Socket);
+handle_uri(_,"/large_404_response.html",_,_,Socket,_) ->
+ %% long body to make sure it will be sent in multiple tcp packets
+ Body = string:copies("other multiple packets ", 200),
+ Head = io_lib:format("HTTP/1.1 404 not found\r\n"
+ "Content-length: ~B\r\n"
+ "Content-type: text/plain\r\n\r\n",
+ [length(Body)]),
+ send(Socket, Head),
+ send(Socket, Body),
+ close(Socket);
+
handle_uri(_,"/once.html",_,_,Socket,_) ->
Head = "HTTP/1.1 200 ok\r\n" ++
"Content-Length:32\r\n\r\n",
diff --git a/lib/inets/test/uri_SUITE.erl b/lib/inets/test/uri_SUITE.erl
index bfcd7bd339..2642b8fd4e 100644
--- a/lib/inets/test/uri_SUITE.erl
+++ b/lib/inets/test/uri_SUITE.erl
@@ -49,7 +49,8 @@ all() ->
queries,
fragments,
escaped,
- hexed_query
+ hexed_query,
+ scheme_validation
].
%%--------------------------------------------------------------------
@@ -175,6 +176,26 @@ hexed_query(Config) when is_list(Config) ->
verify_uri(URI2, Verify2),
verify_uri(URI3, Verify3).
+scheme_validation(Config) when is_list(Config) ->
+ {ok, {http,[],"localhost",80,"/",""}} =
+ http_uri:parse("http://localhost#fragment"),
+
+ ValidationFun =
+ fun("http") -> valid;
+ (_) -> {error, bad_scheme}
+ end,
+
+ {ok, {http,[],"localhost",80,"/",""}} =
+ http_uri:parse("http://localhost#fragment",
+ [{scheme_validation_fun, ValidationFun}]),
+ {error, bad_scheme} =
+ http_uri:parse("https://localhost#fragment",
+ [{scheme_validation_fun, ValidationFun}]),
+ %% non-fun scheme_validation_fun works as no option passed
+ {ok, {https,[],"localhost",443,"/",""}} =
+ http_uri:parse("https://localhost#fragment",
+ [{scheme_validation_fun, none}]).
+
%%--------------------------------------------------------------------
%% Internal Functions ------------------------------------------------
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 12:35:35 +0000