aboutsummaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/inets/doc/src/httpc.xml48
-rw-r--r--lib/inets/doc/src/notes.xml37
-rw-r--r--lib/inets/src/http_client/httpc.erl82
-rw-r--r--lib/inets/src/http_client/httpc_handler.erl18
-rw-r--r--lib/inets/src/http_client/httpc_request.erl102
-rw-r--r--lib/inets/test/httpc_SUITE.erl151
6 files changed, 345 insertions, 93 deletions
diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml
index 8f68087871..532f5eefde 100644
--- a/lib/inets/doc/src/httpc.xml
+++ b/lib/inets/doc/src/httpc.xml
@@ -76,21 +76,29 @@ socket_opt() = See the Options used by gen_tcp(3) and
<p>For more information about HTTP see rfc 2616</p>
<code type="none"><![CDATA[
-method() = head | get | put | post | trace | options | delete
-request() = {url(), headers()} |
- {url(), headers(), content_type(), body()}
-url() = string() - Syntax according to the URI definition in rfc 2396, ex: "http://www.erlang.org"
-status_line() = {http_version(), status_code(), reason_phrase()}
-http_version() = string() ex: "HTTP/1.1"
-status_code() = integer()
-reason_phrase() = string()
-content_type() = string()
-headers() = [header()]
-header() = {field(), value()}
-field() = string()
-value() = string()
-body() = string() | binary()
-filename() = string()
+method() = head | get | put | post | trace | options | delete
+request() = {url(), headers()} |
+ {url(), headers(), content_type(), body()}
+url() = string() - Syntax according to the URI definition in rfc 2396, ex: "http://www.erlang.org"
+status_line() = {http_version(), status_code(), reason_phrase()}
+http_version() = string() ex: "HTTP/1.1"
+status_code() = integer()
+reason_phrase() = string()
+content_type() = string()
+headers() = [header()]
+header() = {field(), value()}
+field() = string()
+value() = string()
+body() = string() |
+ binary() |
+ {fun(accumulator()) -> body_processing_result(),
+ accumulator()} |
+ {chunkify,
+ fun(accumulator()) -> body_processing_result(),
+ accumulator()}
+body_processing_result() = eof | {ok, iolist(), accumulator()}
+accumulator() = term()
+filename() = string()
]]></code>
</section>
@@ -142,8 +150,9 @@ ssl_options() = {verify, code()} |
<fsummary>Sends a get HTTP-request</fsummary>
<type>
<v>Url = url() </v>
- <v>Result = {status_line(), headers(), body()} |
- {status_code(), body()} | request_id() </v>
+ <v>Result = {status_line(), headers(), Body} |
+ {status_code(), Body} | request_id() </v>
+ <v>Body = string() | binary()</v>
<v>Profile = profile()</v>
<v>Reason = term() </v>
</type>
@@ -191,8 +200,9 @@ ssl_options() = {verify, code()} |
<v>Function = atom() </v>
<v>Args = list() </v>
<v>body_format() = string | binary </v>
- <v>Result = {status_line(), headers(), body()} |
- {status_code(), body()} | request_id() </v>
+ <v>Result = {status_line(), headers(), Body} |
+ {status_code(), Body} | request_id() </v>
+ <v>Body = string() | binary()</v>
<v>Profile = profile() </v>
<v>Reason = {connect_failed, term()} |
{send_failed, term()} | term() </v>
diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml
index 5da9d98002..4c572e3a9c 100644
--- a/lib/inets/doc/src/notes.xml
+++ b/lib/inets/doc/src/notes.xml
@@ -32,6 +32,43 @@
<file>notes.xml</file>
</header>
+ <section><title>Inets 5.6</title>
+
+ <section><title>Improvements and New Features</title>
+<!--
+ <p>-</p>
+-->
+ <list>
+ <item>
+ <p>[httpc] Add support for upload body streaming (PUT and POST).</p>
+ <p>For more info,
+ see the definition of the <c>Body</c> argument of the
+ <seealso marker="httpc#request2">request/4,5</seealso>
+ function. </p>
+ <p>Filipe David Manana</p>
+ <p>Own Id: OTP-9094</p>
+ </item>
+ </list>
+ </section>
+
+ <section><title>Fixed Bugs and Malfunctions</title>
+ <p>-</p>
+
+<!--
+ <list>
+ <item>
+ <p>[httpc] Pipelined and queued requests not processed when
+ connection closed remotelly.</p>
+ <p>Own Id: OTP-8906</p>
+ </item>
+ </list>
+-->
+ </section>
+
+
+ </section> <!-- 5.6 -->
+
+
<section><title>Inets 5.5.2</title>
<section><title>Improvements and New Features</title>
diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl
index 04fae13b20..c7f8a02a0b 100644
--- a/lib/inets/src/http_client/httpc.erl
+++ b/lib/inets/src/http_client/httpc.erl
@@ -126,7 +126,10 @@ request(Url, Profile) ->
%% Header = {Field, Value}
%% Field = string()
%% Value = string()
-%% Body = string() | binary() - HTLM-code
+%% Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} |
+%% {chunkify, fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code
+%% SendFunResult = eof | {ok, iolist(), NewSendAcc}
+%% SendAcc = NewSendAcc = term()
%%
%% Description: Sends a HTTP-request. The function can be both
%% syncronus and asynchronous in the later case the function will
@@ -426,26 +429,44 @@ service_info(Pid) ->
handle_request(Method, Url,
{Scheme, UserInfo, Host, Port, Path, Query},
- Headers, ContentType, Body,
+ Headers0, ContentType, Body0,
HTTPOptions0, Options0, Profile) ->
- Started = http_util:timestamp(),
- NewHeaders = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers],
+ Started = http_util:timestamp(),
+ NewHeaders0 = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers0],
try
begin
+ ?hcrt("begin processing", [{started, Started},
+ {new_headers, NewHeaders0}]),
+
+ {NewHeaders, Body} =
+ case Body0 of
+ {chunkify, ProcessBody, Acc}
+ when is_function(ProcessBody, 1) ->
+ NewHeaders1 = ensure_chunked_encoding(NewHeaders0),
+ Body1 = {mk_chunkify_fun(ProcessBody), Acc},
+ {NewHeaders1, Body1};
+ {ProcessBody, _}
+ when is_function(ProcessBody, 1) ->
+ {NewHeaders0, Body0};
+ _ when is_list(Body0) orelse is_binary(Body0) ->
+ {NewHeaders0, Body0};
+ _ ->
+ throw({error, {bad_body, Body0}})
+ end,
+
HTTPOptions = http_options(HTTPOptions0),
Options = request_options(Options0),
Sync = proplists:get_value(sync, Options),
Stream = proplists:get_value(stream, Options),
Host2 = header_host(Scheme, Host, Port),
HeadersRecord = header_record(NewHeaders, Host2, HTTPOptions),
- Receiver = proplists:get_value(receiver, Options),
- SocketOpts = proplists:get_value(socket_opts, Options),
- UrlEncodeBool = HTTPOptions#http_options.url_encode,
- MaybeEscPath = url_encode(Path, UrlEncodeBool),
- MaybeEscQuery = url_encode(Query, UrlEncodeBool),
- AbsUri = url_encode(Url, UrlEncodeBool),
+ Receiver = proplists:get_value(receiver, Options),
+ SocketOpts = proplists:get_value(socket_opts, Options),
+ MaybeEscPath = maybe_encode_uri(HTTPOptions, Path),
+ MaybeEscQuery = maybe_encode_uri(HTTPOptions, Query),
+ AbsUri = maybe_encode_uri(HTTPOptions, Url),
Request = #request{from = Receiver,
scheme = Scheme,
@@ -458,38 +479,71 @@ handle_request(Method, Url,
settings = HTTPOptions,
abs_uri = AbsUri,
userinfo = UserInfo,
- stream = Stream,
- headers_as_is = headers_as_is(Headers, Options),
+ stream = Stream,
+ headers_as_is = headers_as_is(Headers0, Options),
socket_opts = SocketOpts,
started = Started},
+
case httpc_manager:request(Request, profile_name(Profile)) of
{ok, RequestId} ->
handle_answer(RequestId, Sync, Options);
{error, Reason} ->
+ ?hcrd("request failed", [{reason, Reason}]),
{error, Reason}
end
end
catch
error:{noproc, _} ->
+ ?hcrl("noproc", [{profile, Profile}]),
{error, {not_started, Profile}};
throw:Error ->
+ ?hcrl("throw", [{error, Error}]),
Error
end.
-url_encode(URI, true) ->
+ensure_chunked_encoding(Hdrs) ->
+ Key = "transfer-encoding",
+ lists:keystore(Key, 1, Hdrs, {Key, "chunked"}).
+
+maybe_encode_uri(#http_options{url_encode = true}, URI) ->
http_uri:encode(URI);
-url_encode(URI, false) ->
+maybe_encode_uri(_, URI) ->
URI.
+mk_chunkify_fun(ProcessBody) ->
+ fun(eof_body) ->
+ eof;
+ (Acc) ->
+ case ProcessBody(Acc) of
+ eof ->
+ {ok, <<"0\r\n\r\n">>, eof_body};
+ {ok, Data, NewAcc} ->
+ {ok, mk_chunk_bin(Data), NewAcc}
+ end
+ end.
+
+mk_chunk_bin(Data) ->
+ Bin = iolist_to_binary(Data),
+ iolist_to_binary([hex_size(Bin), "\r\n", Bin, "\r\n"]).
+
+hex_size(Bin) ->
+ hd(io_lib:format("~.16B", [size(Bin)])).
+
+
handle_answer(RequestId, false, _) ->
{ok, RequestId};
handle_answer(RequestId, true, Options) ->
receive
{http, {RequestId, saved_to_file}} ->
+ ?hcrt("received saved-to-file", [{request_id, RequestId}]),
{ok, saved_to_file};
{http, {RequestId, {_,_,_} = Result}} ->
+ ?hcrt("received answer", [{request_id, RequestId},
+ {result, Result}]),
return_answer(Options, Result);
{http, {RequestId, {error, Reason}}} ->
+ ?hcrt("received error", [{request_id, RequestId},
+ {reason, Reason}]),
{error, Reason}
end.
diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl
index cb6f3e2841..5e22400fe0 100644
--- a/lib/inets/src/http_client/httpc_handler.erl
+++ b/lib/inets/src/http_client/httpc_handler.erl
@@ -1724,9 +1724,27 @@ handle_verbose(_) ->
%% ok.
+send_raw(#session{socket = Socket, socket_type = SocketType},
+ {ProcessBody, Acc}) when is_function(ProcessBody, 1) ->
+ ?hcrt("send raw", [{acc, Acc}]),
+ send_raw(SocketType, Socket, ProcessBody, Acc);
send_raw(#session{socket = Socket, socket_type = SocketType}, Body) ->
http_transport:send(SocketType, Socket, Body).
+send_raw(SocketType, Socket, ProcessBody, Acc) ->
+ case ProcessBody(Acc) of
+ eof ->
+ ok;
+ {ok, Data, NewAcc} ->
+ DataBin = iolist_to_binary(Data),
+ ?hcrd("send", [{data, DataBin}]),
+ case http_transport:send(SocketType, Socket, DataBin) of
+ ok ->
+ send_raw(SocketType, Socket, ProcessBody, NewAcc);
+ Error ->
+ Error
+ end
+ end.
call(Msg, Pid) ->
diff --git a/lib/inets/src/http_client/httpc_request.erl b/lib/inets/src/http_client/httpc_request.erl
index d4df97ad40..0d602adb11 100644
--- a/lib/inets/src/http_client/httpc_request.erl
+++ b/lib/inets/src/http_client/httpc_request.erl
@@ -79,37 +79,65 @@ send(SendAddr, Socket, SocketType,
{settings, HttpOptions},
{userinfo, UserInfo}]),
- TmpHeaders = handle_user_info(UserInfo, Headers),
+ TmpHdrs = handle_user_info(UserInfo, Headers),
- {TmpHeaders2, Body} =
- post_data(Method, TmpHeaders, Content, HeadersAsIs),
+ {TmpHdrs2, Body} = post_data(Method, TmpHdrs, Content, HeadersAsIs),
- {NewHeaders, Uri} = case Address of
- SendAddr ->
- {TmpHeaders2, Path ++ Query};
- _Proxy ->
- TmpHeaders3 =
- handle_proxy(HttpOptions, TmpHeaders2),
- {TmpHeaders3, AbsUri}
- end,
-
- FinalHeaders = case NewHeaders of
- HeaderList when is_list(HeaderList) ->
- http_headers(HeaderList, []);
- _ ->
- http_request:http_headers(NewHeaders)
- end,
+ {NewHeaders, Uri} =
+ case Address of
+ SendAddr ->
+ {TmpHdrs2, Path ++ Query};
+ _Proxy ->
+ TmpHdrs3 = handle_proxy(HttpOptions, TmpHdrs2),
+ {TmpHdrs3, AbsUri}
+ end,
+
+ FinalHeaders =
+ case NewHeaders of
+ HeaderList when is_list(HeaderList) ->
+ http_headers(HeaderList, []);
+ _ ->
+ http_request:http_headers(NewHeaders)
+ end,
Version = HttpOptions#http_options.version,
- Message = [method(Method), " ", Uri, " ",
- version(Version), ?CRLF,
- headers(FinalHeaders, Version), ?CRLF, Body],
+ do_send_body(SocketType, Socket, Method, Uri, Version, FinalHeaders, Body).
+
+
+do_send_body(SocketType, Socket, Method, Uri, Version, Headers,
+ {ProcessBody, Acc}) when is_function(ProcessBody, 1) ->
+ ?hcrt("send", [{acc, Acc}]),
+ case do_send_body(SocketType, Socket, Method, Uri, Version, Headers, []) of
+ ok ->
+ do_send_body(SocketType, Socket, ProcessBody, Acc);
+ Error ->
+ Error
+ end;
+do_send_body(SocketType, Socket, Method, Uri, Version, Headers, Body) ->
+ ?hcrt("create message", [{body, Body}]),
+ Message = [method(Method), " ", Uri, " ",
+ version(Version), ?CRLF,
+ headers(Headers, Version), ?CRLF, Body],
?hcrd("send", [{message, Message}]),
-
http_transport:send(SocketType, Socket, lists:append(Message)).
+do_send_body(SocketType, Socket, ProcessBody, Acc) ->
+ case ProcessBody(Acc) of
+ eof ->
+ ok;
+ {ok, Data, NewAcc} ->
+ DataBin = iolist_to_binary(Data),
+ ?hcrd("send", [{data, DataBin}]),
+ case http_transport:send(SocketType, Socket, DataBin) of
+ ok ->
+ do_send_body(SocketType, Socket, ProcessBody, NewAcc);
+ Error ->
+ Error
+ end
+ end.
+
%%-------------------------------------------------------------------------
%% is_idempotent(Method) ->
@@ -161,7 +189,6 @@ is_client_closing(Headers) ->
%%%========================================================================
post_data(Method, Headers, {ContentType, Body}, HeadersAsIs)
when (Method =:= post) orelse (Method =:= put) ->
- ContentLength = body_length(Body),
NewBody = case Headers#http_request_h.expect of
"100-continue" ->
"";
@@ -170,14 +197,22 @@ post_data(Method, Headers, {ContentType, Body}, HeadersAsIs)
end,
NewHeaders = case HeadersAsIs of
- [] ->
- Headers#http_request_h{'content-type' =
- ContentType,
- 'content-length' =
- ContentLength};
- _ ->
- HeadersAsIs
- end,
+ [] ->
+ Headers#http_request_h{
+ 'content-type' = ContentType,
+ 'content-length' = case body_length(Body) of
+ undefined ->
+ % on upload streaming the caller must give a
+ % value to the Content-Length header
+ % (or use chunked Transfer-Encoding)
+ Headers#http_request_h.'content-length';
+ Len when is_list(Len) ->
+ Len
+ end
+ };
+ _ ->
+ HeadersAsIs
+ end,
{NewHeaders, NewBody};
@@ -190,7 +225,10 @@ body_length(Body) when is_binary(Body) ->
integer_to_list(size(Body));
body_length(Body) when is_list(Body) ->
- integer_to_list(length(Body)).
+ integer_to_list(length(Body));
+
+body_length({DataFun, _Acc}) when is_function(DataFun, 1) ->
+ undefined.
method(Method) ->
http_util:to_upper(atom_to_list(Method)).
diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl
index 2c8febf5ed..c1a36d23de 100644
--- a/lib/inets/test/httpc_SUITE.erl
+++ b/lib/inets/test/httpc_SUITE.erl
@@ -62,36 +62,82 @@
suite() -> [{ct_hooks,[ts_install_cth]}].
all() ->
- [proxy_options, proxy_head, proxy_get, proxy_trace,
- proxy_post, proxy_put, proxy_delete, proxy_auth,
- proxy_headers, proxy_emulate_lower_versions,
- http_options, http_head, http_get, http_post,
- http_dummy_pipe, http_inets_pipe, http_trace,
- http_async, http_save_to_file, http_save_to_file_async,
- http_headers, http_headers_dummy, http_bad_response,
- ssl_head, ossl_head, essl_head, ssl_get, ossl_get,
- essl_get, ssl_trace, ossl_trace, essl_trace,
- http_redirect, http_redirect_loop,
- http_internal_server_error, http_userinfo, http_cookie,
- http_server_does_not_exist, http_invalid_http,
- http_emulate_lower_versions, http_relaxed,
- page_does_not_exist, proxy_page_does_not_exist,
- proxy_https_not_supported, http_stream,
- http_stream_once, proxy_stream, parse_url, options,
- ipv6, headers_as_is, {group, tickets}].
+ [
+ proxy_options,
+ proxy_head,
+ proxy_get,
+ proxy_trace,
+ proxy_post,
+ proxy_put,
+ proxy_delete,
+ proxy_auth,
+ proxy_headers,
+ proxy_emulate_lower_versions,
+ http_options,
+ http_head,
+ http_get,
+ http_post,
+ http_post_streaming,
+ http_dummy_pipe,
+ http_inets_pipe,
+ http_trace,
+ http_async,
+ http_save_to_file,
+ http_save_to_file_async,
+ http_headers,
+ http_headers_dummy,
+ http_bad_response,
+ ssl_head,
+ ossl_head,
+ essl_head,
+ ssl_get,
+ ossl_get,
+ essl_get,
+ ssl_trace,
+ ossl_trace,
+ essl_trace,
+ http_redirect,
+ http_redirect_loop,
+ http_internal_server_error,
+ http_userinfo, http_cookie,
+ http_server_does_not_exist,
+ http_invalid_http,
+ http_emulate_lower_versions,
+ http_relaxed,
+ page_does_not_exist,
+ proxy_page_does_not_exist,
+ proxy_https_not_supported,
+ http_stream,
+ http_stream_once,
+ proxy_stream,
+ parse_url,
+ options,
+ ipv6,
+ headers_as_is,
+ {group, tickets}
+ ].
groups() ->
- [{tickets, [],
- [hexed_query_otp_6191, empty_body_otp_6243,
- empty_response_header_otp_6830,
- transfer_encoding_otp_6807, proxy_not_modified_otp_6821,
- no_content_204_otp_6982, missing_CR_otp_7304,
- {group, otp_7883}, {group, otp_8154}, {group, otp_8106},
- otp_8056, otp_8352, otp_8371, otp_8739]},
- {otp_7883, [], [otp_7883_1, otp_7883_2]},
+ [{tickets, [], [hexed_query_otp_6191,
+ empty_body_otp_6243,
+ empty_response_header_otp_6830,
+ transfer_encoding_otp_6807,
+ proxy_not_modified_otp_6821,
+ no_content_204_otp_6982,
+ missing_CR_otp_7304,
+ {group, otp_7883},
+ {group, otp_8154},
+ {group, otp_8106},
+ otp_8056,
+ otp_8352,
+ otp_8371,
+ otp_8739]},
+ {otp_7883, [], [otp_7883_1,
+ otp_7883_2]},
{otp_8154, [], [otp_8154_1]},
- {otp_8106, [],
- [otp_8106_pid, otp_8106_fun, otp_8106_mfa]}].
+ {otp_8106, [], [otp_8106_pid,
+ otp_8106_fun,
+ otp_8106_mfa]}].
init_per_group(_GroupName, Config) ->
Config.
@@ -138,6 +184,7 @@ init_per_suite(Config) ->
{local_port, ?IP_PORT},
{local_ssl_port, ?SSL_PORT} | Config].
+
%%--------------------------------------------------------------------
%% Function: end_per_suite(Config) -> _
%% Config - [tuple()]
@@ -151,6 +198,7 @@ end_per_suite(Config) ->
application:stop(ssl),
ok.
+
%%--------------------------------------------------------------------
%% Function: init_per_testcase(Case, Config) -> Config
%% Case - atom()
@@ -395,6 +443,53 @@ http_post(Config) when is_list(Config) ->
end.
%%-------------------------------------------------------------------------
+http_post_streaming(doc) ->
+ ["Test streaming http post request against local server. "
+ "We only care about the client side of the the post. "
+ "The server script will not actually use the post data."];
+http_post_streaming(suite) ->
+ [];
+http_post_streaming(Config) when is_list(Config) ->
+ case ?config(local_server, Config) of
+ ok ->
+ Port = ?config(local_port, Config),
+ URL = case test_server:os_type() of
+ {win32, _} ->
+ ?URL_START ++ integer_to_list(Port) ++
+ "/cgi-bin/cgi_echo.exe";
+ _ ->
+ ?URL_START ++ integer_to_list(Port) ++
+ "/cgi-bin/cgi_echo"
+ end,
+ %% Cgi-script expects the body length to be 100
+ BodyFun = fun(0) ->
+ io:format("~w:http_post_streaming_fun -> "
+ "zero~n", [?MODULE]),
+ eof;
+ (LenLeft) ->
+ io:format("~w:http_post_streaming_fun -> "
+ "LenLeft: ~p~n", [?MODULE, LenLeft]),
+ {ok, lists:duplicate(10, "1"), LenLeft - 10}
+ end,
+
+ {ok, {{_,200,_}, [_ | _], [_ | _]}} =
+ httpc:request(post, {URL,
+ [{"expect", "100-continue"},
+ {"content-length", "100"}],
+ "text/plain", {BodyFun, 100}}, [], []),
+
+ {ok, {{_,504,_}, [_ | _], []}} =
+ httpc:request(post, {URL,
+ [{"expect", "100-continue"},
+ {"content-length", "10"}],
+ "text/plain", {BodyFun, 10}}, [], []);
+
+ _ ->
+ {skip, "Failed to start local http-server"}
+ end.
+
+
+%%-------------------------------------------------------------------------
http_emulate_lower_versions(doc) ->
["Perform request as 0.9 and 1.0 clients."];
http_emulate_lower_versions(suite) ->
@@ -2573,7 +2668,7 @@ otp_8739_dummy_server_init(Parent) ->
Parent ! {port, Port},
otp_8739_dummy_server_main(Parent, ListenSocket).
-otp_8739_dummy_server_main(Parent, ListenSocket) ->
+otp_8739_dummy_server_main(_Parent, ListenSocket) ->
case gen_tcp:accept(ListenSocket) of
{ok, Sock} ->
%% Ignore the request, and simply wait for the socket to close
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-14 07:56:12 +0000