diff options
-rw-r--r-- | lib/inets/doc/src/httpc.xml | 48 | ||||
-rw-r--r-- | lib/inets/doc/src/notes.xml | 24 | ||||
-rw-r--r-- | lib/inets/src/http_client/httpc.erl | 82 | ||||
-rw-r--r-- | lib/inets/src/http_client/httpc_handler.erl | 18 | ||||
-rw-r--r-- | lib/inets/src/http_client/httpc_request.erl | 102 | ||||
-rw-r--r-- | lib/inets/src/inets_app/inets.appup.src | 28 | ||||
-rw-r--r-- | lib/inets/test/httpc_SUITE.erl | 151 |
7 files changed, 343 insertions, 110 deletions
diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index 8f68087871..532f5eefde 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -76,21 +76,29 @@ socket_opt() = See the Options used by gen_tcp(3) and <p>For more information about HTTP see rfc 2616</p> <code type="none"><![CDATA[ -method() = head | get | put | post | trace | options | delete -request() = {url(), headers()} | - {url(), headers(), content_type(), body()} -url() = string() - Syntax according to the URI definition in rfc 2396, ex: "http://www.erlang.org" -status_line() = {http_version(), status_code(), reason_phrase()} -http_version() = string() ex: "HTTP/1.1" -status_code() = integer() -reason_phrase() = string() -content_type() = string() -headers() = [header()] -header() = {field(), value()} -field() = string() -value() = string() -body() = string() | binary() -filename() = string() +method() = head | get | put | post | trace | options | delete +request() = {url(), headers()} | + {url(), headers(), content_type(), body()} +url() = string() - Syntax according to the URI definition in rfc 2396, ex: "http://www.erlang.org" +status_line() = {http_version(), status_code(), reason_phrase()} +http_version() = string() ex: "HTTP/1.1" +status_code() = integer() +reason_phrase() = string() +content_type() = string() +headers() = [header()] +header() = {field(), value()} +field() = string() +value() = string() +body() = string() | + binary() | + {fun(accumulator()) -> body_processing_result(), + accumulator()} | + {chunkify, + fun(accumulator()) -> body_processing_result(), + accumulator()} +body_processing_result() = eof | {ok, iolist(), accumulator()} +accumulator() = term() +filename() = string() ]]></code> </section> @@ -142,8 +150,9 @@ ssl_options() = {verify, code()} | <fsummary>Sends a get HTTP-request</fsummary> <type> <v>Url = url() </v> - <v>Result = {status_line(), headers(), body()} | - {status_code(), body()} | request_id() </v> + <v>Result = {status_line(), headers(), Body} | + {status_code(), Body} | request_id() </v> + <v>Body = string() | binary()</v> <v>Profile = profile()</v> <v>Reason = term() </v> </type> @@ -191,8 +200,9 @@ ssl_options() = {verify, code()} | <v>Function = atom() </v> <v>Args = list() </v> <v>body_format() = string | binary </v> - <v>Result = {status_line(), headers(), body()} | - {status_code(), body()} | request_id() </v> + <v>Result = {status_line(), headers(), Body} | + {status_code(), Body} | request_id() </v> + <v>Body = string() | binary()</v> <v>Profile = profile() </v> <v>Reason = {connect_failed, term()} | {send_failed, term()} | term() </v> diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 72807a27fa..e08d0180b6 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -38,14 +38,22 @@ <!-- <p>-</p> --> - <list> <item> + <p>[httpc] Add support for upload body streaming (PUT and POST).</p> + <p>For more info, + see the definition of the <c>Body</c> argument of the + <seealso marker="httpc#request2">request/4,5</seealso> + function. </p> + <p>Filipe David Manana</p> + <p>Own Id: OTP-9094</p> + </item> + + <item> <p>[ftp] Added (type) spec for all exported functions.</p> <p>Own Id: OTP-9114 Aux Id: seq11799</p> </item> </list> - </section> <section><title>Fixed Bugs and Malfunctions</title> @@ -54,19 +62,15 @@ <!-- <list> <item> - <p>[httpd] httpd_response:send_chunk handles empty list and - empty binary - i.e. no chunk is sent, but it does - not handle a list with an empty binary [<<>>]. - This will be sent as an empty chunk - which in turn - will be encoded by http_chunk to the same as a final - chunk, which will make the http client believe that - the end of the page is reached.</p> - <p>Own Id: OTP-8906</p> + <p>[httpc] Pipelined and queued requests not processed when + connection closed remotelly.</p> + <p>Own Id: OTP-8906</p> </item> </list> --> </section> + </section> <!-- 5.6 --> diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index 04fae13b20..b70b16f57f 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -126,7 +126,10 @@ request(Url, Profile) -> %% Header = {Field, Value} %% Field = string() %% Value = string() -%% Body = string() | binary() - HTLM-code +%% Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} | +%% {chunkify, fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code +%% SendFunResult = eof | {ok, iolist(), NewSendAcc} +%% SendAcc = NewSendAcc = term() %% %% Description: Sends a HTTP-request. The function can be both %% syncronus and asynchronous in the later case the function will @@ -426,26 +429,44 @@ service_info(Pid) -> handle_request(Method, Url, {Scheme, UserInfo, Host, Port, Path, Query}, - Headers, ContentType, Body, + Headers0, ContentType, Body0, HTTPOptions0, Options0, Profile) -> - Started = http_util:timestamp(), - NewHeaders = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers], + Started = http_util:timestamp(), + NewHeaders0 = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers0], try begin + ?hcrt("begin processing", [{started, Started}, + {new_headers, NewHeaders0}]), + + {NewHeaders, Body} = + case Body0 of + {chunkify, ProcessBody, Acc} + when is_function(ProcessBody, 1) -> + NewHeaders1 = ensure_chunked_encoding(NewHeaders0), + Body1 = {mk_chunkify_fun(ProcessBody), Acc}, + {NewHeaders1, Body1}; + {ProcessBody, _} + when is_function(ProcessBody, 1) -> + {NewHeaders0, Body0}; + _ when is_list(Body0) orelse is_binary(Body0) -> + {NewHeaders0, Body0}; + _ -> + throw({error, {bad_body, Body0}}) + end, + HTTPOptions = http_options(HTTPOptions0), Options = request_options(Options0), Sync = proplists:get_value(sync, Options), Stream = proplists:get_value(stream, Options), Host2 = header_host(Scheme, Host, Port), HeadersRecord = header_record(NewHeaders, Host2, HTTPOptions), - Receiver = proplists:get_value(receiver, Options), - SocketOpts = proplists:get_value(socket_opts, Options), - UrlEncodeBool = HTTPOptions#http_options.url_encode, - MaybeEscPath = url_encode(Path, UrlEncodeBool), - MaybeEscQuery = url_encode(Query, UrlEncodeBool), - AbsUri = url_encode(Url, UrlEncodeBool), + Receiver = proplists:get_value(receiver, Options), + SocketOpts = proplists:get_value(socket_opts, Options), + MaybeEscPath = maybe_encode_uri(HTTPOptions, Path), + MaybeEscQuery = maybe_encode_uri(HTTPOptions, Query), + AbsUri = maybe_encode_uri(HTTPOptions, Url), Request = #request{from = Receiver, scheme = Scheme, @@ -458,38 +479,71 @@ handle_request(Method, Url, settings = HTTPOptions, abs_uri = AbsUri, userinfo = UserInfo, - stream = Stream, - headers_as_is = headers_as_is(Headers, Options), + stream = Stream, + headers_as_is = headers_as_is(Headers0, Options), socket_opts = SocketOpts, started = Started}, + case httpc_manager:request(Request, profile_name(Profile)) of {ok, RequestId} -> handle_answer(RequestId, Sync, Options); {error, Reason} -> + ?hcrd("request failed", [{reason, Reason}]), {error, Reason} end end catch error:{noproc, _} -> + ?hcrv("noproc", [{profile, Profile}]), {error, {not_started, Profile}}; throw:Error -> + ?hcrv("throw", [{error, Error}]), Error end. -url_encode(URI, true) -> +ensure_chunked_encoding(Hdrs) -> + Key = "transfer-encoding", + lists:keystore(Key, 1, Hdrs, {Key, "chunked"}). + +maybe_encode_uri(#http_options{url_encode = true}, URI) -> http_uri:encode(URI); -url_encode(URI, false) -> +maybe_encode_uri(_, URI) -> URI. +mk_chunkify_fun(ProcessBody) -> + fun(eof_body) -> + eof; + (Acc) -> + case ProcessBody(Acc) of + eof -> + {ok, <<"0\r\n\r\n">>, eof_body}; + {ok, Data, NewAcc} -> + {ok, mk_chunk_bin(Data), NewAcc} + end + end. + +mk_chunk_bin(Data) -> + Bin = iolist_to_binary(Data), + iolist_to_binary([hex_size(Bin), "\r\n", Bin, "\r\n"]). + +hex_size(Bin) -> + hd(io_lib:format("~.16B", [size(Bin)])). + + handle_answer(RequestId, false, _) -> {ok, RequestId}; handle_answer(RequestId, true, Options) -> receive {http, {RequestId, saved_to_file}} -> + ?hcrt("received saved-to-file", [{request_id, RequestId}]), {ok, saved_to_file}; {http, {RequestId, {_,_,_} = Result}} -> + ?hcrt("received answer", [{request_id, RequestId}, + {result, Result}]), return_answer(Options, Result); {http, {RequestId, {error, Reason}}} -> + ?hcrt("received error", [{request_id, RequestId}, + {reason, Reason}]), {error, Reason} end. diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index cb6f3e2841..5e22400fe0 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -1724,9 +1724,27 @@ handle_verbose(_) -> %% ok. +send_raw(#session{socket = Socket, socket_type = SocketType}, + {ProcessBody, Acc}) when is_function(ProcessBody, 1) -> + ?hcrt("send raw", [{acc, Acc}]), + send_raw(SocketType, Socket, ProcessBody, Acc); send_raw(#session{socket = Socket, socket_type = SocketType}, Body) -> http_transport:send(SocketType, Socket, Body). +send_raw(SocketType, Socket, ProcessBody, Acc) -> + case ProcessBody(Acc) of + eof -> + ok; + {ok, Data, NewAcc} -> + DataBin = iolist_to_binary(Data), + ?hcrd("send", [{data, DataBin}]), + case http_transport:send(SocketType, Socket, DataBin) of + ok -> + send_raw(SocketType, Socket, ProcessBody, NewAcc); + Error -> + Error + end + end. call(Msg, Pid) -> diff --git a/lib/inets/src/http_client/httpc_request.erl b/lib/inets/src/http_client/httpc_request.erl index d4df97ad40..0d602adb11 100644 --- a/lib/inets/src/http_client/httpc_request.erl +++ b/lib/inets/src/http_client/httpc_request.erl @@ -79,37 +79,65 @@ send(SendAddr, Socket, SocketType, {settings, HttpOptions}, {userinfo, UserInfo}]), - TmpHeaders = handle_user_info(UserInfo, Headers), + TmpHdrs = handle_user_info(UserInfo, Headers), - {TmpHeaders2, Body} = - post_data(Method, TmpHeaders, Content, HeadersAsIs), + {TmpHdrs2, Body} = post_data(Method, TmpHdrs, Content, HeadersAsIs), - {NewHeaders, Uri} = case Address of - SendAddr -> - {TmpHeaders2, Path ++ Query}; - _Proxy -> - TmpHeaders3 = - handle_proxy(HttpOptions, TmpHeaders2), - {TmpHeaders3, AbsUri} - end, - - FinalHeaders = case NewHeaders of - HeaderList when is_list(HeaderList) -> - http_headers(HeaderList, []); - _ -> - http_request:http_headers(NewHeaders) - end, + {NewHeaders, Uri} = + case Address of + SendAddr -> + {TmpHdrs2, Path ++ Query}; + _Proxy -> + TmpHdrs3 = handle_proxy(HttpOptions, TmpHdrs2), + {TmpHdrs3, AbsUri} + end, + + FinalHeaders = + case NewHeaders of + HeaderList when is_list(HeaderList) -> + http_headers(HeaderList, []); + _ -> + http_request:http_headers(NewHeaders) + end, Version = HttpOptions#http_options.version, - Message = [method(Method), " ", Uri, " ", - version(Version), ?CRLF, - headers(FinalHeaders, Version), ?CRLF, Body], + do_send_body(SocketType, Socket, Method, Uri, Version, FinalHeaders, Body). + + +do_send_body(SocketType, Socket, Method, Uri, Version, Headers, + {ProcessBody, Acc}) when is_function(ProcessBody, 1) -> + ?hcrt("send", [{acc, Acc}]), + case do_send_body(SocketType, Socket, Method, Uri, Version, Headers, []) of + ok -> + do_send_body(SocketType, Socket, ProcessBody, Acc); + Error -> + Error + end; +do_send_body(SocketType, Socket, Method, Uri, Version, Headers, Body) -> + ?hcrt("create message", [{body, Body}]), + Message = [method(Method), " ", Uri, " ", + version(Version), ?CRLF, + headers(Headers, Version), ?CRLF, Body], ?hcrd("send", [{message, Message}]), - http_transport:send(SocketType, Socket, lists:append(Message)). +do_send_body(SocketType, Socket, ProcessBody, Acc) -> + case ProcessBody(Acc) of + eof -> + ok; + {ok, Data, NewAcc} -> + DataBin = iolist_to_binary(Data), + ?hcrd("send", [{data, DataBin}]), + case http_transport:send(SocketType, Socket, DataBin) of + ok -> + do_send_body(SocketType, Socket, ProcessBody, NewAcc); + Error -> + Error + end + end. + %%------------------------------------------------------------------------- %% is_idempotent(Method) -> @@ -161,7 +189,6 @@ is_client_closing(Headers) -> %%%======================================================================== post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) when (Method =:= post) orelse (Method =:= put) -> - ContentLength = body_length(Body), NewBody = case Headers#http_request_h.expect of "100-continue" -> ""; @@ -170,14 +197,22 @@ post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) end, NewHeaders = case HeadersAsIs of - [] -> - Headers#http_request_h{'content-type' = - ContentType, - 'content-length' = - ContentLength}; - _ -> - HeadersAsIs - end, + [] -> + Headers#http_request_h{ + 'content-type' = ContentType, + 'content-length' = case body_length(Body) of + undefined -> + % on upload streaming the caller must give a + % value to the Content-Length header + % (or use chunked Transfer-Encoding) + Headers#http_request_h.'content-length'; + Len when is_list(Len) -> + Len + end + }; + _ -> + HeadersAsIs + end, {NewHeaders, NewBody}; @@ -190,7 +225,10 @@ body_length(Body) when is_binary(Body) -> integer_to_list(size(Body)); body_length(Body) when is_list(Body) -> - integer_to_list(length(Body)). + integer_to_list(length(Body)); + +body_length({DataFun, _Acc}) when is_function(DataFun, 1) -> + undefined. method(Method) -> http_util:to_upper(atom_to_list(Method)). diff --git a/lib/inets/src/inets_app/inets.appup.src b/lib/inets/src/inets_app/inets.appup.src index 6f32a31c1c..2d12353e00 100644 --- a/lib/inets/src/inets_app/inets.appup.src +++ b/lib/inets/src/inets_app/inets.appup.src @@ -20,13 +20,20 @@ [ {"5.5.2", [ - {load_module, ftp, soft_purge, soft_purge, []} + {load_module, ftp, soft_purge, soft_purge, []}, + {load_module, httpc, soft_purge, soft_purge, [httpc_handler]}, + {load_module, httpc_request, soft_purge, soft_purge, []}, + {update, httpc_handler, soft, soft_purge, soft_purge, [httpc_request]} ] }, {"5.5.1", [ {load_module, ftp, soft_purge, soft_purge, []}, - {load_module, http_chunk, soft_purge, soft_purge, []} + {load_module, http_chunk, soft_purge, soft_purge, []}, + {load_module, httpc, soft_purge, soft_purge, [httpc_handler]}, + {load_module, httpc_request, soft_purge, soft_purge, []}, + {update, httpc_handler, soft, soft_purge, soft_purge, + [httpc_request,http_chunk]} ] }, {"5.5", @@ -40,16 +47,23 @@ ] } ], - [ - {"5.5.2", + [ + {"5.5.2", [ - {load_module, ftp, soft_purge, soft_purge, []} + {load_module, ftp, soft_purge, soft_purge, []}, + {load_module, httpc, soft_purge, soft_purge, [httpc_handler]}, + {load_module, httpc_request, soft_purge, soft_purge, []}, + {update, httpc_handler, soft, soft_purge, soft_purge, [httpc_request]} ] - }, + }, {"5.5.1", [ {load_module, ftp, soft_purge, soft_purge, []}, - {load_module, http_chunk, soft_purge, soft_purge, []} + {load_module, http_chunk, soft_purge, soft_purge, []}, + {load_module, httpc, soft_purge, soft_purge, [httpc_handler]}, + {load_module, httpc_request, soft_purge, soft_purge, []}, + {update, httpc_handler, soft, soft_purge, soft_purge, + [httpc_request,http_chunk]} ] }, {"5.5", diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 2c8febf5ed..c1a36d23de 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -62,36 +62,82 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [proxy_options, proxy_head, proxy_get, proxy_trace, - proxy_post, proxy_put, proxy_delete, proxy_auth, - proxy_headers, proxy_emulate_lower_versions, - http_options, http_head, http_get, http_post, - http_dummy_pipe, http_inets_pipe, http_trace, - http_async, http_save_to_file, http_save_to_file_async, - http_headers, http_headers_dummy, http_bad_response, - ssl_head, ossl_head, essl_head, ssl_get, ossl_get, - essl_get, ssl_trace, ossl_trace, essl_trace, - http_redirect, http_redirect_loop, - http_internal_server_error, http_userinfo, http_cookie, - http_server_does_not_exist, http_invalid_http, - http_emulate_lower_versions, http_relaxed, - page_does_not_exist, proxy_page_does_not_exist, - proxy_https_not_supported, http_stream, - http_stream_once, proxy_stream, parse_url, options, - ipv6, headers_as_is, {group, tickets}]. + [ + proxy_options, + proxy_head, + proxy_get, + proxy_trace, + proxy_post, + proxy_put, + proxy_delete, + proxy_auth, + proxy_headers, + proxy_emulate_lower_versions, + http_options, + http_head, + http_get, + http_post, + http_post_streaming, + http_dummy_pipe, + http_inets_pipe, + http_trace, + http_async, + http_save_to_file, + http_save_to_file_async, + http_headers, + http_headers_dummy, + http_bad_response, + ssl_head, + ossl_head, + essl_head, + ssl_get, + ossl_get, + essl_get, + ssl_trace, + ossl_trace, + essl_trace, + http_redirect, + http_redirect_loop, + http_internal_server_error, + http_userinfo, http_cookie, + http_server_does_not_exist, + http_invalid_http, + http_emulate_lower_versions, + http_relaxed, + page_does_not_exist, + proxy_page_does_not_exist, + proxy_https_not_supported, + http_stream, + http_stream_once, + proxy_stream, + parse_url, + options, + ipv6, + headers_as_is, + {group, tickets} + ]. groups() -> - [{tickets, [], - [hexed_query_otp_6191, empty_body_otp_6243, - empty_response_header_otp_6830, - transfer_encoding_otp_6807, proxy_not_modified_otp_6821, - no_content_204_otp_6982, missing_CR_otp_7304, - {group, otp_7883}, {group, otp_8154}, {group, otp_8106}, - otp_8056, otp_8352, otp_8371, otp_8739]}, - {otp_7883, [], [otp_7883_1, otp_7883_2]}, + [{tickets, [], [hexed_query_otp_6191, + empty_body_otp_6243, + empty_response_header_otp_6830, + transfer_encoding_otp_6807, + proxy_not_modified_otp_6821, + no_content_204_otp_6982, + missing_CR_otp_7304, + {group, otp_7883}, + {group, otp_8154}, + {group, otp_8106}, + otp_8056, + otp_8352, + otp_8371, + otp_8739]}, + {otp_7883, [], [otp_7883_1, + otp_7883_2]}, {otp_8154, [], [otp_8154_1]}, - {otp_8106, [], - [otp_8106_pid, otp_8106_fun, otp_8106_mfa]}]. + {otp_8106, [], [otp_8106_pid, + otp_8106_fun, + otp_8106_mfa]}]. init_per_group(_GroupName, Config) -> Config. @@ -138,6 +184,7 @@ init_per_suite(Config) -> {local_port, ?IP_PORT}, {local_ssl_port, ?SSL_PORT} | Config]. + %%-------------------------------------------------------------------- %% Function: end_per_suite(Config) -> _ %% Config - [tuple()] @@ -151,6 +198,7 @@ end_per_suite(Config) -> application:stop(ssl), ok. + %%-------------------------------------------------------------------- %% Function: init_per_testcase(Case, Config) -> Config %% Case - atom() @@ -395,6 +443,53 @@ http_post(Config) when is_list(Config) -> end. %%------------------------------------------------------------------------- +http_post_streaming(doc) -> + ["Test streaming http post request against local server. " + "We only care about the client side of the the post. " + "The server script will not actually use the post data."]; +http_post_streaming(suite) -> + []; +http_post_streaming(Config) when is_list(Config) -> + case ?config(local_server, Config) of + ok -> + Port = ?config(local_port, Config), + URL = case test_server:os_type() of + {win32, _} -> + ?URL_START ++ integer_to_list(Port) ++ + "/cgi-bin/cgi_echo.exe"; + _ -> + ?URL_START ++ integer_to_list(Port) ++ + "/cgi-bin/cgi_echo" + end, + %% Cgi-script expects the body length to be 100 + BodyFun = fun(0) -> + io:format("~w:http_post_streaming_fun -> " + "zero~n", [?MODULE]), + eof; + (LenLeft) -> + io:format("~w:http_post_streaming_fun -> " + "LenLeft: ~p~n", [?MODULE, LenLeft]), + {ok, lists:duplicate(10, "1"), LenLeft - 10} + end, + + {ok, {{_,200,_}, [_ | _], [_ | _]}} = + httpc:request(post, {URL, + [{"expect", "100-continue"}, + {"content-length", "100"}], + "text/plain", {BodyFun, 100}}, [], []), + + {ok, {{_,504,_}, [_ | _], []}} = + httpc:request(post, {URL, + [{"expect", "100-continue"}, + {"content-length", "10"}], + "text/plain", {BodyFun, 10}}, [], []); + + _ -> + {skip, "Failed to start local http-server"} + end. + + +%%------------------------------------------------------------------------- http_emulate_lower_versions(doc) -> ["Perform request as 0.9 and 1.0 clients."]; http_emulate_lower_versions(suite) -> @@ -2573,7 +2668,7 @@ otp_8739_dummy_server_init(Parent) -> Parent ! {port, Port}, otp_8739_dummy_server_main(Parent, ListenSocket). -otp_8739_dummy_server_main(Parent, ListenSocket) -> +otp_8739_dummy_server_main(_Parent, ListenSocket) -> case gen_tcp:accept(ListenSocket) of {ok, Sock} -> %% Ignore the request, and simply wait for the socket to close |