From f9ec3cbca0f05fd9640bbd5cd3e21942c4512d3d Mon Sep 17 00:00:00 2001 From: Filipe David Manana Date: Sun, 26 Sep 2010 11:58:45 +0100 Subject: httpc: allow streaming of PUT and POST request bodies This is a must when uploading large bodies that are to large to store in a string or binary. Besides a string or binary, a body can now be a function and an accumulator. Example: -module(httpc_post_stream_test). -compile(export_all). -define(LEN, 1024 * 1024). prepare_data() -> {ok, Fd} = file:open("test_data.dat", [binary, write]), ok = file:write(Fd, lists:duplicate(?LEN, "1")), ok = file:close(Fd). test() -> inets:start(), ok = prepare_data(), {ok, Fd1} = file:open("test_data.dat", [binary, read]), BodyFun = fun(Fd) -> case file:read(Fd, 512) of eof -> eof; {ok, Data} -> {ok, Data, Fd} end end, {ok, {{_,200,_}, _, _}} = httpc:request(post, {"http://localhost:8888", [{"content-length", integer_to_list(?LEN)}], "text/plain", {BodyFun, Fd1}}, [], []), ok = file:close(Fd1). --- lib/inets/doc/src/httpc.xml | 4 +- lib/inets/src/http_client/httpc.erl | 4 +- lib/inets/src/http_client/httpc_request.erl | 64 ++++++++++++++++++++++------- lib/inets/test/httpc_SUITE.erl | 40 ++++++++++++++++++ 4 files changed, 96 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index 9c8df28fec..df333074cd 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -89,7 +89,9 @@ headers() = [header()] header() = {field(), value()} field() = string() value() = string() -body() = string() | binary() +body() = string() | binary() | {fun(acc()) -> send_fun_result(), acc()} +send_fun_result() = eof | {ok, iolist(), acc()} +acc() = term() filename() = string() ]]> diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index 851364001c..b82a9db4c9 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -126,7 +126,9 @@ request(Url, Profile) -> %% Header = {Field, Value} %% Field = string() %% Value = string() -%% Body = string() | binary() - HTLM-code +%% Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code +%% SendFunResult = eof | {ok, iolist(), NewSendAcc} +%% SendAcc = NewSendAcc = term() %% %% Description: Sends a HTTP-request. The function can be both %% syncronus and asynchronous in the later case the function will diff --git a/lib/inets/src/http_client/httpc_request.erl b/lib/inets/src/http_client/httpc_request.erl index d4df97ad40..5386d1eb4a 100644 --- a/lib/inets/src/http_client/httpc_request.erl +++ b/lib/inets/src/http_client/httpc_request.erl @@ -101,15 +101,41 @@ send(SendAddr, Socket, SocketType, end, Version = HttpOptions#http_options.version, - Message = [method(Method), " ", Uri, " ", - version(Version), ?CRLF, - headers(FinalHeaders, Version), ?CRLF, Body], + do_send_body(SocketType, Socket, Method, Uri, Version, FinalHeaders, Body). + +do_send_body(SocketType, Socket, Method, Uri, Version, Headers, {DataFun, Acc}) + when is_function(DataFun, 1) -> + case do_send_body(SocketType, Socket, Method, Uri, Version, Headers, []) of + ok -> + data_fun_loop(SocketType, Socket, DataFun, Acc); + Error -> + Error + end; + +do_send_body(SocketType, Socket, Method, Uri, Version, Headers, Body) -> + Message = [method(Method), " ", Uri, " ", + version(Version), ?CRLF, + headers(Headers, Version), ?CRLF, Body], ?hcrd("send", [{message, Message}]), - http_transport:send(SocketType, Socket, lists:append(Message)). +data_fun_loop(SocketType, Socket, DataFun, Acc) -> + case DataFun(Acc) of + eof -> + ok; + {ok, Data, NewAcc} -> + DataBin = iolist_to_binary(Data), + ?hcrd("send", [{message, DataBin}]), + case http_transport:send(SocketType, Socket, DataBin) of + ok -> + data_fun_loop(SocketType, Socket, DataFun, NewAcc); + Error -> + Error + end + end. + %%------------------------------------------------------------------------- %% is_idempotent(Method) -> @@ -161,7 +187,6 @@ is_client_closing(Headers) -> %%%======================================================================== post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) when (Method =:= post) orelse (Method =:= put) -> - ContentLength = body_length(Body), NewBody = case Headers#http_request_h.expect of "100-continue" -> ""; @@ -170,14 +195,22 @@ post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) end, NewHeaders = case HeadersAsIs of - [] -> - Headers#http_request_h{'content-type' = - ContentType, - 'content-length' = - ContentLength}; - _ -> - HeadersAsIs - end, + [] -> + Headers#http_request_h{ + 'content-type' = ContentType, + 'content-length' = case body_length(Body) of + undefined -> + % on upload streaming the caller must give a + % value to the Content-Length header + % (or use chunked Transfer-Encoding) + Headers#http_request_h.'content-length'; + Len when is_list(Len) -> + Len + end + }; + _ -> + HeadersAsIs + end, {NewHeaders, NewBody}; @@ -190,7 +223,10 @@ body_length(Body) when is_binary(Body) -> integer_to_list(size(Body)); body_length(Body) when is_list(Body) -> - integer_to_list(length(Body)). + integer_to_list(length(Body)); + +body_length({DataFun, _Acc}) when is_function(DataFun, 1) -> + undefined. method(Method) -> http_util:to_upper(atom_to_list(Method)). diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 902e440c80..6947f75b3d 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -77,6 +77,7 @@ all(suite) -> http_head, http_get, http_post, + http_post_streaming, http_dummy_pipe, http_inets_pipe, http_trace, @@ -423,6 +424,45 @@ http_post(Config) when is_list(Config) -> {skip, "Failed to start local http-server"} end. +%%------------------------------------------------------------------------- +http_post_streaming(doc) -> + ["Test streaming http post request against local server. We" + " only care about the client side of the the post. The server" + " script will not actually use the post data."]; +http_post_streaming(suite) -> + []; +http_post_streaming(Config) when is_list(Config) -> + case ?config(local_server, Config) of + ok -> + Port = ?config(local_port, Config), + URL = case test_server:os_type() of + {win32, _} -> + ?URL_START ++ integer_to_list(Port) ++ + "/cgi-bin/cgi_echo.exe"; + _ -> + ?URL_START ++ integer_to_list(Port) ++ + "/cgi-bin/cgi_echo" + end, + %% Cgi-script expects the body length to be 100 + BodyFun = fun(0) -> + eof; + (LenLeft) -> + {ok, lists:duplicate(10, "1"), LenLeft - 10} + end, + + {ok, {{_,200,_}, [_ | _], [_ | _]}} = + httpc:request(post, {URL, + [{"expect", "100-continue"}, {"content-length", "100"}], + "text/plain", {BodyFun, 100}}, [], []), + + {ok, {{_,504,_}, [_ | _], []}} = + httpc:request(post, {URL, + [{"expect", "100-continue"}, {"content-length", "10"}], + "text/plain", {BodyFun, 10}}, [], []); + _ -> + {skip, "Failed to start local http-server"} + end. + %%------------------------------------------------------------------------- http_emulate_lower_versions(doc) -> ["Perform request as 0.9 and 1.0 clients."]; -- cgit v1.2.3 From 6951ed1075b8c36d5b6f51e5e5df7bd14602c1d8 Mon Sep 17 00:00:00 2001 From: Filipe David Manana Date: Tue, 5 Oct 2010 00:26:33 +0100 Subject: httpc: add option to do automatic chunked transfer-encoding This is specially useful when a client doesn't know in advance the length of the payload (so that it can't set the Content-Length header). Example: -module(httpc_post_stream_test). -compile(export_all). prepare_data() -> crypto:start(), {ok, Fd} = file:open("test_data.dat", [binary, write]), ok = file:write(Fd, lists:duplicate(crypto:rand_uniform(8182, 32768), "1")), ok = file:close(Fd). test() -> inets:start(), ok = prepare_data(), {ok, Fd1} = file:open("test_data.dat", [binary, read]), BodyFun = fun(Fd) -> case file:read(Fd, 512) of eof -> eof; {ok, Data} -> {ok, Data, Fd} end end, %% header 'Transfer-Encoding: chunked' is added by httpc {ok, {{_,200,_}, _, _}} = httpc:request(post, {"http://localhost:8888", [], "text/plain", {chunkify, BodyFun, Fd1}}, [], []), ok = file:close(Fd1). --- lib/inets/doc/src/httpc.xml | 4 +++- lib/inets/src/http_client/httpc.erl | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index df333074cd..8b04b4c7f3 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -89,7 +89,9 @@ headers() = [header()] header() = {field(), value()} field() = string() value() = string() -body() = string() | binary() | {fun(acc()) -> send_fun_result(), acc()} +body() = string() | binary() | + {fun(acc()) -> send_fun_result(), acc()} | + {chunkify, fun(acc()) -> send_fun_result(), acc()} send_fun_result() = eof | {ok, iolist(), acc()} acc() = term() filename() = string() diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index b82a9db4c9..8cf82df809 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -126,7 +126,8 @@ request(Url, Profile) -> %% Header = {Field, Value} %% Field = string() %% Value = string() -%% Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code +%% Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} | +%% {chunkify, fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code %% SendFunResult = eof | {ok, iolist(), NewSendAcc} %% SendAcc = NewSendAcc = term() %% @@ -428,11 +429,20 @@ service_info(Pid) -> handle_request(Method, Url, {Scheme, UserInfo, Host, Port, Path, Query}, - Headers, ContentType, Body, + Headers0, ContentType, Body0, HTTPOptions0, Options0, Profile) -> Started = http_util:timestamp(), - NewHeaders = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers], + NewHeaders0 = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers0], + + {NewHeaders, Body} = case Body0 of + {chunkify, BodyFun, Acc} -> + NewHeaders1 = lists:keystore("transfer-encoding", 1, + NewHeaders0, {"transfer-encoding", "chunked"}), + {NewHeaders1, {chunkify_fun(BodyFun), Acc}}; + _ -> + {NewHeaders0, Body0} + end, try begin @@ -456,7 +466,7 @@ handle_request(Method, Url, abs_uri = Url, userinfo = UserInfo, stream = Stream, - headers_as_is = headers_as_is(Headers, Options), + headers_as_is = headers_as_is(Headers0, Options), socket_opts = SocketOpts, started = Started}, case httpc_manager:request(Request, profile_name(Profile)) of @@ -473,6 +483,22 @@ handle_request(Method, Url, Error end. +chunkify_fun(BodyFun) -> + fun(eof_body_fun) -> + eof; + (Acc) -> + case BodyFun(Acc) of + eof -> + {ok, <<"0\r\n\r\n">>, eof_body_fun}; + {ok, Data, NewAcc} -> + Bin = iolist_to_binary(Data), + Chunk = [hex_size(Bin), "\r\n", Bin, "\r\n"], + {ok, iolist_to_binary(Chunk), NewAcc} + end + end. + +hex_size(Bin) -> + hd(io_lib:format("~.16B", [size(Bin)])). handle_answer(RequestId, false, _) -> {ok, RequestId}; -- cgit v1.2.3 From db29f9ede14ff5b8d747230fcad8ffa1b157f1e1 Mon Sep 17 00:00:00 2001 From: Micael Karlberg Date: Mon, 7 Mar 2011 14:52:30 +0100 Subject: Adding missing "send loop" for raw sending. Also fixed some of the documentation (types). --- lib/inets/doc/src/httpc.xml | 52 ++++--- lib/inets/doc/src/notes.xml | 85 +++++++---- lib/inets/src/http_client/httpc.erl | 85 +++++++---- lib/inets/src/http_client/httpc_handler.erl | 18 +++ lib/inets/src/http_client/httpc_request.erl | 52 ++++--- lib/inets/test/httpc_SUITE.erl | 229 ++++++++++------------------ 6 files changed, 269 insertions(+), 252 deletions(-) (limited to 'lib') diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index 6dcf2d6d17..12f4fa535e 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -76,25 +76,29 @@ socket_opt() = See the Options used by gen_tcp(3) and

For more information about HTTP see rfc 2616

send_fun_result(), acc()} | - {chunkify, fun(acc()) -> send_fun_result(), acc()} -send_fun_result() = eof | {ok, iolist(), acc()} -acc() = term() -filename() = string() +method() = head | get | put | post | trace | options | delete +request() = {url(), headers()} | + {url(), headers(), content_type(), body()} +url() = string() - Syntax according to the URI definition in rfc 2396, ex: "http://www.erlang.org" +status_line() = {http_version(), status_code(), reason_phrase()} +http_version() = string() ex: "HTTP/1.1" +status_code() = integer() +reason_phrase() = string() +content_type() = string() +headers() = [header()] +header() = {field(), value()} +field() = string() +value() = string() +body() = string() | + binary() | + {fun(accumulator()) -> body_processing_result(), + accumulator()} | + {chunkify, + fun(accumulator()) -> body_processing_result(), + accumulator()} +body_processing_result() = eof | {ok, iolist(), accumulator()} +accumulator() = term() +filename() = string() ]]> @@ -146,8 +150,9 @@ ssl_options() = {verify, code()} | Sends a get HTTP-request Url = url() - Result = {status_line(), headers(), body()} | - {status_code(), body()} | request_id() + Result = {status_line(), headers(), Body} | + {status_code(), Body} | request_id() + Body = string() | binary() Profile = profile() Reason = term() @@ -195,8 +200,9 @@ ssl_options() = {verify, code()} | Function = atom() Args = list() body_format() = string | binary - Result = {status_line(), headers(), body()} | - {status_code(), body()} | request_id() + Result = {status_line(), headers(), Body} | + {status_code(), Body} | request_id() + Body = string() | binary() Profile = profile() Reason = {connect_failed, term()} | {send_failed, term()} | term() diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index 11b0af4310..8c0d683a90 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -1,4 +1,4 @@ - + @@ -32,50 +32,80 @@ notes.xml -
Inets 5.5.1 +
Inets 5.6 -
Fixed Bugs and Malfunctions +
Improvements and New Features + -

Fix format_man_pages so it handles all man sections - and remove warnings/errors in various man pages.

-

- Own Id: OTP-8600

+

[httpc] Add support for upload body streaming (PUT and POST).

+

For more info, + see the definition of the Body argument of the + request/4,5 + function.

+

Filipe David Manana

+

Own Id: OTP-9094

+
+
+ +
Fixed Bugs and Malfunctions +

-

+ +
+
+ + +
Inets 5.5.1 +
Improvements and New Features -

- Miscellaneous inet6 related problems.

-

- Own Id: OTP-8927

+

Miscellaneous inet6 related problems.

+

Own Id: OTP-8927

-

- Updated http-server to make sure URLs in error-messages - are URL-encoded. Added support in http-client to use - URL-encoding. Also added the missing include directory - for the inets application.

-

- Own Id: OTP-8940 Aux Id: seq11735

+

Updated http-server to make sure URLs in error-messages + are URL-encoded. Added support in http-client to use + URL-encoding. Also added the missing include directory + for the inets application.

+

Own Id: OTP-8940, Aux Id: seq11735

-
+
Fixed Bugs and Malfunctions + + +

Fix format_man_pages so it handles all man sections + and remove warnings/errors in various man pages.

+

Own Id: OTP-8600

+
+ +

[httpc] Pipelined and queued requests not processed when + connection closed remotelly.

+

Own Id: OTP-8906

+
+
+
+ +
+ -
Inets 5.5 +
Inets 5.5
Fixed Bugs and Malfunctions @@ -120,9 +150,10 @@
-
+
+ -
Inets 5.4 +
Inets 5.4
Improvements and New Features