aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorFilipe David Manana <[email protected]>2010-09-26 11:58:45 +0100
committerBjörn Gustavsson <[email protected]>2010-10-05 15:08:23 +0200
commitf9ec3cbca0f05fd9640bbd5cd3e21942c4512d3d (patch)
tree0b71168e700abdd029136dbadb3b6b2c89b97d71
parent0a1f48c46cf629af7d3719e94250733d1589efa1 (diff)
downloadotp-f9ec3cbca0f05fd9640bbd5cd3e21942c4512d3d.tar.gz
otp-f9ec3cbca0f05fd9640bbd5cd3e21942c4512d3d.tar.bz2
otp-f9ec3cbca0f05fd9640bbd5cd3e21942c4512d3d.zip
httpc: allow streaming of PUT and POST request bodies
This is a must when uploading large bodies that are to large to store in a string or binary. Besides a string or binary, a body can now be a function and an accumulator. Example: -module(httpc_post_stream_test). -compile(export_all). -define(LEN, 1024 * 1024). prepare_data() -> {ok, Fd} = file:open("test_data.dat", [binary, write]), ok = file:write(Fd, lists:duplicate(?LEN, "1")), ok = file:close(Fd). test() -> inets:start(), ok = prepare_data(), {ok, Fd1} = file:open("test_data.dat", [binary, read]), BodyFun = fun(Fd) -> case file:read(Fd, 512) of eof -> eof; {ok, Data} -> {ok, Data, Fd} end end, {ok, {{_,200,_}, _, _}} = httpc:request(post, {"http://localhost:8888", [{"content-length", integer_to_list(?LEN)}], "text/plain", {BodyFun, Fd1}}, [], []), ok = file:close(Fd1).
-rw-r--r--lib/inets/doc/src/httpc.xml4
-rw-r--r--lib/inets/src/http_client/httpc.erl4
-rw-r--r--lib/inets/src/http_client/httpc_request.erl64
-rw-r--r--lib/inets/test/httpc_SUITE.erl40
4 files changed, 96 insertions, 16 deletions
diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml
index 9c8df28fec..df333074cd 100644
--- a/lib/inets/doc/src/httpc.xml
+++ b/lib/inets/doc/src/httpc.xml
@@ -89,7 +89,9 @@ headers() = [header()]
header() = {field(), value()}
field() = string()
value() = string()
-body() = string() | binary()
+body() = string() | binary() | {fun(acc()) -> send_fun_result(), acc()}
+send_fun_result() = eof | {ok, iolist(), acc()}
+acc() = term()
filename() = string()
]]></code>
diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl
index 851364001c..b82a9db4c9 100644
--- a/lib/inets/src/http_client/httpc.erl
+++ b/lib/inets/src/http_client/httpc.erl
@@ -126,7 +126,9 @@ request(Url, Profile) ->
%% Header = {Field, Value}
%% Field = string()
%% Value = string()
-%% Body = string() | binary() - HTLM-code
+%% Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code
+%% SendFunResult = eof | {ok, iolist(), NewSendAcc}
+%% SendAcc = NewSendAcc = term()
%%
%% Description: Sends a HTTP-request. The function can be both
%% syncronus and asynchronous in the later case the function will
diff --git a/lib/inets/src/http_client/httpc_request.erl b/lib/inets/src/http_client/httpc_request.erl
index d4df97ad40..5386d1eb4a 100644
--- a/lib/inets/src/http_client/httpc_request.erl
+++ b/lib/inets/src/http_client/httpc_request.erl
@@ -101,15 +101,41 @@ send(SendAddr, Socket, SocketType,
end,
Version = HttpOptions#http_options.version,
- Message = [method(Method), " ", Uri, " ",
- version(Version), ?CRLF,
- headers(FinalHeaders, Version), ?CRLF, Body],
+ do_send_body(SocketType, Socket, Method, Uri, Version, FinalHeaders, Body).
+
+do_send_body(SocketType, Socket, Method, Uri, Version, Headers, {DataFun, Acc})
+ when is_function(DataFun, 1) ->
+ case do_send_body(SocketType, Socket, Method, Uri, Version, Headers, []) of
+ ok ->
+ data_fun_loop(SocketType, Socket, DataFun, Acc);
+ Error ->
+ Error
+ end;
+
+do_send_body(SocketType, Socket, Method, Uri, Version, Headers, Body) ->
+ Message = [method(Method), " ", Uri, " ",
+ version(Version), ?CRLF,
+ headers(Headers, Version), ?CRLF, Body],
?hcrd("send", [{message, Message}]),
-
http_transport:send(SocketType, Socket, lists:append(Message)).
+data_fun_loop(SocketType, Socket, DataFun, Acc) ->
+ case DataFun(Acc) of
+ eof ->
+ ok;
+ {ok, Data, NewAcc} ->
+ DataBin = iolist_to_binary(Data),
+ ?hcrd("send", [{message, DataBin}]),
+ case http_transport:send(SocketType, Socket, DataBin) of
+ ok ->
+ data_fun_loop(SocketType, Socket, DataFun, NewAcc);
+ Error ->
+ Error
+ end
+ end.
+
%%-------------------------------------------------------------------------
%% is_idempotent(Method) ->
@@ -161,7 +187,6 @@ is_client_closing(Headers) ->
%%%========================================================================
post_data(Method, Headers, {ContentType, Body}, HeadersAsIs)
when (Method =:= post) orelse (Method =:= put) ->
- ContentLength = body_length(Body),
NewBody = case Headers#http_request_h.expect of
"100-continue" ->
"";
@@ -170,14 +195,22 @@ post_data(Method, Headers, {ContentType, Body}, HeadersAsIs)
end,
NewHeaders = case HeadersAsIs of
- [] ->
- Headers#http_request_h{'content-type' =
- ContentType,
- 'content-length' =
- ContentLength};
- _ ->
- HeadersAsIs
- end,
+ [] ->
+ Headers#http_request_h{
+ 'content-type' = ContentType,
+ 'content-length' = case body_length(Body) of
+ undefined ->
+ % on upload streaming the caller must give a
+ % value to the Content-Length header
+ % (or use chunked Transfer-Encoding)
+ Headers#http_request_h.'content-length';
+ Len when is_list(Len) ->
+ Len
+ end
+ };
+ _ ->
+ HeadersAsIs
+ end,
{NewHeaders, NewBody};
@@ -190,7 +223,10 @@ body_length(Body) when is_binary(Body) ->
integer_to_list(size(Body));
body_length(Body) when is_list(Body) ->
- integer_to_list(length(Body)).
+ integer_to_list(length(Body));
+
+body_length({DataFun, _Acc}) when is_function(DataFun, 1) ->
+ undefined.
method(Method) ->
http_util:to_upper(atom_to_list(Method)).
diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl
index 902e440c80..6947f75b3d 100644
--- a/lib/inets/test/httpc_SUITE.erl
+++ b/lib/inets/test/httpc_SUITE.erl
@@ -77,6 +77,7 @@ all(suite) ->
http_head,
http_get,
http_post,
+ http_post_streaming,
http_dummy_pipe,
http_inets_pipe,
http_trace,
@@ -424,6 +425,45 @@ http_post(Config) when is_list(Config) ->
end.
%%-------------------------------------------------------------------------
+http_post_streaming(doc) ->
+ ["Test streaming http post request against local server. We"
+ " only care about the client side of the the post. The server"
+ " script will not actually use the post data."];
+http_post_streaming(suite) ->
+ [];
+http_post_streaming(Config) when is_list(Config) ->
+ case ?config(local_server, Config) of
+ ok ->
+ Port = ?config(local_port, Config),
+ URL = case test_server:os_type() of
+ {win32, _} ->
+ ?URL_START ++ integer_to_list(Port) ++
+ "/cgi-bin/cgi_echo.exe";
+ _ ->
+ ?URL_START ++ integer_to_list(Port) ++
+ "/cgi-bin/cgi_echo"
+ end,
+ %% Cgi-script expects the body length to be 100
+ BodyFun = fun(0) ->
+ eof;
+ (LenLeft) ->
+ {ok, lists:duplicate(10, "1"), LenLeft - 10}
+ end,
+
+ {ok, {{_,200,_}, [_ | _], [_ | _]}} =
+ httpc:request(post, {URL,
+ [{"expect", "100-continue"}, {"content-length", "100"}],
+ "text/plain", {BodyFun, 100}}, [], []),
+
+ {ok, {{_,504,_}, [_ | _], []}} =
+ httpc:request(post, {URL,
+ [{"expect", "100-continue"}, {"content-length", "10"}],
+ "text/plain", {BodyFun, 10}}, [], []);
+ _ ->
+ {skip, "Failed to start local http-server"}
+ end.
+
+%%-------------------------------------------------------------------------
http_emulate_lower_versions(doc) ->
["Perform request as 0.9 and 1.0 clients."];
http_emulate_lower_versions(suite) ->