httpc: allow streaming of PUT and POST request bodies

This is a must when uploading large bodies that are to large to store
in a string or binary.

Besides a string or binary, a body can now be a function and
an accumulator.

Example:

-module(httpc_post_stream_test).
-compile(export_all).

-define(LEN, 1024 * 1024).

prepare_data() ->
    {ok, Fd} = file:open("test_data.dat", [binary, write]),
    ok = file:write(Fd, lists:duplicate(?LEN, "1")),
    ok = file:close(Fd).

test() ->
    inets:start(),
    ok = prepare_data(),
    {ok, Fd1} = file:open("test_data.dat", [binary, read]),
    BodyFun = fun(Fd) ->
        case file:read(Fd, 512) of
        eof ->
            eof;
        {ok, Data} ->
            {ok, Data, Fd}
        end
    end,
    {ok, {{_,200,_}, _, _}} = httpc:request(post, {"http://localhost:8888",
        [{"content-length", integer_to_list(?LEN)}], "text/plain", {BodyFun, Fd1}}, [], []),
    ok = file:close(Fd1).

1 files changed, 50 insertions, 14 deletions

diff --git a/lib/inets/src/http_client/httpc_request.erl b/lib/inets/src/http_client/httpc_request.erl
index d4df97ad40..5386d1eb4a 100644
--- a/lib/inets/src/http_client/httpc_request.erl
+++ b/lib/inets/src/http_client/httpc_request.erl
@@ -101,15 +101,41 @@ send(SendAddr, Socket, SocketType,
 		   end,
     Version = HttpOptions#http_options.version,
 
-    Message = [method(Method), " ", Uri, " ", 
-	       version(Version), ?CRLF, 
-	       headers(FinalHeaders, Version), ?CRLF, Body],
+    do_send_body(SocketType, Socket, Method, Uri, Version, FinalHeaders, Body).
 
+
+do_send_body(SocketType, Socket, Method, Uri, Version, Headers, {DataFun, Acc})
+    when is_function(DataFun, 1) ->
+    case do_send_body(SocketType, Socket, Method, Uri, Version, Headers, []) of
+        ok ->
+            data_fun_loop(SocketType, Socket, DataFun, Acc);
+        Error ->
+            Error
+    end;
+
+do_send_body(SocketType, Socket, Method, Uri, Version, Headers, Body) ->
+    Message = [method(Method), " ", Uri, " ",
+	       version(Version), ?CRLF,
+	       headers(Headers, Version), ?CRLF, Body],
     ?hcrd("send", [{message, Message}]),
-    
     http_transport:send(SocketType, Socket, lists:append(Message)).
 
 
+data_fun_loop(SocketType, Socket, DataFun, Acc) ->
+    case DataFun(Acc) of
+        eof ->
+            ok;
+        {ok, Data, NewAcc} ->
+            DataBin = iolist_to_binary(Data),
+            ?hcrd("send", [{message, DataBin}]),
+            case http_transport:send(SocketType, Socket, DataBin) of
+                ok ->
+                    data_fun_loop(SocketType, Socket, DataFun, NewAcc);
+                Error ->
+                    Error
+            end
+    end.
+
 
 %%-------------------------------------------------------------------------
 %% is_idempotent(Method) ->
@@ -161,7 +187,6 @@ is_client_closing(Headers) ->
 %%%========================================================================
 post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) 
   when (Method =:= post) orelse (Method =:= put) ->
-    ContentLength = body_length(Body),	      
     NewBody = case Headers#http_request_h.expect of
 		  "100-continue" ->
 		      "";
@@ -170,14 +195,22 @@ post_data(Method, Headers, {ContentType, Body}, HeadersAsIs)
 	      end,
     
     NewHeaders = case HeadersAsIs of
-		     [] ->
-			 Headers#http_request_h{'content-type' = 
-						ContentType, 
-						'content-length' = 
-						ContentLength};
-		     _ ->
-			 HeadersAsIs
-		 end,
+        [] ->
+            Headers#http_request_h{
+                'content-type' = ContentType,
+                'content-length' = case body_length(Body) of
+                    undefined ->
+                        % on upload streaming the caller must give a
+                        % value to the Content-Length header
+                        % (or use chunked Transfer-Encoding)
+                        Headers#http_request_h.'content-length';
+                    Len when is_list(Len) ->
+                        Len
+                    end
+            };
+        _ ->
+            HeadersAsIs
+    end,
     
     {NewHeaders, NewBody};
 
@@ -190,7 +223,10 @@ body_length(Body) when is_binary(Body) ->
    integer_to_list(size(Body));
 
 body_length(Body) when is_list(Body) ->
-  integer_to_list(length(Body)).
+  integer_to_list(length(Body));
+
+body_length({DataFun, _Acc}) when is_function(DataFun, 1) ->
+  undefined.
 
 method(Method) ->
     http_util:to_upper(atom_to_list(Method)).