Merge branch 'fm/httpc-upload-body-streaming' into bmk/inets/httpc/support_upload_body_streaming/OTP-OTP-9094

Conflicts:
	lib/inets/src/http_client/httpc.erl
	lib/inets/test/httpc_SUITE.erl

4 files changed, 249 insertions, 20 deletions

diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml
index bcdd2913e0..6dcf2d6d17 100644
--- a/lib/inets/doc/src/httpc.xml
+++ b/lib/inets/doc/src/httpc.xml
@@ -89,7 +89,11 @@ headers()       = [header()]
 header()        = {field(), value()}
 field()         = string() 
 value()         = string() 
-body()          = string() | binary() 
+body()            = string() | binary() |
+                    {fun(acc()) -> send_fun_result(), acc()} |
+                    {chunkify, fun(acc()) -> send_fun_result(), acc()}
+send_fun_result() = eof | {ok, iolist(), acc()}
+acc()             = term()
 filename()      = string() 
     ]]></code>
 
diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl
index 04fae13b20..7490bb0e46 100644
--- a/lib/inets/src/http_client/httpc.erl
+++ b/lib/inets/src/http_client/httpc.erl
@@ -126,7 +126,10 @@ request(Url, Profile) ->
 %%      Header = {Field, Value}
 %%	Field = string()
 %%	Value = string()
-%%	Body = string() | binary() - HTLM-code
+%%	Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} |
+%%              {chunkify, fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code
+%%      SendFunResult = eof | {ok, iolist(), NewSendAcc}
+%%      SendAcc = NewSendAcc = term()
 %%
 %% Description: Sends a HTTP-request. The function can be both
 %% syncronus and asynchronous in the later case the function will
@@ -426,11 +429,20 @@ service_info(Pid) ->
 
 handle_request(Method, Url, 
 	       {Scheme, UserInfo, Host, Port, Path, Query}, 
-	       Headers, ContentType, Body, 
+	       Headers0, ContentType, Body0,
 	       HTTPOptions0, Options0, Profile) ->
 
     Started    = http_util:timestamp(), 
-    NewHeaders = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers],
+    NewHeaders0 = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers0],
+
+    {NewHeaders, Body} = case Body0 of
+        {chunkify, BodyFun, Acc} ->
+            NewHeaders1 = lists:keystore("transfer-encoding", 1,
+                NewHeaders0, {"transfer-encoding", "chunked"}),
+            {NewHeaders1, {chunkify_fun(BodyFun), Acc}};
+        _ ->
+            {NewHeaders0, Body0}
+    end,
 
     try
 	begin
@@ -458,8 +470,8 @@ handle_request(Method, Url,
 			       settings      = HTTPOptions, 
 			       abs_uri       = AbsUri,
 			       userinfo      = UserInfo, 
-			       stream        = Stream,
-			       headers_as_is = headers_as_is(Headers, Options),
+			       stream        = Stream, 
+			       headers_as_is = headers_as_is(Headers0, Options),
 			       socket_opts   = SocketOpts, 
 			       started       = Started},
 	    case httpc_manager:request(Request, profile_name(Profile)) of
@@ -481,6 +493,23 @@ url_encode(URI, true) ->
 url_encode(URI, false) ->
     URI.
 
+chunkify_fun(BodyFun) ->
+    fun(eof_body_fun) ->
+        eof;
+    (Acc) ->
+        case BodyFun(Acc) of
+            eof ->
+                {ok, <<"0\r\n\r\n">>, eof_body_fun};
+            {ok, Data, NewAcc} ->
+                Bin = iolist_to_binary(Data),
+                Chunk = [hex_size(Bin), "\r\n", Bin, "\r\n"],
+                {ok, iolist_to_binary(Chunk), NewAcc}
+        end
+    end.
+
+hex_size(Bin) ->
+    hd(io_lib:format("~.16B", [size(Bin)])).
+
 handle_answer(RequestId, false, _) ->
     {ok, RequestId};
 handle_answer(RequestId, true, Options) ->
diff --git a/lib/inets/src/http_client/httpc_request.erl b/lib/inets/src/http_client/httpc_request.erl
index d4df97ad40..5386d1eb4a 100644
--- a/lib/inets/src/http_client/httpc_request.erl
+++ b/lib/inets/src/http_client/httpc_request.erl
@@ -101,15 +101,41 @@ send(SendAddr, Socket, SocketType,
 		   end,
     Version = HttpOptions#http_options.version,
 
-    Message = [method(Method), " ", Uri, " ", 
-	       version(Version), ?CRLF, 
-	       headers(FinalHeaders, Version), ?CRLF, Body],
+    do_send_body(SocketType, Socket, Method, Uri, Version, FinalHeaders, Body).
 
+
+do_send_body(SocketType, Socket, Method, Uri, Version, Headers, {DataFun, Acc})
+    when is_function(DataFun, 1) ->
+    case do_send_body(SocketType, Socket, Method, Uri, Version, Headers, []) of
+        ok ->
+            data_fun_loop(SocketType, Socket, DataFun, Acc);
+        Error ->
+            Error
+    end;
+
+do_send_body(SocketType, Socket, Method, Uri, Version, Headers, Body) ->
+    Message = [method(Method), " ", Uri, " ",
+	       version(Version), ?CRLF,
+	       headers(Headers, Version), ?CRLF, Body],
     ?hcrd("send", [{message, Message}]),
-    
     http_transport:send(SocketType, Socket, lists:append(Message)).
 
 
+data_fun_loop(SocketType, Socket, DataFun, Acc) ->
+    case DataFun(Acc) of
+        eof ->
+            ok;
+        {ok, Data, NewAcc} ->
+            DataBin = iolist_to_binary(Data),
+            ?hcrd("send", [{message, DataBin}]),
+            case http_transport:send(SocketType, Socket, DataBin) of
+                ok ->
+                    data_fun_loop(SocketType, Socket, DataFun, NewAcc);
+                Error ->
+                    Error
+            end
+    end.
+
 
 %%-------------------------------------------------------------------------
 %% is_idempotent(Method) ->
@@ -161,7 +187,6 @@ is_client_closing(Headers) ->
 %%%========================================================================
 post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) 
   when (Method =:= post) orelse (Method =:= put) ->
-    ContentLength = body_length(Body),	      
     NewBody = case Headers#http_request_h.expect of
 		  "100-continue" ->
 		      "";
@@ -170,14 +195,22 @@ post_data(Method, Headers, {ContentType, Body}, HeadersAsIs)
 	      end,
     
     NewHeaders = case HeadersAsIs of
-		     [] ->
-			 Headers#http_request_h{'content-type' = 
-						ContentType, 
-						'content-length' = 
-						ContentLength};
-		     _ ->
-			 HeadersAsIs
-		 end,
+        [] ->
+            Headers#http_request_h{
+                'content-type' = ContentType,
+                'content-length' = case body_length(Body) of
+                    undefined ->
+                        % on upload streaming the caller must give a
+                        % value to the Content-Length header
+                        % (or use chunked Transfer-Encoding)
+                        Headers#http_request_h.'content-length';
+                    Len when is_list(Len) ->
+                        Len
+                    end
+            };
+        _ ->
+            HeadersAsIs
+    end,
     
     {NewHeaders, NewBody};
 
@@ -190,7 +223,10 @@ body_length(Body) when is_binary(Body) ->
    integer_to_list(size(Body));
 
 body_length(Body) when is_list(Body) ->
-  integer_to_list(length(Body)).
+  integer_to_list(length(Body));
+
+body_length({DataFun, _Acc}) when is_function(DataFun, 1) ->
+  undefined.
 
 method(Method) ->
     http_util:to_upper(atom_to_list(Method)).
diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl
index 7aa11ebc27..41737a0d74 100644
--- a/lib/inets/test/httpc_SUITE.erl
+++ b/lib/inets/test/httpc_SUITE.erl
@@ -59,6 +59,7 @@
 %%		or a skip tuple if the platform is not supported.  
 %%--------------------------------------------------------------------
 
+<<<<<<< variant A
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() -> 
@@ -100,6 +101,126 @@ end_per_group(_GroupName, Config) ->
     Config.
 
 
+>>>>>>> variant B
+all(doc) ->
+    ["Test the http client in the intes application."];
+all(suite) ->
+    [
+     proxy_options, 
+     proxy_head, 
+     proxy_get, 
+     proxy_trace, 
+     proxy_post,
+     proxy_put, 
+     proxy_delete,
+     proxy_auth,
+     proxy_headers,
+     proxy_emulate_lower_versions,
+     http_options, 
+     http_head, 
+     http_get, 
+     http_post,
+     http_post_streaming,
+     http_dummy_pipe,
+     http_inets_pipe,
+     http_trace,
+     http_async,
+     http_save_to_file,
+     http_save_to_file_async,
+     http_headers,
+     http_headers_dummy,
+     http_bad_response,
+     ssl_head, 
+     ossl_head, 
+     essl_head, 
+     ssl_get, 
+     ossl_get, 
+     essl_get, 
+     ssl_trace, 
+     ossl_trace, 
+     essl_trace, 
+     http_redirect, 
+     http_redirect_loop,
+     http_internal_server_error,
+     http_userinfo,
+     http_cookie,
+     http_server_does_not_exist,
+     http_invalid_http,
+     http_emulate_lower_versions,
+     http_relaxed, 
+     page_does_not_exist, 
+     proxy_page_does_not_exist, 
+     proxy_https_not_supported,
+     http_stream,
+     http_stream_once,
+     proxy_stream,
+     parse_url,
+     options,
+     ipv6,
+     headers_as_is,
+     tickets
+    ].
+ 
+####### Ancestor
+all(doc) ->
+    ["Test the http client in the intes application."];
+all(suite) ->
+    [
+     proxy_options, 
+     proxy_head, 
+     proxy_get, 
+     proxy_trace, 
+     proxy_post,
+     proxy_put, 
+     proxy_delete,
+     proxy_auth,
+     proxy_headers,
+     proxy_emulate_lower_versions,
+     http_options, 
+     http_head, 
+     http_get, 
+     http_post,
+     http_dummy_pipe,
+     http_inets_pipe,
+     http_trace,
+     http_async,
+     http_save_to_file,
+     http_save_to_file_async,
+     http_headers,
+     http_headers_dummy,
+     http_bad_response,
+     ssl_head, 
+     ossl_head, 
+     essl_head, 
+     ssl_get, 
+     ossl_get, 
+     essl_get, 
+     ssl_trace, 
+     ossl_trace, 
+     essl_trace, 
+     http_redirect, 
+     http_redirect_loop,
+     http_internal_server_error,
+     http_userinfo,
+     http_cookie,
+     http_server_does_not_exist,
+     http_invalid_http,
+     http_emulate_lower_versions,
+     http_relaxed, 
+     page_does_not_exist, 
+     proxy_page_does_not_exist, 
+     proxy_https_not_supported,
+     http_stream,
+     http_stream_once,
+     proxy_stream,
+     parse_url,
+     options,
+     ipv6,
+     headers_as_is,
+     tickets
+    ].
+ 
+======= end
 %%--------------------------------------------------------------------
 %% Function: init_per_suite(Config) -> Config
 %% Config - [tuple()]
@@ -395,6 +516,45 @@ http_post(Config) when is_list(Config) ->
   end.  
 
 %%-------------------------------------------------------------------------
+http_post_streaming(doc) ->
+    ["Test streaming http post request against local server. We"
+    " only care about the client side of the the post. The server"
+    " script will not actually use the post data."];
+http_post_streaming(suite) ->
+    [];
+http_post_streaming(Config) when is_list(Config) ->
+    case ?config(local_server, Config) of
+        ok ->
+            Port = ?config(local_port, Config),
+            URL = case test_server:os_type() of
+                {win32, _} ->
+                    ?URL_START ++ integer_to_list(Port) ++
+                        "/cgi-bin/cgi_echo.exe";
+                 _ ->
+                    ?URL_START ++ integer_to_list(Port) ++
+                        "/cgi-bin/cgi_echo"
+            end,
+            %% Cgi-script expects the body length to be 100
+            BodyFun = fun(0) ->
+                    eof;
+                (LenLeft) ->
+                    {ok, lists:duplicate(10, "1"), LenLeft - 10}
+            end,
+
+            {ok, {{_,200,_}, [_ | _], [_ | _]}} =
+            httpc:request(post, {URL,
+                [{"expect", "100-continue"}, {"content-length", "100"}],
+                "text/plain", {BodyFun, 100}}, [], []),
+
+            {ok, {{_,504,_}, [_ | _], []}} =
+            httpc:request(post, {URL,
+                [{"expect", "100-continue"}, {"content-length", "10"}],
+                "text/plain", {BodyFun, 10}}, [], []);
+      _ ->
+          {skip, "Failed to start local http-server"}
+    end.
+
+%%-------------------------------------------------------------------------
 http_emulate_lower_versions(doc) ->
     ["Perform request as 0.9 and 1.0 clients."];
 http_emulate_lower_versions(suite) ->