From f9ec3cbca0f05fd9640bbd5cd3e21942c4512d3d Mon Sep 17 00:00:00 2001
From: Filipe David Manana <fdmanana@gmail.com>
Date: Sun, 26 Sep 2010 11:58:45 +0100
Subject: httpc: allow streaming of PUT and POST request bodies

This is a must when uploading large bodies that are to large to store
in a string or binary.

Besides a string or binary, a body can now be a function and
an accumulator.

Example:

-module(httpc_post_stream_test).
-compile(export_all).

-define(LEN, 1024 * 1024).

prepare_data() ->
    {ok, Fd} = file:open("test_data.dat", [binary, write]),
    ok = file:write(Fd, lists:duplicate(?LEN, "1")),
    ok = file:close(Fd).

test() ->
    inets:start(),
    ok = prepare_data(),
    {ok, Fd1} = file:open("test_data.dat", [binary, read]),
    BodyFun = fun(Fd) ->
        case file:read(Fd, 512) of
        eof ->
            eof;
        {ok, Data} ->
            {ok, Data, Fd}
        end
    end,
    {ok, {{_,200,_}, _, _}} = httpc:request(post, {"http://localhost:8888",
        [{"content-length", integer_to_list(?LEN)}], "text/plain", {BodyFun, Fd1}}, [], []),
    ok = file:close(Fd1).
---
 lib/inets/src/http_client/httpc.erl | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib/inets/src/http_client/httpc.erl')

diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl
index 851364001c..b82a9db4c9 100644
--- a/lib/inets/src/http_client/httpc.erl
+++ b/lib/inets/src/http_client/httpc.erl
@@ -126,7 +126,9 @@ request(Url, Profile) ->
 %%      Header = {Field, Value}
 %%	Field = string()
 %%	Value = string()
-%%	Body = string() | binary() - HTLM-code
+%%	Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code
+%%      SendFunResult = eof | {ok, iolist(), NewSendAcc}
+%%      SendAcc = NewSendAcc = term()
 %%
 %% Description: Sends a HTTP-request. The function can be both
 %% syncronus and asynchronous in the later case the function will
-- 
cgit v1.2.3


From 6951ed1075b8c36d5b6f51e5e5df7bd14602c1d8 Mon Sep 17 00:00:00 2001
From: Filipe David Manana <fdmanana@gmail.com>
Date: Tue, 5 Oct 2010 00:26:33 +0100
Subject: httpc: add option to do automatic chunked transfer-encoding

This is specially useful when a client doesn't know in advance the
length of the payload (so that it can't set the
Content-Length header).

Example:

-module(httpc_post_stream_test).
-compile(export_all).

prepare_data() ->
    crypto:start(),
    {ok, Fd} = file:open("test_data.dat", [binary, write]),
    ok = file:write(Fd, lists:duplicate(crypto:rand_uniform(8182, 32768), "1")),
    ok = file:close(Fd).

test() ->
    inets:start(),
    ok = prepare_data(),
    {ok, Fd1} = file:open("test_data.dat", [binary, read]),
    BodyFun = fun(Fd) ->
        case file:read(Fd, 512) of
        eof ->
            eof;
        {ok, Data} ->
            {ok, Data, Fd}
        end
    end,
    %% header 'Transfer-Encoding: chunked' is added by httpc
    {ok, {{_,200,_}, _, _}} = httpc:request(post, {"http://localhost:8888",
        [], "text/plain", {chunkify, BodyFun, Fd1}}, [], []),
    ok = file:close(Fd1).
---
 lib/inets/src/http_client/httpc.erl | 34 ++++++++++++++++++++++++++++++----
 1 file changed, 30 insertions(+), 4 deletions(-)

(limited to 'lib/inets/src/http_client/httpc.erl')

diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl
index b82a9db4c9..8cf82df809 100644
--- a/lib/inets/src/http_client/httpc.erl
+++ b/lib/inets/src/http_client/httpc.erl
@@ -126,7 +126,8 @@ request(Url, Profile) ->
 %%      Header = {Field, Value}
 %%	Field = string()
 %%	Value = string()
-%%	Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code
+%%	Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} |
+%%              {chunkify, fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code
 %%      SendFunResult = eof | {ok, iolist(), NewSendAcc}
 %%      SendAcc = NewSendAcc = term()
 %%
@@ -428,11 +429,20 @@ service_info(Pid) ->
 
 handle_request(Method, Url, 
 	       {Scheme, UserInfo, Host, Port, Path, Query}, 
-	       Headers, ContentType, Body, 
+	       Headers0, ContentType, Body0,
 	       HTTPOptions0, Options0, Profile) ->
 
     Started    = http_util:timestamp(), 
-    NewHeaders = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers],
+    NewHeaders0 = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers0],
+
+    {NewHeaders, Body} = case Body0 of
+        {chunkify, BodyFun, Acc} ->
+            NewHeaders1 = lists:keystore("transfer-encoding", 1,
+                NewHeaders0, {"transfer-encoding", "chunked"}),
+            {NewHeaders1, {chunkify_fun(BodyFun), Acc}};
+        _ ->
+            {NewHeaders0, Body0}
+    end,
 
     try
 	begin
@@ -456,7 +466,7 @@ handle_request(Method, Url,
 			       abs_uri       = Url, 
 			       userinfo      = UserInfo, 
 			       stream        = Stream, 
-			       headers_as_is = headers_as_is(Headers, Options),
+			       headers_as_is = headers_as_is(Headers0, Options),
 			       socket_opts   = SocketOpts, 
 			       started       = Started},
 	    case httpc_manager:request(Request, profile_name(Profile)) of
@@ -473,6 +483,22 @@ handle_request(Method, Url,
 	    Error
     end.
 
+chunkify_fun(BodyFun) ->
+    fun(eof_body_fun) ->
+        eof;
+    (Acc) ->
+        case BodyFun(Acc) of
+            eof ->
+                {ok, <<"0\r\n\r\n">>, eof_body_fun};
+            {ok, Data, NewAcc} ->
+                Bin = iolist_to_binary(Data),
+                Chunk = [hex_size(Bin), "\r\n", Bin, "\r\n"],
+                {ok, iolist_to_binary(Chunk), NewAcc}
+        end
+    end.
+
+hex_size(Bin) ->
+    hd(io_lib:format("~.16B", [size(Bin)])).
 
 handle_answer(RequestId, false, _) ->
     {ok, RequestId};
-- 
cgit v1.2.3