From f9ec3cbca0f05fd9640bbd5cd3e21942c4512d3d Mon Sep 17 00:00:00 2001 From: Filipe David Manana Date: Sun, 26 Sep 2010 11:58:45 +0100 Subject: httpc: allow streaming of PUT and POST request bodies This is a must when uploading large bodies that are to large to store in a string or binary. Besides a string or binary, a body can now be a function and an accumulator. Example: -module(httpc_post_stream_test). -compile(export_all). -define(LEN, 1024 * 1024). prepare_data() -> {ok, Fd} = file:open("test_data.dat", [binary, write]), ok = file:write(Fd, lists:duplicate(?LEN, "1")), ok = file:close(Fd). test() -> inets:start(), ok = prepare_data(), {ok, Fd1} = file:open("test_data.dat", [binary, read]), BodyFun = fun(Fd) -> case file:read(Fd, 512) of eof -> eof; {ok, Data} -> {ok, Data, Fd} end end, {ok, {{_,200,_}, _, _}} = httpc:request(post, {"http://localhost:8888", [{"content-length", integer_to_list(?LEN)}], "text/plain", {BodyFun, Fd1}}, [], []), ok = file:close(Fd1). --- lib/inets/src/http_client/httpc.erl | 4 +- lib/inets/src/http_client/httpc_request.erl | 64 ++++++++++++++++++++++------- 2 files changed, 53 insertions(+), 15 deletions(-) (limited to 'lib/inets/src') diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index 851364001c..b82a9db4c9 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -126,7 +126,9 @@ request(Url, Profile) -> %% Header = {Field, Value} %% Field = string() %% Value = string() -%% Body = string() | binary() - HTLM-code +%% Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code +%% SendFunResult = eof | {ok, iolist(), NewSendAcc} +%% SendAcc = NewSendAcc = term() %% %% Description: Sends a HTTP-request. The function can be both %% syncronus and asynchronous in the later case the function will diff --git a/lib/inets/src/http_client/httpc_request.erl b/lib/inets/src/http_client/httpc_request.erl index d4df97ad40..5386d1eb4a 100644 --- a/lib/inets/src/http_client/httpc_request.erl +++ b/lib/inets/src/http_client/httpc_request.erl @@ -101,15 +101,41 @@ send(SendAddr, Socket, SocketType, end, Version = HttpOptions#http_options.version, - Message = [method(Method), " ", Uri, " ", - version(Version), ?CRLF, - headers(FinalHeaders, Version), ?CRLF, Body], + do_send_body(SocketType, Socket, Method, Uri, Version, FinalHeaders, Body). + +do_send_body(SocketType, Socket, Method, Uri, Version, Headers, {DataFun, Acc}) + when is_function(DataFun, 1) -> + case do_send_body(SocketType, Socket, Method, Uri, Version, Headers, []) of + ok -> + data_fun_loop(SocketType, Socket, DataFun, Acc); + Error -> + Error + end; + +do_send_body(SocketType, Socket, Method, Uri, Version, Headers, Body) -> + Message = [method(Method), " ", Uri, " ", + version(Version), ?CRLF, + headers(Headers, Version), ?CRLF, Body], ?hcrd("send", [{message, Message}]), - http_transport:send(SocketType, Socket, lists:append(Message)). +data_fun_loop(SocketType, Socket, DataFun, Acc) -> + case DataFun(Acc) of + eof -> + ok; + {ok, Data, NewAcc} -> + DataBin = iolist_to_binary(Data), + ?hcrd("send", [{message, DataBin}]), + case http_transport:send(SocketType, Socket, DataBin) of + ok -> + data_fun_loop(SocketType, Socket, DataFun, NewAcc); + Error -> + Error + end + end. + %%------------------------------------------------------------------------- %% is_idempotent(Method) -> @@ -161,7 +187,6 @@ is_client_closing(Headers) -> %%%======================================================================== post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) when (Method =:= post) orelse (Method =:= put) -> - ContentLength = body_length(Body), NewBody = case Headers#http_request_h.expect of "100-continue" -> ""; @@ -170,14 +195,22 @@ post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) end, NewHeaders = case HeadersAsIs of - [] -> - Headers#http_request_h{'content-type' = - ContentType, - 'content-length' = - ContentLength}; - _ -> - HeadersAsIs - end, + [] -> + Headers#http_request_h{ + 'content-type' = ContentType, + 'content-length' = case body_length(Body) of + undefined -> + % on upload streaming the caller must give a + % value to the Content-Length header + % (or use chunked Transfer-Encoding) + Headers#http_request_h.'content-length'; + Len when is_list(Len) -> + Len + end + }; + _ -> + HeadersAsIs + end, {NewHeaders, NewBody}; @@ -190,7 +223,10 @@ body_length(Body) when is_binary(Body) -> integer_to_list(size(Body)); body_length(Body) when is_list(Body) -> - integer_to_list(length(Body)). + integer_to_list(length(Body)); + +body_length({DataFun, _Acc}) when is_function(DataFun, 1) -> + undefined. method(Method) -> http_util:to_upper(atom_to_list(Method)). -- cgit v1.2.3 From 6951ed1075b8c36d5b6f51e5e5df7bd14602c1d8 Mon Sep 17 00:00:00 2001 From: Filipe David Manana Date: Tue, 5 Oct 2010 00:26:33 +0100 Subject: httpc: add option to do automatic chunked transfer-encoding This is specially useful when a client doesn't know in advance the length of the payload (so that it can't set the Content-Length header). Example: -module(httpc_post_stream_test). -compile(export_all). prepare_data() -> crypto:start(), {ok, Fd} = file:open("test_data.dat", [binary, write]), ok = file:write(Fd, lists:duplicate(crypto:rand_uniform(8182, 32768), "1")), ok = file:close(Fd). test() -> inets:start(), ok = prepare_data(), {ok, Fd1} = file:open("test_data.dat", [binary, read]), BodyFun = fun(Fd) -> case file:read(Fd, 512) of eof -> eof; {ok, Data} -> {ok, Data, Fd} end end, %% header 'Transfer-Encoding: chunked' is added by httpc {ok, {{_,200,_}, _, _}} = httpc:request(post, {"http://localhost:8888", [], "text/plain", {chunkify, BodyFun, Fd1}}, [], []), ok = file:close(Fd1). --- lib/inets/src/http_client/httpc.erl | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) (limited to 'lib/inets/src') diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index b82a9db4c9..8cf82df809 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -126,7 +126,8 @@ request(Url, Profile) -> %% Header = {Field, Value} %% Field = string() %% Value = string() -%% Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code +%% Body = string() | binary() | {fun(SendAcc) -> SendFunResult, SendAcc} | +%% {chunkify, fun(SendAcc) -> SendFunResult, SendAcc} - HTLM-code %% SendFunResult = eof | {ok, iolist(), NewSendAcc} %% SendAcc = NewSendAcc = term() %% @@ -428,11 +429,20 @@ service_info(Pid) -> handle_request(Method, Url, {Scheme, UserInfo, Host, Port, Path, Query}, - Headers, ContentType, Body, + Headers0, ContentType, Body0, HTTPOptions0, Options0, Profile) -> Started = http_util:timestamp(), - NewHeaders = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers], + NewHeaders0 = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers0], + + {NewHeaders, Body} = case Body0 of + {chunkify, BodyFun, Acc} -> + NewHeaders1 = lists:keystore("transfer-encoding", 1, + NewHeaders0, {"transfer-encoding", "chunked"}), + {NewHeaders1, {chunkify_fun(BodyFun), Acc}}; + _ -> + {NewHeaders0, Body0} + end, try begin @@ -456,7 +466,7 @@ handle_request(Method, Url, abs_uri = Url, userinfo = UserInfo, stream = Stream, - headers_as_is = headers_as_is(Headers, Options), + headers_as_is = headers_as_is(Headers0, Options), socket_opts = SocketOpts, started = Started}, case httpc_manager:request(Request, profile_name(Profile)) of @@ -473,6 +483,22 @@ handle_request(Method, Url, Error end. +chunkify_fun(BodyFun) -> + fun(eof_body_fun) -> + eof; + (Acc) -> + case BodyFun(Acc) of + eof -> + {ok, <<"0\r\n\r\n">>, eof_body_fun}; + {ok, Data, NewAcc} -> + Bin = iolist_to_binary(Data), + Chunk = [hex_size(Bin), "\r\n", Bin, "\r\n"], + {ok, iolist_to_binary(Chunk), NewAcc} + end + end. + +hex_size(Bin) -> + hd(io_lib:format("~.16B", [size(Bin)])). handle_answer(RequestId, false, _) -> {ok, RequestId}; -- cgit v1.2.3 From db29f9ede14ff5b8d747230fcad8ffa1b157f1e1 Mon Sep 17 00:00:00 2001 From: Micael Karlberg Date: Mon, 7 Mar 2011 14:52:30 +0100 Subject: Adding missing "send loop" for raw sending. Also fixed some of the documentation (types). --- lib/inets/src/http_client/httpc.erl | 85 +++++++++++++++++++---------- lib/inets/src/http_client/httpc_handler.erl | 18 ++++++ lib/inets/src/http_client/httpc_request.erl | 52 +++++++++--------- 3 files changed, 100 insertions(+), 55 deletions(-) (limited to 'lib/inets/src') diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index 7490bb0e46..c7f8a02a0b 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -432,32 +432,41 @@ handle_request(Method, Url, Headers0, ContentType, Body0, HTTPOptions0, Options0, Profile) -> - Started = http_util:timestamp(), + Started = http_util:timestamp(), NewHeaders0 = [{http_util:to_lower(Key), Val} || {Key, Val} <- Headers0], - {NewHeaders, Body} = case Body0 of - {chunkify, BodyFun, Acc} -> - NewHeaders1 = lists:keystore("transfer-encoding", 1, - NewHeaders0, {"transfer-encoding", "chunked"}), - {NewHeaders1, {chunkify_fun(BodyFun), Acc}}; - _ -> - {NewHeaders0, Body0} - end, - try begin + ?hcrt("begin processing", [{started, Started}, + {new_headers, NewHeaders0}]), + + {NewHeaders, Body} = + case Body0 of + {chunkify, ProcessBody, Acc} + when is_function(ProcessBody, 1) -> + NewHeaders1 = ensure_chunked_encoding(NewHeaders0), + Body1 = {mk_chunkify_fun(ProcessBody), Acc}, + {NewHeaders1, Body1}; + {ProcessBody, _} + when is_function(ProcessBody, 1) -> + {NewHeaders0, Body0}; + _ when is_list(Body0) orelse is_binary(Body0) -> + {NewHeaders0, Body0}; + _ -> + throw({error, {bad_body, Body0}}) + end, + HTTPOptions = http_options(HTTPOptions0), Options = request_options(Options0), Sync = proplists:get_value(sync, Options), Stream = proplists:get_value(stream, Options), Host2 = header_host(Scheme, Host, Port), HeadersRecord = header_record(NewHeaders, Host2, HTTPOptions), - Receiver = proplists:get_value(receiver, Options), - SocketOpts = proplists:get_value(socket_opts, Options), - UrlEncodeBool = HTTPOptions#http_options.url_encode, - MaybeEscPath = url_encode(Path, UrlEncodeBool), - MaybeEscQuery = url_encode(Query, UrlEncodeBool), - AbsUri = url_encode(Url, UrlEncodeBool), + Receiver = proplists:get_value(receiver, Options), + SocketOpts = proplists:get_value(socket_opts, Options), + MaybeEscPath = maybe_encode_uri(HTTPOptions, Path), + MaybeEscQuery = maybe_encode_uri(HTTPOptions, Query), + AbsUri = maybe_encode_uri(HTTPOptions, Url), Request = #request{from = Receiver, scheme = Scheme, @@ -474,51 +483,67 @@ handle_request(Method, Url, headers_as_is = headers_as_is(Headers0, Options), socket_opts = SocketOpts, started = Started}, + case httpc_manager:request(Request, profile_name(Profile)) of {ok, RequestId} -> handle_answer(RequestId, Sync, Options); {error, Reason} -> + ?hcrd("request failed", [{reason, Reason}]), {error, Reason} end end catch error:{noproc, _} -> + ?hcrl("noproc", [{profile, Profile}]), {error, {not_started, Profile}}; throw:Error -> + ?hcrl("throw", [{error, Error}]), Error end. -url_encode(URI, true) -> +ensure_chunked_encoding(Hdrs) -> + Key = "transfer-encoding", + lists:keystore(Key, 1, Hdrs, {Key, "chunked"}). + +maybe_encode_uri(#http_options{url_encode = true}, URI) -> http_uri:encode(URI); -url_encode(URI, false) -> +maybe_encode_uri(_, URI) -> URI. -chunkify_fun(BodyFun) -> - fun(eof_body_fun) -> - eof; - (Acc) -> - case BodyFun(Acc) of - eof -> - {ok, <<"0\r\n\r\n">>, eof_body_fun}; - {ok, Data, NewAcc} -> - Bin = iolist_to_binary(Data), - Chunk = [hex_size(Bin), "\r\n", Bin, "\r\n"], - {ok, iolist_to_binary(Chunk), NewAcc} - end +mk_chunkify_fun(ProcessBody) -> + fun(eof_body) -> + eof; + (Acc) -> + case ProcessBody(Acc) of + eof -> + {ok, <<"0\r\n\r\n">>, eof_body}; + {ok, Data, NewAcc} -> + {ok, mk_chunk_bin(Data), NewAcc} + end end. +mk_chunk_bin(Data) -> + Bin = iolist_to_binary(Data), + iolist_to_binary([hex_size(Bin), "\r\n", Bin, "\r\n"]). + hex_size(Bin) -> hd(io_lib:format("~.16B", [size(Bin)])). + handle_answer(RequestId, false, _) -> {ok, RequestId}; handle_answer(RequestId, true, Options) -> receive {http, {RequestId, saved_to_file}} -> + ?hcrt("received saved-to-file", [{request_id, RequestId}]), {ok, saved_to_file}; {http, {RequestId, {_,_,_} = Result}} -> + ?hcrt("received answer", [{request_id, RequestId}, + {result, Result}]), return_answer(Options, Result); {http, {RequestId, {error, Reason}}} -> + ?hcrt("received error", [{request_id, RequestId}, + {reason, Reason}]), {error, Reason} end. diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index cb6f3e2841..5e22400fe0 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -1724,9 +1724,27 @@ handle_verbose(_) -> %% ok. +send_raw(#session{socket = Socket, socket_type = SocketType}, + {ProcessBody, Acc}) when is_function(ProcessBody, 1) -> + ?hcrt("send raw", [{acc, Acc}]), + send_raw(SocketType, Socket, ProcessBody, Acc); send_raw(#session{socket = Socket, socket_type = SocketType}, Body) -> http_transport:send(SocketType, Socket, Body). +send_raw(SocketType, Socket, ProcessBody, Acc) -> + case ProcessBody(Acc) of + eof -> + ok; + {ok, Data, NewAcc} -> + DataBin = iolist_to_binary(Data), + ?hcrd("send", [{data, DataBin}]), + case http_transport:send(SocketType, Socket, DataBin) of + ok -> + send_raw(SocketType, Socket, ProcessBody, NewAcc); + Error -> + Error + end + end. call(Msg, Pid) -> diff --git a/lib/inets/src/http_client/httpc_request.erl b/lib/inets/src/http_client/httpc_request.erl index 5386d1eb4a..0d602adb11 100644 --- a/lib/inets/src/http_client/httpc_request.erl +++ b/lib/inets/src/http_client/httpc_request.erl @@ -79,41 +79,43 @@ send(SendAddr, Socket, SocketType, {settings, HttpOptions}, {userinfo, UserInfo}]), - TmpHeaders = handle_user_info(UserInfo, Headers), + TmpHdrs = handle_user_info(UserInfo, Headers), - {TmpHeaders2, Body} = - post_data(Method, TmpHeaders, Content, HeadersAsIs), + {TmpHdrs2, Body} = post_data(Method, TmpHdrs, Content, HeadersAsIs), - {NewHeaders, Uri} = case Address of - SendAddr -> - {TmpHeaders2, Path ++ Query}; - _Proxy -> - TmpHeaders3 = - handle_proxy(HttpOptions, TmpHeaders2), - {TmpHeaders3, AbsUri} - end, - - FinalHeaders = case NewHeaders of - HeaderList when is_list(HeaderList) -> - http_headers(HeaderList, []); - _ -> - http_request:http_headers(NewHeaders) - end, + {NewHeaders, Uri} = + case Address of + SendAddr -> + {TmpHdrs2, Path ++ Query}; + _Proxy -> + TmpHdrs3 = handle_proxy(HttpOptions, TmpHdrs2), + {TmpHdrs3, AbsUri} + end, + + FinalHeaders = + case NewHeaders of + HeaderList when is_list(HeaderList) -> + http_headers(HeaderList, []); + _ -> + http_request:http_headers(NewHeaders) + end, Version = HttpOptions#http_options.version, do_send_body(SocketType, Socket, Method, Uri, Version, FinalHeaders, Body). -do_send_body(SocketType, Socket, Method, Uri, Version, Headers, {DataFun, Acc}) - when is_function(DataFun, 1) -> +do_send_body(SocketType, Socket, Method, Uri, Version, Headers, + {ProcessBody, Acc}) when is_function(ProcessBody, 1) -> + ?hcrt("send", [{acc, Acc}]), case do_send_body(SocketType, Socket, Method, Uri, Version, Headers, []) of ok -> - data_fun_loop(SocketType, Socket, DataFun, Acc); + do_send_body(SocketType, Socket, ProcessBody, Acc); Error -> Error end; do_send_body(SocketType, Socket, Method, Uri, Version, Headers, Body) -> + ?hcrt("create message", [{body, Body}]), Message = [method(Method), " ", Uri, " ", version(Version), ?CRLF, headers(Headers, Version), ?CRLF, Body], @@ -121,16 +123,16 @@ do_send_body(SocketType, Socket, Method, Uri, Version, Headers, Body) -> http_transport:send(SocketType, Socket, lists:append(Message)). -data_fun_loop(SocketType, Socket, DataFun, Acc) -> - case DataFun(Acc) of +do_send_body(SocketType, Socket, ProcessBody, Acc) -> + case ProcessBody(Acc) of eof -> ok; {ok, Data, NewAcc} -> DataBin = iolist_to_binary(Data), - ?hcrd("send", [{message, DataBin}]), + ?hcrd("send", [{data, DataBin}]), case http_transport:send(SocketType, Socket, DataBin) of ok -> - data_fun_loop(SocketType, Socket, DataFun, NewAcc); + do_send_body(SocketType, Socket, ProcessBody, NewAcc); Error -> Error end -- cgit v1.2.3 From f10de199e5aad1aecbff03a5e7af9e1c5d017301 Mon Sep 17 00:00:00 2001 From: Micael Karlberg Date: Fri, 18 Mar 2011 10:20:16 +0100 Subject: Fixed debug macro. Fixed appup file (not updated). Fixed application version. --- lib/inets/src/http_client/httpc.erl | 4 ++-- lib/inets/src/inets_app/inets.appup.src | 28 +++++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 5 deletions(-) (limited to 'lib/inets/src') diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index c7f8a02a0b..b70b16f57f 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -494,10 +494,10 @@ handle_request(Method, Url, end catch error:{noproc, _} -> - ?hcrl("noproc", [{profile, Profile}]), + ?hcrv("noproc", [{profile, Profile}]), {error, {not_started, Profile}}; throw:Error -> - ?hcrl("throw", [{error, Error}]), + ?hcrv("throw", [{error, Error}]), Error end. diff --git a/lib/inets/src/inets_app/inets.appup.src b/lib/inets/src/inets_app/inets.appup.src index 07da8ca961..65ce0cf0eb 100644 --- a/lib/inets/src/inets_app/inets.appup.src +++ b/lib/inets/src/inets_app/inets.appup.src @@ -18,9 +18,20 @@ {"%VSN%", [ + {"5.5.2", + [ + {load_module, httpc, soft_purge, soft_purge, [httpc_handler]}, + {load_module, httpc_request, soft_purge, soft_purge, []}, + {update, httpc_handler, soft, soft_purge, soft_purge, [httpc_request]} + ] + }, {"5.5.1", [ - {load_module, http_chunk, soft_purge, soft_purge, []} + {load_module, http_chunk, soft_purge, soft_purge, []}, + {load_module, httpc, soft_purge, soft_purge, [httpc_handler]}, + {load_module, httpc_request, soft_purge, soft_purge, []}, + {update, httpc_handler, soft, soft_purge, soft_purge, + [httpc_request,http_chunk]} ] }, {"5.5", @@ -34,10 +45,21 @@ ] } ], - [ + [ + {"5.5.2", + [ + {load_module, httpc, soft_purge, soft_purge, [httpc_handler]}, + {load_module, httpc_request, soft_purge, soft_purge, []}, + {update, httpc_handler, soft, soft_purge, soft_purge, [httpc_request]} + ] + }, {"5.5.1", [ - {load_module, http_chunk, soft_purge, soft_purge, []} + {load_module, http_chunk, soft_purge, soft_purge, []}, + {load_module, httpc, soft_purge, soft_purge, [httpc_handler]}, + {load_module, httpc_request, soft_purge, soft_purge, []}, + {update, httpc_handler, soft, soft_purge, soft_purge, + [httpc_request,http_chunk]} ] }, {"5.5", -- cgit v1.2.3