From ed5d51442c4f151b1deaf99a5927fd01fdd5f5e3 Mon Sep 17 00:00:00 2001 From: Ingela Anderton Andin Date: Thu, 1 Jun 2017 16:09:11 +0200 Subject: inets: httpd - Add chunk handling of client data If the client uses PUT or POST to send a HTTP body the server currently will gather the whole body in memory before passing it on to the mod-callback. For legacy reasons it also converts the binary body to a list, this is really bad for memory usage! Add new option max_client_body_chunk to enable chunked handling of content-length requests. Also make it possible for chunked-encoded data to be propagated in portions to the mod-callback. Conflicts: lib/inets/test/httpd_SUITE.erl --- lib/inets/doc/src/httpd.xml | 13 +- lib/inets/doc/src/mod_esi.xml | 69 ++++++--- lib/inets/src/http_server/httpd_example.erl | 24 ++- lib/inets/src/http_server/httpd_request.erl | 48 +++++- .../src/http_server/httpd_request_handler.erl | 162 +++++++++++++++++---- lib/inets/src/http_server/httpd_response.erl | 24 ++- lib/inets/src/http_server/mod_esi.erl | 40 ++--- lib/inets/test/httpd_SUITE.erl | 56 ++++++- 8 files changed, 329 insertions(+), 107 deletions(-) (limited to 'lib/inets') diff --git a/lib/inets/doc/src/httpd.xml b/lib/inets/doc/src/httpd.xml index d74635fc01..edf8731a82 100644 --- a/lib/inets/doc/src/httpd.xml +++ b/lib/inets/doc/src/httpd.xml @@ -279,7 +279,18 @@ requests defined by max_keep_alive_requests, the server closes the connection. The server closes it even if there are queued request. Default is no limit.

- + + + + {max_client_body_chunk, integer()} + +

Enforces chunking of a HTTP PUT or POST body data to be deliverd + to the mod_esi callback. Note this is not supported for mod_cgi. + Default is no limit e.i the whole body is deliverd as one entity, which could + be very memory consuming. mod_esi(3). +

+ + diff --git a/lib/inets/doc/src/mod_esi.xml b/lib/inets/doc/src/mod_esi.xml index 46cc796c8a..e2f0758cdf 100644 --- a/lib/inets/doc/src/mod_esi.xml +++ b/lib/inets/doc/src/mod_esi.xml @@ -121,35 +121,61 @@ - Module:Function(SessionID, Env, Input)-> _ + Module:Function(SessionID, Env, Input)-> {continue, State} | _ Creates a dynamic web page and returns it chunk by chunk to the server process by calling mod_esi:deliver/2. SessionID = term() Env = env() - Input = string() + Input = string() | chunked_data() + chunked_data() = {first, Data::binary()} | + {continue, Data::binary(), State::term()} | + {last, Data::binary(), State::term()} + State = term()

Module must be found in the code path and export Function with an arity of three. An erlScriptAlias must also be set up in the configuration file for the web server.

-

If the HTTP request is a 'post' request and a body is sent, - content_length is the length of the posted - data. If 'get' is used, query_string is the data after - ? in the URL.

-

ParsedHeader is the HTTP request as a key-value tuple - list. The keys in ParsedHeader are in lower case.

-

SessionID is an identifier - the server uses when deliver/2 is called. Do not - assume anything about the datatype.

-

Use this callback function to generate dynamic web - content dynamically. When a part of the page is generated, send the - data back to the client through deliver/2. Notice - that the first chunk of data sent to the client must at - least contain all HTTP header fields that the response - will generate. If the first chunk does not contain the - end of HTTP header, that is, "\r\n\r\n", - the server assumes that no HTTP header fields will be generated.

+ +

mod_esi:deliver/2 shall be used to generate the response + to the client and SessionID is an identifier that shall by used when + calling this function, do not assume anything about + the datatype. This function may be called + several times to chunk the the respons data. Notice that the + first chunk of data sent to the client must at least contain + all HTTP header fields that the response will generate. If the + first chunk does not contain the end of HTTP header, + that is, "\r\n\r\n", the server assumes that no HTTP + header fields will be generated.

+ +

Env environment data of the request see description above.

+ +

Input is query data of a GET request or the body of + a PUT or POST request. The default behavior (legacy reasons) + for delivering the body, is that the whole body is gathered and + converted to a string. But if the httpd config parameter + max_client_body_chunk + is set, the body will be delivered as binary chunks + instead. The maximum size of the chunks is either max_client_body_chunk + or decide by the client if it uses HTTP chunked encoding + to send the body. When using the chunking + mechanism this callback must return {continue, State::term()} + for all calls where Input is {first, + Data::binary()} or {continue, Data::binary(), + State::term()}. When Input is {last, + Data::binary(), State::term()} the return value will be ignored. +

Note that if the body is + small all data may be delivered in only one chunk and then the + callback will be called with {last, Data::binary(), undefined} + without getting called with {first, + Data::binary()}.

The input State is + the last returned State, in it the callback can include + any data that it needs to keep track of when handling the chunks. +

+

@@ -159,14 +185,13 @@ This function is deprecated and is only kept for backwards compatibility. Env = env() - Input = string() + Input = string() Response = string()

This callback format consumes much memory, as the whole response must be generated before it is sent to the - user. This function is deprecated and is only kept for backwards - compatibility. + user. This callback format is deprecated. For new development, use Module:Function/3.

 diff --git a/lib/inets/src/http_server/httpd_example.erl b/lib/inets/src/http_server/httpd_example.erl index c893b10dca..45b6deba97 100644 --- a/lib/inets/src/http_server/httpd_example.erl +++ b/lib/inets/src/http_server/httpd_example.erl @@ -22,7 +22,7 @@ -export([print/1]). -export([get/2, put/2, post/2, yahoo/2, test1/2, get_bin/2, peer/2]). --export([newformat/3]). +-export([newformat/3, post_chunked/3]). %% These are used by the inets test-suite -export([delay/1, chunk_timeout/3]). @@ -131,15 +131,31 @@ footer() -> " \n". - -newformat(SessionID, _Env, _Input)-> +post_chunked(_SessionID, _Env, {first, _Body} = _Bodychunk) -> + {continue, {state, 1}}; +post_chunked(_SessionID, _Env, {continue, _Body, {state, N}} = _Bodychunk) -> + {continue, {state, N+1}}; +post_chunked(SessionID, _Env, {last, _Body, {state, N}} = _Bodychunk) -> + mod_esi:deliver(SessionID, "Content-Type:text/html\r\n\r\n"), + mod_esi:deliver(SessionID, top("Received chunked body")), + mod_esi:deliver(SessionID, "Received" ++ integer_to_list(N) ++ "chunks"), + mod_esi:deliver(SessionID, footer()); +post_chunked(SessionID, _Env, {last, _Body, undefined} = _Bodychunk) -> + mod_esi:deliver(SessionID, "Content-Type:text/html\r\n\r\n"), + mod_esi:deliver(SessionID, top("Received chunked body")), + mod_esi:deliver(SessionID, "Received 1 chunk"), + mod_esi:deliver(SessionID, footer()); +post_chunked(_, _, _Body) -> + exit(body_not_chunked). + +newformat(SessionID,_,_) -> mod_esi:deliver(SessionID, "Content-Type:text/html\r\n\r\n"), mod_esi:deliver(SessionID, top("new esi format test")), mod_esi:deliver(SessionID, "This new format is nice
"), mod_esi:deliver(SessionID, "This new format is nice
"), mod_esi:deliver(SessionID, "This new format is nice
"), mod_esi:deliver(SessionID, footer()). - + %% ------------------------------------------------------ delay(Time) when is_integer(Time) -> diff --git a/lib/inets/src/http_server/httpd_request.erl b/lib/inets/src/http_server/httpd_request.erl index 749f58c197..0eaf073255 100644 --- a/lib/inets/src/http_server/httpd_request.erl +++ b/lib/inets/src/http_server/httpd_request.erl @@ -36,7 +36,7 @@ %% little at a time on a socket. -export([ parse_method/1, parse_uri/1, parse_version/1, parse_headers/1, - whole_body/1 + whole_body/1, body_chunk_first/3, body_chunk/3, add_chunk/1 ]). @@ -76,13 +76,12 @@ body_data(Headers, Body) -> ContentLength = list_to_integer(Headers#http_request_h.'content-length'), case size(Body) - ContentLength of 0 -> - {binary_to_list(Body), <<>>}; + {Body, <<>>}; _ -> <> = Body, - {binary_to_list(BodyThisReq), Next} + {BodyThisReq, Next} end. - %%------------------------------------------------------------------------- %% validate(Method, Uri, Version) -> ok | {error, {bad_request, Reason} | %% {error, {not_supported, {Method, Uri, Version}} @@ -292,10 +291,46 @@ parse_headers(<>, Header, Headers, Current, parse_headers(Rest, [Octet | Header], Headers, Current + 1, Max, Options, Result). +body_chunk_first(Body, 0 = Length, _) -> + whole_body(Body, Length); +body_chunk_first(Body, Length, MaxChunk) -> + case body_chunk(Body, Length, MaxChunk) of + {ok, {last, NewBody}} -> + {ok, NewBody}; + Other -> + Other + end. +%% Used to chunk non chunk decoded post/put data +add_chunk([<<>>, Body, Length, MaxChunk]) -> + body_chunk(Body, Length, MaxChunk); +add_chunk([More, Body, Length, MaxChunk]) -> + body_chunk(<>, Length, MaxChunk). + +body_chunk(<<>> = Body, Length, MaxChunk) -> + {ok, {continue, ?MODULE, add_chunk, [Body, Length, MaxChunk]}}; +body_chunk(Body, Length, nolimit) -> + whole_body(Body, Length); + +body_chunk(Body, Length, MaxChunk) when Length > MaxChunk -> + case size(Body) >= MaxChunk of + true -> + <> = Body, + {ok, {{continue, Chunk}, ?MODULE, add_chunk, [Rest, Length - MaxChunk, MaxChunk]}}; + false -> + {ok, {continue, ?MODULE, add_chunk, [Body, Length, MaxChunk]}} + end; +body_chunk(Body, Length, MaxChunk) -> + case size(Body) of + Length -> + {ok, {last, Body}}; + _ -> + {ok, {continue, ?MODULE, add_chunk, [Body, Length, MaxChunk]}} + end. + whole_body(Body, Length) -> case size(Body) of N when N < Length, Length > 0 -> - {?MODULE, whole_body, [Body, Length]}; + {?MODULE, add_chunk, [Body, Length, nolimit]}; N when N >= Length, Length >= 0 -> %% When a client uses pipelining trailing data %% may be part of the next request! @@ -443,6 +478,3 @@ check_header({"content-length", Value}, Maxsizes) -> end; check_header(_, _) -> ok. - - - diff --git a/lib/inets/src/http_server/httpd_request_handler.erl b/lib/inets/src/http_server/httpd_request_handler.erl index 538d52b98d..bd4fdd3832 100644 --- a/lib/inets/src/http_server/httpd_request_handler.erl +++ b/lib/inets/src/http_server/httpd_request_handler.erl @@ -49,7 +49,8 @@ headers, %% #http_request_h{} body, %% binary() data, %% The total data received in bits, checked after 10s - byte_limit %% Bit limit per second before kick out + byte_limit, %% Bit limit per second before kick out + chunk }). %%==================================================================== @@ -124,7 +125,8 @@ continue_init(Manager, ConfigDB, SocketType, Socket, TimeOut) -> NrOfRequest = max_keep_alive_request(ConfigDB), MaxContentLen = max_content_length(ConfigDB), Customize = customize(ConfigDB), - + MaxChunk = max_client_body_chunk(ConfigDB), + {_, Status} = httpd_manager:new_connection(Manager), MFA = {httpd_request, parse, [[{max_uri, MaxURISize}, {max_header, MaxHeaderSize}, @@ -139,7 +141,8 @@ continue_init(Manager, ConfigDB, SocketType, Socket, TimeOut) -> status = Status, timeout = TimeOut, max_keep_alive_request = NrOfRequest, - mfa = MFA}, + mfa = MFA, + chunk = chunk_start(MaxChunk)}, http_transport:setopts(SocketType, Socket, [binary, {packet, 0}, {active, once}]), @@ -194,6 +197,7 @@ handle_cast(Msg, #state{mod = ModData} = State) -> %%-------------------------------------------------------------------- handle_info({Proto, Socket, Data}, #state{mfa = {Module, Function, Args}, + chunk = {ChunkState, _}, mod = #mod{socket_type = SockType, socket = Socket} = ModData} = State) when (((Proto =:= tcp) orelse @@ -207,7 +211,8 @@ handle_info({Proto, Socket, Data}, _ -> State#state.data + byte_size(Data) end, - case PROCESSED of + + case PROCESSED of {ok, Result} -> NewState = case NewDataSize of undefined -> @@ -215,7 +220,7 @@ handle_info({Proto, Socket, Data}, _ -> set_new_data_size(cancel_request_timeout(State), NewDataSize) end, - handle_http_msg(Result, NewState); + handle_msg(Result, NewState); {error, {size_error, MaxSize, ErrCode, ErrStr}, Version} -> NewModData = ModData#mod{http_version = Version}, httpd_response:send_status(NewModData, ErrCode, ErrStr), @@ -224,7 +229,10 @@ handle_info({Proto, Socket, Data}, error_log(Reason, NewModData), {stop, normal, State#state{response_sent = true, mod = NewModData}}; - + + {http_chunk = Module, Function, Args} when ChunkState =/= undefined -> + NewState = handle_chunk(Module, Function, Args, State), + {noreply, NewState}; NewMFA -> http_transport:setopts(SockType, Socket, [{active, once}]), case NewDataSize of @@ -349,6 +357,34 @@ await_socket_ownership_transfer(AcceptTimeout) -> exit(accept_socket_timeout) end. + +%%% Internal chunking of client body +handle_msg({{continue, Chunk}, Module, Function, Args}, #state{chunk = {_, CbState}} = State) -> + handle_internal_chunk(State#state{chunk = {continue, CbState}, + body = Chunk}, Module, Function, Args); +handle_msg({continue, Module, Function, Args}, #state{mod = ModData} = State) -> + http_transport:setopts(ModData#mod.socket_type, + ModData#mod.socket, + [{active, once}]), + {noreply, State#state{mfa = {Module, Function, Args}}}; +handle_msg({last, Body}, #state{headers = Headers, chunk = {_, CbState}} = State) -> + NewHeaders = Headers#http_request_h{'content-length' = integer_to_list(size(Body))}, + handle_response(State#state{chunk = {last, CbState}, + headers = NewHeaders, + body = Body}); +%%% Last data chunked by client +handle_msg({ChunkedHeaders, Body}, #state{headers = Headers , chunk = {ChunkState, CbState}} = State) when ChunkState =/= undefined -> + NewHeaders = http_chunk:handle_headers(Headers, ChunkedHeaders), + handle_response(State#state{chunk = {last, CbState}, + headers = NewHeaders, + body = Body}); +handle_msg({ChunkedHeaders, Body}, #state{headers = Headers , chunk = {undefined, _}} = State) -> + NewHeaders = http_chunk:handle_headers(Headers, ChunkedHeaders), + handle_response(State#state{headers = NewHeaders, + body = Body}); +handle_msg(Result, State) -> + handle_http_msg(Result, State). + handle_http_msg({_, _, Version, {_, _}, _}, #state{status = busy, mod = ModData} = State) -> handle_manager_busy(State#state{mod = @@ -405,10 +441,6 @@ handle_http_msg({Method, Uri, Version, {RecordHeaders, Headers}, Body}, error_log(Reason, ModData), {stop, normal, State#state{response_sent = true}} end; -handle_http_msg({ChunkedHeaders, Body}, - State = #state{headers = Headers}) -> - NewHeaders = http_chunk:handle_headers(Headers, ChunkedHeaders), - handle_response(State#state{headers = NewHeaders, body = Body}); handle_http_msg(Body, State) -> handle_response(State#state{body = Body}). @@ -443,22 +475,25 @@ handle_body(#state{mod = #mod{config_db = ConfigDB}} = State) -> end. -handle_body(#state{headers = Headers, body = Body, mod = ModData} = State, +handle_body(#state{headers = Headers, body = Body, + chunk = {ChunkState, CbState}, mod = #mod{config_db = ConfigDB} = ModData} = State, MaxHeaderSize, MaxBodySize) -> + MaxChunk = max_client_body_chunk(ConfigDB), case Headers#http_request_h.'transfer-encoding' of "chunked" -> try http_chunk:decode(Body, MaxBodySize, MaxHeaderSize) of - {Module, Function, Args} -> + {Module, Function, Args} -> http_transport:setopts(ModData#mod.socket_type, ModData#mod.socket, [{active, once}]), {noreply, State#state{mfa = - {Module, Function, Args}}}; - {ok, {ChunkedHeaders, NewBody}} -> - NewHeaders = - http_chunk:handle_headers(Headers, ChunkedHeaders), - handle_response(State#state{headers = NewHeaders, - body = NewBody}) + {Module, Function, Args}, + chunk = chunk_start(MaxChunk)}}; + {ok, {ChunkedHeaders, NewBody}} -> + NewHeaders = http_chunk:handle_headers(Headers, ChunkedHeaders), + handle_response(State#state{headers = NewHeaders, + body = NewBody, + chunk = chunk_finish(ChunkState, CbState, MaxChunk)}) catch throw:Error -> httpd_response:send_status(ModData, 400, @@ -476,21 +511,25 @@ handle_body(#state{headers = Headers, body = Body, mod = ModData} = State, error_log(Reason, ModData), {stop, normal, State#state{response_sent = true}}; _ -> - Length = list_to_integer(Headers#http_request_h.'content-length'), + Length = list_to_integer(Headers#http_request_h.'content-length'), + MaxChunk = max_client_body_chunk(ConfigDB), case ((Length =< MaxBodySize) or (MaxBodySize == nolimit)) of true -> - case httpd_request:whole_body(Body, Length) of - {Module, Function, Args} -> - http_transport:setopts(ModData#mod.socket_type, + case httpd_request:body_chunk_first(Body, Length, MaxChunk) of + {ok, {continue, Module, Function, Args}} -> + http_transport:setopts(ModData#mod.socket_type, ModData#mod.socket, [{active, once}]), {noreply, State#state{mfa = {Module, Function, Args}}}; - - {ok, NewBody} -> - handle_response( - State#state{headers = Headers, - body = NewBody}) + {ok, {{continue, Chunk}, Module, Function, Args}} -> + handle_internal_chunk(State#state{chunk = chunk_start(MaxChunk), + body = Chunk}, Module, Function, Args); + {ok, NewBody} -> + handle_response(State#state{chunk = chunk_finish(ChunkState, + CbState, MaxChunk), + headers = Headers, + body = NewBody}) end; false -> httpd_response:send_status(ModData, 413, "Body too long"), @@ -550,15 +589,61 @@ expect(Headers, _, ConfigDB) -> end end. +handle_chunk(http_chunk = Module, decode_data = Function, + [ChunkSize, TotalChunk, {MaxBodySize, BodySoFar, _AccLength, MaxHeaderSize}], + #state{chunk = {_, CbState}, + mod = #mod{socket_type = SockType, + socket = Socket} = ModData} = State) -> + {continue, NewCbState} = httpd_response:handle_continuation(ModData#mod{entity_body = + {continue, BodySoFar, CbState}}), + http_transport:setopts(SockType, Socket, [{active, once}]), + State#state{chunk = {continue, NewCbState}, mfa = {Module, Function, [ChunkSize, TotalChunk, {MaxBodySize, <<>>, 0, MaxHeaderSize}]}}; + +handle_chunk(http_chunk = Module, decode_size = Function, + [Data, HexList, _AccSize, {MaxBodySize, BodySoFar, _AccLength, MaxHeaderSize}], + #state{chunk = {_, CbState}, + mod = #mod{socket_type = SockType, + socket = Socket} = ModData} = State) -> + {continue, NewCbState} = httpd_response:handle_continuation(ModData#mod{entity_body = {continue, BodySoFar, CbState}}), + http_transport:setopts(SockType, Socket, [{active, once}]), + State#state{chunk = {continue, NewCbState}, mfa = {Module, Function, [Data, HexList, 0, {MaxBodySize, <<>>, 0, MaxHeaderSize}]}}; +handle_chunk(Module, Function, Args, #state{mod = #mod{socket_type = SockType, + socket = Socket}} = State) -> + http_transport:setopts(SockType, Socket, [{active, once}]), + State#state{mfa = {Module, Function, Args}}. + +handle_internal_chunk(#state{chunk = {ChunkState, CbState}, body = Chunk, + mod = #mod{socket_type = SockType, + socket = Socket} = ModData} = State, Module, Function, Args)-> + Bodychunk = body_chunk(ChunkState, CbState, Chunk), + {continue, NewCbState} = httpd_response:handle_continuation(ModData#mod{entity_body = Bodychunk}), + case Args of + [<<>> | _] -> + http_transport:setopts(SockType, Socket, [{active, once}]), + {noreply, State#state{chunk = {continue, NewCbState}, mfa = {Module, Function, Args}}}; + _ -> + handle_info({dummy, Socket, <<>>}, State#state{chunk = {continue, NewCbState}, + mfa = {Module, Function, Args}}) + end. + +handle_response(#state{body = Body, + headers = Headers, + mod = ModData, + chunk = {last, CbState}, + max_keep_alive_request = Max} = State) when Max > 0 -> + {NewBody, Data} = httpd_request:body_data(Headers, Body), + ok = httpd_response:generate_and_send_response( + ModData#mod{entity_body = {last, NewBody, CbState}}), + handle_next_request(State#state{response_sent = true}, Data); handle_response(#state{body = Body, mod = ModData, headers = Headers, max_keep_alive_request = Max} = State) when Max > 0 -> {NewBody, Data} = httpd_request:body_data(Headers, Body), + %% Backwards compatible, may cause memory explosion ok = httpd_response:generate_and_send_response( - ModData#mod{entity_body = NewBody}), + ModData#mod{entity_body = binary_to_list(NewBody)}), handle_next_request(State#state{response_sent = true}, Data); - handle_response(#state{body = Body, headers = Headers, mod = ModData} = State) -> @@ -578,6 +663,7 @@ handle_next_request(#state{mod = #mod{connection = true} = ModData, MaxURISize = max_uri_size(ModData#mod.config_db), MaxContentLen = max_content_length(ModData#mod.config_db), Customize = customize(ModData#mod.config_db), + MaxChunk = max_client_body_chunk(ModData#mod.config_db), MFA = {httpd_request, parse, [[{max_uri, MaxURISize}, {max_header, MaxHeaderSize}, {max_version, ?HTTP_MAX_VERSION_STRING}, @@ -590,6 +676,7 @@ handle_next_request(#state{mod = #mod{connection = true} = ModData, max_keep_alive_request = decrease(Max), headers = undefined, body = undefined, + chunk = chunk_start(MaxChunk), response_sent = false}, NewState = activate_request_timeout(TmpState), @@ -647,6 +734,9 @@ error_log(ReasonString, #mod{config_db = ConfigDB}) -> max_header_size(ConfigDB) -> httpd_util:lookup(ConfigDB, max_header_size, ?HTTP_MAX_HEADER_SIZE). +max_client_body_chunk(ConfigDB) -> + httpd_util:lookup(ConfigDB, max_client_body_chunk, nolimit). + max_uri_size(ConfigDB) -> httpd_util:lookup(ConfigDB, max_uri_size, ?HTTP_MAX_URI_SIZE). @@ -661,3 +751,17 @@ max_content_length(ConfigDB) -> customize(ConfigDB) -> httpd_util:lookup(ConfigDB, customize, httpd_custom). + +chunk_start(nolimit) -> + {undefined, undefined}; +chunk_start(_) -> + {first, undefined}. +chunk_finish(_, _, nolimit) -> + {undefined, undefined}; +chunk_finish(_, CbState, _) -> + {last, CbState}. + +body_chunk(first, _, Chunk) -> + {first, Chunk}; +body_chunk(ChunkState, CbState, Chunk) -> + {ChunkState, Chunk, CbState}. diff --git a/lib/inets/src/http_server/httpd_response.erl b/lib/inets/src/http_server/httpd_response.erl index effa273e92..6b9053fda6 100644 --- a/lib/inets/src/http_server/httpd_response.erl +++ b/lib/inets/src/http_server/httpd_response.erl @@ -21,7 +21,7 @@ -module(httpd_response). -export([generate_and_send_response/1, send_status/3, send_header/3, send_body/3, send_chunk/3, send_final_chunk/2, send_final_chunk/3, - split_header/2, is_disable_chunked_send/1, cache_headers/2]). + split_header/2, is_disable_chunked_send/1, cache_headers/2, handle_continuation/1]). -export([map_status_code/2]). -include_lib("inets/src/inets_app/inets_internal.hrl"). @@ -31,6 +31,9 @@ -define(VMODULE,"RESPONSE"). +handle_continuation(Mod) -> + generate_and_send_response(Mod). + %% If peername does not exist the client already discarded the %% request so we do not need to send a reply. generate_and_send_response(#mod{init_data = @@ -39,6 +42,8 @@ generate_and_send_response(#mod{init_data = generate_and_send_response(#mod{config_db = ConfigDB} = ModData) -> Modules = httpd_util:lookup(ConfigDB, modules, ?DEFAULT_MODS), case traverse_modules(ModData, Modules) of + {continue, _} = Continue -> + Continue; done -> ok; {proceed, Data} -> @@ -69,17 +74,15 @@ generate_and_send_response(#mod{config_db = ConfigDB} = ModData) -> traverse_modules(ModData,[]) -> {proceed,ModData#mod.data}; traverse_modules(ModData,[Module|Rest]) -> - ?hdrd("traverse modules", [{callback_module, Module}]), try apply(Module, do, [ModData]) of + {continue, _} = Continue -> + Continue; done -> - ?hdrt("traverse modules - done", []), - done; + done; {break, NewData} -> - ?hdrt("traverse modules - break", [{new_data, NewData}]), - {proceed, NewData}; + {proceed, NewData}; {proceed, NewData} -> - ?hdrt("traverse modules - proceed", [{new_data, NewData}]), - traverse_modules(ModData#mod{data = NewData}, Rest) + traverse_modules(ModData#mod{data = NewData}, Rest) catch T:E -> String = @@ -104,15 +107,10 @@ send_status(#mod{socket_type = SocketType, socket = Socket, config_db = ConfigDB} = ModData, StatusCode, PhraseArgs) -> - ?hdrd("send status", [{status_code, StatusCode}, - {phrase_args, PhraseArgs}]), - ReasonPhrase = httpd_util:reason_phrase(StatusCode), Message = httpd_util:message(StatusCode, PhraseArgs, ConfigDB), Body = get_body(ReasonPhrase, Message), - ?hdrt("send status - header", [{reason_phrase, ReasonPhrase}, - {message, Message}]), send_header(ModData, StatusCode, [{content_type, "text/html"}, {content_length, integer_to_list(length(Body))}]), diff --git a/lib/inets/src/http_server/mod_esi.erl b/lib/inets/src/http_server/mod_esi.erl index b21af1418c..3a589ca5f0 100644 --- a/lib/inets/src/http_server/mod_esi.erl +++ b/lib/inets/src/http_server/mod_esi.erl @@ -31,7 +31,6 @@ -include("httpd.hrl"). -include("httpd_internal.hrl"). --include("inets_internal.hrl"). -define(VMODULE,"ESI"). -define(DEFAULT_ERL_TIMEOUT,15000). @@ -69,7 +68,6 @@ deliver(_SessionID, _Data) -> %% Description: See httpd(3) ESWAPI CALLBACK FUNCTIONS %%------------------------------------------------------------------------- do(ModData) -> - ?hdrt("do", []), case proplists:get_value(status, ModData#mod.data) of {_StatusCode, _PhraseArgs, _Reason} -> {proceed, ModData#mod.data}; @@ -190,7 +188,6 @@ store({erl_script_nocache, Value}, _) -> %%% Internal functions %%%======================================================================== generate_response(ModData) -> - ?hdrt("generate response", []), case scheme(ModData#mod.request_uri, ModData#mod.config_db) of {eval, ESIBody, Modules} -> eval(ModData, ESIBody, Modules); @@ -242,7 +239,6 @@ alias_match_str(Alias, eval_script_alias) -> erl(#mod{method = Method} = ModData, ESIBody, Modules) when (Method =:= "GET") orelse (Method =:= "HEAD") orelse (Method =:= "DELETE") -> - ?hdrt("erl", [{method, Method}]), case httpd_util:split(ESIBody,":|%3A|/",2) of {ok, [ModuleName, FuncAndInput]} -> case httpd_util:split(FuncAndInput,"[\?/]",2) of @@ -273,14 +269,12 @@ erl(#mod{method = "PUT", entity_body = Body} = ModData, generate_webpage(ModData, ESIBody, Modules, list_to_atom(ModuleName), FunctionName, {Input,Body}, - [{entity_body, Body} | - script_elements(FuncAndInput, Input)]); + script_elements(FuncAndInput, Input)); {ok, [FunctionName]} -> generate_webpage(ModData, ESIBody, Modules, list_to_atom(ModuleName), FunctionName, {undefined,Body}, - [{entity_body, Body} | - script_elements(FuncAndInput, "")]); + script_elements(FuncAndInput, "")); {ok, BadRequest} -> {proceed,[{status,{400,none, BadRequest}} | ModData#mod.data]} @@ -290,12 +284,11 @@ erl(#mod{method = "PUT", entity_body = Body} = ModData, end; erl(#mod{method = "POST", entity_body = Body} = ModData, ESIBody, Modules) -> - ?hdrt("erl", [{method, post}]), case httpd_util:split(ESIBody,":|%3A|/",2) of {ok,[ModuleName, Function]} -> generate_webpage(ModData, ESIBody, Modules, list_to_atom(ModuleName), - Function, Body, [{entity_body, Body}]); + Function, Body, []); {ok, BadRequest} -> {proceed,[{status, {400, none, BadRequest}} | ModData#mod.data]} end; @@ -304,7 +297,6 @@ erl(#mod{request_uri = ReqUri, method = "PATCH", http_version = Version, data = Data}, _ESIBody, _Modules) -> - ?hdrt("erl", [{method, patch}]), {proceed, [{status,{501,{"PATCH", ReqUri, Version}, ?NICE("Erl mechanism doesn't support method PATCH")}}| Data]}. @@ -315,7 +307,6 @@ generate_webpage(ModData, ESIBody, [all], Module, FunctionName, FunctionName, Input, ScriptElements); generate_webpage(ModData, ESIBody, Modules, Module, FunctionName, Input, ScriptElements) -> - ?hdrt("generate webpage", []), Function = list_to_atom(FunctionName), case lists:member(Module, Modules) of true -> @@ -337,7 +328,6 @@ generate_webpage(ModData, ESIBody, Modules, Module, FunctionName, %% Old API that waits for the dymnamic webpage to be totally generated %% before anythig is sent back to the client. erl_scheme_webpage_whole(Mod, Func, Env, Input, ModData) -> - ?hdrt("erl_scheme_webpage_whole", [{module, Mod}, {function, Func}]), case (catch Mod:Func(Env, Input)) of {'EXIT',{undef, _}} -> {proceed, [{status, {404, ModData#mod.request_uri, "Not found"}} @@ -375,7 +365,6 @@ erl_scheme_webpage_whole(Mod, Func, Env, Input, ModData) -> %% in small chunks at the time during generation. erl_scheme_webpage_chunk(Mod, Func, Env, Input, ModData) -> process_flag(trap_exit, true), - ?hdrt("erl_scheme_webpage_chunk", [{module, Mod}, {function, Func}]), Self = self(), %% Spawn worker that generates the webpage. %% It would be nicer to use erlang:function_exported/3 but if the @@ -386,7 +375,9 @@ erl_scheme_webpage_chunk(Mod, Func, Env, Input, ModData) -> {'EXIT', {undef,_}} -> %% Will force fallback on the old API exit(erl_scheme_webpage_chunk_undefined); - _ -> + {continue, _} = Continue -> + exit(Continue); + _ -> ok end end), @@ -400,13 +391,12 @@ deliver_webpage_chunk(#mod{config_db = Db} = ModData, Pid) -> deliver_webpage_chunk(ModData, Pid, Timeout). deliver_webpage_chunk(#mod{config_db = Db} = ModData, Pid, Timeout) -> - ?hdrt("deliver_webpage_chunk", [{timeout, Timeout}]), case receive_headers(Timeout) of {error, Reason} -> %% Happens when webpage generator callback/3 is undefined - ?hdrv("deliver_webpage_chunk - failed receiving headers", - [{reason, Reason}]), {error, Reason}; + {continue, _} = Continue -> + Continue; {Headers, Body} -> case httpd_esi:handle_headers(Headers) of {proceed, AbsPath} -> @@ -430,7 +420,6 @@ deliver_webpage_chunk(#mod{config_db = Db} = ModData, Pid, Timeout) -> IsDisableChunkedSend) end; timeout -> - ?hdrv("deliver_webpage_chunk - timeout", []), send_headers(ModData, 504, [{"connection", "close"}]), httpd_socket:close(ModData#mod.socket_type, ModData#mod.socket), {proceed,[{response, {already_sent, 200, 0}} | ModData#mod.data]} @@ -439,16 +428,14 @@ deliver_webpage_chunk(#mod{config_db = Db} = ModData, Pid, Timeout) -> receive_headers(Timeout) -> receive {esi_data, Chunk} -> - ?hdrt("receive_headers - received esi data (esi)", []), httpd_esi:parse_headers(lists:flatten(Chunk)); {ok, Chunk} -> - ?hdrt("receive_headers - received esi data (ok)", []), httpd_esi:parse_headers(lists:flatten(Chunk)); {'EXIT', Pid, erl_scheme_webpage_chunk_undefined} when is_pid(Pid) -> - ?hdrd("receive_headers - exit:chunk-undef", []), {error, erl_scheme_webpage_chunk_undefined}; - {'EXIT', Pid, Reason} when is_pid(Pid) -> - ?hdrv("receive_headers - exit", [{reason, Reason}]), + {'EXIT', Pid, {continue, _} = Continue} when is_pid(Pid) -> + Continue; + {'EXIT', Pid, Reason} when is_pid(Pid) -> exit({mod_esi_linked_process_died, Pid, Reason}) after Timeout -> timeout @@ -463,7 +450,6 @@ handle_body(_, #mod{method = "HEAD"} = ModData, _, _, Size, _) -> {proceed, [{response, {already_sent, 200, Size}} | ModData#mod.data]}; handle_body(Pid, ModData, Body, Timeout, Size, IsDisableChunkedSend) -> - ?hdrt("handle_body - send chunk", [{timeout, Timeout}, {size, Size}]), httpd_response:send_chunk(ModData, Body, IsDisableChunkedSend), receive {esi_data, Data} when is_binary(Data) -> @@ -543,7 +529,6 @@ eval(#mod{request_uri = ReqUri, method = "PUT", http_version = Version, data = Data}, _ESIBody, _Modules) -> - ?hdrt("eval", [{method, put}]), {proceed,[{status,{501,{"PUT", ReqUri, Version}, ?NICE("Eval mechanism doesn't support method PUT")}}| Data]}; @@ -552,7 +537,6 @@ eval(#mod{request_uri = ReqUri, method = "DELETE", http_version = Version, data = Data}, _ESIBody, _Modules) -> - ?hdrt("eval", [{method, delete}]), {proceed,[{status,{501,{"DELETE", ReqUri, Version}, ?NICE("Eval mechanism doesn't support method DELETE")}}| Data]}; @@ -561,14 +545,12 @@ eval(#mod{request_uri = ReqUri, method = "POST", http_version = Version, data = Data}, _ESIBody, _Modules) -> - ?hdrt("eval", [{method, post}]), {proceed,[{status,{501,{"POST", ReqUri, Version}, ?NICE("Eval mechanism doesn't support method POST")}}| Data]}; eval(#mod{method = Method} = ModData, ESIBody, Modules) when (Method =:= "GET") orelse (Method =:= "HEAD") -> - ?hdrt("eval", [{method, Method}]), case is_authorized(ESIBody, Modules) of true -> case generate_webpage(ESIBody) of diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index b4f0f2aa7d..6c8728470b 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -74,6 +74,7 @@ all() -> {group, https_reload}, {group, http_mime_types}, {group, http_logging}, + {group, http_post}, mime_types_format ]. @@ -100,6 +101,7 @@ groups() -> {http_logging, [], [{group, logging}]}, {http_reload, [], [{group, reload}]}, {https_reload, [], [{group, reload}]}, + {http_post, [], [{group, post}]}, {http_mime_types, [], [alias_1_1, alias_1_0, alias_0_9]}, {limit, [], [max_clients_1_1, max_clients_1_0, max_clients_0_9]}, {custom, [], [customize, add_default]}, @@ -112,6 +114,7 @@ groups() -> disturbing_1_0, disturbing_0_9 ]}, + {post, [], [chunked_post, chunked_chunked_encoded_post]}, {basic_auth, [], [basic_auth_1_1, basic_auth_1_0, basic_auth_0_9]}, {auth_api, [], [auth_api_1_1, auth_api_1_0, auth_api_0_9 ]}, @@ -152,6 +155,7 @@ http_get() -> ipv6 ]. + load() -> [light, medium %%,heavy @@ -218,6 +222,7 @@ init_per_group(Group, Config0) when Group == http_basic; Group == http_auth_api_mnesia; Group == http_security; Group == http_reload; + Group == http_post; Group == http_mime_types -> ok = start_apps(Group), @@ -275,6 +280,7 @@ end_per_group(Group, _Config) when Group == http_basic; Group == http_htaccess; Group == http_security; Group == http_reload; + Group == http_post; Group == http_mime_types -> inets:stop(); @@ -299,7 +305,7 @@ end_per_group(_, _) -> %%-------------------------------------------------------------------- init_per_testcase(Case, Config) when Case == host; Case == trace -> - ct:timetrap({seconds, 20}), + ct:timetrap({seconds, 40}), Prop = proplists:get_value(tc_group_properties, Config), Name = proplists:get_value(name, Prop), Cb = case Name of @@ -676,6 +682,51 @@ ipv6(Config) when is_list(Config) -> {skip, "Host does not support IPv6"} end. +%%------------------------------------------------------------------------- +chunked_post() -> + [{doc,"Test option max_client_body_chunk"}]. +chunked_post(Config) when is_list(Config) -> + ok = http_status("POST /cgi-bin/erl/httpd_example:post_chunked ", + {"Content-Length:833 \r\n", + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ" + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ"}, + [{http_version, "HTTP/1.1"} |Config], + [{statuscode, 200}]), + ok = http_status("POST /cgi-bin/erl/httpd_example:post_chunked ", + {"Content-Length:2 \r\n", + "ZZ" + }, + [{http_version, "HTTP/1.1"} |Config], + [{statuscode, 200}]). + +chunked_chunked_encoded_post() -> + [{doc,"Test option max_client_body_chunk with chunked client encoding"}]. +chunked_chunked_encoded_post(Config) when is_list(Config) -> + Chunk = http_chunk:encode("ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ"), + LastChunk = http_chunk:encode_last(), + Chunks = lists:duplicate(10000, Chunk), + ok = http_status("POST /cgi-bin/erl/httpd_example:post_chunked ", + {"Transfer-Encoding:chunked \r\n", + [Chunks | LastChunk]}, + [{http_version, "HTTP/1.1"} | Config], + [{statuscode, 200}]). + + %%------------------------------------------------------------------------- htaccess_1_1(Config) when is_list(Config) -> htaccess([{http_version, "HTTP/1.1"} | Config]). @@ -1685,6 +1736,7 @@ start_apps(Group) when Group == http_basic; Group == http_security; Group == http_logging; Group == http_reload; + Group == http_post; Group == http_mime_types-> inets_test_lib:start_apps([inets]). @@ -1731,6 +1783,8 @@ server_config(https_basic, Config) -> basic_conf() ++ server_config(https, Config); server_config(http_reload, Config) -> [{keep_alive_timeout, 2}] ++ server_config(http, Config); +server_config(http_post, Config) -> + [{max_client_body_chunk, 10}] ++ server_config(http, Config); server_config(https_reload, Config) -> [{keep_alive_timeout, 2}] ++ server_config(https, Config); server_config(http_limit, Config) -> -- cgit v1.2.3