diff options
Diffstat (limited to 'lib/inets')
| -rw-r--r-- | lib/inets/doc/src/httpc.xml | 23 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_handler.erl | 65 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_manager.erl | 32 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_request.erl | 67 | ||||
| -rw-r--r-- | lib/inets/src/http_client/httpc_response.erl | 1 | ||||
| -rw-r--r-- | lib/inets/test/httpc_SUITE.erl | 177 |
6 files changed, 280 insertions, 85 deletions
diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index 14662f257c..521ad6a015 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -312,8 +312,7 @@ <v>Body = string() | binary()</v> <v>Profile = profile() | pid()</v> <d>When started <c>stand_alone</c> only the pid can be used.</d> - <v>Reason = {connect_failed, term()} | - {send_failed, term()} | term()</v> + <v>Reason = term()</v> </type> <desc> @@ -442,17 +441,22 @@ <tag><c><![CDATA[socket_opts]]></c></tag> <item> - <p>Socket options to be used for this and subsequent - requests.</p> + <p>Socket options to be used for this request.</p> <p>Overrides any value set by function <seealso marker="#set_options-1">set_options</seealso>.</p> <p>The validity of the options is <em>not</em> checked by the HTTP client they are assumed to be correct and passed on to ssl application and inet driver, which may reject - them if they are not correct. Note that the current - implementation assumes the requests to the same host, port - combination will use the same socket options. + them if they are not correct. </p> + <note> + <p> + Persistent connections are not supported when setting the + <c>socket_opts</c> option. When <c>socket_opts</c> is not + set the current implementation assumes the requests to the + same host, port combination will use the same socket options. + </p> + </note> <p>By default the socket options set by function <seealso marker="#set_options-1">set_options/[1,2]</seealso> @@ -625,8 +629,11 @@ to complete. The HTTP/1.1 specification suggests a limit of two persistent connections per server, which is the default value of option <c>max_sessions</c>.</p> + <p> + The current implementation assumes the requests to the same host, port + combination will use the same socket options. + </p> </note> - <marker id="get_options"></marker> </desc> </func> diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index 9b09832eb8..eeb08ce0ee 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -48,19 +48,17 @@ queue_timer :: reference() | 'undefined' }). --type session_failed() :: {'connect_failed',term()} | {'send_failed',term()}. - -record(state, { request :: request() | 'undefined', - session :: session() | session_failed() | 'undefined', + session :: session() | 'undefined', status_line, % {Version, StatusCode, ReasonPharse} headers :: http_response_h() | 'undefined', body :: binary() | 'undefined', mfa, % {Module, Function, Args} pipeline = queue:new() :: queue:queue(), keep_alive = queue:new() :: queue:queue(), - status, % undefined | new | pipeline | keep_alive | close | {ssl_tunnel, Request} + status :: undefined | new | pipeline | keep_alive | close | {ssl_tunnel, request()}, canceled = [], % [RequestId] max_header_size = nolimit :: nolimit | integer(), max_body_size = nolimit :: nolimit | integer(), @@ -255,8 +253,8 @@ handle_call(Request, From, State) -> Result -> Result catch - _:Reason -> - {stop, {shutdown, Reason} , State} + Class:Reason:ST -> + {stop, {shutdown, {{Class, Reason}, ST}}, State} end. @@ -271,8 +269,8 @@ handle_cast(Msg, State) -> Result -> Result catch - _:Reason -> - {stop, {shutdown, Reason} , State} + Class:Reason:ST -> + {stop, {shutdown, {{Class, Reason}, ST}}, State} end. %%-------------------------------------------------------------------- @@ -286,8 +284,8 @@ handle_info(Info, State) -> Result -> Result catch - _:Reason -> - {stop, {shutdown, Reason} , State} + Class:Reason:ST -> + {stop, {shutdown, {{Class, Reason}, ST}}, State} end. %%-------------------------------------------------------------------- @@ -295,23 +293,6 @@ handle_info(Info, State) -> %% Description: Shutdown the httpc_handler %%-------------------------------------------------------------------- -%% Init error there is no socket to be closed. -terminate(normal, - #state{request = Request, - session = {send_failed, _} = Reason} = State) -> - maybe_send_answer(Request, - httpc_response:error(Request, Reason), - State), - ok; - -terminate(normal, - #state{request = Request, - session = {connect_failed, _} = Reason} = State) -> - maybe_send_answer(Request, - httpc_response:error(Request, Reason), - State), - ok; - terminate(normal, #state{session = undefined}) -> ok; @@ -588,11 +569,11 @@ do_handle_info({Proto, _Socket, Data}, activate_once(Session), {noreply, State#state{mfa = NewMFA}} catch - _:Reason -> + Class:Reason:ST -> ClientReason = {could_not_parse_as_http, Data}, ClientErrMsg = httpc_response:error(Request, ClientReason), NewState = answer_request(Request, ClientErrMsg, State), - {stop, {shutdown, Reason}, NewState} + {stop, {shutdown, {{Class, Reason}, ST}}, NewState} end; do_handle_info({Proto, Socket, Data}, @@ -1058,15 +1039,15 @@ handle_response(#state{status = new} = State) -> ?hcrd("handle response - status = new", []), handle_response(try_to_enable_pipeline_or_keep_alive(State)); -handle_response(#state{request = Request, - status = Status, - session = Session, - status_line = StatusLine, - headers = Headers, - body = Body, - options = Options, - profile_name = ProfileName} = State) - when Status =/= new -> +handle_response(#state{status = Status0} = State0) when Status0 =/= new -> + State = handle_server_closing(State0), + #state{request = Request, + session = Session, + status_line = StatusLine, + headers = Headers, + body = Body, + options = Options, + profile_name = ProfileName} = State, handle_cookies(Headers, Request, Options, ProfileName), case httpc_response:result({StatusLine, Headers, Body}, Request) of %% 100-continue @@ -1330,6 +1311,14 @@ try_to_enable_pipeline_or_keep_alive( State#state{status = close} end. +handle_server_closing(State = #state{status = close}) -> State; +handle_server_closing(State = #state{headers = undefined}) -> State; +handle_server_closing(State = #state{headers = Headers}) -> + case httpc_response:is_server_closing(Headers) of + true -> State#state{status = close}; + false -> State + end. + answer_request(#request{id = RequestId, from = From} = Request, Msg, #state{session = Session, timers = Timers, diff --git a/lib/inets/src/http_client/httpc_manager.erl b/lib/inets/src/http_client/httpc_manager.erl index 7b8d7875de..c3404dbb37 100644 --- a/lib/inets/src/http_client/httpc_manager.erl +++ b/lib/inets/src/http_client/httpc_manager.erl @@ -750,8 +750,26 @@ handle_request(#request{settings = start_handler(NewRequest#request{headers = NewHeaders}, State), {reply, {ok, NewRequest#request.id}, State}; -handle_request(Request, State = #state{options = Options}) -> +%% Simple socket options handling (ERL-441). +%% +%% TODO: Refactor httpc to enable sending socket options in requests +%% using persistent connections. This workaround opens a new +%% connection for each request with non-empty socket_opts. +handle_request(Request0 = #request{socket_opts = SocketOpts}, + State0 = #state{options = Options0}) + when is_list(SocketOpts) andalso length(SocketOpts) > 0 -> + Request = handle_cookies(generate_request_id(Request0), State0), + Options = convert_options(SocketOpts, Options0), + State = State0#state{options = Options}, + Headers = + (Request#request.headers)#http_request_h{connection + = "close"}, + %% Reset socket_opts to avoid setopts failure. + start_handler(Request#request{headers = Headers, socket_opts = []}, State), + %% Do not change the state + {reply, {ok, Request#request.id}, State0}; +handle_request(Request, State = #state{options = Options}) -> NewRequest = handle_cookies(generate_request_id(Request), State), SessionType = session_type(Options), case select_session(Request#request.method, @@ -775,6 +793,18 @@ handle_request(Request, State = #state{options = Options}) -> {reply, {ok, NewRequest#request.id}, State}. +%% Convert Request options to State options +convert_options([], Options) -> + Options; +convert_options([{ipfamily, Value}|T], Options) -> + convert_options(T, Options#options{ipfamily = Value}); +convert_options([{ip, Value}|T], Options) -> + convert_options(T, Options#options{ip = Value}); +convert_options([{port, Value}|T], Options) -> + convert_options(T, Options#options{port = Value}); +convert_options([Option|T], Options = #options{socket_opts = SocketOpts}) -> + convert_options(T, Options#options{socket_opts = SocketOpts ++ [Option]}). + start_handler(#request{id = Id, from = From} = Request, #state{profile_name = ProfileName, diff --git a/lib/inets/src/http_client/httpc_request.erl b/lib/inets/src/http_client/httpc_request.erl index 89872a3831..641b6559de 100644 --- a/lib/inets/src/http_client/httpc_request.erl +++ b/lib/inets/src/http_client/httpc_request.erl @@ -190,35 +190,11 @@ is_client_closing(Headers) -> %%%======================================================================== post_data(Method, Headers, {ContentType, Body}, HeadersAsIs) when (Method =:= post) - orelse (Method =:= put) - orelse (Method =:= patch) - orelse (Method =:= delete) -> - - NewBody = case Headers#http_request_h.expect of - "100-continue" -> - ""; - _ -> - Body - end, - - NewHeaders = case HeadersAsIs of - [] -> - Headers#http_request_h{ - 'content-type' = ContentType, - 'content-length' = case body_length(Body) of - undefined -> - % on upload streaming the caller must give a - % value to the Content-Length header - % (or use chunked Transfer-Encoding) - Headers#http_request_h.'content-length'; - Len when is_list(Len) -> - Len - end - }; - _ -> - HeadersAsIs - end, - + orelse (Method =:= put) + orelse (Method =:= patch) + orelse (Method =:= delete) -> + NewBody = update_body(Headers, Body), + NewHeaders = update_headers(Headers, ContentType, Body, HeadersAsIs), {NewHeaders, NewBody}; post_data(_, Headers, _, []) -> @@ -226,14 +202,39 @@ post_data(_, Headers, _, []) -> post_data(_, _, _, HeadersAsIs = [_|_]) -> {HeadersAsIs, ""}. +update_body(Headers, Body) -> + case Headers#http_request_h.expect of + "100-continue" -> + ""; + _ -> + Body + end. + +update_headers(Headers, ContentType, Body, []) -> + case Body of + [] -> + Headers#http_request_h{'content-length' = "0"}; + <<>> -> + Headers#http_request_h{'content-length' = "0"}; + {Fun, _Acc} when is_function(Fun, 1) -> + %% A client MUST NOT generate a 100-continue expectation in a request + %% that does not include a message body. This implies that either the + %% Content-Length or the Transfer-Encoding header MUST be present. + %% DO NOT send content-type when Body is empty. + Headers#http_request_h{'content-type' = ContentType}; + _ -> + Headers#http_request_h{ + 'content-length' = body_length(Body), + 'content-type' = ContentType} + end; +update_headers(_, _, _, HeadersAsIs) -> + HeadersAsIs. + body_length(Body) when is_binary(Body) -> integer_to_list(size(Body)); body_length(Body) when is_list(Body) -> - integer_to_list(length(Body)); - -body_length({DataFun, _Acc}) when is_function(DataFun, 1) -> - undefined. + integer_to_list(length(Body)). method(Method) -> http_util:to_upper(atom_to_list(Method)). diff --git a/lib/inets/src/http_client/httpc_response.erl b/lib/inets/src/http_client/httpc_response.erl index 58ab9144df..92dc9b0e02 100644 --- a/lib/inets/src/http_client/httpc_response.erl +++ b/lib/inets/src/http_client/httpc_response.erl @@ -83,7 +83,6 @@ whole_body(Body, Length) -> %% result(Response, Request) -> %% Response - {StatusLine, Headers, Body} %% Request - #request{} -%% Session - #tcp_session{} %% %% Description: Checks the status code ... %%------------------------------------------------------------------------- diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 38705372c9..47c7ffd190 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -53,6 +53,7 @@ suite() -> all() -> [ {group, http}, + {group, http_ipv6}, {group, sim_http}, {group, http_internal}, {group, http_unix_socket}, @@ -64,10 +65,11 @@ all() -> groups() -> [ {http, [], real_requests()}, + {http_ipv6, [], [request_options]}, %% process_leak_on_keepalive is depending on stream_fun_server_close %% and it shall be the last test case in the suite otherwise cookie %% will fail. - {sim_http, [], only_simulated() ++ [process_leak_on_keepalive]}, + {sim_http, [], only_simulated() ++ server_closing_connection() ++ [process_leak_on_keepalive]}, {http_internal, [], real_requests_esi()}, {http_unix_socket, [], simulated_unix_socket()}, {https, [], real_requests()}, @@ -151,9 +153,16 @@ only_simulated() -> relaxed, multipart_chunks, get_space, + delete_no_body, stream_fun_server_close ]. +server_closing_connection() -> + [ + server_closing_connection_on_first_response, + server_closing_connection_on_second_response + ]. + misc() -> [ server_does_not_exist, @@ -207,6 +216,16 @@ init_per_group(http_unix_socket = Group, Config0) -> Port = server_start(Group, server_config(Group, Config)), [{port, Port} | Config] end; +init_per_group(http_ipv6 = Group, Config0) -> + case is_ipv6_supported() of + true -> + start_apps(Group), + Config = proplists:delete(port, Config0), + Port = server_start(Group, server_config(Group, Config)), + [{port, Port} | Config]; + false -> + {skip, "Host does not support IPv6"} + end; init_per_group(Group, Config0) -> start_apps(Group), Config = proplists:delete(port, Config0), @@ -233,7 +252,7 @@ init_per_testcase(pipeline, Config) -> init_per_testcase(persistent_connection, Config) -> inets:start(httpc, [{profile, persistent}]), httpc:set_options([{keep_alive_timeout, 50000}, - {max_keep_alive_length, 3}], persistent_connection), + {max_keep_alive_length, 3}], persistent), Config; init_per_testcase(wait_for_whole_response, Config) -> @@ -252,10 +271,38 @@ end_per_testcase(pipeline, _Config) -> inets:stop(httpc, pipeline); end_per_testcase(persistent_connection, _Config) -> inets:stop(httpc, persistent); +end_per_testcase(Case, Config) + when Case == server_closing_connection_on_first_response; + Case == server_closing_connection_on_second_response -> + %% Test case uses at most one session. Ensure no leftover + %% sessions left behind. + {_, Status} = proplists:lookup(tc_status, Config), + ShallCleanup = case Status of + ok -> true; + {failed, _} -> true; + {skipped, _} -> false + end, + if ShallCleanup =:= true -> + httpc:request(url(group_name(Config), "/just_close.html", Config)), + ok; + true -> + ct:pal("Not cleaning up because test case status was ~p", [Status]), + ok + end; end_per_testcase(_Case, _Config) -> ok. +is_ipv6_supported() -> + case gen_udp:open(0, [inet6]) of + {ok, Socket} -> + gen_udp:close(Socket), + true; + _ -> + false + end. + + %%-------------------------------------------------------------------- %% Test Cases -------------------------------------------------------- %%-------------------------------------------------------------------- @@ -1275,6 +1322,53 @@ stream_fun_server_close(Config) when is_list(Config) -> end. %%-------------------------------------------------------------------- +server_closing_connection_on_first_response() -> + [{doc, "Client receives \"Connection: close\" on first response." + "A client that receives a \"close\" connection option MUST cease sending" + "requests on that connection and close the connection after reading" + "the response message containing the \"close\""}]. +server_closing_connection_on_first_response(Config) when is_list(Config) -> + ReqSrvSendOctFun = + fun(V, U, S) -> + {ok, {{V, S, _}, Headers0, []}} = + httpc:request(get, {U, []}, [{version, V}], []), + {_, SendOctStr} = + proplists:lookup("x-socket-stat-send-oct", Headers0), + list_to_integer(SendOctStr) + end, + V = "HTTP/1.1", + Url0 = url(group_name(Config), "/http_1_1_send_oct.html", Config), + Url1 = url(group_name(Config), "/http_1_1_send_oct_and_connection_close.html", Config), + %% Test case assumes at most one reusable past session. + _ = ReqSrvSendOctFun(V, Url1, 204), + 0 = ReqSrvSendOctFun(V, Url0, 204), + ok. + +%%-------------------------------------------------------------------- +server_closing_connection_on_second_response() -> + [{doc, "Client receives \"Connection: close\" on second response." + "A client that receives a \"close\" connection option MUST cease sending" + "requests on that connection and close the connection after reading" + "the response message containing the \"close\""}]. +server_closing_connection_on_second_response(Config) when is_list(Config) -> + ReqSrvSendOctFun = + fun(V, U, S) -> + {ok, {{V, S, _}, Headers0, []}} = + httpc:request(get, {U, []}, [{version, V}], []), + {_, SendOctStr} = + proplists:lookup("x-socket-stat-send-oct", Headers0), + list_to_integer(SendOctStr) + end, + V = "HTTP/1.1", + Url0 = url(group_name(Config), "/http_1_1_send_oct.html", Config), + Url1 = url(group_name(Config), "/http_1_1_send_oct_and_connection_close.html", Config), + %% Test case assumes no reusable past sessions. + SendOct0 = 0 = ReqSrvSendOctFun(V, Url0, 204), + case ReqSrvSendOctFun(V, Url1, 204) of SendOct1 when SendOct1 > SendOct0 -> ok end, + 0 = ReqSrvSendOctFun(V, Url0, 204), + ok. + +%%-------------------------------------------------------------------- slow_connection() -> [{doc, "Test that a request on a slow keep-alive connection won't crash the httpc_manager"}]. slow_connection(Config) when is_list(Config) -> @@ -1305,6 +1399,26 @@ unix_domain_socket(Config) when is_list(Config) -> {ok, {{_,200,_}, [_ | _], _}} = httpc:request(get, {URL, []}, [], []). +%%------------------------------------------------------------------------- +delete_no_body(doc) -> + ["Test that a DELETE request without Body does not send a Content-Type header - Solves ERL-536"]; +delete_no_body(Config) when is_list(Config) -> + URL = url(group_name(Config), "/delete_no_body.html", Config), + %% Simulated server replies 500 if 'Content-Type' header is present + {ok, {{_,200,_}, _, _}} = + httpc:request(delete, {URL, []}, [], []), + {ok, {{_,500,_}, _, _}} = + httpc:request(delete, {URL, [], "text/plain", "TEST"}, [], []). + +%%-------------------------------------------------------------------- +request_options() -> + [{doc, "Test http get request with socket options against local server (IPv6)"}]. +request_options(Config) when is_list(Config) -> + Request = {url(group_name(Config), "/dummy.html", Config), []}, + {ok, {{_,200,_}, [_ | _], _ = [_ | _]}} = httpc:request(get, Request, [], + [{socket_opts,[{ipfamily, inet6}]}]), + {error,{failed_connect,_ }} = httpc:request(get, Request, [], []). + %%-------------------------------------------------------------------- @@ -1394,6 +1508,9 @@ url(http, End, Config) -> Port = proplists:get_value(port, Config), {ok,Host} = inet:gethostname(), ?URL_START ++ Host ++ ":" ++ integer_to_list(Port) ++ End; +url(http_ipv6, End, Config) -> + Port = proplists:get_value(port, Config), + ?URL_START ++ "[::1]" ++ ":" ++ integer_to_list(Port) ++ End; url(https, End, Config) -> Port = proplists:get_value(port, Config), {ok,Host} = inet:gethostname(), @@ -1438,7 +1555,11 @@ server_start(http_unix_socket, Config) -> {_Pid, Port} = http_test_lib:dummy_server(unix_socket, Inet, [{content_cb, ?MODULE}, {unix_socket, Socket}]), Port; - +server_start(http_ipv6, HttpdConfig) -> + {ok, Pid} = inets:start(httpd, HttpdConfig), + Serv = inets:services_info(), + {value, {_, _, Info}} = lists:keysearch(Pid, 2, Serv), + proplists:get_value(port, Info); server_start(_, HttpdConfig) -> {ok, Pid} = inets:start(httpd, HttpdConfig), Serv = inets:services_info(), @@ -1457,6 +1578,17 @@ server_config(http, Config) -> {mime_type, "text/plain"}, {script_alias, {"/cgi-bin/", filename:join(ServerRoot, "cgi-bin") ++ "/"}} ]; +server_config(http_ipv6, Config) -> + ServerRoot = proplists:get_value(server_root, Config), + [{port, 0}, + {server_name,"httpc_test"}, + {server_root, ServerRoot}, + {document_root, proplists:get_value(doc_root, Config)}, + {bind_address, {0,0,0,0,0,0,0,1}}, + {ipfamily, inet6}, + {mime_type, "text/plain"}, + {script_alias, {"/cgi-bin/", filename:join(ServerRoot, "cgi-bin") ++ "/"}} + ]; server_config(http_internal, Config) -> ServerRoot = proplists:get_value(server_root, Config), [{port, 0}, @@ -1811,6 +1943,13 @@ auth_header([{"authorization", Value} | _]) -> auth_header([_ | Tail]) -> auth_header(Tail). +content_type_header([]) -> + not_found; +content_type_header([{"content-type", Value}|_]) -> + {ok, string:strip(Value)}; +content_type_header([_|T]) -> + content_type_header(T). + handle_auth("Basic " ++ UserInfo, Challange, DefaultResponse) -> case string:tokens(base64:decode_to_string(UserInfo), ":") of ["alladin", "sesame"] = Auth -> @@ -2232,10 +2371,40 @@ handle_uri("GET","/v1/kv/foo",_,_,_,_) -> "Content-Length: 24\r\n" ++ "Content-Type: application/json\r\n\r\n" ++ "[{\"Value\": \"aGVsbG8=\"}]\n"; - +handle_uri(_,"/http_1_1_send_oct.html",_,_,Socket,_) -> + "HTTP/1.1 204 No Content\r\n" ++ + "X-Socket-Stat-Send-Oct: " ++ integer_to_list(get_stat(Socket, send_oct)) ++ "\r\n" ++ + "\r\n"; +handle_uri(_,"/http_1_1_send_oct_and_connection_close.html",_,_,Socket,_) -> + "HTTP/1.1 204 No Content\r\n" ++ + "X-Socket-Stat-Send-Oct: " ++ integer_to_list(get_stat(Socket, send_oct)) ++ "\r\n" ++ + "Connection: close\r\n" ++ + "\r\n"; +handle_uri(_,"/delete_no_body.html", _,Headers,_, DefaultResponse) -> + Error = "HTTP/1.1 500 Internal Server Error\r\n" ++ + "Content-Length:0\r\n\r\n", + case content_type_header(Headers) of + {ok, _} -> + Error; + not_found -> + DefaultResponse + end; handle_uri(_,_,_,_,_,DefaultResponse) -> DefaultResponse. +get_stat(S, Opt) -> + case getstat(S, [Opt]) of + {ok, [{Opt, V}]} when is_integer(V) -> + V; + {error, _} = E -> + E + end. + +getstat(#sslsocket{} = S, Opts) -> + ssl:getstat(S, Opts); +getstat(S, Opts) -> + inet:getstat(S, Opts). + url_start(#sslsocket{}) -> {ok,Host} = inet:gethostname(), ?TLS_URL_START ++ Host ++ ":"; |