diff options
Diffstat (limited to 'lib/inets')
22 files changed, 746 insertions, 141 deletions
diff --git a/lib/inets/doc/src/http_uri.xml b/lib/inets/doc/src/http_uri.xml index 696b7dfa31..20c042c202 100644 --- a/lib/inets/doc/src/http_uri.xml +++ b/lib/inets/doc/src/http_uri.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2012</year><year>2016</year> + <year>2012</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -45,6 +45,7 @@ this module:</p> <p><c>boolean() = true | false</c></p> <p><c>string()</c> = list of ASCII characters</p> + <p><c>unicode_binary()</c> = binary() with characters encoded in the UTF-8 coding standard</p> </section> @@ -53,22 +54,22 @@ <p>Type definitions that are related to URI:</p> <taglist> - <tag><c>uri() = string()</c></tag> + <tag><c>uri() = string() | unicode:unicode_binary()</c></tag> <item><p>Syntax according to the URI definition in RFC 3986, for example, "http://www.erlang.org/"</p></item> - <tag><c>user_info() = string()</c></tag> + <tag><c>user_info() = string() | unicode:unicode_binary()</c></tag> <item><p></p></item> <tag><c>scheme() = atom()</c></tag> <item><p>Example: http, https</p></item> - <tag><c>host() = string()</c></tag> + <tag><c>host() = string() | unicode:unicode_binary()</c></tag> <item><p></p></item> <tag><c>port() = pos_integer()</c></tag> <item><p></p></item> - <tag><c>path() = string()</c></tag> + <tag><c>path() = string() | unicode:unicode_binary()</c></tag> <item><p>Represents a file path or directory path</p></item> - <tag><c>query() = string()</c></tag> + <tag><c>query() = string() | unicode:unicode_binary()</c></tag> <item><p></p></item> - <tag><c>fragment() = string()</c></tag> + <tag><c>fragment() = string() | unicode:unicode_binary()</c></tag> <item><p></p></item> </taglist> @@ -83,7 +84,7 @@ <fsummary>Decodes a hexadecimal encoded URI.</fsummary> <type> - <v>HexEncodedURI = string() - A possibly hexadecimal encoded URI</v> + <v>HexEncodedURI = string() | unicode:unicode_binary() - A possibly hexadecimal encoded URI</v> <v>URI = uri()</v> </type> @@ -98,7 +99,7 @@ <fsummary>Encodes a hexadecimal encoded URI.</fsummary> <type> <v>URI = uri()</v> - <v>HexEncodedURI = string() - Hexadecimal encoded URI</v> + <v>HexEncodedURI = string() | unicode:unicode_binary() - Hexadecimal encoded URI</v> </type> <desc> @@ -145,7 +146,7 @@ <p>Scheme validation fun is to be defined as follows:</p> <code> -fun(SchemeStr :: string()) -> +fun(SchemeStr :: string() | unicode:unicode_binary()) -> valid | {error, Reason :: term()}. </code> diff --git a/lib/inets/doc/src/httpc.xml b/lib/inets/doc/src/httpc.xml index 4217b3c4fb..29e4b22632 100644 --- a/lib/inets/doc/src/httpc.xml +++ b/lib/inets/doc/src/httpc.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2004</year><year>2016</year> + <year>2004</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -344,7 +344,7 @@ <tag><c><![CDATA[ssl]]></c></tag> <item> <p>This is the <c>SSL/TLS</c> connectin configuration option.</p> - <p>Defaults to <c>[]</c>. See <seealso marker="ssl:ssl">ssl:connect/[2, 3,4]</seealso> for availble options.</p> + <p>Defaults to <c>[]</c>. See <seealso marker="ssl:ssl">ssl:connect/[2,3,4]</seealso> for available options.</p> </item> <tag><c><![CDATA[autoredirect]]></c></tag> @@ -408,7 +408,7 @@ <c>{self, once}</c>, the first message has an extra element, that is, <c>{http, {RequestId, stream_start, Headers, Pid}}</c>. This is the process id to be used as an argument to - <c>http:stream_next/1</c> to trigger the next message to be sent to + <c>httpc:stream_next/1</c> to trigger the next message to be sent to the calling process.</p> <p>Notice that chunked encoding can add headers so that there are more headers in the <c>stream_end</c> diff --git a/lib/inets/doc/src/notes.xml b/lib/inets/doc/src/notes.xml index e102ad4826..2f4f20347a 100644 --- a/lib/inets/doc/src/notes.xml +++ b/lib/inets/doc/src/notes.xml @@ -33,7 +33,98 @@ <file>notes.xml</file> </header> - <section><title>Inets 6.3.6</title> + <section><title>Inets 6.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + httpd_util:rfc1123_date/1 gracefully handle invalid DST + dates by returning the original time in the expected + rfc1123 format.</p> + <p> + Own Id: OTP-14394</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Add unicode binary support to http_uri functions</p> + <p> + Own Id: OTP-14404</p> + </item> + <item> + <p> + httpc - Change timeout handling so the redirects cause a + new timer to be set. This means that a simple redirected + request could return after 2*timeout milliseconds.</p> + <p> + Own Id: OTP-14429</p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 6.3.9</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The close of a chunked file reception crashed in a + certain timing sequence.</p> + <p> + Own Id: OTP-14391 Aux Id: seq13306 </p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 6.3.8</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Added missing release note for inets-6.3.7</p> + <p> + Own Id: OTP-14383</p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 6.3.7</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed a bug in ftp that made further operations after a + recv_chunk operation impossible.</p> + <p> + Own Id: OTP-14242</p> + </item> + + <item> + <p>Make default port, 80 and 443, implicit in automatic redirection. + </p> + <p> Own Id: OTP-14301 + </p> + </item> + </list> + </section> + +</section> + +<section><title>Inets 6.3.6</title> <section><title>Fixed Bugs and Malfunctions</title> <list> @@ -345,6 +436,21 @@ </section> + <section><title>Inets 6.1.1.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Mend ipv6_host_with_brackets option in httpc</p> + <p> + Own Id: OTP-13417</p> + </item> + </list> + </section> + + </section> + <section><title>Inets 6.1.1</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -751,7 +857,7 @@ <list> <item> <p> - Gracefully handle invalid content-lenght headers instead + Gracefully handle invalid content-length headers instead of crashing in list_to_integer.</p> <p> Own Id: OTP-12429</p> diff --git a/lib/inets/src/ftp/ftp.erl b/lib/inets/src/ftp/ftp.erl index 9d0c59eda8..e0430654eb 100644 --- a/lib/inets/src/ftp/ftp.erl +++ b/lib/inets/src/ftp/ftp.erl @@ -100,6 +100,12 @@ ftp_extension = ?FTP_EXT_DEFAULT }). +-record(recv_chunk_closing, { + dconn_closed = false, + pos_compl_received = false, + client_called_us = false + }). + -type shortage_reason() :: 'etnospc' | 'epnospc'. -type restriction_reason() :: 'epath' | 'efnamena' | 'elogin' | 'enotbinary'. @@ -108,7 +114,7 @@ -define(DBG(F,A), 'n/a'). %%-define(DBG(F,A), io:format(F,A)). -%%-define(DBG(F,A), if is_list(F) -> ct:pal(F,A); is_atom(F)->ct:pal(atom_to_list(F),A) end). +%%-define(DBG(F,A), ct:pal("~p:~p " ++ if is_list(F) -> F; is_atom(F) -> atom_to_list(F) end, [?MODULE,?LINE|A])). %%%========================================================================= %%% API - CLIENT FUNCTIONS @@ -1343,6 +1349,25 @@ handle_call({_,{recv_chunk_start, RemoteFile}}, From, #state{chunk = false} handle_call({_, recv_chunk}, _, #state{chunk = false} = State) -> {reply, {error, "ftp:recv_chunk_start/2 not called"}, State}; +handle_call({_, recv_chunk}, _From, #state{chunk = true, + caller = #recv_chunk_closing{dconn_closed = true, + pos_compl_received = true + } + } = State0) -> + %% The ftp:recv_chunk call was the last event we waited for, finnish and clean up + ?DBG("recv_chunk_closing ftp:recv_chunk, last event",[]), + activate_ctrl_connection(State0), + {reply, ok, State0#state{caller = undefined, + chunk = false, + client = undefined}}; + +handle_call({_, recv_chunk}, From, #state{chunk = true, + caller = #recv_chunk_closing{} = R + } = State) -> + %% Waiting for more, don't care what + ?DBG("recv_chunk_closing ftp:recv_chunk, get more",[]), + {noreply, State#state{client = From, caller = R#recv_chunk_closing{client_called_us=true}}}; + handle_call({_, recv_chunk}, From, #state{chunk = true} = State0) -> State = activate_data_connection(State0), {noreply, State#state{client = From, caller = recv_chunk}}; @@ -1480,19 +1505,24 @@ handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, file_close(Fd), progress_report({transfer_size, 0}, State), activate_ctrl_connection(State), + ?DBG("Data channel close",[]), {noreply, State#state{dsock = undefined, data = <<>>}}; -handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, client = From, +handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, + client = Client, caller = recv_chunk} = State) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> - gen_server:reply(From, ok), - {noreply, State#state{dsock = undefined, client = undefined, - data = <<>>, caller = undefined, - chunk = false}}; + ?DBG("Data channel close recv_chunk",[]), + activate_ctrl_connection(State), + {noreply, State#state{dsock = undefined, + caller = #recv_chunk_closing{dconn_closed = true, + client_called_us = Client =/= undefined} + }}; handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, caller = recv_bin, data = Data} = State) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> + ?DBG("Data channel close",[]), activate_ctrl_connection(State), {noreply, State#state{dsock = undefined, data = <<>>, caller = {recv_bin, Data}}}; @@ -1500,6 +1530,7 @@ handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, caller = recv_bin, handle_info({Cls, Socket}, #state{dsock = {Trpt,Socket}, data = Data, caller = {handle_dir_result, Dir}} = State) when {Cls,Trpt}=={tcp_closed,tcp} ; {Cls,Trpt}=={ssl_closed,ssl} -> + ?DBG("Data channel close",[]), activate_ctrl_connection(State), {noreply, State#state{dsock = undefined, caller = {handle_dir_result, Dir, Data}, @@ -1601,13 +1632,13 @@ terminate(normal, State) -> %% If terminate reason =/= normal the progress reporting process will %% be killed by the exit signal. progress_report(stop, State), - do_termiante({error, econn}, State); + do_terminate({error, econn}, State); terminate(Reason, State) -> Report = io_lib:format("Ftp connection closed due to: ~p~n", [Reason]), error_logger:error_report(Report), - do_termiante({error, eclosed}, State). + do_terminate({error, eclosed}, State). -do_termiante(ErrorMsg, State) -> +do_terminate(ErrorMsg, State) -> close_data_connection(State), close_ctrl_connection(State), case State#state.client of @@ -2046,6 +2077,30 @@ handle_ctrl_result({pos_prel, _}, #state{client = From, end; %%-------------------------------------------------------------------------- +%% File handling - chunk_transfer complete + +handle_ctrl_result({pos_compl, _}, #state{client = From, + caller = #recv_chunk_closing{dconn_closed = true, + client_called_us = true, + pos_compl_received = false + }} + = State0) when From =/= undefined -> + %% The pos_compl was the last event we waited for, finnish and clean up + ?DBG("recv_chunk_closing pos_compl, last event",[]), + gen_server:reply(From, ok), + activate_ctrl_connection(State0), + {noreply, State0#state{caller = undefined, + chunk = false, + client = undefined}}; + +handle_ctrl_result({pos_compl, _}, #state{caller = #recv_chunk_closing{}=R} + = State0) -> + %% Waiting for more, don't care what + ?DBG("recv_chunk_closing pos_compl, wait more",[]), + {noreply, State0#state{caller = R#recv_chunk_closing{pos_compl_received=true}}}; + + +%%-------------------------------------------------------------------------- %% File handling - recv_file handle_ctrl_result({pos_prel, _}, #state{caller = {recv_file, _}} = State0) -> case accept_data_connection(State0) of diff --git a/lib/inets/src/http_client/httpc.erl b/lib/inets/src/http_client/httpc.erl index bd5f6df39e..bf2da82603 100644 --- a/lib/inets/src/http_client/httpc.erl +++ b/lib/inets/src/http_client/httpc.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% Copyright Ericsson AB 2009-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -524,7 +524,7 @@ handle_request(Method, Url, Options = request_options(Options0), Sync = proplists:get_value(sync, Options), Stream = proplists:get_value(stream, Options), - Host2 = header_host(Scheme, Host, Port), + Host2 = http_request:normalize_host(Scheme, Host, Port), HeadersRecord = header_record(NewHeaders, Host2, HTTPOptions), Receiver = proplists:get_value(receiver, Options), SocketOpts = proplists:get_value(socket_opts, Options), @@ -1035,14 +1035,6 @@ bad_option(Option, BadValue) -> throw({error, {bad_option, Option, BadValue}}). -header_host(https, Host, 443 = _Port) -> - Host; -header_host(http, Host, 80 = _Port) -> - Host; -header_host(_Scheme, Host, Port) -> - Host ++ ":" ++ integer_to_list(Port). - - header_record(NewHeaders, Host, #http_options{version = Version}) -> header_record(NewHeaders, #http_request_h{}, Host, Version). diff --git a/lib/inets/src/http_client/httpc_handler.erl b/lib/inets/src/http_client/httpc_handler.erl index c99200777b..bd1d2e833a 100644 --- a/lib/inets/src/http_client/httpc_handler.erl +++ b/lib/inets/src/http_client/httpc_handler.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2016. All Rights Reserved. +%% Copyright Ericsson AB 2002-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1175,17 +1175,20 @@ handle_empty_queue(Session, ProfileName, TimeOut, State) -> %% If a pipline | keep_alive session has been idle for some time is not %% closed by the server, the client may want to close it. NewState = activate_queue_timeout(TimeOut, State), - update_session(ProfileName, Session, #session.queue_length, 0), - %% Note mfa will be initialized when a new request - %% arrives. - {noreply, - NewState#state{request = undefined, - mfa = undefined, - status_line = undefined, - headers = undefined, - body = undefined - } - }. + case update_session(ProfileName, Session, #session.queue_length, 0) of + {stop, Reason} -> + {stop, {shutdown, Reason}, State}; + _ -> + %% Note mfa will be initialized when a new request + %% arrives. + {noreply, + NewState#state{request = undefined, + mfa = undefined, + status_line = undefined, + headers = undefined, + body = undefined + }} + end. receive_response(Request, Session, Data, State) -> NewState = init_wait_for_response_state(Request, State), @@ -1224,7 +1227,7 @@ close_socket(#session{socket = Socket, socket_type = SocketType}) -> http_transport:close(SocketType, Socket). activate_request_timeout( - #state{request = #request{timer = undefined} = Request} = State) -> + #state{request = #request{timer = OldRef} = Request} = State) -> Timeout = (Request#request.settings)#http_options.timeout, case Timeout of infinity -> @@ -1232,17 +1235,21 @@ activate_request_timeout( _ -> ReqId = Request#request.id, Msg = {timeout, ReqId}, + case OldRef of + undefined -> + ok; + _ -> + %% Timer is already running! This is the case for a redirect or retry + %% We need to restart the timer because the handler pid has changed + cancel_timer(OldRef, Msg) + end, Ref = erlang:send_after(Timeout, self(), Msg), Request2 = Request#request{timer = Ref}, ReqTimers = [{Request#request.id, Ref} | (State#state.timers)#timers.request_timers], Timers = #timers{request_timers = ReqTimers}, State#state{request = Request2, timers = Timers} - end; - -%% Timer is already running! This is the case for a redirect or retry -activate_request_timeout(State) -> - State. + end. activate_queue_timeout(infinity, State) -> State; @@ -1673,7 +1680,7 @@ update_session(ProfileName, #session{id = SessionId} = Session, Pos, Value) -> Session2 = erlang:setelement(Pos, Session, Value), insert_session(Session2, ProfileName); error:badarg -> - exit(normal); %% Manager has been shutdown + {stop, normal}; T:E -> %% Unexpected this must be an error! Stacktrace = erlang:get_stacktrace(), @@ -1693,14 +1700,14 @@ update_session(ProfileName, #session{id = SessionId} = Session, Pos, Value) -> Session, (catch httpc_manager:lookup_session(SessionId, ProfileName)), T, E]), - exit({failed_updating_session, - [{profile, ProfileName}, - {session_id, SessionId}, - {pos, Pos}, - {value, Value}, - {etype, T}, - {error, E}, - {stacktrace, Stacktrace}]}) + {stop, {failed_updating_session, + [{profile, ProfileName}, + {session_id, SessionId}, + {pos, Pos}, + {value, Value}, + {etype, T}, + {error, E}, + {stacktrace, Stacktrace}]}} end. diff --git a/lib/inets/src/http_client/httpc_response.erl b/lib/inets/src/http_client/httpc_response.erl index 0fd5faa466..b3b11b74ab 100644 --- a/lib/inets/src/http_client/httpc_response.erl +++ b/lib/inets/src/http_client/httpc_response.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -362,8 +362,9 @@ redirect(Response = {StatusLine, Headers, Body}, Request) -> {ok, error(Request, Reason), Data}; %% Automatic redirection {ok, {Scheme, _, Host, Port, Path, Query}} -> + HostPort = http_request:normalize_host(Scheme, Host, Port), NewHeaders = - (Request#request.headers)#http_request_h{host = Host++":"++integer_to_list(Port)}, + (Request#request.headers)#http_request_h{host = HostPort}, NewRequest = Request#request{redircount = Request#request.redircount+1, @@ -434,7 +435,7 @@ format_response({StatusLine, Headers, Body}) -> Length = list_to_integer(Headers#http_response_h.'content-length'), {NewBody, Data} = case Length of - -1 -> % When no lenght indicator is provided + -1 -> % When no length indicator is provided {Body, <<>>}; Length when (Length =< size(Body)) -> <<BodyThisReq:Length/binary, Next/binary>> = Body, diff --git a/lib/inets/src/http_lib/http_request.erl b/lib/inets/src/http_lib/http_request.erl index c77b616f0d..f68b233e10 100644 --- a/lib/inets/src/http_lib/http_request.erl +++ b/lib/inets/src/http_lib/http_request.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2015. All Rights Reserved. +%% Copyright Ericsson AB 2005-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ -include("http_internal.hrl"). --export([headers/2, http_headers/1, is_absolut_uri/1, key_value/1]). +-export([headers/2, http_headers/1, is_absolut_uri/1, key_value/1, normalize_host/3]). key_value(KeyValueStr) -> @@ -85,6 +85,22 @@ is_absolut_uri("https://" ++ _) -> is_absolut_uri(_) -> false. +%%------------------------------------------------------------------------- +%% normalize_host(Scheme, Host, Port) -> string() +%% Scheme - http | https +%% Host - string() +%% Port - integer() +%% +%% Description: returns a normalized Host header value, with the port +%% number omitted for well-known ports +%%------------------------------------------------------------------------- +normalize_host(https, Host, 443 = _Port) -> + Host; +normalize_host(http, Host, 80 = _Port) -> + Host; +normalize_host(_Scheme, Host, Port) -> + Host ++ ":" ++ integer_to_list(Port). + %%%======================================================================== %%% Internal functions %%%======================================================================== diff --git a/lib/inets/src/http_lib/http_uri.erl b/lib/inets/src/http_lib/http_uri.erl index cb3e107ccf..c4be5abd7c 100644 --- a/lib/inets/src/http_lib/http_uri.erl +++ b/lib/inets/src/http_lib/http_uri.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2016. All Rights Reserved. +%% Copyright Ericsson AB 2006-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -102,16 +102,23 @@ parse(AbsURI, Opts) -> reserved() -> sets:from_list([$;, $:, $@, $&, $=, $+, $,, $/, $?, - $#, $[, $], $<, $>, $\", ${, $}, $|, + $#, $[, $], $<, $>, $\", ${, $}, $|, %" $\\, $', $^, $%, $ ]). -encode(URI) -> +encode(URI) when is_list(URI) -> Reserved = reserved(), - lists:append([uri_encode(Char, Reserved) || Char <- URI]). + lists:append([uri_encode(Char, Reserved) || Char <- URI]); +encode(URI) when is_binary(URI) -> + Reserved = reserved(), + << <<(uri_encode_binary(Char, Reserved))/binary>> || <<Char>> <= URI >>. -decode(String) -> - do_decode(String). +decode(String) when is_list(String) -> + do_decode(String); +decode(String) when is_binary(String) -> + do_decode_binary(String). +do_decode([$+|Rest]) -> + [$ |do_decode(Rest)]; do_decode([$%,Hex1,Hex2|Rest]) -> [hex2dec(Hex1)*16+hex2dec(Hex2)|do_decode(Rest)]; do_decode([First|Rest]) -> @@ -119,6 +126,14 @@ do_decode([First|Rest]) -> do_decode([]) -> []. +do_decode_binary(<<$+, Rest/bits>>) -> + <<$ , (do_decode_binary(Rest))/binary>>; +do_decode_binary(<<$%, Hex:2/binary, Rest/bits>>) -> + <<(binary_to_integer(Hex, 16)), (do_decode_binary(Rest))/binary>>; +do_decode_binary(<<First:1/binary, Rest/bits>>) -> + <<First/binary, (do_decode_binary(Rest))/binary>>; +do_decode_binary(<<>>) -> + <<>>. %%%======================================================================== %%% Internal functions @@ -162,9 +177,30 @@ extract_scheme(Str, Opts) -> {error, Error} end; _ -> - {ok, list_to_atom(http_util:to_lower(Str))} + {ok, to_atom(http_util:to_lower(Str))} end. +to_atom(S) when is_list(S) -> + list_to_atom(S); +to_atom(S) when is_binary(S) -> + binary_to_atom(S, unicode). + +parse_uri_rest(Scheme, DefaultPort, <<"//", URIPart/binary>>, Opts) -> + {Authority, PathQueryFragment} = + split_uri(URIPart, "[/?#]", {URIPart, <<"">>}, 1, 0), + {RawPath, QueryFragment} = + split_uri(PathQueryFragment, "[?#]", {PathQueryFragment, <<"">>}, 1, 0), + {Query, Fragment} = + split_uri(QueryFragment, "#", {QueryFragment, <<"">>}, 1, 0), + {UserInfo, HostPort} = split_uri(Authority, "@", {<<"">>, Authority}, 1, 1), + {Host, Port} = parse_host_port(Scheme, DefaultPort, HostPort, Opts), + Path = path(RawPath), + case lists:keyfind(fragment, 1, Opts) of + {fragment, true} -> + {ok, {Scheme, UserInfo, Host, Port, Path, Query, Fragment}}; + _ -> + {ok, {Scheme, UserInfo, Host, Port, Path, Query}} + end; parse_uri_rest(Scheme, DefaultPort, "//" ++ URIPart, Opts) -> {Authority, PathQueryFragment} = split_uri(URIPart, "[/?#]", {URIPart, ""}, 1, 0), @@ -185,6 +221,11 @@ parse_uri_rest(Scheme, DefaultPort, "//" ++ URIPart, Opts) -> %% In this version of the function, we no longer need %% the Scheme argument, but just in case... +parse_host_port(_Scheme, DefaultPort, <<"[", HostPort/binary>>, Opts) -> %ipv6 + {Host, ColonPort} = split_uri(HostPort, "\\]", {HostPort, <<"">>}, 1, 1), + Host2 = maybe_ipv6_host_with_brackets(Host, Opts), + {_, Port} = split_uri(ColonPort, ":", {<<"">>, DefaultPort}, 0, 1), + {Host2, int_port(Port)}; parse_host_port(_Scheme, DefaultPort, "[" ++ HostPort, Opts) -> %ipv6 {Host, ColonPort} = split_uri(HostPort, "\\]", {HostPort, ""}, 1, 1), Host2 = maybe_ipv6_host_with_brackets(Host, Opts), @@ -198,12 +239,19 @@ parse_host_port(_Scheme, DefaultPort, HostPort, _Opts) -> split_uri(UriPart, SplitChar, NoMatchResult, SkipLeft, SkipRight) -> case re:run(UriPart, SplitChar, [{capture, first}]) of {match, [{Match, _}]} -> - {string:substr(UriPart, 1, Match + 1 - SkipLeft), - string:substr(UriPart, Match + 1 + SkipRight, length(UriPart))}; + {string:slice(UriPart, 0, Match + 1 - SkipLeft), + string:slice(UriPart, Match + SkipRight, string:length(UriPart))}; nomatch -> NoMatchResult end. +maybe_ipv6_host_with_brackets(Host, Opts) when is_binary(Host) -> + case lists:keysearch(ipv6_host_with_brackets, 1, Opts) of + {value, {ipv6_host_with_brackets, true}} -> + <<"[", Host/binary, "]">>; + _ -> + Host + end; maybe_ipv6_host_with_brackets(Host, Opts) -> case lists:keysearch(ipv6_host_with_brackets, 1, Opts) of {value, {ipv6_host_with_brackets, true}} -> @@ -212,15 +260,18 @@ maybe_ipv6_host_with_brackets(Host, Opts) -> Host end. - int_port(Port) when is_integer(Port) -> Port; +int_port(Port) when is_binary(Port) -> + binary_to_integer(Port); int_port(Port) when is_list(Port) -> list_to_integer(Port); %% This is the case where no port was found and there was no default port int_port(no_default_port) -> throw({error, no_default_port}). +path(<<"">>) -> + <<"/">>; path("") -> "/"; path(Path) -> @@ -234,6 +285,14 @@ uri_encode(Char, Reserved) -> [Char] end. +uri_encode_binary(Char, Reserved) -> + case sets:is_element(Char, Reserved) of + true -> + << $%, (integer_to_binary(Char, 16))/binary >>; + false -> + <<Char>> + end. + hex2dec(X) when (X>=$0) andalso (X=<$9) -> X-$0; hex2dec(X) when (X>=$A) andalso (X=<$F) -> X-$A+10; hex2dec(X) when (X>=$a) andalso (X=<$f) -> X-$a+10. diff --git a/lib/inets/src/http_lib/http_util.erl b/lib/inets/src/http_lib/http_util.erl index 20c342dd66..487d04f7aa 100644 --- a/lib/inets/src/http_lib/http_util.erl +++ b/lib/inets/src/http_lib/http_util.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2016. All Rights Reserved. +%% Copyright Ericsson AB 2005-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -35,10 +35,10 @@ %%% Internal application API %%%========================================================================= to_upper(Str) -> - string:to_upper(Str). + string:uppercase(Str). to_lower(Str) -> - string:to_lower(Str). + string:lowercase(Str). %% Example: Mon, 09-Dec-2002 13:46:00 GMT convert_netscapecookie_date([_D,_A,_Y, $,, $ , diff --git a/lib/inets/src/http_server/httpd_util.erl b/lib/inets/src/http_server/httpd_util.erl index a647f04ddc..4a2eff4770 100644 --- a/lib/inets/src/http_server/httpd_util.erl +++ b/lib/inets/src/http_server/httpd_util.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2016. All Rights Reserved. +%% Copyright Ericsson AB 1997-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -333,7 +333,9 @@ rfc1123_date(LocalTime) -> {{YYYY,MM,DD},{Hour,Min,Sec}} = case calendar:local_time_to_universal_time_dst(LocalTime) of [Gmt] -> Gmt; - [_,Gmt] -> Gmt + [_,Gmt] -> Gmt; + % Should not happen, but handle the empty list to prevent an error. + [] -> LocalTime end, DayNumber = calendar:day_of_the_week({YYYY,MM,DD}), lists:flatten( diff --git a/lib/inets/src/http_server/mod_disk_log.erl b/lib/inets/src/http_server/mod_disk_log.erl index 3be5f2dd74..2023546f01 100644 --- a/lib/inets/src/http_server/mod_disk_log.erl +++ b/lib/inets/src/http_server/mod_disk_log.erl @@ -363,17 +363,21 @@ create_disk_log(Filename, MaxBytes, MaxFiles, ConfigList) -> %%---------------------------------------------------------------------- open(Filename, MaxBytes, MaxFiles, internal) -> - Opts = [{format, internal}, {repair, truncate}], - open1(Filename, MaxBytes, MaxFiles, Opts); + Opt0 = {format, internal}, + Opts1 = [Opt0, {repair, true}], + Opts2 = [Opt0, {repair, truncate}], + open1(Filename, MaxBytes, MaxFiles, Opts1, Opts2); open(Filename, MaxBytes, MaxFiles, _) -> Opts = [{format, external}], - open1(Filename, MaxBytes, MaxFiles, Opts). + open1(Filename, MaxBytes, MaxFiles, Opts, Opts). -open1(Filename, MaxBytes, MaxFiles, Opts0) -> - Opts1 = [{name, Filename}, {file, Filename}, {type, wrap}] ++ Opts0, - case open2(Opts1, {MaxBytes, MaxFiles}) of +open1(Filename, MaxBytes, MaxFiles, Opts1, Opts2) -> + Opts0 = [{name, Filename}, {file, Filename}, {type, wrap}], + case open2(Opts0 ++ Opts1, Opts0 ++ Opts2, {MaxBytes, MaxFiles}) of {ok, LogDB} -> {ok, LogDB}; + {repaired, LogDB, {recovered, _}, {badbytes, _}} -> + {ok, LogDB}; {error, Reason} -> {error, ?NICE("Can't create " ++ Filename ++ @@ -382,11 +386,16 @@ open1(Filename, MaxBytes, MaxFiles, Opts0) -> {error, ?NICE("Can't create "++Filename)} end. -open2(Opts, Size) -> - case disk_log:open(Opts) of +open2(Opts1, Opts2, Size) -> + case disk_log:open(Opts1) of {error, {badarg, size}} -> %% File did not exist, add the size option and try again - disk_log:open([{size, Size} | Opts]); + disk_log:open([{size, Size} | Opts1]); + {error, {Reason, _}} when + Reason == not_a_log_file; + Reason == invalid_index_file -> + %% File was corrupt, add the truncate option and try again + disk_log:open([{size, Size} | Opts2]); Else -> Else end. diff --git a/lib/inets/src/http_server/mod_log.erl b/lib/inets/src/http_server/mod_log.erl index ad7e9713d9..ec570504be 100644 --- a/lib/inets/src/http_server/mod_log.erl +++ b/lib/inets/src/http_server/mod_log.erl @@ -105,8 +105,8 @@ do(Info) -> Code = proplists:get_value(code,Head,unknown), transfer_log(Info, "-", AuthUser, Date, Code, Size), {proceed, Info#mod.data}; - {_StatusCode, Response} -> - transfer_log(Info,"-",AuthUser,Date,200, + {StatusCode, Response} -> + transfer_log(Info, "-", AuthUser, Date, StatusCode, httpd_util:flatlength(Response)), {proceed,Info#mod.data}; undefined -> diff --git a/lib/inets/test/ftp_SUITE.erl b/lib/inets/test/ftp_SUITE.erl index e2dec0c42a..3dfec01ba2 100644 --- a/lib/inets/test/ftp_SUITE.erl +++ b/lib/inets/test/ftp_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -93,8 +93,11 @@ ftp_tests()-> append_chunk, recv, recv_3, - recv_bin, + recv_bin, + recv_bin_twice, recv_chunk, + recv_chunk_twice, + recv_chunk_three_times, type, quote, error_elogin, @@ -191,9 +194,22 @@ end_per_suite(Config) -> ok. %%-------------------------------------------------------------------- -init_per_group(_Group, Config) -> Config. - -end_per_group(_Group, Config) -> Config. +init_per_group(Group, Config) when Group == ftps_active, + Group == ftps_passive -> + catch crypto:stop(), + try crypto:start() of + ok -> + Config + catch + _:_ -> + {skip, "Crypto did not start"} + end; + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, Config) -> + Config. %%-------------------------------------------------------------------- init_per_testcase(Case, Config0) -> @@ -533,15 +549,19 @@ append_chunk(Config0) -> recv() -> [{doc, "Receive a file using recv/2"}]. recv(Config0) -> - File = "f_dst.txt", + File1 = "f_dst1.txt", + File2 = "f_dst2.txt", SrcDir = "a_dir", - Contents = <<"ftp_SUITE test ...">>, - Config = set_state([reset, {mkfile,[SrcDir,File],Contents}], Config0), + Contents1 = <<"1 ftp_SUITE test ...">>, + Contents2 = <<"2 ftp_SUITE test ...">>, + Config = set_state([reset, {mkfile,[SrcDir,File1],Contents1}, {mkfile,[SrcDir,File2],Contents2}], Config0), Pid = proplists:get_value(ftp, Config), ok = ftp:cd(Pid, id2ftp(SrcDir,Config)), ok = ftp:lcd(Pid, id2ftp("",Config)), - ok = ftp:recv(Pid, File), - chk_file(File, Contents, Config), + ok = ftp:recv(Pid, File1), + chk_file(File1, Contents1, Config), + ok = ftp:recv(Pid, File2), + chk_file(File2, Contents2, Config), {error,epath} = ftp:recv(Pid, "non_existing_file"), ok. @@ -572,6 +592,25 @@ recv_bin(Config0) -> ok. %%------------------------------------------------------------------------- +recv_bin_twice() -> + [{doc, "Receive two files as a binaries."}]. +recv_bin_twice(Config0) -> + File1 = "f_dst1.txt", + File2 = "f_dst2.txt", + Contents1 = <<"1 ftp_SUITE test ...">>, + Contents2 = <<"2 ftp_SUITE test ...">>, + Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}], Config0), + ct:log("First transfer",[]), + Pid = proplists:get_value(ftp, Config), + {ok,Received1} = ftp:recv_bin(Pid, id2ftp(File1,Config)), + find_diff(Received1, Contents1), + ct:log("Second transfer",[]), + {ok,Received2} = ftp:recv_bin(Pid, id2ftp(File2,Config)), + find_diff(Received2, Contents2), + ct:log("Transfers ready!",[]), + {error,epath} = ftp:recv_bin(Pid, id2ftp("non_existing_file",Config)), + ok. +%%------------------------------------------------------------------------- recv_chunk() -> [{doc, "Receive a file using chunk-wise."}]. recv_chunk(Config0) -> @@ -584,13 +623,74 @@ recv_chunk(Config0) -> {ok, ReceivedContents, _Ncunks} = recv_chunk(Pid, <<>>), find_diff(ReceivedContents, Contents). +recv_chunk_twice() -> + [{doc, "Receive two files using chunk-wise."}]. +recv_chunk_twice(Config0) -> + File1 = "big_file1.txt", + File2 = "big_file2.txt", + Contents1 = list_to_binary( lists:duplicate(1000, lists:seq(0,255)) ), + Contents2 = crypto:strong_rand_bytes(1200), + Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}], Config0), + Pid = proplists:get_value(ftp, Config), + {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), + {ok, ReceivedContents1, _Ncunks1} = recv_chunk(Pid, <<>>), + ok = ftp:recv_chunk_start(Pid, id2ftp(File2,Config)), + {ok, ReceivedContents2, _Ncunks2} = recv_chunk(Pid, <<>>), + find_diff(ReceivedContents1, Contents1), + find_diff(ReceivedContents2, Contents2). + +recv_chunk_three_times() -> + [{doc, "Receive two files using chunk-wise."}, + {timetrap,{seconds,120}}]. +recv_chunk_three_times(Config0) -> + File1 = "big_file1.txt", + File2 = "big_file2.txt", + File3 = "big_file3.txt", + Contents1 = list_to_binary( lists:duplicate(1000, lists:seq(0,255)) ), + Contents2 = crypto:strong_rand_bytes(1200), + Contents3 = list_to_binary( lists:duplicate(1000, lists:seq(255,0,-1)) ), + + Config = set_state([reset, {mkfile,File1,Contents1}, {mkfile,File2,Contents2}, {mkfile,File3,Contents3}], Config0), + Pid = proplists:get_value(ftp, Config), + {{error, "ftp:recv_chunk_start/2 not called"},_} = recv_chunk(Pid, <<>>), + + ok = ftp:recv_chunk_start(Pid, id2ftp(File1,Config)), + {ok, ReceivedContents1, Nchunks1} = recv_chunk(Pid, <<>>), + + ok = ftp:recv_chunk_start(Pid, id2ftp(File2,Config)), + {ok, ReceivedContents2, _Nchunks2} = recv_chunk(Pid, <<>>), + + ok = ftp:recv_chunk_start(Pid, id2ftp(File3,Config)), + {ok, ReceivedContents3, _Nchunks3} = recv_chunk(Pid, <<>>, 10000, 0, Nchunks1), + + find_diff(ReceivedContents1, Contents1), + find_diff(ReceivedContents2, Contents2), + find_diff(ReceivedContents3, Contents3). + + + recv_chunk(Pid, Acc) -> - recv_chunk(Pid, Acc, 0). + recv_chunk(Pid, Acc, 0, 0, undefined). + + + +%% ExpectNchunks :: integer() | undefined +recv_chunk(Pid, Acc, DelayMilliSec, N, ExpectNchunks) when N+1 < ExpectNchunks -> + %% for all I in integer(), I < undefined + recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks); + +recv_chunk(Pid, Acc, DelayMilliSec, N, ExpectNchunks) -> + %% N >= ExpectNchunks-1 + timer:sleep(DelayMilliSec), + recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks). + -recv_chunk(Pid, Acc, N) -> +recv_chunk1(Pid, Acc, DelayMilliSec, N, ExpectNchunks) -> + ct:log("Call ftp:recv_chunk",[]), case ftp:recv_chunk(Pid) of ok -> {ok, Acc, N}; - {ok, Bin} -> recv_chunk(Pid, <<Acc/binary, Bin/binary>>, N+1); + {ok, Bin} -> recv_chunk(Pid, <<Acc/binary, Bin/binary>>, DelayMilliSec, N+1, ExpectNchunks); Error -> {Error, N} end. diff --git a/lib/inets/test/httpc_SUITE.erl b/lib/inets/test/httpc_SUITE.erl index 8aea38037d..e6dcd2285f 100644 --- a/lib/inets/test/httpc_SUITE.erl +++ b/lib/inets/test/httpc_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -108,6 +108,7 @@ only_simulated() -> tolerate_missing_CR, userinfo, bad_response, + timeout_redirect, internal_server_error, invalid_http, invalid_chunk_size, @@ -163,21 +164,17 @@ init_per_group(misc = Group, Config) -> ok = httpc:set_options([{ipfamily, Inet}]), Config; + init_per_group(Group, Config0) when Group =:= sim_https; Group =:= https-> - ct:timetrap({seconds, 30}), - start_apps(Group), - StartSsl = try ssl:start() + catch crypto:stop(), + try crypto:start() of + ok -> + ct:timetrap({seconds, 30}), + start_apps(Group), + do_init_per_group(Group, Config0) catch - Error:Reason -> - {skip, lists:flatten(io_lib:format("Failed to start apps for https Error=~p Reason=~p", [Error, Reason]))} - end, - case StartSsl of - {error, {already_started, _}} -> - do_init_per_group(Group, Config0); - ok -> - do_init_per_group(Group, Config0); - _ -> - StartSsl + _:_ -> + {skip, "Crypto did not start"} end; init_per_group(Group, Config0) -> @@ -789,6 +786,14 @@ bad_response(Config) when is_list(Config) -> ct:print("Wrong Statusline: ~p~n", [Reason]). %%------------------------------------------------------------------------- +timeout_redirect() -> + [{doc, "Test that timeout works for redirects, check ERL-420."}]. +timeout_redirect(Config) when is_list(Config) -> + URL = url(group_name(Config), "/redirect_to_missing_crlf.html", Config), + {error, timeout} = httpc:request(get, {URL, []}, [{timeout, 400}], []). + +%%------------------------------------------------------------------------- + internal_server_error(doc) -> ["Test 50X codes"]; internal_server_error(Config) when is_list(Config) -> @@ -1919,6 +1924,16 @@ handle_uri(_,"/missing_crlf.html",_,_,_,_) -> "Content-Length:32\r\n" ++ "<HTML><BODY>foobar</BODY></HTML>"; +handle_uri(_,"/redirect_to_missing_crlf.html",Port,_,Socket,_) -> + NewUri = url_start(Socket) ++ + integer_to_list(Port) ++ "/missing_crlf.html", + Body = "<HTML><BODY><a href=" ++ NewUri ++ + ">New place</a></BODY></HTML>", + "HTTP/1.1 303 See Other \r\n" ++ + "Location:" ++ NewUri ++ "\r\n" ++ + "Content-Length:" ++ integer_to_list(length(Body)) + ++ "\r\n\r\n" ++ Body; + handle_uri(_,"/wrong_statusline.html",_,_,_,_) -> "ok 200 HTTP/1.1\r\n\r\n" ++ "Content-Length:32\r\n\r\n" ++ diff --git a/lib/inets/test/httpd_1_1.erl b/lib/inets/test/httpd_1_1.erl index 3755ed117b..ce9f7acc4d 100644 --- a/lib/inets/test/httpd_1_1.erl +++ b/lib/inets/test/httpd_1_1.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2016. All Rights Reserved. +%% Copyright Ericsson AB 2005-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -405,11 +405,11 @@ getRangeSize(Head)-> {multiPart, BoundaryString}; _X1 -> case re:run(Head, ?CONTENT_RANGE "bytes=.*\r\n", [{capture, first}]) of - {match, [{Start, Lenght}]} -> + {match, [{Start, Length}]} -> %% Get the range data remove the fieldname and the %% end of line. RangeInfo = string:substr(Head, Start + 1 + 20, - Lenght - (20 +2)), + Length - (20 +2)), rangeSize(string:strip(RangeInfo)); _X2 -> error diff --git a/lib/inets/test/httpd_SUITE.erl b/lib/inets/test/httpd_SUITE.erl index aae4ce5256..b4f0f2aa7d 100644 --- a/lib/inets/test/httpd_SUITE.erl +++ b/lib/inets/test/httpd_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2016. All Rights Reserved. +%% Copyright Ericsson AB 2013-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -73,6 +73,7 @@ all() -> {group, http_reload}, {group, https_reload}, {group, http_mime_types}, + {group, http_logging}, mime_types_format ]. @@ -96,6 +97,7 @@ groups() -> {https_htaccess, [], [{group, htaccess}]}, {http_security, [], [{group, security}]}, {https_security, [], [{group, security}]}, + {http_logging, [], [{group, logging}]}, {http_reload, [], [{group, reload}]}, {https_reload, [], [{group, reload}]}, {http_mime_types, [], [alias_1_1, alias_1_0, alias_0_9]}, @@ -119,6 +121,8 @@ groups() -> ]}, {htaccess, [], [htaccess_1_1, htaccess_1_0, htaccess_0_9]}, {security, [], [security_1_1, security_1_0]}, %% Skip 0.9 as causes timing issus in test code + {logging, [], [disk_log_internal, disk_log_exists, + disk_log_bad_size, disk_log_bad_file]}, {http_1_1, [], [host, chunked, expect, cgi, cgi_chunked_encoding_test, trace, range, if_modified_since, mod_esi_chunk_timeout, @@ -197,7 +201,14 @@ init_per_group(Group, Config0) when Group == https_basic; Group == https_security; Group == https_reload -> - init_ssl(Group, Config0); + catch crypto:stop(), + try crypto:start() of + ok -> + init_ssl(Group, Config0) + catch + _:_ -> + {skip, "Crypto did not start"} + end; init_per_group(Group, Config0) when Group == http_basic; Group == http_limit; Group == http_custom; @@ -232,7 +243,14 @@ init_per_group(https_htaccess = Group, Config) -> Path = proplists:get_value(doc_root, Config), catch remove_htaccess(Path), create_htaccess_data(Path, proplists:get_value(address, Config)), - init_ssl(Group, Config); + catch crypto:stop(), + try crypto:start() of + ok -> + init_ssl(Group, Config) + catch + _:_ -> + {skip, "Crypto did not start"} + end; init_per_group(auth_api, Config) -> [{auth_prefix, ""} | Config]; init_per_group(auth_api_dets, Config) -> @@ -240,6 +258,11 @@ init_per_group(auth_api_dets, Config) -> init_per_group(auth_api_mnesia, Config) -> start_mnesia(proplists:get_value(node, Config)), [{auth_prefix, "mnesia_"} | Config]; +init_per_group(http_logging, Config) -> + Config1 = [{http_version, "HTTP/1.1"} | Config], + ServerRoot = proplists:get_value(server_root, Config1), + Path = ServerRoot ++ "/httpd_log_transfer", + [{transfer_log, Path} | Config1]; init_per_group(_, Config) -> Config. @@ -296,10 +319,60 @@ init_per_testcase(range, Config) -> create_range_data(DocRoot), dbg(range, Config, init); +init_per_testcase(disk_log_internal, Config0) -> + ok = start_apps(http_logging), + Config1 = init_httpd(http_logging, [{type, ip_comm} | Config0]), + ct:timetrap({seconds, 20}), + dbg(disk_log_internal, Config1, init); + +init_per_testcase(disk_log_exists, Config0) -> + ServerRoot = proplists:get_value(server_root, Config0), + Filename = ServerRoot ++ "/httpd_log_transfer", + {ok, Log} = disk_log:open([{name, Filename}, {file, Filename}, + {repair, truncate}, {format, internal}, + {type, wrap}, {size, {1048576, 5}}]), + ok = disk_log:log(Log, {bogus, node(), self()}), + ok = disk_log:close(Log), + ok = start_apps(http_logging), + Config1 = init_httpd(http_logging, [{type, ip_comm} | Config0]), + ct:timetrap({seconds, 20}), + dbg(disk_log_internal, Config1, init); + +init_per_testcase(disk_log_bad_size, Config0) -> + ServerRoot = proplists:get_value(server_root, Config0), + Filename = ServerRoot ++ "/httpd_log_transfer", + {ok, Log} = disk_log:open([{name, Filename}, {file, Filename}, + {repair, truncate}, {format, internal}, + {type, wrap}, {size, {1048576, 5}}]), + ok = disk_log:log(Log, {bogus, node(), self()}), + ok = disk_log:close(Log), + ok = file:delete(Filename ++ ".siz"), + ok = start_apps(http_logging), + Config1 = init_httpd(http_logging, [{type, ip_comm} | Config0]), + ct:timetrap({seconds, 20}), + dbg(disk_log_internal, Config1, init); + +init_per_testcase(disk_log_bad_file, Config0) -> + ServerRoot = proplists:get_value(server_root, Config0), + Filename = ServerRoot ++ "/httpd_log_transfer", + ok = file:write_file(Filename ++ ".1", <<>>), + ok = start_apps(http_logging), + Config1 = init_httpd(http_logging, [{type, ip_comm} | Config0]), + ct:timetrap({seconds, 20}), + dbg(disk_log_internal, Config1, init); + init_per_testcase(Case, Config) -> ct:timetrap({seconds, 20}), dbg(Case, Config, init). +end_per_testcase(Case, Config) when + Case == disk_log_internal; + Case == disk_log_exists; + Case == disk_log_bad_size; + Case == disk_log_bad_file -> + inets:stop(), + dbg(Case, Config, 'end'); + end_per_testcase(Case, Config) -> dbg(Case, Config, 'end'). @@ -1243,6 +1316,63 @@ security(Config) -> true = unblock_user(Node, "two", Port, OpenDir). %%------------------------------------------------------------------------- + +disk_log_internal() -> + ["Test mod_disk_log"]. + +disk_log_internal(Config) -> + Version = proplists:get_value(http_version, Config), + Request = "GET /" ++ integer_to_list(rand:uniform(1000000)) ++ " ", + ok = http_status(Request, Config, [{statuscode, 404}]), + Log = proplists:get_value(transfer_log, Config), + Match = list_to_binary(Request ++ Version), + disk_log_internal1(Log, Match, disk_log:chunk(Log, start)). +disk_log_internal1(_, _, eof) -> + ct:fail(eof); +disk_log_internal1(Log, Match, {Cont, [H | T]}) -> + case binary:match(H, Match) of + nomatch -> + disk_log_internal1(Log, Match, {Cont, T}); + _ -> + ok + end; +disk_log_internal1(Log, Match, {Cont, []}) -> + disk_log_internal1(Log, Match, disk_log:chunk(Log, Cont)). + +disk_log_exists() -> + ["Test mod_disk_log with existing logs"]. + +disk_log_exists(Config) -> + Log = proplists:get_value(transfer_log, Config), + Self = self(), + Node = node(), + Log = proplists:get_value(transfer_log, Config), + {_, [{bogus, Node, Self} | _]} = disk_log:chunk(Log, start). + +disk_log_bad_size() -> + ["Test mod_disk_log with existing log, missing .siz"]. + +disk_log_bad_size(Config) -> + Log = proplists:get_value(transfer_log, Config), + Self = self(), + Node = node(), + Log = proplists:get_value(transfer_log, Config), + {_, [{bogus, Node, Self} | _]} = disk_log:chunk(Log, start). + +disk_log_bad_file() -> + ["Test mod_disk_log with bad file"]. + +disk_log_bad_file(Config) -> + Log = proplists:get_value(transfer_log, Config), + Version = proplists:get_value(http_version, Config), + Request = "GET /" ++ integer_to_list(rand:uniform(1000000)) ++ " ", + ok = http_status(Request, Config, [{statuscode, 404}]), + Log = proplists:get_value(transfer_log, Config), + Match = list_to_binary(Request ++ Version), + {_, [H | _]} = disk_log:chunk(Log, start), + {_, _} = binary:match(H, Match). + +%%------------------------------------------------------------------------- non_disturbing_reconfiger_dies(Config) when is_list(Config) -> do_reconfiger_dies([{http_version, "HTTP/1.1"} | Config], non_disturbing). disturbing_reconfiger_dies(Config) when is_list(Config) -> @@ -1553,6 +1683,7 @@ start_apps(Group) when Group == http_basic; Group == http_auth_api_mnesia; Group == http_htaccess; Group == http_security; + Group == http_logging; Group == http_reload; Group == http_mime_types-> inets_test_lib:start_apps([inets]). @@ -1648,6 +1779,8 @@ server_config(http_security, Config) -> server_config(https_security, Config) -> ServerRoot = proplists:get_value(server_root, Config), tl(auth_conf(ServerRoot)) ++ security_conf(ServerRoot) ++ server_config(https, Config); +server_config(http_logging, Config) -> + log_conf() ++ server_config(http, Config); server_config(http_mime_types, Config0) -> Config1 = basic_conf() ++ server_config(http, Config0), ServerRoot = proplists:get_value(server_root, Config0), @@ -1849,6 +1982,16 @@ mod_security_conf(SecFile, Dir) -> {path, Dir} %% This is should not be needed, but is atm, awful design! ]. +log_conf() -> + [{modules, [mod_alias, mod_dir, mod_get, mod_head, mod_disk_log]}, + {transfer_disk_log, "httpd_log_transfer"}, + {security_disk_log, "httpd_log_security"}, + {error_disk_log, "httpd_log_error"}, + {transfer_disk_log_size, {1048576, 5}}, + {error_disk_log_size, {1048576, 5}}, + {error_disk_log_size, {1048576, 5}}, + {security_disk_log_size, {1048576, 5}}, + {disk_log_format, internal}]. http_status(Request, Config, Expected) -> Version = proplists:get_value(http_version, Config), diff --git a/lib/inets/test/httpd_basic_SUITE.erl b/lib/inets/test/httpd_basic_SUITE.erl index f413248092..931cd076cc 100644 --- a/lib/inets/test/httpd_basic_SUITE.erl +++ b/lib/inets/test/httpd_basic_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2016. All Rights Reserved. +%% Copyright Ericsson AB 2007-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -42,7 +42,8 @@ all() -> escaped_url_in_error_body, script_timeout, slowdose, - keep_alive_timeout + keep_alive_timeout, + invalid_rfc1123_date ]. groups() -> @@ -383,6 +384,16 @@ slowdose(Config) when is_list(Config) -> end. %%------------------------------------------------------------------------- + +invalid_rfc1123_date() -> + [{doc, "Test that a non-DST date is handled correcly"}]. +invalid_rfc1123_date(Config) when is_list(Config) -> + Rfc1123FormattedDate = "Sun, 26 Mar 2017 01:00:00 GMT", + NonDSTDateTime = {{2017, 03, 26},{1, 0, 0}}, + Rfc1123FormattedDate =:= httpd_util:rfc1123_date(NonDSTDateTime). + + +%%------------------------------------------------------------------------- %% Internal functions %%------------------------------------------------------------------------- diff --git a/lib/inets/test/httpd_test_data/server_root/conf/httpd.conf b/lib/inets/test/httpd_test_data/server_root/conf/httpd.conf index 3f9fde03b5..3add93cd73 100644 --- a/lib/inets/test/httpd_test_data/server_root/conf/httpd.conf +++ b/lib/inets/test/httpd_test_data/server_root/conf/httpd.conf @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2016. All Rights Reserved. +# Copyright Ericsson AB 1997-2017. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -128,7 +128,7 @@ SecurityDiskLogSize 200000 10 MaxClients 50 -# KeepAlive set the flag for persistent connections. For peristent connections +# KeepAlive set the flag for persistent connections. For persistent connections # set KeepAlive to on. To use One request per connection set the flag to off # Note: The value has changed since previous version of INETS. KeepAlive on diff --git a/lib/inets/test/old_httpd_SUITE_data/server_root/conf/httpd.conf b/lib/inets/test/old_httpd_SUITE_data/server_root/conf/httpd.conf index 3f9fde03b5..3add93cd73 100644 --- a/lib/inets/test/old_httpd_SUITE_data/server_root/conf/httpd.conf +++ b/lib/inets/test/old_httpd_SUITE_data/server_root/conf/httpd.conf @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2016. All Rights Reserved. +# Copyright Ericsson AB 1997-2017. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -128,7 +128,7 @@ SecurityDiskLogSize 200000 10 MaxClients 50 -# KeepAlive set the flag for persistent connections. For peristent connections +# KeepAlive set the flag for persistent connections. For persistent connections # set KeepAlive to on. To use One request per connection set the flag to off # Note: The value has changed since previous version of INETS. KeepAlive on diff --git a/lib/inets/test/uri_SUITE.erl b/lib/inets/test/uri_SUITE.erl index b26c645821..3e7799141c 100644 --- a/lib/inets/test/uri_SUITE.erl +++ b/lib/inets/test/uri_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ -module(uri_SUITE). +-include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include("inets_test_lib.hrl"). @@ -50,7 +51,8 @@ all() -> fragments, escaped, hexed_query, - scheme_validation + scheme_validation, + encode_decode ]. %%-------------------------------------------------------------------- @@ -73,7 +75,10 @@ end_per_testcase(_Case, _Config) -> ipv4(Config) when is_list(Config) -> {ok, {http,[],"127.0.0.1",80,"/foobar.html",[]}} = - http_uri:parse("http://127.0.0.1/foobar.html"). + http_uri:parse("http://127.0.0.1/foobar.html"), + + {ok, {http,<<>>,<<"127.0.0.1">>,80,<<"/foobar.html">>,<<>>}} = + http_uri:parse(<<"http://127.0.0.1/foobar.html">>). ipv6(Config) when is_list(Config) -> {ok, {http,[],"2010:836B:4179::836B:4179",80,"/foobar.html",[]}} = @@ -89,24 +94,52 @@ ipv6(Config) when is_list(Config) -> [{foo, false}]), {error, {malformed_url, _, "http://2010:836B:4179::836B:4179/foobar.html"}} = - http_uri:parse("http://2010:836B:4179::836B:4179/foobar.html"). + http_uri:parse("http://2010:836B:4179::836B:4179/foobar.html"), + + {ok, {http,<<>>,<<"2010:836B:4179::836B:4179">>,80,<<"/foobar.html">>,<<>>}} = + http_uri:parse(<<"http://[2010:836B:4179::836B:4179]/foobar.html">>), + {ok, {http,<<>>,<<"[2010:836B:4179::836B:4179]">>,80,<<"/foobar.html">>,<<>>}} = + http_uri:parse(<<"http://[2010:836B:4179::836B:4179]/foobar.html">>, + [{ipv6_host_with_brackets, true}]), + {ok, {http,<<>>,<<"2010:836B:4179::836B:4179">>,80,<<"/foobar.html">>,<<>>}} = + http_uri:parse(<<"http://[2010:836B:4179::836B:4179]/foobar.html">>, + [{ipv6_host_with_brackets, false}]), + {ok, {http,<<>>,<<"2010:836B:4179::836B:4179">>,80,<<"/foobar.html">>,<<>>}} = + http_uri:parse(<<"http://[2010:836B:4179::836B:4179]/foobar.html">>, + [{foo, false}]), + {error, + {malformed_url, _, <<"http://2010:836B:4179::836B:4179/foobar.html">>}} = + http_uri:parse(<<"http://2010:836B:4179::836B:4179/foobar.html">>). host(Config) when is_list(Config) -> {ok, {http,[],"localhost",8888,"/foobar.html",[]}} = - http_uri:parse("http://localhost:8888/foobar.html"). + http_uri:parse("http://localhost:8888/foobar.html"), + + {ok, {http,<<>>,<<"localhost">>,8888,<<"/foobar.html">>,<<>>}} = + http_uri:parse(<<"http://localhost:8888/foobar.html">>). userinfo(Config) when is_list(Config) -> {ok, {http,"nisse:foobar","localhost",8888,"/foobar.html",[]}} = - http_uri:parse("http://nisse:foobar@localhost:8888/foobar.html"). + http_uri:parse("http://nisse:foobar@localhost:8888/foobar.html"), + + {ok, {http,<<"nisse:foobar">>,<<"localhost">>,8888,<<"/foobar.html">>,<<>>}} = + http_uri:parse(<<"http://nisse:foobar@localhost:8888/foobar.html">>). scheme(Config) when is_list(Config) -> {error, no_scheme} = http_uri:parse("localhost/foobar.html"), {error, {malformed_url, _, _}} = - http_uri:parse("localhost:8888/foobar.html"). + http_uri:parse("localhost:8888/foobar.html"), + + {error, no_scheme} = http_uri:parse(<<"localhost/foobar.html">>), + {error, {malformed_url, _, _}} = + http_uri:parse(<<"localhost:8888/foobar.html">>). queries(Config) when is_list(Config) -> {ok, {http,[],"localhost",8888,"/foobar.html","?foo=bar&foobar=42"}} = - http_uri:parse("http://localhost:8888/foobar.html?foo=bar&foobar=42"). + http_uri:parse("http://localhost:8888/foobar.html?foo=bar&foobar=42"), + + {ok, {http,<<>>,<<"localhost">>,8888,<<"/foobar.html">>,<<"?foo=bar&foobar=42">>}} = + http_uri:parse(<<"http://localhost:8888/foobar.html?foo=bar&foobar=42">>). fragments(Config) when is_list(Config) -> {ok, {http,[],"localhost",80,"/",""}} = @@ -142,6 +175,41 @@ fragments(Config) when is_list(Config) -> http_uri:parse("http://localhost?query#", [{fragment,true}]), {ok, {http,[],"localhost",80,"/path","?query","#"}} = http_uri:parse("http://localhost/path?query#", [{fragment,true}]), + + + {ok, {http,<<>>,<<"localhost">>,80,<<"/">>,<<"">>}} = + http_uri:parse(<<"http://localhost#fragment">>), + {ok, {http,<<>>,<<"localhost">>,80,<<"/path">>,<<"">>}} = + http_uri:parse(<<"http://localhost/path#fragment">>), + {ok, {http,<<>>,<<"localhost">>,80,<<"/">>,<<"?query">>}} = + http_uri:parse(<<"http://localhost?query#fragment">>), + {ok, {http,<<>>,<<"localhost">>,80,<<"/path">>,<<"?query">>}} = + http_uri:parse(<<"http://localhost/path?query#fragment">>), + {ok, {http,<<>>,<<"localhost">>,80,<<"/">>,<<"">>,<<"#fragment">>}} = + http_uri:parse(<<"http://localhost#fragment">>, [{fragment,true}]), + {ok, {http,<<>>,<<"localhost">>,80,<<"/path">>,<<"">>,<<"#fragment">>}} = + http_uri:parse(<<"http://localhost/path#fragment">>, [{fragment,true}]), + {ok, {http,<<>>,<<"localhost">>,80,<<"/">>,<<"?query">>,<<"#fragment">>}} = + http_uri:parse(<<"http://localhost?query#fragment">>, [{fragment,true}]), + {ok, {http,<<>>,<<"localhost">>,80,<<"/path">>,<<"?query">>,<<"#fragment">>}} = + http_uri:parse(<<"http://localhost/path?query#fragment">>, + [{fragment,true}]), + {ok, {http,<<>>,<<"localhost">>,80,<<"/">>,<<"">>,<<"">>}} = + http_uri:parse(<<"http://localhost">>, [{fragment,true}]), + {ok, {http,<<>>,<<"localhost">>,80,<<"/path">>,<<"">>,<<"">>}} = + http_uri:parse(<<"http://localhost/path">>, [{fragment,true}]), + {ok, {http,<<>>,<<"localhost">>,80,<<"/">>,<<"?query">>,<<"">>}} = + http_uri:parse(<<"http://localhost?query">>, [{fragment,true}]), + {ok, {http,<<>>,<<"localhost">>,80,<<"/path">>,<<"?query">>,<<"">>}} = + http_uri:parse(<<"http://localhost/path?query">>, [{fragment,true}]), + {ok, {http,<<>>,<<"localhost">>,80,<<"/">>,<<"">>,<<"#">>}} = + http_uri:parse(<<"http://localhost#">>, [{fragment,true}]), + {ok, {http,<<>>,<<"localhost">>,80,<<"/path">>,<<"">>,<<"#">>}} = + http_uri:parse(<<"http://localhost/path#">>, [{fragment,true}]), + {ok, {http,<<>>,<<"localhost">>,80,<<"/">>,<<"?query">>,<<"#">>}} = + http_uri:parse(<<"http://localhost?query#">>, [{fragment,true}]), + {ok, {http,<<>>,<<"localhost">>,80,<<"/path">>,<<"?query">>,<<"#">>}} = + http_uri:parse(<<"http://localhost/path?query#">>, [{fragment,true}]), ok. escaped(Config) when is_list(Config) -> @@ -152,7 +220,16 @@ escaped(Config) when is_list(Config) -> {ok, {http,[],"www.somedomain.com",80,"/%25abc",[]}} = http_uri:parse("http://www.somedomain.com/%25abc"), {ok, {http,[],"www.somedomain.com",80,"/%25abc", "?foo=bar"}} = - http_uri:parse("http://www.somedomain.com/%25abc?foo=bar"). + http_uri:parse("http://www.somedomain.com/%25abc?foo=bar"), + + {ok, {http,<<>>,<<"www.somedomain.com">>,80,<<"/%2Eabc">>,<<>>}} = + http_uri:parse(<<"http://www.somedomain.com/%2Eabc">>), + {ok, {http,<<>>,<<"www.somedomain.com">>,80,<<"/%252Eabc">>,<<>>}} = + http_uri:parse(<<"http://www.somedomain.com/%252Eabc">>), + {ok, {http,<<>>,<<"www.somedomain.com">>,80,<<"/%25abc">>,<<>>}} = + http_uri:parse(<<"http://www.somedomain.com/%25abc">>), + {ok, {http,<<>>,<<"www.somedomain.com">>,80,<<"/%25abc">>, <<"?foo=bar">>}} = + http_uri:parse(<<"http://www.somedomain.com/%25abc?foo=bar">>). hexed_query(doc) -> [{doc, "Solves OTP-6191"}]; @@ -196,6 +273,17 @@ scheme_validation(Config) when is_list(Config) -> http_uri:parse("https://localhost#fragment", [{scheme_validation_fun, none}]). +encode_decode(Config) when is_list(Config) -> + ?assertEqual("foo%20bar", http_uri:encode("foo bar")), + ?assertEqual(<<"foo%20bar">>, http_uri:encode(<<"foo bar">>)), + + ?assertEqual("foo bar", http_uri:decode("foo+bar")), + ?assertEqual(<<"foo bar">>, http_uri:decode(<<"foo+bar">>)), + ?assertEqual("foo bar", http_uri:decode("foo%20bar")), + ?assertEqual(<<"foo bar">>, http_uri:decode(<<"foo%20bar">>)), + ?assertEqual("foo\r\n", http_uri:decode("foo%0D%0A")), + ?assertEqual(<<"foo\r\n">>, http_uri:decode(<<"foo%0D%0A">>)). + %%-------------------------------------------------------------------- %% Internal Functions ------------------------------------------------ diff --git a/lib/inets/vsn.mk b/lib/inets/vsn.mk index f2288466f8..96796f11c0 100644 --- a/lib/inets/vsn.mk +++ b/lib/inets/vsn.mk @@ -19,6 +19,6 @@ # %CopyrightEnd% APPLICATION = inets -INETS_VSN = 6.3.6 +INETS_VSN = 6.4 PRE_VSN = APP_VSN = "$(APPLICATION)-$(INETS_VSN)$(PRE_VSN)" |