From 67b38c36eaa9b6d3edb80df75637f0e8cd1823f3 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Mon, 28 Sep 2015 15:37:41 +0200 Subject: Speed up receive of many small packages When data from the netconf server was split into many ssh packages, the netconf client performed really bad. This is now improved. --- lib/common_test/src/ct_netconfc.erl | 22 ++++++++++++-------- .../ct_netconfc_SUITE_data/netconfc1_SUITE.erl | 24 ++++++++++++++++------ lib/common_test/test/ct_netconfc_SUITE_data/ns.erl | 15 ++++++++++++++ 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl index 0de7bf03af..799d795ed0 100644 --- a/lib/common_test/src/ct_netconfc.erl +++ b/lib/common_test/src/ct_netconfc.erl @@ -1366,11 +1366,18 @@ to_xml_doc(Simple) -> %%% Parse and handle received XML data handle_data(NewData,#state{connection=Connection,buff=Buff0} = State0) -> log(Connection,recv,NewData), - Data = append_wo_initial_nl(Buff0,NewData), - case binary:split(Data,[?END_TAG],[]) of + {Start,AddSz} = + case byte_size(Buff0) of + BSz when BSz<5 -> {0,BSz}; + BSz -> {BSz-5,5} + end, + Length = byte_size(NewData) + AddSz, + Data = <>, + case binary:split(Data,?END_TAG,[{scope,{Start,Length}}]) of [_NoEndTagFound] -> {noreply, State0#state{buff=Data}}; - [FirstMsg,Buff1] -> + [FirstMsg0,Buff1] -> + FirstMsg = remove_initial_nl(FirstMsg0), SaxArgs = [{event_fun,fun sax_event/3}, {event_state,[]}], case xmerl_sax_parser:stream(FirstMsg, SaxArgs) of {ok, Simple, _Thrash} -> @@ -1392,11 +1399,10 @@ handle_data(NewData,#state{connection=Connection,buff=Buff0} = State0) -> %% xml does not accept a leading nl and some netconf server add a nl after %% each ?END_TAG, ignore them -append_wo_initial_nl(<<>>,NewData) -> NewData; -append_wo_initial_nl(<<"\n", Data/binary>>, NewData) -> - append_wo_initial_nl(Data, NewData); -append_wo_initial_nl(Data, NewData) -> - <>. +remove_initial_nl(<<"\n", Data/binary>>) -> + remove_initial_nl(Data); +remove_initial_nl(Data) -> + Data. handle_error(Reason, State) -> Pending1 = case State#state.pending of diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl index 64ebfbc463..5f84634f74 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl @@ -36,7 +36,8 @@ -compile(export_all). suite() -> - [{ct_hooks, [{cth_conn_log, + [{timetrap,?default_timeout}, + {ct_hooks, [{cth_conn_log, [{ct_netconfc,[{log_type,html}, %will be overwritten by config {hosts,[my_named_connection,netconf1]}] }] @@ -72,6 +73,7 @@ all() -> invalid_opt, timeout_close_session, get, + get_a_lot, timeout_get, get_xpath, get_config, @@ -112,12 +114,9 @@ end_per_group(_GroupName, Config) -> init_per_testcase(_Case, Config) -> ets:delete_all_objects(ns_tab), - Dog = test_server:timetrap(?default_timeout), - [{watchdog, Dog}|Config]. + Config. -end_per_testcase(_Case, Config) -> - Dog=?config(watchdog, Config), - test_server:timetrap_cancel(Dog), +end_per_testcase(_Case, _Config) -> ok. init_per_suite(Config) -> @@ -351,6 +350,19 @@ get(Config) -> ?ok = ct_netconfc:close_session(Client), ok. +get_a_lot(Config) -> + DataDir = ?config(data_dir,Config), + {ok,Client} = open_success(DataDir), + Descr = lists:append(lists:duplicate(1000,"Description of myserver! ")), + Server = {server,[{xmlns,"myns"}],[{name,[],["myserver"]}, + {description,[],[Descr]}]}, + Data = lists:duplicate(100,Server), + ?NS:expect_reply('get',{fragmented,{data,Data}}), + {ok,Data} = ct_netconfc:get(Client,{server,[{xmlns,"myns"}],[]}), + ?NS:expect_do_reply('close-session',close,ok), + ?ok = ct_netconfc:close_session(Client), + ok. + timeout_get(Config) -> DataDir = ?config(data_dir,Config), {ok,Client} = open_success(DataDir), diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl index 8c30383343..3fc99e5486 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl @@ -277,6 +277,18 @@ hupp_kill(State = #session{connection = ConnRef}) -> send({CM,Ch},Data) -> ssh_connection:send(CM, Ch, Data). +%%% Split into many small parts and send to client +send_frag({CM,Ch},Data) -> + Sz = rand:uniform(2000), + case Data of + <> -> + ssh_connection:send(CM, Ch, Chunk), + send_frag({CM,Ch},Rest); + Chunk -> + ssh_connection:send(CM, Ch, Chunk) + end. + + %%% Kill ssh connection kill({CM,_Ch}) -> ssh:close(CM). @@ -424,6 +436,9 @@ do(_, undefined) -> reply(_,undefined) -> ?dbg("no reply~n",[]), ok; +reply(ConnRef,{fragmented,Reply}) -> + ?dbg("Reply fragmented: ~p~n",[Reply]), + send_frag(ConnRef,make_msg(Reply)); reply(ConnRef,Reply) -> ?dbg("Reply: ~p~n",[Reply]), send(ConnRef, make_msg(Reply)). -- cgit v1.2.3 From 4cf832f1ad163f5b25dd8a6f2d314c169c23c82f Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Mon, 28 Sep 2015 16:45:05 +0200 Subject: Don't log headings without content The netconf server collects data until an XML tag is completed before pretty printing received data. Each time data is logged, a heading like the following is printed: = CT_NETCONFC ==== 28-Sep-2015::16:43:46,842 =================================== = Client <0.194.0> <----- {"127.0.0.1",2060} =================================== This commit removes printing of this header if there is no data to be printed below - i.e. if the XML tag is not yet complete and we are waiting for more data. --- lib/common_test/src/ct_conn_log_h.erl | 11 ++++++++--- lib/common_test/src/ct_netconfc.erl | 25 +++++++++++++++++++------ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/lib/common_test/src/ct_conn_log_h.erl b/lib/common_test/src/ct_conn_log_h.erl index 2dd4fdac05..5eb67853d9 100644 --- a/lib/common_test/src/ct_conn_log_h.erl +++ b/lib/common_test/src/ct_conn_log_h.erl @@ -109,9 +109,14 @@ write_report(_Time,#conn_log{header=false,module=ConnMod}=Info,Data,GL,State) -> write_report(Time,#conn_log{module=ConnMod}=Info,Data,GL,State) -> {LogType,Fd} = get_log(Info,GL,State), - io:format(Fd,"~n~ts~ts~ts",[format_head(ConnMod,LogType,Time), - format_title(LogType,Info), - format_data(ConnMod,LogType,Data)]). + case format_data(ConnMod,LogType,Data) of + [] -> + ok; + FormattedData -> + io:format(Fd,"~n~ts~ts~ts",[format_head(ConnMod,LogType,Time), + format_title(LogType,Info), + FormattedData]) + end. write_error(Time,#conn_log{module=ConnMod}=Info,Report,GL,State) -> case get_log(Info,GL,State) of diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl index 799d795ed0..ed805e4d14 100644 --- a/lib/common_test/src/ct_netconfc.erl +++ b/lib/common_test/src/ct_netconfc.erl @@ -1768,9 +1768,14 @@ format_data(How,Data) -> do_format_data(raw,Data) -> io_lib:format("~n~ts~n",[hide_password(Data)]); do_format_data(pretty,Data) -> - io_lib:format("~n~ts~n",[indent(Data)]); + maybe_io_lib_format(indent(Data)); do_format_data(html,Data) -> - io_lib:format("~n~ts~n",[html_format(Data)]). + maybe_io_lib_format(html_format(Data)). + +maybe_io_lib_format(<<>>) -> + []; +maybe_io_lib_format(String) -> + io_lib:format("~n~ts~n",[String]). %%%----------------------------------------------------------------- %%% Hide password elements from XML data @@ -1809,13 +1814,21 @@ indent1(" Line++indent1(Rest2,Indent2); indent1(" %% Stop tag - {Line,Rest2,Indent2} = indent_line1(Rest1,Indent1,[$/,$<]), - "\n"++Line++indent1(Rest2,Indent2); + case indent_line1(Rest1,Indent1,[$/,$<]) of + {[],[],_} -> + []; + {Line,Rest2,Indent2} -> + "\n"++Line++indent1(Rest2,Indent2) + end; indent1("<"++Rest1,Indent1) -> %% Start- or empty tag put(tag,get_tag(Rest1)), - {Line,Rest2,Indent2} = indent_line(Rest1,Indent1,[$<]), - "\n"++Line++indent1(Rest2,Indent2); + case indent_line(Rest1,Indent1,[$<]) of + {[],[],_} -> + []; + {Line,Rest2,Indent2} -> + "\n"++Line++indent1(Rest2,Indent2) + end; indent1([H|T],Indent) -> [H|indent1(T,Indent)]; indent1([],_Indent) -> -- cgit v1.2.3 From 976214f8d738d4852348496df79f84264d899aba Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Fri, 27 Nov 2015 13:04:58 +0100 Subject: Extended table_trans timer in order to handle big data on slow machines The test case netconfc1_SUITE:get_a_lot often fails with table_trans_timeout in the netconf server (ns.erl) on virtual machines. The correction is to overcome this problem. Amount of data used in the test case is also reduced a bit. --- lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl | 2 +- lib/common_test/test/ct_netconfc_SUITE_data/ns.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl index 5f84634f74..2a8f8237bc 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl @@ -353,7 +353,7 @@ get(Config) -> get_a_lot(Config) -> DataDir = ?config(data_dir,Config), {ok,Client} = open_success(DataDir), - Descr = lists:append(lists:duplicate(1000,"Description of myserver! ")), + Descr = lists:append(lists:duplicate(100,"Description of myserver! ")), Server = {server,[{xmlns,"myns"}],[{name,[],["myserver"]}, {description,[],[Descr]}]}, Data = lists:duplicate(100,Server), diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl index 3fc99e5486..07893faabc 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl @@ -306,7 +306,7 @@ table_trans(Fun,Args) -> receive {table_trans_done,Result} -> Result - after 5000 -> + after 20000 -> exit(table_trans_timeout) end end. -- cgit v1.2.3