Merge branch 'siri/ct_netconfc/slow-down/OTP-13007' into maint

* siri/ct_netconfc/slow-down/OTP-13007:
  Extended table_trans timer in order to handle big data on slow machines
  Don't log headings without content
  Speed up receive of many small packages

Conflicts:
	lib/common_test/src/ct_conn_log_h.erl

4 files changed, 75 insertions, 24 deletions

diff --git a/lib/common_test/src/ct_conn_log_h.erl b/lib/common_test/src/ct_conn_log_h.erl
index 5239ec1ff8..f7615fdc14 100644
--- a/lib/common_test/src/ct_conn_log_h.erl
+++ b/lib/common_test/src/ct_conn_log_h.erl
@@ -116,9 +116,14 @@ write_report(Time,#conn_log{module=ConnMod}=Info,Data,GL,State) ->
 	{silent,_} ->
 	    ok;
 	{LogType,Fd} ->
-	    io:format(Fd,"~n~ts~ts~ts",[format_head(ConnMod,LogType,Time),
-					format_title(LogType,Info),
-					format_data(ConnMod,LogType,Data)])
+	    case format_data(ConnMod,LogType,Data) of
+		[] ->
+		    ok;
+		FormattedData ->
+		    io:format(Fd,"~n~ts~ts~ts",[format_head(ConnMod,LogType,Time),
+						format_title(LogType,Info),
+						FormattedData])
+	    end
     end.
 
 write_error(Time,#conn_log{module=ConnMod}=Info,Report,GL,State) ->
diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl
index 05977e5649..6e3d1ab1d8 100644
--- a/lib/common_test/src/ct_netconfc.erl
+++ b/lib/common_test/src/ct_netconfc.erl
@@ -1374,11 +1374,18 @@ to_xml_doc(Simple) ->
 %%% Parse and handle received XML data
 handle_data(NewData,#state{connection=Connection,buff=Buff0} = State0) ->
     log(Connection,recv,NewData),
-    Data = append_wo_initial_nl(Buff0,NewData),
-    case binary:split(Data,[?END_TAG],[]) of
+    {Start,AddSz} =
+	case byte_size(Buff0) of
+	    BSz when BSz<5 -> {0,BSz};
+	    BSz -> {BSz-5,5}
+	end,
+    Length = byte_size(NewData) + AddSz,
+    Data = <<Buff0/binary, NewData/binary>>,
+    case binary:split(Data,?END_TAG,[{scope,{Start,Length}}]) of
 	[_NoEndTagFound] ->
 	    {noreply, State0#state{buff=Data}};
-	[FirstMsg,Buff1] ->
+	[FirstMsg0,Buff1] ->
+	    FirstMsg = remove_initial_nl(FirstMsg0),
 	    SaxArgs = [{event_fun,fun sax_event/3}, {event_state,[]}],
 	    case xmerl_sax_parser:stream(FirstMsg, SaxArgs) of
 		{ok, Simple, _Thrash} ->
@@ -1400,11 +1407,10 @@ handle_data(NewData,#state{connection=Connection,buff=Buff0} = State0) ->
 
 %% xml does not accept a leading nl and some netconf server add a nl after
 %% each ?END_TAG, ignore them
-append_wo_initial_nl(<<>>,NewData) -> NewData;
-append_wo_initial_nl(<<"\n", Data/binary>>, NewData) ->
-    append_wo_initial_nl(Data, NewData);
-append_wo_initial_nl(Data, NewData) ->
-    <<Data/binary, NewData/binary>>.
+remove_initial_nl(<<"\n", Data/binary>>) ->
+    remove_initial_nl(Data);
+remove_initial_nl(Data) ->
+    Data.
 
 handle_error(Reason, State) ->
     Pending1 = case State#state.pending of
@@ -1770,9 +1776,14 @@ format_data(How,Data) ->
 do_format_data(raw,Data) ->
     io_lib:format("~n~ts~n",[hide_password(Data)]);
 do_format_data(pretty,Data) ->
-    io_lib:format("~n~ts~n",[indent(Data)]);
+    maybe_io_lib_format(indent(Data));
 do_format_data(html,Data) ->
-    io_lib:format("~n~ts~n",[html_format(Data)]).
+    maybe_io_lib_format(html_format(Data)).
+
+maybe_io_lib_format(<<>>) ->
+    [];
+maybe_io_lib_format(String) ->
+    io_lib:format("~n~ts~n",[String]).
 
 %%%-----------------------------------------------------------------
 %%% Hide password elements from XML data
@@ -1811,13 +1822,21 @@ indent1("<?"++Rest1,Indent1) ->
     Line++indent1(Rest2,Indent2);
 indent1("</"++Rest1,Indent1) ->
     %% Stop tag
-    {Line,Rest2,Indent2} = indent_line1(Rest1,Indent1,[$/,$<]),
-    "\n"++Line++indent1(Rest2,Indent2);
+    case indent_line1(Rest1,Indent1,[$/,$<]) of
+	{[],[],_} ->
+	    [];
+	{Line,Rest2,Indent2} ->
+	    "\n"++Line++indent1(Rest2,Indent2)
+    end;
 indent1("<"++Rest1,Indent1) ->
     %% Start- or empty tag
     put(tag,get_tag(Rest1)),
-    {Line,Rest2,Indent2} = indent_line(Rest1,Indent1,[$<]),
-    "\n"++Line++indent1(Rest2,Indent2);
+    case indent_line(Rest1,Indent1,[$<]) of
+	{[],[],_} ->
+	    [];
+	{Line,Rest2,Indent2} ->
+	    "\n"++Line++indent1(Rest2,Indent2)
+    end;
 indent1([H|T],Indent) ->
     [H|indent1(T,Indent)];
 indent1([],_Indent) ->
diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl
index aaa0723488..ea49e36608 100644
--- a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl
+++ b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl
@@ -36,7 +36,8 @@
 -compile(export_all).
 
 suite() ->
-    [{ct_hooks, [{cth_conn_log,
+    [{timetrap,?default_timeout},
+     {ct_hooks, [{cth_conn_log,
 		  [{ct_netconfc,[{log_type,html}, %will be overwritten by config
 				 {hosts,[my_named_connection,netconf1]}]
 		   }]
@@ -72,6 +73,7 @@ all() ->
 	     invalid_opt,
 	     timeout_close_session,
 	     get,
+	     get_a_lot,
 	     timeout_get,
 	     flush_timeout_get,
 	     get_xpath,
@@ -113,12 +115,9 @@ end_per_group(_GroupName, Config) ->
 
 init_per_testcase(_Case, Config) ->
     ets:delete_all_objects(ns_tab),
-    Dog = test_server:timetrap(?default_timeout),
-    [{watchdog, Dog}|Config].
+    Config.
 
-end_per_testcase(_Case, Config) ->
-    Dog=?config(watchdog, Config),
-    test_server:timetrap_cancel(Dog),
+end_per_testcase(_Case, _Config) ->
     ok.
 
 init_per_suite(Config) ->
@@ -352,6 +351,19 @@ get(Config) ->
     ?ok = ct_netconfc:close_session(Client),
     ok.
 
+get_a_lot(Config) ->
+    DataDir = ?config(data_dir,Config),
+    {ok,Client} = open_success(DataDir),
+    Descr = lists:append(lists:duplicate(100,"Description of myserver! ")),
+    Server = {server,[{xmlns,"myns"}],[{name,[],["myserver"]},
+				       {description,[],[Descr]}]},
+    Data = lists:duplicate(100,Server),
+    ?NS:expect_reply('get',{fragmented,{data,Data}}),
+    {ok,Data} = ct_netconfc:get(Client,{server,[{xmlns,"myns"}],[]}),
+    ?NS:expect_do_reply('close-session',close,ok),
+    ?ok = ct_netconfc:close_session(Client),
+    ok.
+
 timeout_get(Config) ->
     DataDir = ?config(data_dir,Config),
     {ok,Client} = open_success(DataDir),
diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl
index 8c30383343..07893faabc 100644
--- a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl
+++ b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl
@@ -277,6 +277,18 @@ hupp_kill(State = #session{connection = ConnRef}) ->
 send({CM,Ch},Data) ->
     ssh_connection:send(CM, Ch, Data).
 
+%%% Split into many small parts and send to client
+send_frag({CM,Ch},Data) ->
+    Sz = rand:uniform(2000),
+    case Data of
+	<<Chunk:Sz/binary,Rest/binary>> ->
+	    ssh_connection:send(CM, Ch, Chunk),
+	    send_frag({CM,Ch},Rest);
+	Chunk ->
+	    ssh_connection:send(CM, Ch, Chunk)
+    end.
+
+
 %%% Kill ssh connection
 kill({CM,_Ch}) ->
     ssh:close(CM).
@@ -294,7 +306,7 @@ table_trans(Fun,Args) ->
 	    receive
 		{table_trans_done,Result} ->
 		    Result
-	    after 5000 ->
+	    after 20000 ->
 		    exit(table_trans_timeout)
 	    end
     end.
@@ -424,6 +436,9 @@ do(_, undefined) ->
 reply(_,undefined) ->
     ?dbg("no reply~n",[]),
     ok;
+reply(ConnRef,{fragmented,Reply}) ->
+    ?dbg("Reply fragmented: ~p~n",[Reply]),
+    send_frag(ConnRef,make_msg(Reply));
 reply(ConnRef,Reply) ->
     ?dbg("Reply: ~p~n",[Reply]),
     send(ConnRef, make_msg(Reply)).