Speed up receive of many small packages

When data from the netconf server was split into many ssh packages,
the netconf client performed really bad. This is now improved.

3 files changed, 47 insertions, 14 deletions

diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl
index 0de7bf03af..799d795ed0 100644
--- a/lib/common_test/src/ct_netconfc.erl
+++ b/lib/common_test/src/ct_netconfc.erl
@@ -1366,11 +1366,18 @@ to_xml_doc(Simple) ->
 %%% Parse and handle received XML data
 handle_data(NewData,#state{connection=Connection,buff=Buff0} = State0) ->
     log(Connection,recv,NewData),
-    Data = append_wo_initial_nl(Buff0,NewData),
-    case binary:split(Data,[?END_TAG],[]) of
+    {Start,AddSz} =
+	case byte_size(Buff0) of
+	    BSz when BSz<5 -> {0,BSz};
+	    BSz -> {BSz-5,5}
+	end,
+    Length = byte_size(NewData) + AddSz,
+    Data = <<Buff0/binary, NewData/binary>>,
+    case binary:split(Data,?END_TAG,[{scope,{Start,Length}}]) of
 	[_NoEndTagFound] ->
 	    {noreply, State0#state{buff=Data}};
-	[FirstMsg,Buff1] ->
+	[FirstMsg0,Buff1] ->
+	    FirstMsg = remove_initial_nl(FirstMsg0),
 	    SaxArgs = [{event_fun,fun sax_event/3}, {event_state,[]}],
 	    case xmerl_sax_parser:stream(FirstMsg, SaxArgs) of
 		{ok, Simple, _Thrash} ->
@@ -1392,11 +1399,10 @@ handle_data(NewData,#state{connection=Connection,buff=Buff0} = State0) ->
 
 %% xml does not accept a leading nl and some netconf server add a nl after
 %% each ?END_TAG, ignore them
-append_wo_initial_nl(<<>>,NewData) -> NewData;
-append_wo_initial_nl(<<"\n", Data/binary>>, NewData) ->
-    append_wo_initial_nl(Data, NewData);
-append_wo_initial_nl(Data, NewData) ->
-    <<Data/binary, NewData/binary>>.
+remove_initial_nl(<<"\n", Data/binary>>) ->
+    remove_initial_nl(Data);
+remove_initial_nl(Data) ->
+    Data.
 
 handle_error(Reason, State) ->
     Pending1 = case State#state.pending of
diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl
index 64ebfbc463..5f84634f74 100644
--- a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl
+++ b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl
@@ -36,7 +36,8 @@
 -compile(export_all).
 
 suite() ->
-    [{ct_hooks, [{cth_conn_log,
+    [{timetrap,?default_timeout},
+     {ct_hooks, [{cth_conn_log,
 		  [{ct_netconfc,[{log_type,html}, %will be overwritten by config
 				 {hosts,[my_named_connection,netconf1]}]
 		   }]
@@ -72,6 +73,7 @@ all() ->
 	     invalid_opt,
 	     timeout_close_session,
 	     get,
+	     get_a_lot,
 	     timeout_get,
 	     get_xpath,
 	     get_config,
@@ -112,12 +114,9 @@ end_per_group(_GroupName, Config) ->
 
 init_per_testcase(_Case, Config) ->
     ets:delete_all_objects(ns_tab),
-    Dog = test_server:timetrap(?default_timeout),
-    [{watchdog, Dog}|Config].
+    Config.
 
-end_per_testcase(_Case, Config) ->
-    Dog=?config(watchdog, Config),
-    test_server:timetrap_cancel(Dog),
+end_per_testcase(_Case, _Config) ->
     ok.
 
 init_per_suite(Config) ->
@@ -351,6 +350,19 @@ get(Config) ->
     ?ok = ct_netconfc:close_session(Client),
     ok.
 
+get_a_lot(Config) ->
+    DataDir = ?config(data_dir,Config),
+    {ok,Client} = open_success(DataDir),
+    Descr = lists:append(lists:duplicate(1000,"Description of myserver! ")),
+    Server = {server,[{xmlns,"myns"}],[{name,[],["myserver"]},
+				       {description,[],[Descr]}]},
+    Data = lists:duplicate(100,Server),
+    ?NS:expect_reply('get',{fragmented,{data,Data}}),
+    {ok,Data} = ct_netconfc:get(Client,{server,[{xmlns,"myns"}],[]}),
+    ?NS:expect_do_reply('close-session',close,ok),
+    ?ok = ct_netconfc:close_session(Client),
+    ok.
+
 timeout_get(Config) ->
     DataDir = ?config(data_dir,Config),
     {ok,Client} = open_success(DataDir),
diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl
index 8c30383343..3fc99e5486 100644
--- a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl
+++ b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl
@@ -277,6 +277,18 @@ hupp_kill(State = #session{connection = ConnRef}) ->
 send({CM,Ch},Data) ->
     ssh_connection:send(CM, Ch, Data).
 
+%%% Split into many small parts and send to client
+send_frag({CM,Ch},Data) ->
+    Sz = rand:uniform(2000),
+    case Data of
+	<<Chunk:Sz/binary,Rest/binary>> ->
+	    ssh_connection:send(CM, Ch, Chunk),
+	    send_frag({CM,Ch},Rest);
+	Chunk ->
+	    ssh_connection:send(CM, Ch, Chunk)
+    end.
+
+
 %%% Kill ssh connection
 kill({CM,_Ch}) ->
     ssh:close(CM).
@@ -424,6 +436,9 @@ do(_, undefined) ->
 reply(_,undefined) ->
     ?dbg("no reply~n",[]),
     ok;
+reply(ConnRef,{fragmented,Reply}) ->
+    ?dbg("Reply fragmented: ~p~n",[Reply]),
+    send_frag(ConnRef,make_msg(Reply));
 reply(ConnRef,Reply) ->
     ?dbg("Reply: ~p~n",[Reply]),
     send(ConnRef, make_msg(Reply)).