Merge branch 'pd/ttb-cleanup' into major

OTP-9430
OTP-9403
OTP-9431

5 files changed, 922 insertions, 57 deletions

diff --git a/lib/observer/test/Makefile b/lib/observer/test/Makefile
index 6073e6ea00..bf99f07081 100644
--- a/lib/observer/test/Makefile
+++ b/lib/observer/test/Makefile
@@ -22,7 +22,10 @@ MODULES =  \
 	observer_SUITE \
 	crashdump_viewer_SUITE \
 	etop_SUITE \
+	ttb_helper \
 	ttb_SUITE \
+	client \
+	server \
 	crashdump_helper
 
 ERL_FILES= $(MODULES:%=%.erl)
diff --git a/lib/observer/test/client.erl b/lib/observer/test/client.erl
new file mode 100644
index 0000000000..e756f9d6e8
--- /dev/null
+++ b/lib/observer/test/client.erl
@@ -0,0 +1,28 @@
+-module(client).
+-compile(export_all).
+
+init(Node) ->
+    application:start(runtime_tools),
+    net_kernel:connect_node(Node).
+
+init() ->
+    init(server_node()).
+
+restart() ->
+    init:restart().
+
+server_node() ->
+    {ok,HostName} = inet:gethostname(),
+    list_to_atom("server@" ++ HostName).
+
+get() ->
+    erlang:send({server,server_node()}, {get,self()}),
+    receive Data -> Data
+    after 1000   -> no_reply
+    end.
+
+put(Thing) ->
+    erlang:send({server,server_node()}, {put,self(),Thing}),
+    receive ok -> ok
+    after 1000 -> no_reply
+    end.
diff --git a/lib/observer/test/server.erl b/lib/observer/test/server.erl
new file mode 100644
index 0000000000..c1b1fea562
--- /dev/null
+++ b/lib/observer/test/server.erl
@@ -0,0 +1,43 @@
+-module(server).
+-compile(export_all).
+
+start() ->
+    application:start(runtime_tools),
+    Pid = spawn(?MODULE,loop,[[], 0]),
+    register(server,Pid).
+
+stop() ->
+    case lists:member(server, registered()) of
+	true ->
+	    server ! stop;
+	false ->
+	    ok
+    end.
+
+loop(Data, Num) ->
+    receive
+	{put,From,Ting} -> From ! ok,
+			   received(From,Ting),
+			   loop([Ting|Data], Num+1);
+	{get,From}      -> From ! Data,
+			   loop(Data, Num+1);
+	stop            -> stopped;
+	clear           -> loop([], Num+1);
+	{cnt, From}     -> From ! Num,
+			   loop(Data, Num)
+    end.
+
+counter() ->
+    server ! {cnt, self()},
+    receive
+	Num ->
+	    Num
+    end.
+
+received(From, Thing) ->
+    case Thing of
+	never_send_this_atom ->
+	    loop(Thing, 0);
+	_ ->
+	    {return, 27, Thing, From}
+    end.
diff --git a/lib/observer/test/ttb_SUITE.erl b/lib/observer/test/ttb_SUITE.erl
index 24b4a22aa9..1fd8b4c892 100644
--- a/lib/observer/test/ttb_SUITE.erl
+++ b/lib/observer/test/ttb_SUITE.erl
@@ -1,7 +1,7 @@
-%%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2002-2011. All Rights Reserved.
+%%
+%% Copyright Ericsson AB 2002-2010. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -33,9 +33,17 @@
 -include_lib("test_server/include/test_server.hrl").
 
 -define(default_timeout, ?t:minutes(1)).
+-define(OUTPUT, "handler_output").
+-define(FNAME, "temptest").
+-define(DIRNAME, "ddtemp").
 
 init_per_testcase(_Case, Config) ->
     ttb:stop(),
+    os:cmd("rm -rf " ++ ?OUTPUT),
+    os:cmd("rm -rf ttb_upload*"),
+    os:cmd("rm -rf " ++ ?DIRNAME),
+    os:cmd("rm -rf *@*"),
+    os:cmd("rm -rf ttb_last_config"),
     ?line Dog=test_server:timetrap(?default_timeout),
     [{watchdog, Dog}|Config].
 end_per_testcase(_Case, Config) ->
@@ -49,7 +57,25 @@ all() ->
     [file, file_no_pi, file_fetch, wrap, wrap_merge,
      wrap_merge_fetch_format, write_config1, write_config2,
      write_config3, history, write_trace_info, seq_trace,
-     diskless, otp_4967_1, otp_4967_2].
+     diskless, diskless_wrap, otp_4967_1, otp_4967_2,
+     fetch_when_no_option_given, basic_ttb_run_ip_port, basic_ttb_run_file_port,
+     return_fetch_dir_implies_fetch, logfile_name_in_fetch_dir, upload_to_my_logdir,
+     upload_to_my_existing_logdir, fetch_with_options_not_as_list,
+     error_when_formatting_multiple_files_4393, format_on_trace_stop,
+     trace_to_remote_files_on_localhost_with_different_pwd,
+     trace_to_local_files_on_localhost_with_different_pwd,
+     trace_to_remote_files_on_localhost_with_different_pwd_abs,
+     changing_cwd_on_control_node, changing_cwd_on_remote_node,
+     changing_cwd_on_control_node_with_local_trace,
+     one_command_trace_setup, dbg_style_fetch, shell_tracing_init,
+     only_one_state_for_format_handler, only_one_state_with_default_format_handler,
+     only_one_state_with_initial_format_handler, run_trace_with_shortcut1,
+     run_trace_with_shortcut2, run_trace_with_shortcut3, run_trace_with_shortcut4,
+     cant_specify_local_and_flush, trace_sorted_by_default,disable_sorting,
+     trace_resumed_after_node_restart, trace_resumed_after_node_restart_ip,
+     trace_resumed_after_node_restart_wrap,
+     trace_resumed_after_node_restart_wrap_mult
+].
 
 groups() -> 
     [].
@@ -92,15 +118,15 @@ file(Config) when is_list(Config) ->
     ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
     ?line ?MODULE:foo(),
     ?line rpc:call(OtherNode,?MODULE,foo,[]),
-    ?line ttb:stop(),
+    ?line ttb:stop([nofetch]),
     ?line ?t:stop_node(OtherNode),
     ?line ok = ttb:format(filename:join(Privdir,atom_to_list(Node)++"-file")),
     ?line ok = ttb:format(filename:join(Privdir,
 					atom_to_list(OtherNode)++"-file")),
 
-    ?line [{trace,{S,_,Node},call,{?MODULE,foo,[]}},
+    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
 	   end_of_trace,
-	   {trace,{_,_,OtherNode},call,{?MODULE,foo,[]}},
+	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
 	   end_of_trace] = flush(),
     ok.
 
@@ -123,15 +149,15 @@ file_no_pi(Config) when is_list(Config) ->
     ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
     ?line ?MODULE:foo(),
     ?line rpc:call(OtherNode,?MODULE,foo,[]),
-    ?line ttb:stop(),
+    ?line ttb:stop([nofetch]),
     ?line ?t:stop_node(OtherNode),
     ?line ok = ttb:format(filename:join(Privdir,atom_to_list(Node)++"-file")),
     ?line ok = ttb:format(filename:join(Privdir,
 					atom_to_list(OtherNode)++"-file")),
 
-    ?line [{trace,LocalProc,call,{?MODULE,foo,[]}},
+    ?line [{trace_ts,LocalProc,call,{?MODULE,foo,[]}, {_,_,_}},
 	   end_of_trace,
-	   {trace,RemoteProc,call,{?MODULE,foo,[]}},
+	   {trace_ts,RemoteProc,call,{?MODULE,foo,[]},{_,_,_}},
 	   end_of_trace] = flush(),
     ?line true = is_pid(LocalProc),
     ?line true = is_pid(RemoteProc),
@@ -170,7 +196,7 @@ file_fetch(Config) when is_list(Config) ->
     ?line ?MODULE:foo(),
     ?line rpc:call(OtherNode,?MODULE,foo,[]),
     ?line ?t:capture_start(),
-    ?line ttb:stop([fetch]),
+    ?line ttb:stop([return_fetch_dir]),
     ?line ?t:capture_stop(),
     ?line [StoreString] = ?t:capture_get(),
     ?line UploadDir =
@@ -194,9 +220,9 @@ file_fetch(Config) when is_list(Config) ->
     ?line ok = ttb:format(filename:join(UploadDir,
 					atom_to_list(OtherNode)++"-file_fetch")),
 
-    ?line [{trace,{S,_,Node},call,{?MODULE,foo,[]}},
+    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
 	   end_of_trace,
-	   {trace,{_,_,OtherNode},call,{?MODULE,foo,[]}},
+	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
 	   end_of_trace] = flush(),
 
     ?line ok = file:set_cwd(Cwd),
@@ -224,19 +250,19 @@ wrap(Config) when is_list(Config) ->
     ?line rpc:call(OtherNode,?MODULE,foo,[]),
     ?line ?MODULE:foo(),
     ?line rpc:call(OtherNode,?MODULE,foo,[]),
-    ?line ttb:stop(),
+    ?line ttb:stop([nofetch]),
     ?line ?t:stop_node(OtherNode),
     ?line ok = ttb:format(filename:join(Privdir,
 					atom_to_list(Node)++"-wrap.*.wrp")),
-    ?line [{trace,{S,_,Node},call,{?MODULE,foo,[]}},
-	   {trace,{S,_,Node},call,{?MODULE,foo,[]}},
-	   {trace,{S,_,Node},call,{?MODULE,foo,[]}},
+    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
+	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
+	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
 	   end_of_trace] = flush(),
     ?line ok = ttb:format(filename:join(Privdir,
 					atom_to_list(OtherNode)++"-wrap.*.wrp")),
-    ?line [{trace,{_,_,OtherNode},call,{?MODULE,foo,[]}},
-	   {trace,{_,_,OtherNode},call,{?MODULE,foo,[]}},
-	   {trace,{_,_,OtherNode},call,{?MODULE,foo,[]}},
+    ?line [{trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
+	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
+	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
 	   end_of_trace] = flush(),
 
     %% Check that merge does not crash even if the timestamp flag is not on.
@@ -244,14 +270,13 @@ wrap(Config) when is_list(Config) ->
 		 [filename:join(Privdir,
 				atom_to_list(Node)++"-wrap.*.wrp"),
 		  filename:join(Privdir,
-				atom_to_list(OtherNode)++"-wrap.*.wrp")]),
-    ?line [{trace,{S,_,Node},call,{?MODULE,foo,[]}},
-	   {trace,{S,_,Node},call,{?MODULE,foo,[]}},
-	   {trace,{S,_,Node},call,{?MODULE,foo,[]}},
-	   end_of_trace,
-	   {trace,{_,_,OtherNode},call,{?MODULE,foo,[]}},
-	   {trace,{_,_,OtherNode},call,{?MODULE,foo,[]}},
-	   {trace,{_,_,OtherNode},call,{?MODULE,foo,[]}},
+				atom_to_list(OtherNode)++"-wrap.*.wrp")],[{disable_sort,true}]),
+    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
+	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
+	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
+	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
+	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
+	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
 	   end_of_trace] = flush(),
     ok.
 
@@ -277,7 +302,7 @@ wrap_merge(Config) when is_list(Config) ->
     ?line rpc:call(OtherNode,?MODULE,foo,[]),
     ?line ?MODULE:foo(),
     ?line rpc:call(OtherNode,?MODULE,foo,[]),
-    ?line ttb:stop(),
+    ?line ttb:stop([nofetch]),
     ?line ?t:stop_node(OtherNode),
     ?line ok = ttb:format(
 		 [filename:join(Privdir,
@@ -289,7 +314,6 @@ wrap_merge(Config) when is_list(Config) ->
 	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
 	   {trace_ts,_,call,{?MODULE,foo,[]},_},
 	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
-	   end_of_trace,
 	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},_},
 	   end_of_trace] = flush(),
     ok.
@@ -330,7 +354,6 @@ wrap_merge_fetch_format(Config) when is_list(Config) ->
 	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
 	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},_},
 	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
-	   end_of_trace,
 	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},_},
 	   end_of_trace] = flush(),
 
@@ -360,16 +383,15 @@ write_config1(Config) when is_list(Config) ->
     ?line ok = ttb:run_config(File),
     ?line ?MODULE:foo(),
     ?line rpc:call(OtherNode,?MODULE,foo,[]),
-    ?line ttb:stop(),
+    ?line ttb:stop([nofetch]),
     ?line ?t:stop_node(OtherNode),
     ?line ok = ttb:format(
 		 [filename:join(Privdir,
 				atom_to_list(Node)++"-write_config1"),
 		  filename:join(Privdir,
 				atom_to_list(OtherNode)++"-write_config1")]),
-    ?line [{trace,{S,_,Node},call,{?MODULE,foo,[]}},
-	   end_of_trace,
-	   {trace,Other,call,{?MODULE,foo,[]}},
+    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
+	   {trace_ts,Other,call,{?MODULE,foo,[]},{_,_,_}},
 	   end_of_trace] = flush(),
 
     case metatest(Other,OtherNode,Privdir,"-write_config1.ti") of
@@ -410,16 +432,15 @@ write_config2(Config) when is_list(Config) ->
     ?line ok = ttb:run_config(File),
     ?line ?MODULE:foo(),
     ?line rpc:call(OtherNode,?MODULE,foo,[]),
-    ?line ttb:stop(),
+    ?line ttb:stop([nofetch]),
     ?line ?t:stop_node(OtherNode),
     ?line ok = ttb:format(
 		 [filename:join(Privdir,
 				atom_to_list(Node)++"-write_config2"),
 		  filename:join(Privdir,
 				atom_to_list(OtherNode)++"-write_config2")]),
-    ?line [{trace,{S,_,Node},call,{?MODULE,foo,[]}},
-	   end_of_trace,
-	   {trace,Other,call,{?MODULE,foo,[]}},
+    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
+	   {trace_ts,Other,call,{?MODULE,foo,[]},{_,_,_}},
 	   end_of_trace] = flush(),
 
     case metatest(Other,OtherNode,Privdir,"-write_config2.ti") of
@@ -455,18 +476,18 @@ write_config3(Config) when is_list(Config) ->
     ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
     ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
     ?line ok = ttb:write_config(File,[1,2]),
-    ?line ttb:stop(),
+    ?line ttb:stop([nofetch]),
     ?line [_,_] = ttb:list_config(File),
     ?line ok = ttb:run_config(File),
     ?line ?MODULE:foo(),
     ?line rpc:call(OtherNode,?MODULE,foo,[]),
-    ?line ttb:stop(),
+    ?line ttb:stop([nofetch]),
     ?line ok = ttb:format(
 		 [filename:join(Privdir,
 				atom_to_list(Node)++"-write_config3"),
 		  filename:join(Privdir,
 				atom_to_list(OtherNode)++"-write_config3")]),
-    ?line [] = flush(), %foo is not traced
+    ?line [end_of_trace] = flush(), %foo is not traced
 
     ?line ok = ttb:write_config(File,[{ttb,tp,[?MODULE,foo,[]]}],
 			      [append]),
@@ -474,16 +495,15 @@ write_config3(Config) when is_list(Config) ->
     ?line ok = ttb:run_config(File),
     ?line ?MODULE:foo(),
     ?line rpc:call(OtherNode,?MODULE,foo,[]),
-    ?line ttb:stop(),
+    ?line ttb:stop([nofetch]),
     ?line ?t:stop_node(OtherNode),
     ?line ok = ttb:format(
 		 [filename:join(Privdir,
 				atom_to_list(Node)++"-write_config3"),
 		  filename:join(Privdir,
 				atom_to_list(OtherNode)++"-write_config3")]),
-    ?line [{trace,{S,_,Node},call,{?MODULE,foo,[]}},
-	   end_of_trace,
-	   {trace,Other,call,{?MODULE,foo,[]}},
+    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
+	   {trace_ts,Other,call,{?MODULE,foo,[]},{_,_,_}},
 	   end_of_trace] = flush(),
 
     case metatest(Other,OtherNode,Privdir,"-write_config3.ti") of
@@ -531,12 +551,12 @@ history(Config) when is_list(Config) ->
     ?line ?MODULE:foo(),
     ?line ok = ttb:run_history([3,4]),
     ?line ?MODULE:foo(),
-    ?line ttb:stop(),
+    ?line ttb:stop([nofetch]),
     ?line ?t:stop_node(OtherNode),
     ?line ok = ttb:format(
 		 [filename:join(Privdir,atom_to_list(Node)++"-history"),
 		  filename:join(Privdir,atom_to_list(OtherNode)++"-history")]),
-    ?line [{trace,{S,_,Node},call,{?MODULE,foo,[]}},
+    ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
 	   end_of_trace] = flush(),
     ok.
     
@@ -561,17 +581,16 @@ write_trace_info(Config) when is_list(Config) ->
     ?line ok = ttb:write_trace_info(mytraceinfo,fun() -> node() end),
     ?line ?MODULE:foo(),
     ?line rpc:call(OtherNode,?MODULE,foo,[]),
-    ?line ttb:stop(),
+    ?line ttb:stop([nofetch]),
     ?line ?t:stop_node(OtherNode),
     ?line ok = ttb:format(
 		 [filename:join(Privdir,atom_to_list(Node)++"-write_trace_info"),
 		  filename:join(Privdir,
 				atom_to_list(OtherNode)++"-write_trace_info")],
 		 [{handler,{fun otherhandler/4,S}}]),
-    ?line [{{trace,{S,_,Node},call,{?MODULE,foo,[]}},[Node]},
-	   {end_of_trace,[Node]},
-	   {{trace,{_,_,OtherNode},call,{?MODULE,foo,[]}},[OtherNode]},
-	   {end_of_trace,[OtherNode]}] = flush(),
+    ?line [{{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},[Node]},
+	   {{trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},[OtherNode]},
+	   end_of_trace] = flush(),
 
     ok.
 
@@ -593,10 +612,10 @@ seq_trace(Config) when is_list(Config) ->
 
     ?line Start = spawn(fun() -> seq() end),
     ?line timer:sleep(300),
-    ?line ttb:stop(),
+    ?line ttb:stop([nofetch]),
     ?line ok = ttb:format(
 		 [filename:join(Privdir,atom_to_list(Node)++"-seq_trace")]),
-    ?line [{trace,StartProc,call,{?MODULE,seq,[]}},
+    ?line [{trace_ts,StartProc,call,{?MODULE,seq,[]},{_,_,_}},
 	   {seq_trace,0,{send,{0,1},StartProc,P1Proc,{Start,P2}}},
 	   {seq_trace,0,{send,{1,2},P1Proc,P2Proc,{P1,Start}}},
 	   {seq_trace,0,{send,{2,3},P2Proc,StartProc,{P2,P1}}},
@@ -660,15 +679,41 @@ diskless(Config) when is_list(Config) ->
 
     ?line rpc:call(RemoteNode,?MODULE,foo,[]),
     ?line timer:sleep(500), % needed for the IP port to flush
-    ?line ttb:stop(),
+    ?line ttb:stop([nofetch]),
     ?line ?t:stop_node(RemoteNode),
     ?line ok = ttb:format(filename:join(Privdir,
 					atom_to_list(RemoteNode)++"-diskless")),
 
-    ?line [{trace,{_,_,RemoteNode},call,{?MODULE,foo,[]}},
+    ?line [{trace_ts,{_,_,RemoteNode},call,{?MODULE,foo,[]},{_,_,_}},
 	   end_of_trace] = flush(),
     ok.
 
+diskless_wrap(suite) ->
+    [];
+diskless_wrap(doc) ->
+    ["Start tracing on diskless remote node, save to local wrapped file"];
+diskless_wrap(Config) when is_list(Config) ->
+    ?line {ok,RemoteNode} = ?t:start_node(node2,slave,[]),
+    ?line c:nl(?MODULE),
+    ?line S = self(),
+    ?line Privdir=?config(priv_dir, Config),
+    ?line File = filename:join(Privdir,"diskless"),
+    ?line {ok,[RemoteNode]} =
+	ttb:tracer([RemoteNode],[{file, {local, {wrap,File,200,3}}},
+				 {handler,{fun myhandler/4, S}}]),
+    ?line {ok,[{all,[{matched,RemoteNode,_}]}]} = ttb:p(all,call),
+    ?line {ok,[{matched,RemoteNode,1}]} = ttb:tp(?MODULE,foo,[]),
+
+    ?line rpc:call(RemoteNode,?MODULE,foo,[]),
+    ?line timer:sleep(500), % needed for the IP port to flush
+    ?line ttb:stop([nofetch]),
+    ?line ?t:stop_node(RemoteNode),
+    ?line ok = ttb:format(filename:join(Privdir,
+					atom_to_list(RemoteNode)++"-diskless.*.wrp")),
+
+    ?line [{trace_ts,{_,_,RemoteNode},call,{?MODULE,foo,[]},{_,_,_}},
+	   end_of_trace] = flush(),
+    ok.
 
 otp_4967_1(suite) ->
     [];
@@ -715,7 +760,7 @@ otp_4967_2(Config) when is_list(Config) ->
     io:format("11: ~p",[now()]),
     ?line true = lists:member(heihopp,Msgs), % the heihopp message itself
     io:format("13: ~p",[now()]),
-    ?line {value,{trace,_,send,heihopp,{_,otp_4967,Node}}} = 
+    ?line {value,{trace_ts,_,send,heihopp,{_,otp_4967,Node},{_,_,_}}} =
 	lists:keysearch(heihopp,4,Msgs), % trace trace of the heihopp message
     io:format("14: ~p",[now()]),
     ?line end_of_trace = lists:last(Msgs), % end of the trace
@@ -728,6 +773,30 @@ myhandler(_Fd,Trace,_,Relay) ->
     Relay ! Trace,
     Relay.
 
+simple_call_handler() ->
+    {fun(A, {trace_ts, _, call, _, _} ,_,_) -> io:format(A, "ok.~n", []);
+	(_, end_of_trace, _, _) -> ok end, []}.
+
+marking_call_handler() ->
+    {fun(_, _, _, initial) -> file:write_file("HANDLER_OK", []);
+	(_,_,_,_) -> ok end, initial}.
+
+counter_call_handler() ->
+    {fun(_, {trace_ts, _, call, _, _} ,_,State) -> State + 1;
+	(A, end_of_trace, _, State) -> io:format(A,"~p.~n", [State]) end, 0}.
+
+ret_caller_call_handler() ->
+    {fun(A, {trace_ts, _, call, _, _, _} ,_,_) -> io:format(A, "ok.~n", []);
+	(A, {trace_ts, _, return_from, _, _, _}, _, _) -> io:format(A, "ok.~n", []);
+	(_, _, _, _) -> ok end, []}.
+
+node_call_handler() ->
+    {fun(A, {trace_ts, {_,_,Node}, call, _, _} ,_,_) -> io:format(A, "~p.~n", [Node]);
+	(_, end_of_trace, _, _) -> ok end, []}.
+
+otherhandler(_Fd,_,end_of_trace,Relay) ->
+    Relay ! end_of_trace,
+    Relay;
 otherhandler(_Fd,Trace,TI,Relay) ->
     {value,{mytraceinfo,I}} = lists:keysearch(mytraceinfo,1,TI),
     Relay ! {Trace,I},
@@ -794,3 +863,568 @@ check_gone(Dir,File) ->
 	      false -> 
 		  ok
 	  end.
+
+start_client_and_server() ->
+    ?line {ok,ClientNode} = ?t:start_node(client,slave,[]),
+    ?line ok = ttb_helper:c(code, add_paths, [code:get_path()]),
+    ?line {ok,ServerNode} = ?t:start_node(server,slave,[]),
+    ?line ok = ttb_helper:s(code, add_paths, [code:get_path()]),
+    ?line ttb_helper:clear(),
+    {ServerNode, ClientNode}.
+
+begin_trace(ServerNode, ClientNode, Dest) ->
+    ?line {ok, _} =
+	ttb:tracer([ServerNode,ClientNode],[{file, Dest}]),
+    ?line ttb:p(all, call),
+    ?line ttb:tp(server, received, []),
+    ?line ttb:tp(client, put, []),
+    ?line ttb:tp(client, get, []).
+
+begin_trace_local(ServerNode, ClientNode, Dest) ->
+    ?line {ok, _} =
+	ttb:tracer([ServerNode,ClientNode],[{file, Dest}]),
+    ?line ttb:p(all, call),
+    ?line ttb:tpl(server, received, []),
+    ?line ttb:tpl(client, put, []),
+    ?line ttb:tpl(client, get, []).
+
+check_size(N, Dest, Output, ServerNode, ClientNode) ->
+    ?line begin_trace(ServerNode, ClientNode, Dest),
+    ?line case Dest of
+        {local, _} ->
+            ?line ttb_helper:msgs_ip(N);
+        _ ->
+            ?line ttb_helper:msgs(N)
+    end,
+    ?line {_, D} = ttb:stop([fetch, return_fetch_dir]),
+    ?line ttb:format(D, [{out, Output}, {handler, simple_call_handler()}]),
+    ?line {ok, Ret} = file:consult(Output),
+    ?line true = (N + 1 == length(Ret)).
+
+fetch_when_no_option_given(suite) ->
+    [];
+fetch_when_no_option_given(doc) ->
+    ["Fetch when no option given"];
+fetch_when_no_option_given(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line {ok, Privdir} = file:get_cwd(),
+    ?line [] = filelib:wildcard(filename:join(Privdir,"ttb_upload_temptest*")),
+    begin_trace(ServerNode, ClientNode, ?FNAME),
+    ?line ttb_helper:msgs(4),
+    ?line stopped = ttb:stop(),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line [_] = filelib:wildcard(filename:join(Privdir,"ttb_upload_temptest*")).
+
+basic_ttb_run_ip_port(suite) ->
+    [];
+basic_ttb_run_ip_port(doc) ->
+    ["Basic ttb run ip port"];
+basic_ttb_run_ip_port(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line check_size(1, {local, ?FNAME}, ?OUTPUT, ServerNode, ClientNode),
+    ?line check_size(2, {local, ?FNAME}, ?OUTPUT, ServerNode, ClientNode),
+    ?line check_size(10, {local, ?FNAME}, ?OUTPUT, ServerNode, ClientNode),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode).
+
+basic_ttb_run_file_port(suite) ->
+    [];
+basic_ttb_run_file_port(doc) ->
+    ["Basic ttb run file port"];
+basic_ttb_run_file_port(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line check_size(1, ?FNAME, ?OUTPUT, ServerNode, ClientNode),
+    ?line check_size(2, ?FNAME, ?OUTPUT, ServerNode, ClientNode),
+    ?line check_size(10, ?FNAME, ?OUTPUT, ServerNode, ClientNode),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode).
+
+return_fetch_dir_implies_fetch(suite) ->
+    [];
+return_fetch_dir_implies_fetch(doc) ->
+    ["Return_fetch_dir implies fetch"];
+return_fetch_dir_implies_fetch(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace(ServerNode, ClientNode, ?FNAME),
+    ?line ttb_helper:msgs(2),
+    ?line {_,_} = ttb:stop([return_fetch_dir]),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode).
+
+logfile_name_in_fetch_dir(suite) ->
+    [];
+logfile_name_in_fetch_dir(doc) ->
+    ["Logfile name in fetch dir"];
+logfile_name_in_fetch_dir(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace(ServerNode, ClientNode, {local, ?FNAME}),
+    ?line {_,Dir} = ttb:stop([return_fetch_dir]),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line P1 = lists:nth(3, string:tokens(filename:basename(Dir), "_")),
+    ?line P2 = hd(string:tokens(P1, "-")),
+    ?line _File = P2.
+
+upload_to_my_logdir(suite) ->
+    [];
+upload_to_my_logdir(doc) ->
+    ["Upload to my logdir"];
+upload_to_my_logdir(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line {ok, _} =
+	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}]),
+    ?line {stopped,_} = ttb:stop([return_fetch_dir, {fetch_dir, ?DIRNAME}]),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line true = filelib:is_file(?DIRNAME),
+    ?line [] = filelib:wildcard("ttb_upload_"++?FNAME).
+
+upload_to_my_existing_logdir(suite) ->
+    [];
+upload_to_my_existing_logdir(doc) ->
+    ["Upload to my existing logdir"];
+upload_to_my_existing_logdir(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line ok = file:make_dir(?DIRNAME),
+    ?line {ok, _} =
+	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}]),
+    ?line {error,_,_} = (catch ttb:stop([return_fetch_dir, {fetch_dir, ?DIRNAME}])),
+    ?line {stopped,_} = ttb:stop(return_fetch_dir),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode).
+
+fetch_with_options_not_as_list(suite) ->
+    [];
+fetch_with_options_not_as_list(doc) ->
+    ["Fetch with options not as list"];
+fetch_with_options_not_as_list(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line {ok, _} =
+	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}]),
+    ?line {stopped, D} = ttb:stop(return_fetch_dir),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line false = filelib:is_file(?OUTPUT),
+    ?line ttb:format(D, {out, ?OUTPUT}),
+    ?line true = filelib:is_file(?OUTPUT).
+
+error_when_formatting_multiple_files_4393(suite) ->
+    [];
+error_when_formatting_multiple_files_4393(doc) ->
+    ["Error when formatting multiple files"];
+error_when_formatting_multiple_files_4393(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace(ServerNode, ClientNode, ?FNAME),
+    ?line ttb_helper:msgs(2),
+    ?line {_, Dir} = ttb:stop(return_fetch_dir),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line Files = [filename:join(Dir, atom_to_list(ttb_helper:get_node(server)) ++ "-" ++ ?FNAME),
+             filename:join(Dir, atom_to_list(ttb_helper:get_node(client)) ++ "-" ++ ?FNAME)],
+    ?line ok = ttb:format(Files).
+
+format_on_trace_stop(suite) ->
+    [];
+format_on_trace_stop(doc) ->
+    ["Format on trace stop"];
+format_on_trace_stop(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace(ServerNode, ClientNode, {local, ?FNAME}),
+    ?line ttb_helper:msgs_ip(2),
+    ?line file:delete("HANDLER_OK"),
+    ?line {_,_} = ttb:stop([fetch, return_fetch_dir, {format, {handler, marking_call_handler()}}]),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line true = filelib:is_file("HANDLER_OK"),
+    ?line ok = file:delete("HANDLER_OK").
+
+%% The following three tests are for the issue "fixes fetch fail when nodes on the same host
+%% have different cwd"
+trace_to_remote_files_on_localhost_with_different_pwd(suite) ->
+    [];
+trace_to_remote_files_on_localhost_with_different_pwd(doc) ->
+    ["Trace to remote files on localhost with different pwd"];
+trace_to_remote_files_on_localhost_with_different_pwd(Config) when is_list(Config) ->
+    ?line {ok, OldDir} = file:get_cwd(),
+    ?line ok = file:set_cwd(".."),
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line check_size(2, ?FNAME, ?OUTPUT, ServerNode, ClientNode),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line ok = file:set_cwd(OldDir).
+
+trace_to_local_files_on_localhost_with_different_pwd(suite) ->
+    [];
+trace_to_local_files_on_localhost_with_different_pwd(doc) ->
+    ["Trace to local files on localhost with different pwd"];
+trace_to_local_files_on_localhost_with_different_pwd(Config) when is_list(Config) ->
+    ?line {ok, OldDir} = file:get_cwd(),
+    ?line ok = file:set_cwd(".."),
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line check_size(2, {local, ?FNAME}, ?OUTPUT, ServerNode, ClientNode),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line ok = file:set_cwd(OldDir).
+
+trace_to_remote_files_on_localhost_with_different_pwd_abs(suite) ->
+    [];
+trace_to_remote_files_on_localhost_with_different_pwd_abs(doc) ->
+    ["Trace to remote files on localhost with different pwd abs"];
+trace_to_remote_files_on_localhost_with_different_pwd_abs(Config) when is_list(Config) ->
+    ?line {ok, OldDir} = file:get_cwd(),
+    ?line ok = file:set_cwd(".."),
+    ?line {ok, Path} = file:get_cwd(),
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line File = filename:join(Path, ?FNAME),
+    ?line check_size(2, File, ?OUTPUT, ServerNode, ClientNode),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line ok = file:set_cwd(OldDir).
+
+%% Trace is not affected by changes of cwd on control node or remote nodes during tracing
+%% (three tests)
+changing_cwd_on_control_node(suite) ->
+    [];
+changing_cwd_on_control_node(doc) ->
+    ["Changing cwd on control node during tracing is safe"];
+changing_cwd_on_control_node(Config) when is_list(Config) ->
+    ?line {ok, OldDir} = file:get_cwd(),
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace(ServerNode, ClientNode, ?FNAME),
+    ?line NumMsgs = 3,
+    ?line ttb_helper:msgs(NumMsgs),
+    ?line ok = file:set_cwd(".."),
+    ?line ttb_helper:msgs(NumMsgs),
+    ?line {_, D} = ttb:stop([fetch, return_fetch_dir]),
+    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, simple_call_handler()}]),
+    ?line {ok, Ret} = file:consult(?OUTPUT),
+    ?line true = (2*(NumMsgs + 1) == length(Ret)),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line ok = file:set_cwd(OldDir).
+
+changing_cwd_on_control_node_with_local_trace(suite) ->
+    [];
+changing_cwd_on_control_node_with_local_trace(doc) ->
+    ["Changing cwd on control node during local tracing is safe"];
+changing_cwd_on_control_node_with_local_trace(Config) when is_list(Config) ->
+    ?line {ok, OldDir} = file:get_cwd(),
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace(ServerNode, ClientNode, {local, ?FNAME}),
+    ?line NumMsgs = 3,
+    ?line ttb_helper:msgs_ip(NumMsgs),
+    ?line ok = file:set_cwd(".."),
+    ?line ttb_helper:msgs_ip(NumMsgs),
+    ?line {_, D} = ttb:stop([fetch, return_fetch_dir]),
+    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, simple_call_handler()}]),
+    ?line {ok, Ret} = file:consult(?OUTPUT),
+    ?line true = (2*(NumMsgs + 1) == length(Ret)),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line ok = file:set_cwd(OldDir).
+
+changing_cwd_on_remote_node(suite) ->
+    [];
+changing_cwd_on_remote_node(doc) ->
+    ["Changing cwd on remote node during tracing is safe"];
+changing_cwd_on_remote_node(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace(ServerNode, ClientNode, ?FNAME),
+    ?line NumMsgs = 2,
+    ?line ttb_helper:msgs(NumMsgs),
+    ?line ok = rpc:call(ClientNode, file, set_cwd, [".."]),
+    ?line ttb_helper:msgs(NumMsgs),
+    ?line {_, D} = ttb:stop([fetch, return_fetch_dir]),
+    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, simple_call_handler()}]),
+    ?line {ok, Ret} = file:consult(?OUTPUT),
+    ?line true = (2*(NumMsgs + 1) == length(Ret)),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode).
+
+one_command_trace_setup(suite) ->
+    [];
+one_command_trace_setup(doc) ->
+    ["One command trace setup"];
+one_command_trace_setup(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line ttb:start_trace([ttb_helper:get_node(client), ttb_helper:get_node(server)],
+		     [{server, received, '_', []},
+		      {client, put, 1, []},
+		      {client, get, '_', []}],
+		     {all, call},
+		     [{file, ?FNAME}]),
+    ?line ttb_helper:msgs(2),
+    ?line {_, D} = ttb:stop(return_fetch_dir),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, simple_call_handler()}]),
+    ?line {ok, Ret} = file:consult(?OUTPUT),
+    ?line 5 = length(Ret).
+
+dbg_style_fetch(suite) ->
+    [];
+dbg_style_fetch(doc) ->
+    ["Dbg style fetch"];
+dbg_style_fetch(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line DirSize = length(element(2, file:list_dir("."))),
+    ?line ttb:start_trace([ttb_helper:get_node(client), ttb_helper:get_node(server)],
+			   [{server, received, '_', []},
+		      {client, put, 1, []},
+		      {client, get, '_', []}],
+			   {all, call},
+			   [{shell, only}]),
+    ?line DirSize = length(element(2, file:list_dir("."))),
+    ?line ttb_helper:msgs(2),
+    ?line DirSize = length(element(2, file:list_dir("."))),
+    ?line stopped, ttb:stop(format),
+    %%+1 -> ttb_last_trace
+    ?line true = (DirSize + 1 == length(element(2, file:list_dir(".")))),
+    ?line {ok,[{all, [{matched,_,_}, {matched,_,_}]}]} =
+	ttb:start_trace([ttb_helper:get_node(client), ttb_helper:get_node(server)],
+			[{server, received, '_', []},
+			  {client, put, 1, []},
+			  {client, get, '_', []}],
+			{all, call},
+			[{shell, only}]),
+    ?line ttb:stop(),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode).
+
+shell_tracing_init(suite) ->
+    [];
+shell_tracing_init(doc) ->
+    ["Shell tracing init"];
+shell_tracing_init(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line ttb:tracer([ttb_helper:get_node(client), ttb_helper:get_node(server)], shell),
+    ?line ttb:stop(),
+    ?line ttb:tracer([ttb_helper:get_node(client), ttb_helper:get_node(server)],
+		     [{file, {local, ?FNAME}}, shell]),
+    ?line ttb:stop(),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line local_client_required_on_shell_tracing = try  ttb:tracer([ttb_helper:get_node(client), ttb_helper:get_node(server)],
+								   [{file, ?FNAME}, shell])
+						   catch
+						       exit:local_client_required_on_shell_tracing ->
+							   local_client_required_on_shell_tracing
+						   end.
+
+only_one_state_for_format_handler(suite) ->
+    [];
+only_one_state_for_format_handler(doc) ->
+    ["Only one state for format handler"];
+only_one_state_for_format_handler(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace_local(ServerNode, ClientNode, ?FNAME),
+    ?line ttb_helper:msgs(2),
+    ?line {_, D} = ttb:stop([return_fetch_dir]),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, counter_call_handler()}]),
+    ?line {ok, Ret} = file:consult(?OUTPUT),
+    ?line [5] = Ret.
+
+only_one_state_with_default_format_handler(suite) ->
+    [];
+only_one_state_with_default_format_handler(doc) ->
+    ["Only one state with default format handler"];
+only_one_state_with_default_format_handler(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace_local(ServerNode, ClientNode, ?FNAME),
+    ?line ttb_helper:msgs(2),
+    ?line {_, D} = ttb:stop([return_fetch_dir]),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line ttb:format(D, [{out, ?OUTPUT}]),
+    ?line true = filelib:is_file(?OUTPUT).
+
+only_one_state_with_initial_format_handler(suite) ->
+    [];
+only_one_state_with_initial_format_handler(doc) ->
+    ["Only one state with initial format handler"];
+only_one_state_with_initial_format_handler(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line {ok, _} =
+	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}, {handler, counter_call_handler()}]),
+    ?line ttb:p(all, call),
+    ?line ttb:tpl(server, received, []),
+    ?line ttb:tpl(client, put, []),
+    ?line ttb:tpl(client, get, []),
+    ?line ttb_helper:msgs(2),
+    ?line {_, D} = ttb:stop([return_fetch_dir]),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line ttb:format(D, [{out, ?OUTPUT}]),
+    ?line {ok, Ret} = file:consult(?OUTPUT),
+    ?line [5] = Ret.
+
+run_trace_with_shortcut(Shortcut, Ret, F) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line {ok, _} =
+	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}]),
+    ?line ttb:p(all, call),
+    ?line ttb:F(client, put, Shortcut),
+    ?line ttb_helper:msgs(2),
+    ?line {_, D} = ttb:stop([return_fetch_dir]),
+    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, ret_caller_call_handler()}]),
+    ?line {ok, Ret} =file:consult(?OUTPUT),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode).
+
+fun_for(return) ->
+    {codestr, "fun(_) -> return_trace() end"};
+fun_for(msg_false) ->
+    {codestr, "fun(_) -> message(false) end"}.
+
+run_trace_with_shortcut1(suite) ->
+    [];
+run_trace_with_shortcut1(doc) ->
+    ["Run trace with shortcut 1"];
+run_trace_with_shortcut1(Config) when is_list(Config) ->
+    ?line run_trace_with_shortcut(caller, [ok,ok], tp),
+    ?line run_trace_with_shortcut(caller, [ok,ok], tpl).
+
+run_trace_with_shortcut2(suite) ->
+    [];
+run_trace_with_shortcut2(doc) ->
+    ["Run trace with shortcut 2"];
+run_trace_with_shortcut2(Config) when is_list(Config) ->
+    ?line run_trace_with_shortcut(return, [ok,ok], tp),
+    ?line run_trace_with_shortcut(return, [ok,ok], tpl).
+
+run_trace_with_shortcut3(suite) ->
+    [];
+run_trace_with_shortcut3(doc) ->
+    ["Run trace with shortcut 3"];
+run_trace_with_shortcut3(Config) when is_list(Config) ->
+    ?line run_trace_with_shortcut(fun_for(return), [ok,ok], tp),
+    ?line run_trace_with_shortcut(fun_for(return), [ok,ok], tpl).
+
+run_trace_with_shortcut4(suite) ->
+    [];
+run_trace_with_shortcut4(doc) ->
+    ["Run trace with shortcut 4"];
+run_trace_with_shortcut4(Config) when is_list(Config) ->
+    ?line run_trace_with_shortcut(fun_for(msg_false), [], tp),
+    ?line run_trace_with_shortcut(fun_for(msg_false), [], tpl).
+
+cant_specify_local_and_flush(suite) ->
+    [];
+cant_specify_local_and_flush(doc) ->
+    ["Can't specify local and flush"];
+cant_specify_local_and_flush(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line flush_unsupported_with_ip_trace_port = try ttb:tracer([ServerNode, ClientNode], [{flush, 1000}, {file, {local, ?FNAME}}])
+						 catch
+						     exit:flush_unsupported_with_ip_trace_port ->
+							 flush_unsupported_with_ip_trace_port
+						 end,
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode).
+
+trace_sorted_by_default(suite) ->
+    [];
+trace_sorted_by_default(doc) ->
+    ["Trace sorted by default"];
+trace_sorted_by_default(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace_local(ServerNode, ClientNode, ?FILE),
+    ?line ttb_helper:msgs(2),
+    ?line {_, D} = ttb:stop([return_fetch_dir]),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, node_call_handler()}, {disable_sort, false}]),
+    {ok, Ret} = file:consult(?OUTPUT),
+    ?line [ClientNode,ServerNode,ClientNode,ServerNode,ServerNode] = Ret.
+
+disable_sorting(suite) ->
+    [];
+disable_sorting(doc) ->
+    ["Disable sorting"];
+disable_sorting(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace_local(ServerNode, ClientNode, ?FILE),
+    ?line ttb_helper:msgs(2),
+    ?line {_, D} = ttb:stop([return_fetch_dir]),
+    ?line ?t:stop_node(ServerNode),
+    ?line ?t:stop_node(ClientNode),
+    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, node_call_handler()}, {disable_sort, true}]),
+    {ok, Ret} = file:consult(?OUTPUT),
+    ?line [ClientNode,ClientNode,ServerNode,ServerNode,ServerNode] = Ret.
+
+%% -----------------------------------------------------------------------------
+%% tests for autoresume of tracing
+%% -----------------------------------------------------------------------------
+
+trace_resumed_after_node_restart(suite) ->
+    [];
+trace_resumed_after_node_restart(doc) ->
+    ["Test trace resumed after node restart, trace to files on remote node."];
+trace_resumed_after_node_restart(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace_with_resume(ServerNode, ClientNode, ?FNAME),
+    ?line logic(2,6,file).
+
+trace_resumed_after_node_restart_ip(suite) ->
+    [];
+trace_resumed_after_node_restart_ip(doc) ->
+    ["Test trace resumed after node restart, trace via tcp/ip to local node."];
+trace_resumed_after_node_restart_ip(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace_with_resume(ServerNode, ClientNode, {local, ?FNAME}),
+    ?line logic(2,6,local).
+
+trace_resumed_after_node_restart_wrap(suite) ->
+    [];
+trace_resumed_after_node_restart_wrap(doc) ->
+    ["Test trace resumed after node restart, wrap option."];
+trace_resumed_after_node_restart_wrap(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace_with_resume(ServerNode, ClientNode, {wrap, ?FNAME, 10, 4}),
+    ?line logic(1,4,file).
+
+trace_resumed_after_node_restart_wrap_mult(suite) ->
+    [];
+trace_resumed_after_node_restart_wrap_mult(doc) ->
+    ["Test trace resumed after node restart, wrap option, multiple files."];
+trace_resumed_after_node_restart_wrap_mult(Config) when is_list(Config) ->
+    ?line {ServerNode, ClientNode} = start_client_and_server(),
+    ?line begin_trace_with_resume(ServerNode, ClientNode, {wrap, ?FNAME, 10, 4}),
+    ?line logic(20,8,file).
+
+logic(N, M, TracingType) ->
+    helper_msgs(N, TracingType),
+    ?t:stop_node(ttb_helper:get_node(client)),
+    timer:sleep(2500),
+    ?line {ok,ClientNode} = ?t:start_node(client,slave,[]),
+    ?line ok = ttb_helper:c(code, add_paths, [code:get_path()]),
+    ?line ttb_helper:c(client, init, []),
+    ?line helper_msgs(N, TracingType),
+    ?line {_, D} = ttb:stop([return_fetch_dir]),
+    ?line ?t:stop_node(ttb_helper:get_node(server)),
+    ?line ?t:stop_node(ClientNode),
+    ?line ttb:format(D, [{out, ?OUTPUT}, {handler, ret_caller_call_handler2()}]),
+    ?line {ok, Ret} = file:consult(?OUTPUT),
+    ?line M = length(Ret).
+
+begin_trace_with_resume(ServerNode, ClientNode, Dest) ->
+    ?line {ok, _} = ttb:tracer([ServerNode,ClientNode], [{file, Dest}, resume]),
+    ?line ttb:p(all, [call, timestamp]),
+    ?line ttb:tp(server, received, []),
+    ?line ttb:tp(client, put, []),
+    ?line ttb:tp(client, get, []).
+
+ret_caller_call_handler2() ->
+    {fun(A, {trace_ts, _, call, _, _} ,_,_) -> io:format(A, "ok.~n", []);
+	(_, _, _, _) -> ok end, []}.
+
+helper_msgs(N, TracingType) ->
+    case TracingType of
+        local ->
+            ttb_helper:msgs_ip(N);
+        _ ->
+            ttb_helper:msgs(N)
+    end.
diff --git a/lib/observer/test/ttb_helper.erl b/lib/observer/test/ttb_helper.erl
new file mode 100644
index 0000000000..19fdc0e159
--- /dev/null
+++ b/lib/observer/test/ttb_helper.erl
@@ -0,0 +1,157 @@
+-module(ttb_helper). %%Nodes control
+-compile(export_all).
+
+%%API
+%%get() -> client:get()
+%%put(X) -> client:put(X)
+%%msgs(N) -> N times client:put(test_msg)
+%%clear() -> restart server
+%%ensure_running() / stop() -> start/stop nodes
+%%get_node(atom) -> return atom@hostname
+
+-define(NODE_CMD(Name),
+	"erl -sname " ++ atom_to_list(Name) ++
+	" -pa .. -pa . -detached -run ttb_helper send_ok").
+-define(REG_NAME, nc_testing).
+
+new_fun() ->
+    fun(_, end_of_trace, _, Dict) -> io:format("~p~n", [dict:to_list(Dict)]);
+       (_, T, _, Dict) -> case element(2, T) of
+                              {Pid, _, _} ->
+                                  dict:update_counter(Pid, 1, Dict);
+                              Pid ->
+                                  dict:update_counter(Pid, 1, Dict)
+                          end
+    end.
+
+new_fun_2() ->
+    fun(_, end_of_trace, _, Dict) -> io:format("~p~n", [dict:to_list(Dict)]);
+       (_, T, _, Dict) ->  case element(2, T) of
+                               {_, Name, _} when is_atom(Name)->
+                                  dict:update_counter(Name, 1, Dict);
+                              Pid ->
+                                  dict:update_counter(Pid, 1, Dict)
+                          end
+
+    end.
+
+
+ensure_running() ->
+    try_start_node(server),
+    try_start_node(client),
+    clear().
+
+try_start_node(Node) ->
+    global:unregister_name(?REG_NAME),
+    global:register_name(?REG_NAME, self()),
+    global:sync(),
+    N = get_node(Node),
+    case net_adm:ping(N) of
+	pong ->
+	    io:format("Node ~p already running~n", [N]);
+	_ ->
+	    io:format("Starting node ~p... ~p ", [Node, os:cmd(?NODE_CMD(Node))]),
+	    recv()
+    end.
+
+clear() ->
+    s(server, stop, []),
+    init().
+
+stop() ->
+    s(init, stop, []),
+    c(init, stop, []).
+
+msgs(N) ->
+    [c(client, put, [test_msg]) || _ <- lists:seq(1, N)],
+    s(server, received, [a,b]),
+    [dbg:flush_trace_port(Node) || Node <- [get_node(client), get_node(server)]].
+
+msgs_ip(N) ->
+    [c(client, put, [test_msg]) || _ <- lists:seq(1, N)],
+    s(server, received, [a,b]),
+    timer:sleep(100). %% allow trace messages to arrive over tcp/ip
+
+run() ->
+    ttb({local, "A"}),
+    msgs(2),
+    c(erlang, whereis, [ttbt]).
+
+get() -> c(client, get, []).
+put(Thing) -> c(client, put, [Thing]).
+
+get_node(Node) ->
+    {ok, Host} = inet:gethostname(),
+    list_to_atom(atom_to_list(Node) ++ "@" ++ Host).
+
+trace_setup() ->
+    ttb:p(all, call),
+    ttb:tp(server, received, []),
+    ttb:tp(client, put, []),
+    ttb:tp(client, get, []).
+
+ttb() -> ttb("A").
+ttb(File) ->
+    ttb:tracer([get_node(client), get_node(server)], [{file, File}, resume]),
+    ttb:p(all, [call, timestamp]),
+    ttb:tp(client, put, []),
+    ttb:tp(client, get, []),
+    ttb:tp(server, received, []).
+
+tc() ->
+    TC = example_config_gen:create_trace_case("dummy comment"),
+    Patterns = example_config_gen:create_pattern(client, put, 1, return),
+    Flags = example_config_gen:create_flags(all, call),
+    Merge = example_config_gen:create_merge_conf(show_handler(), "dummy merge comment"),
+    Merge2 = example_config_gen:create_merge_conf(undefined, "dummy merge comment"),
+    TC2 = example_config_gen:add_pattern(Patterns, TC),
+    TC3 = example_config_gen:add_flags(Flags, TC2),
+    TC4 = example_config_gen:add_merge_conf(Merge, TC3),
+    TC5 = example_config_gen:add_merge_conf(Merge2, TC4),
+    example_config_gen:add_nodes([get_node(client), get_node(server)], TC5).
+
+
+show(X) ->
+    io:format(user, "Showing: ~p~n", [X]).
+
+state_handler() ->
+    {fun(_,_,I,S) -> io:format(user, "Got from ~p: ~p~n", [I,S]), S+1 end, 0}.
+
+show_handler() ->
+    {fun(A,B,_,_) -> io:format(A, "~p~n", [B]) end, []}.
+
+opts() ->
+    [[get_node(client), get_node(server)],
+     [{server, received, '_', []},
+      {client, put, '_', []},
+      {client, get, '_', []}],
+     {all, call},
+     [{file, "TEST"}]].
+
+overload_check(check) ->
+    true;
+overload_check(_) ->
+    ok.
+%%%Internal
+s(M, F, A) -> rpc:call(get_node(server), M, F, A).
+c(M, F, A) -> rpc:call(get_node(client), M, F, A).
+
+send_ok() ->
+    pong = net_adm:ping(get_node(test)),
+    global:sync(),
+    global:send(?REG_NAME, node()).
+
+init() ->
+    True = s(server, start, []),
+    io:format("ok1: ~p~n", [True]),
+    true = c(client, init, [get_node(server)]).
+
+recv() ->
+    receive
+	Node ->
+	    io:format("Node ~p ready.~n", [Node]),
+            ok
+    after 5000 ->
+	    io:format("Startup failed~n",[]),
+	    throw(startup_failed)
+    end.