1: %%
    2: %% %CopyrightBegin%
    3: %%
    4: %%
    5: %% Copyright Ericsson AB 2002-2013. All Rights Reserved.
    6: %%
    7: %% The contents of this file are subject to the Erlang Public License,
    8: %% Version 1.1, (the "License"); you may not use this file except in
    9: %% compliance with the License. You should have received a copy of the
   10: %% Erlang Public License along with this software. If not, it can be
   11: %% retrieved online at http://www.erlang.org/.
   12: %%
   13: %% Software distributed under the License is distributed on an "AS IS"
   14: %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
   15: %% the License for the specific language governing rights and limitations
   16: %% under the License.
   17: %%
   18: %% %CopyrightEnd%
   19: %%
   20: 
   21: -module(ttb_SUITE).
   22: 
   23: -compile(export_all).
   24: %% Test functions
   25: -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
   26: 	 init_per_group/2,end_per_group/2,
   27: 	 file/1,file_no_pi/1,file_fetch/1,wrap/1,wrap_merge/1,
   28: 	 wrap_merge_fetch_format/1,write_config1/1,write_config2/1,
   29: 	 write_config3/1,history/1,write_trace_info/1,seq_trace/1,
   30: 	 diskless/1,otp_4967_1/1,otp_4967_2/1]).
   31: -export([init_per_testcase/2, end_per_testcase/2]).
   32: -export([foo/0]).
   33: 
   34: -include_lib("test_server/include/test_server.hrl").
   35: 
   36: -define(default_timeout, ?t:minutes(1)).
   37: -define(OUTPUT, "handler_output").
   38: -define(FNAME, "temptest").
   39: -define(DIRNAME, "ddtemp").
   40: 
   41: init_per_testcase(Case, Config) ->
   42:     ?line Dog=test_server:timetrap(?default_timeout),
   43:     ttb:stop(),
   44:     rm(?OUTPUT),
   45:     [rm(Upload) || Upload<-filelib:wildcard("ttb_upload*")],
   46:     rm(?DIRNAME),
   47:     [rm(At) || At <- filelib:wildcard("*@*")],
   48:     rm("ttb_last_config"),
   49:     %% Workaround for bug(?) in test_server - if the test case fails
   50:     %% with a timetrap timeout, then end_per_testcase will run with
   51:     %% faulty group_leader - which in turn makes test_server:stop_node
   52:     %% hang (stop_node is called by most of the cleanup functions).
   53:     %% Therefore we do the cleanup before each testcase instead - this
   54:     %% is obviously not 100% correct, but it will at least make sure
   55:     %% that the nodes which are to be started in a test case at are
   56:     %% terminated.
   57:     try apply(?MODULE,Case,[cleanup,Config])
   58:     catch error:undef -> ok
   59:     end,
   60:     [{watchdog, Dog}|Config].
   61: end_per_testcase(_Case, Config) ->
   62:     %% try apply(?MODULE,Case,[cleanup,Config])
   63:     %% catch error:undef -> ok
   64:     %% end,
   65:     Dog=?config(watchdog, Config),
   66:     ?t:timetrap_cancel(Dog),
   67:     ok.
   68: 
   69: suite() -> [{ct_hooks,[ts_install_cth]}].
   70: 
   71: all() -> 
   72:     [file, file_no_pi, file_fetch, wrap, wrap_merge,
   73:      wrap_merge_fetch_format, write_config1, write_config2,
   74:      write_config3, history, write_trace_info, seq_trace,
   75:      diskless, diskless_wrap, otp_4967_1, otp_4967_2,
   76:      fetch_when_no_option_given, basic_ttb_run_ip_port, basic_ttb_run_file_port,
   77:      return_fetch_dir_implies_fetch, logfile_name_in_fetch_dir, upload_to_my_logdir,
   78:      upload_to_my_existing_logdir, fetch_with_options_not_as_list,
   79:      error_when_formatting_multiple_files_4393, format_on_trace_stop,
   80:      trace_to_remote_files_on_localhost_with_different_pwd,
   81:      trace_to_local_files_on_localhost_with_different_pwd,
   82:      trace_to_remote_files_on_localhost_with_different_pwd_abs,
   83:      changing_cwd_on_control_node, changing_cwd_on_remote_node,
   84:      changing_cwd_on_control_node_with_local_trace,
   85:      one_command_trace_setup, dbg_style_fetch, shell_tracing_init,
   86:      only_one_state_for_format_handler, only_one_state_with_default_format_handler,
   87:      only_one_state_with_initial_format_handler, run_trace_with_shortcut1,
   88:      run_trace_with_shortcut2, run_trace_with_shortcut3, run_trace_with_shortcut4,
   89:      cant_specify_local_and_flush, trace_sorted_by_default,disable_sorting,
   90:      trace_resumed_after_node_restart, trace_resumed_after_node_restart_ip,
   91:      trace_resumed_after_node_restart_wrap,
   92:      trace_resumed_after_node_restart_wrap_mult
   93: ].
   94: 
   95: groups() -> 
   96:     [].
   97: 
   98: init_per_suite(Config) ->
   99:     clean_priv_dir(Config),
  100:     Config.
  101: 
  102: end_per_suite(_Config) ->
  103:     ok.
  104: 
  105: init_per_group(_GroupName, Config) ->
  106:     Config.
  107: 
  108: end_per_group(_GroupName, Config) ->
  109:     Config.
  110: 
  111: 
  112: file(suite) ->
  113:     [];
  114: file(doc) ->
  115:     ["Start tracing on multiple nodes, single file"];
  116: file(Config) when is_list(Config) ->
  117:     ?line Node = node(),
  118:     ?line {ok,OtherNode} = ?t:start_node(node2,slave,[]),
  119:     ?line c:nl(?MODULE),
  120:     ?line S = self(),
  121:     ?line Privdir=priv_dir(Config),
  122:     ?line File = filename:join(Privdir,"file"),
  123:     ?line {ok,[Node]} =
  124: 	ttb:tracer(Node,[{file, File},
  125: 			 {handler,{fun myhandler/4, S}}]),
  126:     ?line {ok,[{S,[{matched,Node,_}]}]} = ttb:p(S,call),
  127:     ?line {ok,[OtherNode]} = 
  128: 	ttb:tracer([Node,OtherNode],[{file, File},
  129: 				     {handler,{fun myhandler/4, S}}]),
  130:     ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
  131:     ?line {ok,[]} = ttb:tracer([Node,OtherNode],
  132: 				  [{file, File},
  133: 				   {handler,{fun myhandler/4, S}}]),
  134:     ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
  135:     ?line ?MODULE:foo(),
  136:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  137:     ?line ttb:stop([nofetch]),
  138:     ?line ok = ttb:format(filename:join(Privdir,atom_to_list(Node)++"-file")),
  139:     ?line ok = ttb:format(filename:join(Privdir,
  140: 					atom_to_list(OtherNode)++"-file")),
  141: 
  142:     ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
  143: 	   end_of_trace,
  144: 	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
  145: 	   end_of_trace] = flush(),
  146:     ok.
  147: 
  148: file(cleanup,_Config) ->
  149:     ?line ?t:stop_node(ttb_helper:get_node(node2)).
  150: 
  151: file_no_pi(suite) ->
  152:     [];
  153: file_no_pi(doc) ->
  154:     ["Start tracing on multiple nodes, single file, no process information"];
  155: file_no_pi(Config) when is_list(Config) ->
  156:     ?line Node = node(),
  157:     ?line {ok,OtherNode} = ?t:start_node(node2,slave,[]),
  158:     ?line c:nl(?MODULE),
  159:     ?line S = self(),
  160:     ?line Privdir=priv_dir(Config),
  161:     ?line File = filename:join(Privdir,"file"),
  162:     ?line {ok,[_,_]} =
  163: 	ttb:tracer([Node,OtherNode],[{file, File},
  164: 				     {handler,{fun myhandler/4, S}},
  165: 				     {process_info,false}]),
  166:     ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
  167:     ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
  168:     ?line ?MODULE:foo(),
  169:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  170:     ?line ttb:stop([nofetch]),
  171:     ?line ok = ttb:format(filename:join(Privdir,atom_to_list(Node)++"-file")),
  172:     ?line ok = ttb:format(filename:join(Privdir,
  173: 					atom_to_list(OtherNode)++"-file")),
  174: 
  175:     ?line [{trace_ts,LocalProc,call,{?MODULE,foo,[]}, {_,_,_}},
  176: 	   end_of_trace,
  177: 	   {trace_ts,RemoteProc,call,{?MODULE,foo,[]},{_,_,_}},
  178: 	   end_of_trace] = flush(),
  179:     ?line true = is_pid(LocalProc),
  180:     ?line true = is_pid(RemoteProc),
  181:     ok.
  182: 
  183: file_no_pi(cleanup,_Config) ->
  184:     ?line ?t:stop_node(ttb_helper:get_node(node2)).
  185: 
  186: 
  187: file_fetch(suite) ->
  188:     [];
  189: file_fetch(doc) ->
  190:     ["stop with the fetch option, i.e. collect all files when ttb is stopped"];
  191: file_fetch(Config) when is_list(Config) ->
  192:     ?line Node = node(),
  193:     ?line {ok,OtherNode} = ?t:start_node(node2,slave,[]),
  194:     ?line c:nl(?MODULE),
  195:     ?line S = self(),
  196:     ?line Privdir=priv_dir(Config),
  197:     ?line ThisDir = filename:join(Privdir,this),
  198:     ?line ok = file:make_dir(ThisDir),
  199:     ?line OtherDir = filename:join(Privdir,other),
  200:     ?line ok = file:make_dir(OtherDir),
  201:     ?line ThisFile = filename:join(ThisDir,"file_fetch"),
  202:     ?line OtherFile = filename:join(OtherDir,"file_fetch"),
  203: 
  204:     %% I'm setting priv_dir as cwd, so ttb_upload directory is created there
  205:     %% and not in any other strange place!
  206:     ?line {ok,Cwd} = file:get_cwd(),
  207:     ?line ok = file:set_cwd(Privdir),
  208: 
  209:     ?line {ok,[Node]} =
  210: 	ttb:tracer(Node,[{file, ThisFile},
  211: 			 {handler,{fun myhandler/4, S}}]),
  212:     ?line {ok,[OtherNode]} = 
  213: 	ttb:tracer([OtherNode],[{file, OtherFile},
  214: 				     {handler,{fun myhandler/4, S}}]),
  215:     ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
  216:     ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
  217:     ?line ?MODULE:foo(),
  218:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  219:     ?line ?t:capture_start(),
  220:     ?line ttb:stop([return_fetch_dir]),
  221:     ?line ?t:capture_stop(),
  222:     ?line [StoreString] = ?t:capture_get(),
  223:     ?line UploadDir =
  224: 	lists:last(string:tokens(lists:flatten(StoreString),"$ \n")),
  225: 
  226:     %% check that files are no longer in original directories...
  227:     ?line ok = check_gone(ThisDir,atom_to_list(Node)++"-file_fetch"),
  228:     ?line ok = check_gone(ThisDir,atom_to_list(Node)++"-file_fetch.ti"),
  229: %    ?line false = lists:member(TrcLog,ThisList),
  230: %    ?line false = lists:member(TIFile,ThisList),
  231:     
  232:     ?line {ok,OtherList} = file:list_dir(OtherDir),
  233:     ?line false = lists:member(atom_to_list(OtherNode)++"-file_fetch",OtherList),
  234:     ?line false = lists:member(atom_to_list(OtherNode)++"-file_fetch.ti",
  235: 			       OtherList),
  236:     
  237:     %% but instead in ttb_upload directory, where they can be formatted
  238:     ?line ok = ttb:format(filename:join(UploadDir,
  239: 					atom_to_list(Node)++"-file_fetch")),
  240:     ?line ok = ttb:format(filename:join(UploadDir,
  241: 					atom_to_list(OtherNode)++"-file_fetch")),
  242: 
  243:     ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
  244: 	   end_of_trace,
  245: 	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
  246: 	   end_of_trace] = flush(),
  247: 
  248:     ?line ok = file:set_cwd(Cwd),
  249:     ok.
  250: 
  251: file_fetch(cleanup,_Config) ->
  252:     ?line ?t:stop_node(ttb_helper:get_node(node2)).
  253: 
  254: 
  255: wrap(suite) ->
  256:     [];
  257: wrap(doc) ->
  258:     ["Start tracing on multiple nodes, wrap files"];
  259: wrap(Config) when is_list(Config) ->
  260:     ?line Node = node(),
  261:     ?line {ok,OtherNode} = ?t:start_node(node2,slave,[]),
  262:     ?line c:nl(?MODULE),
  263:     ?line S = self(),
  264:     ?line Privdir=priv_dir(Config),
  265:     ?line File = filename:join(Privdir,"wrap"),
  266:     ?line {ok,[_,_]} = 
  267: 	ttb:tracer([Node,OtherNode],[{file, {wrap,File,200,3}},
  268: 				     {handler,{fun myhandler/4, S}}]),
  269:     ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
  270:     ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
  271:     ?line ?MODULE:foo(),
  272:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  273:     ?line ?MODULE:foo(),
  274:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  275:     ?line ?MODULE:foo(),
  276:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  277:     ?line ttb:stop([nofetch]),
  278:     ?line ok = ttb:format(filename:join(Privdir,
  279: 					atom_to_list(Node)++"-wrap.*.wrp")),
  280:     ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
  281: 	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
  282: 	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
  283: 	   end_of_trace] = flush(),
  284:     ?line ok = ttb:format(filename:join(Privdir,
  285: 					atom_to_list(OtherNode)++"-wrap.*.wrp")),
  286:     ?line [{trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
  287: 	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
  288: 	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
  289: 	   end_of_trace] = flush(),
  290: 
  291:     %% Check that merge does not crash even if the timestamp flag is not on.
  292:     ?line ok = ttb:format(
  293: 		 [filename:join(Privdir,
  294: 				atom_to_list(Node)++"-wrap.*.wrp"),
  295: 		  filename:join(Privdir,
  296: 				atom_to_list(OtherNode)++"-wrap.*.wrp")],[{disable_sort,true}]),
  297:     ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
  298: 	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
  299: 	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
  300: 	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
  301: 	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
  302: 	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},
  303: 	   end_of_trace] = flush(),
  304:     ok.
  305: 
  306: wrap(cleanup,_Config) ->
  307:     ?line ?t:stop_node(ttb_helper:get_node(node2)).
  308: 
  309: wrap_merge(suite) ->
  310:     [];
  311: wrap_merge(doc) ->
  312:     ["Start tracing on multiple nodes, wrap files, merge logs from both nodes"];
  313: wrap_merge(Config) when is_list(Config) ->
  314:     ?line Node = node(),
  315:     ?line {ok,OtherNode} = ?t:start_node(node2,slave,[]),
  316:     ?line c:nl(?MODULE),
  317:     ?line S = self(),
  318:     ?line Privdir=priv_dir(Config),
  319:     ?line File = filename:join(Privdir,"wrap_merge"),
  320:     ?line {ok,[_,_]} = 
  321: 	ttb:tracer([Node,OtherNode],[{file, {wrap,File,200,3}},
  322: 				     {handler,{fun myhandler/4, S}}]),
  323:     ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]}=ttb:p(all,[call,timestamp]),
  324:     ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
  325:     ?line ?MODULE:foo(),
  326:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  327:     ?line ?MODULE:foo(),
  328:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  329:     ?line ?MODULE:foo(),
  330:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  331:     ?line ttb:stop([nofetch]),
  332:     ?line ok = ttb:format(
  333: 		 [filename:join(Privdir,
  334: 				atom_to_list(Node)++"-wrap_merge.*.wrp"),
  335: 		  filename:join(Privdir,
  336: 				atom_to_list(OtherNode)++"-wrap_merge.*.wrp")]),
  337:     ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
  338: 	   {trace_ts,_,call,{?MODULE,foo,[]},_},
  339: 	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
  340: 	   {trace_ts,_,call,{?MODULE,foo,[]},_},
  341: 	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
  342: 	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},_},
  343: 	   end_of_trace] = flush(),
  344:     ok.
  345: 
  346: wrap_merge(cleanup,_Config) ->
  347:     ?line ?t:stop_node(ttb_helper:get_node(node2)).
  348: 
  349: 
  350: wrap_merge_fetch_format(suite) ->
  351:     [];
  352: wrap_merge_fetch_format(doc) ->
  353:     ["Start tracing on multiple nodes, wrap files, fetch and format at stop"];
  354: wrap_merge_fetch_format(Config) when is_list(Config) ->
  355:     ?line Node = node(),
  356:     ?line {ok,OtherNode} = ?t:start_node(node2,slave,[]),
  357:     ?line c:nl(?MODULE),
  358:     ?line S = self(),
  359:     ?line Privdir=priv_dir(Config),
  360:     ?line File = filename:join(Privdir,"wrap_merge_fetch_format"),
  361: 
  362:     %% I'm setting priv_dir as cwd, so ttb_upload directory is created there
  363:     %% and not in any other strange place!
  364:     ?line {ok,Cwd} = file:get_cwd(),
  365:     ?line ok = file:set_cwd(Privdir),
  366: 
  367:     ?line {ok,[_,_]} = 
  368: 	ttb:tracer([Node,OtherNode],[{file, {wrap,File,200,3}},
  369: 				     {handler,{fun myhandler/4, S}}]),
  370:     ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]}=ttb:p(all,[call,timestamp]),
  371:     ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
  372:     ?line ?MODULE:foo(),
  373:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  374:     ?line ?MODULE:foo(),
  375:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  376:     ?line ?MODULE:foo(),
  377:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  378:     ?line ttb:stop([format]),
  379:     ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
  380: 	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},_},
  381: 	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
  382: 	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},_},
  383: 	   {trace_ts,{S,_,Node},call,{?MODULE,foo,[]},_},
  384: 	   {trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},_},
  385: 	   end_of_trace] = flush(),
  386: 
  387:     ?line ok = file:set_cwd(Cwd),
  388:     ok.
  389: 
  390: wrap_merge_fetch_format(cleanup,_Config) ->
  391:     ?line ?t:stop_node(ttb_helper:get_node(node2)).
  392: 
  393: write_config1(suite) ->
  394:     [];
  395: write_config1(doc) ->
  396:     ["Write config given commands"];
  397: write_config1(Config) when is_list(Config) ->
  398:     ?line Node = node(),
  399:     ?line {ok,OtherNode} = ?t:start_node(node2,slave,[]),
  400:     ?line c:nl(?MODULE),
  401:     ?line S = self(),
  402: 
  403:     ?line Privdir=priv_dir(Config),
  404:     ?line File = filename:join(Privdir,"write_config1"),
  405:     ?line ok = ttb:write_config(File,
  406: 				[{ttb,tracer,[[Node,OtherNode],
  407: 					      [{file, File},
  408: 					       {handler,{fun myhandler/4,S}}]]},
  409: 				 {ttb,p,[all,call]},
  410: 				 {ttb,tp,[?MODULE,foo,[]]}]),
  411:     ?line [_,_,_] = ttb:list_config(File),
  412:     ?line ok = ttb:run_config(File),
  413:     ?line ?MODULE:foo(),
  414:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  415:     ?line ttb:stop([nofetch]),
  416:     ?line ok = ttb:format(
  417: 		 [filename:join(Privdir,
  418: 				atom_to_list(Node)++"-write_config1"),
  419: 		  filename:join(Privdir,
  420: 				atom_to_list(OtherNode)++"-write_config1")]),
  421:     ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
  422: 	   {trace_ts,Other,call,{?MODULE,foo,[]},{_,_,_}},
  423: 	   end_of_trace] = flush(),
  424: 
  425:     case metatest(Other,OtherNode,Privdir,"-write_config1.ti") of
  426: 	{error,Reason} ->
  427: 	    timer:sleep(5000),
  428: 	    ?line ok = ttb:format(
  429: 			 [filename:join(Privdir,
  430: 					atom_to_list(Node)++"-write_config1"),
  431: 			  filename:join(Privdir,
  432: 					atom_to_list(OtherNode)++
  433: 					"-write_config1")]),
  434: 	    ?line io:format("\nTrying again: ~p\n",[flush()]),
  435: 	    ?line ?t:fail(Reason);
  436: 	ok ->
  437: 	    ok
  438:     end,	    
  439:     ok.
  440: 
  441: 
  442: write_config1(cleanup,_Config) ->
  443:     ?line ?t:stop_node(ttb_helper:get_node(node2)).
  444: 
  445: write_config2(suite) ->
  446:     [];
  447: write_config2(doc) ->
  448:     ["Write config from history (all)"];
  449: write_config2(Config) when is_list(Config) ->
  450:     ?line Node = node(),
  451:     ?line {ok,OtherNode} = ?t:start_node(node2,slave,[]),
  452:     ?line c:nl(?MODULE),
  453:     ?line S = self(),
  454:     ?line Privdir=priv_dir(Config),
  455:     ?line File = filename:join(Privdir,"write_config2"),
  456:     ?line {ok,[_,_]} = 
  457: 	ttb:tracer([Node,OtherNode],[{file, File},
  458: 				     {handler,{fun myhandler/4, S}}]),
  459:     ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
  460:     ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
  461:     ?line ok = ttb:write_config(File,all),
  462:     ?line ttb:stop(),
  463:     ?line [_,_,_] = ttb:list_config(File),
  464:     ?line ok = ttb:run_config(File),
  465:     ?line ?MODULE:foo(),
  466:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  467:     ?line ttb:stop([nofetch]),
  468:     ?line ok = ttb:format(
  469: 		 [filename:join(Privdir,
  470: 				atom_to_list(Node)++"-write_config2"),
  471: 		  filename:join(Privdir,
  472: 				atom_to_list(OtherNode)++"-write_config2")]),
  473:     ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
  474: 	   {trace_ts,Other,call,{?MODULE,foo,[]},{_,_,_}},
  475: 	   end_of_trace] = flush(),
  476: 
  477:     case metatest(Other,OtherNode,Privdir,"-write_config2.ti") of
  478: 	{error,Reason} ->
  479: 	    timer:sleep(5000),
  480: 	    ?line ok = ttb:format(
  481: 			 [filename:join(Privdir,
  482: 					atom_to_list(Node)++"-write_config2"),
  483: 			  filename:join(Privdir,
  484: 					atom_to_list(OtherNode)++
  485: 					"-write_config2")]),
  486: 	    ?line io:format("\nTrying again: ~p\n",[flush()]),
  487: 	    ?line ?t:fail(Reason);
  488: 	ok ->
  489: 	    ok
  490:     end,
  491:     ok.
  492: 
  493: write_config2(cleanup,_Config) ->
  494:     ?line ?t:stop_node(ttb_helper:get_node(node2)).
  495: 
  496: 
  497: write_config3(suite) ->
  498:     [];
  499: write_config3(doc) ->
  500:     ["Write config from history (selected and append)"];
  501: write_config3(Config) when is_list(Config) ->
  502:     ?line Node = node(),
  503:     ?line {ok,OtherNode} = ?t:start_node(node2,slave,[]),
  504:     ?line c:nl(?MODULE),
  505:     ?line S = self(),
  506:     ?line Privdir=priv_dir(Config),
  507:     ?line File = filename:join(Privdir,"write_config3"),
  508:     ?line {ok,[_,_]} = 
  509: 	ttb:tracer([Node,OtherNode],[{file, File},
  510: 				     {handler,{fun myhandler/4, S}}]),
  511:     ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
  512:     ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
  513:     ?line ok = ttb:write_config(File,[1,2]),
  514:     ?line ttb:stop([nofetch]),
  515:     ?line [_,_] = ttb:list_config(File),
  516:     ?line ok = ttb:run_config(File),
  517:     ?line ?MODULE:foo(),
  518:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  519:     ?line ttb:stop([nofetch]),
  520:     ?line ok = ttb:format(
  521: 		 [filename:join(Privdir,
  522: 				atom_to_list(Node)++"-write_config3"),
  523: 		  filename:join(Privdir,
  524: 				atom_to_list(OtherNode)++"-write_config3")]),
  525:     ?line [end_of_trace] = flush(), %foo is not traced
  526: 
  527:     ?line ok = ttb:write_config(File,[{ttb,tp,[?MODULE,foo,[]]}],
  528: 			      [append]),
  529:     ?line [_,_,_] = ttb:list_config(File),
  530:     ?line ok = ttb:run_config(File),
  531:     ?line ?MODULE:foo(),
  532:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  533:     ?line ttb:stop([nofetch]),
  534:     ?line ok = ttb:format(
  535: 		 [filename:join(Privdir,
  536: 				atom_to_list(Node)++"-write_config3"),
  537: 		  filename:join(Privdir,
  538: 				atom_to_list(OtherNode)++"-write_config3")]),
  539:     ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
  540: 	   {trace_ts,Other,call,{?MODULE,foo,[]},{_,_,_}},
  541: 	   end_of_trace] = flush(),
  542: 
  543:     case metatest(Other,OtherNode,Privdir,"-write_config3.ti") of
  544: 	{error,Reason} ->
  545: 	    timer:sleep(5000),
  546: 	    ?line ok = ttb:format(
  547: 			 [filename:join(Privdir,
  548: 					atom_to_list(Node)++"-write_config3"),
  549: 			  filename:join(Privdir,
  550: 					atom_to_list(OtherNode)++
  551: 					"-write_config3")]),
  552: 	    ?line io:format("\nTrying again: ~p\n",[flush()]),
  553: 	    ?line ?t:fail(Reason);
  554: 	ok ->
  555: 	    ok
  556:     end,
  557:     ok.
  558: 
  559: write_config3(cleanup,_Config) ->
  560:     ?line ?t:stop_node(ttb_helper:get_node(node2)).
  561: 
  562: 
  563: 
  564: history(suite) ->
  565:     [];
  566: history(doc) ->
  567:     ["List history and execute entry from history"];
  568: history(Config) when is_list(Config) ->
  569:     ?line Node = node(),
  570:     ?line {ok,OtherNode} = ?t:start_node(node2,slave,[]),
  571:     ?line c:nl(?MODULE),
  572:     ?line S = self(),
  573: 
  574:     ?line Nodes = [Node,OtherNode],
  575:     ?line Privdir=priv_dir(Config),
  576:     ?line File = filename:join(Privdir,"history"),
  577:     ?line StartOpts = [{file, File},
  578: 		       {handler,{fun myhandler/4, S}}],
  579:     ?line {ok,[_,_]} = ttb:tracer(Nodes,StartOpts),
  580:     ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
  581:     ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
  582:     ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:ctp(?MODULE,foo),
  583:     ?line [{1,{ttb,tracer,[Nodes,StartOpts]}},
  584: 	   {2,{ttb,p,[all,call]}},
  585: 	   {3,{ttb,tp,[?MODULE,foo,[]]}},
  586: 	   {4,{ttb,ctp,[?MODULE,foo]}}] = ttb:list_history(),
  587:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  588:     ?line ok = ttb:run_history(3),
  589:     ?line ?MODULE:foo(),
  590:     ?line ok = ttb:run_history([3,4]),
  591:     ?line ?MODULE:foo(),
  592:     ?line ttb:stop([nofetch]),
  593:     ?line ok = ttb:format(
  594: 		 [filename:join(Privdir,atom_to_list(Node)++"-history"),
  595: 		  filename:join(Privdir,atom_to_list(OtherNode)++"-history")]),
  596:     ?line [{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},
  597: 	   end_of_trace] = flush(),
  598:     ok.
  599: 
  600: history(cleanup,_Config) ->
  601:     ?line ?t:stop_node(ttb_helper:get_node(node2)).
  602: 
  603: 
  604: write_trace_info(suite) ->
  605:     [];
  606: write_trace_info(doc) ->
  607:     ["Write trace info and give handler explicitly in format command"];
  608: write_trace_info(Config) when is_list(Config) ->
  609:     ?line Node = node(),
  610:     ?line {ok,OtherNode} = ?t:start_node(node2,slave,[]),
  611:     ?line c:nl(?MODULE),
  612:     ?line S = self(),
  613:     ?line Privdir=priv_dir(Config),
  614:     ?line File = filename:join(Privdir,"write_trace_info"),
  615:     ?line {ok,[_,_]} = 
  616: 	ttb:tracer([Node,OtherNode],[{file, File},
  617: 				     {handler,{fun myhandler/4, S}}]),
  618:     ?line {ok,[{all,[{matched,_,_},{matched,_,_}]}]} = ttb:p(all,call),
  619:     ?line {ok,[{matched,_,1},{matched,_,1}]} = ttb:tp(?MODULE,foo,[]),
  620:     ?line ok = ttb:write_trace_info(mytraceinfo,fun() -> node() end),
  621:     ?line ?MODULE:foo(),
  622:     ?line rpc:call(OtherNode,?MODULE,foo,[]),
  623:     ?line ttb:stop([nofetch]),
  624:     ?line ok = ttb:format(
  625: 		 [filename:join(Privdir,atom_to_list(Node)++"-write_trace_info"),
  626: 		  filename:join(Privdir,
  627: 				atom_to_list(OtherNode)++"-write_trace_info")],
  628: 		 [{handler,{fun otherhandler/4,S}}]),
  629:     ?line [{{trace_ts,{S,_,Node},call,{?MODULE,foo,[]},{_,_,_}},[Node]},
  630: 	   {{trace_ts,{_,_,OtherNode},call,{?MODULE,foo,[]},{_,_,_}},[OtherNode]},
  631: 	   end_of_trace] = flush(),
  632: 
  633:     ok.
  634: 
  635: write_trace_info(cleanup,_Config) ->
  636:     ?line ?t:stop_node(ttb_helper:get_node(node2)).
  637: 
  638: 
  639: seq_trace(suite) ->
  640:     [];
  641: seq_trace(doc) ->
  642:     ["Test sequential tracing"];
  643: seq_trace(Config) when is_list(Config) ->
  644:     ?line S = self(),
  645: 
  646:     ?line Privdir=priv_dir(Config),
  647:     ?line File = filename:join(Privdir,"seq_trace"),
  648:     ?line {ok,[Node]} = ttb:tracer(node(),[{file,File},
  649: 				     {handler,{fun myhandler/4, S}}]),
  650:     ?line {ok,[{new,[{matched,Node,0}]}]} = ttb:p(new,call),
  651:     ?line {ok,[{matched,Node,1},{saved,1}]} = 
  652: 	   ttb:tpl(?MODULE,seq,0,ttb:seq_trigger_ms(send)),
  653: 
  654:     ?line Start = spawn(fun() -> seq() end),
  655:     ?line timer:sleep(300),
  656:     ?line ttb:stop([nofetch]),
  657:     ?line ok = ttb:format(
  658: 		 [filename:join(Privdir,atom_to_list(Node)++"-seq_trace")]),
  659:     ?line [{trace_ts,StartProc,call,{?MODULE,seq,[]},{_,_,_}},
  660: 	   {seq_trace,0,{send,{0,1},StartProc,P1Proc,{Start,P2}}},
  661: 	   {seq_trace,0,{send,{1,2},P1Proc,P2Proc,{P1,Start}}},
  662: 	   {seq_trace,0,{send,{2,3},P2Proc,StartProc,{P2,P1}}},
  663: 	   end_of_trace] = flush(),
  664: 
  665:    %% Additional test for metatrace
  666:     case StartProc of
  667: 	{Start,_,_} -> ok;
  668: 	Pid when is_pid(Pid) ->
  669: 	    io:format("\n\nProcinfo was pid: ~p.\n"
  670: 		      "Should have been {Pid,Name,Node}\n",
  671: 		      [Pid]),
  672: 	    io:format("Trace information file:\n~p\n",
  673: 		      [ttb:dump_ti(
  674: 			 filename:join(Privdir,
  675: 				       atom_to_list(Node)++"-seq_trace.ti"))]),
  676: 	    ?t:fail("metatrace failed for startproc")
  677:     end,
  678:     case P1Proc of
  679: 	{P1,_,_} -> ok;
  680: 	P1 when is_pid(P1) ->
  681: 	    io:format("\n\nProcinfo was pid: ~p.\n"
  682: 		      "Should have been {Pid,Name,Node}\n",
  683: 		      [P1]),
  684: 	    io:format("Trace information file:\n~p\n",
  685: 		      [ttb:dump_ti(
  686: 			 filename:join(Privdir,
  687: 				       atom_to_list(Node)++"-seq_trace.ti"))]),
  688: 	    ?t:fail("metatrace failed for P1")
  689:     end,
  690:     case P2Proc of
  691: 	{P2,_,_} -> ok;
  692: 	P2 when is_pid(P2) ->
  693: 	    io:format("\n\nProcinfo was pid: ~p.\n"
  694: 		      "Should have been {Pid,Name,Node}\n",
  695: 		      [P2]),
  696: 	    io:format("Trace information file:\n~p\n",
  697: 		      [ttb:dump_ti(
  698: 			 filename:join(Privdir,
  699: 				       atom_to_list(Node)++"-seq_trace.ti"))]),
  700: 	    ?t:fail("metatrace failed for P2")
  701:     end,
  702:     ok.
  703: 
  704: 
  705: diskless(suite) ->
  706:     [];
  707: diskless(doc) ->
  708:     ["Start tracing on diskless remote node"];
  709: diskless(Config) when is_list(Config) ->
  710:     ?line {ok,RemoteNode} = ?t:start_node(node2,slave,[]),
  711:     ?line c:nl(?MODULE),
  712:     ?line S = self(),
  713:     ?line Privdir=priv_dir(Config),
  714:     ?line File = filename:join(Privdir,"diskless"),
  715:     ?line {ok,[RemoteNode]} = 
  716: 	ttb:tracer([RemoteNode],[{file, {local, File}},
  717: 				 {handler,{fun myhandler/4, S}}]),
  718:     ?line {ok,[{all,[{matched,RemoteNode,_}]}]} = ttb:p(all,call),
  719:     ?line {ok,[{matched,RemoteNode,1}]} = ttb:tp(?MODULE,foo,[]),
  720: 
  721:     ?line rpc:call(RemoteNode,?MODULE,foo,[]),
  722:     ?line timer:sleep(500), % needed for the IP port to flush
  723:     ?line ttb:stop([nofetch]),
  724:     ?line ok = ttb:format(filename:join(Privdir,
  725: 					atom_to_list(RemoteNode)++"-diskless")),
  726: 
  727:     ?line [{trace_ts,{_,_,RemoteNode},call,{?MODULE,foo,[]},{_,_,_}},
  728: 	   end_of_trace] = flush(),
  729:     ok.
  730: 
  731: diskless(cleanup,_Config) ->
  732:     ?line ?t:stop_node(ttb_helper:get_node(node2)).
  733: 
  734: diskless_wrap(suite) ->
  735:     [];
  736: diskless_wrap(doc) ->
  737:     ["Start tracing on diskless remote node, save to local wrapped file"];
  738: diskless_wrap(Config) when is_list(Config) ->
  739:     ?line {ok,RemoteNode} = ?t:start_node(node2,slave,[]),
  740:     ?line c:nl(?MODULE),
  741:     ?line S = self(),
  742:     ?line Privdir=priv_dir(Config),
  743:     ?line File = filename:join(Privdir,"diskless"),
  744:     ?line {ok,[RemoteNode]} =
  745: 	ttb:tracer([RemoteNode],[{file, {local, {wrap,File,200,3}}},
  746: 				 {handler,{fun myhandler/4, S}}]),
  747:     ?line {ok,[{all,[{matched,RemoteNode,_}]}]} = ttb:p(all,call),
  748:     ?line {ok,[{matched,RemoteNode,1}]} = ttb:tp(?MODULE,foo,[]),
  749: 
  750:     ?line rpc:call(RemoteNode,?MODULE,foo,[]),
  751:     ?line timer:sleep(500), % needed for the IP port to flush
  752:     ?line ttb:stop([nofetch]),
  753:     ?line ok = ttb:format(filename:join(Privdir,
  754: 					atom_to_list(RemoteNode)++"-diskless.*.wrp")),
  755: 
  756:     ?line [{trace_ts,{_,_,RemoteNode},call,{?MODULE,foo,[]},{_,_,_}},
  757: 	   end_of_trace] = flush(),
  758:     ok.
  759: 
  760: diskless_wrap(cleanup,_Config) ->
  761:     ?line ?t:stop_node(ttb_helper:get_node(node2)).
  762: 
  763: otp_4967_1(suite) ->
  764:     [];
  765: otp_4967_1(doc) ->
  766:     ["OTP-4967: clear flag"];
  767: otp_4967_1(Config) when is_list(Config) ->
  768:     ?line {ok,[Node]} = ttb:tracer(),
  769:     ?line {ok,[{all,[{matched,Node,_}]}]} =  ttb:p(all,call),
  770:     ?line {ok,[{all,[{matched,Node,_}]}]} =  ttb:p(all,clear),
  771:     ?line stopped = ttb:stop(),
  772:     ok.
  773: 
  774: 
  775: otp_4967_2(suite) ->
  776:     [];
  777: otp_4967_2(doc) ->
  778:     ["OTP-4967: Trace message sent to {Name, Node}"];
  779: otp_4967_2(Config) when is_list(Config) ->
  780:     io:format("1: ~p",[now()]),
  781:     ?line Privdir = priv_dir(Config),
  782:     io:format("2: ~p",[now()]),
  783:     ?line File = filename:join(Privdir,"otp_4967"),
  784:     io:format("3: ~p",[now()]),
  785:     ?line S = self(),
  786:     io:format("4: ~p",[now()]),
  787:     ?line {ok,[Node]} =
  788: 	ttb:tracer(node(),[{file, File},
  789: 			   {handler,{fun myhandler/4, S}}]),
  790: 
  791:     io:format("5: ~p",[now()]),
  792:     %% Test that delayed registration of a process works.
  793:     receive after 200 -> ok end,
  794:     ?line register(otp_4967,self()),
  795:     io:format("6: ~p",[now()]),
  796:     ?line {ok,[{S,[{matched,Node,1}]}]} =  ttb:p(self(),s),
  797:     io:format("7: ~p",[now()]),
  798:     ?line {otp_4967,node()} ! heihopp,
  799:     io:format("8: ~p",[now()]),
  800:     ?line stopped = ttb:stop([format]),
  801:     io:format("9: ~p",[now()]),
  802:     ?line Msgs = flush(),
  803:     io:format("10: ~p",[now()]),
  804:     ?line io:format("Messages received: \n~p\n",[Msgs]),
  805:     io:format("11: ~p",[now()]),
  806:     ?line true = lists:member(heihopp,Msgs), % the heihopp message itself
  807:     io:format("13: ~p",[now()]),
  808:     ?line {value,{trace_ts,_,send,heihopp,{_,otp_4967,Node},{_,_,_}}} =
  809: 	lists:keysearch(heihopp,4,Msgs), % trace trace of the heihopp message
  810:     io:format("14: ~p",[now()]),
  811:     ?line end_of_trace = lists:last(Msgs), % end of the trace
  812:     ok.
  813:     
  814: 
  815: myhandler(_Fd,Trace,_,Relay) ->
  816:     Relay ! Trace,
  817:     Relay.
  818: 
  819: simple_call_handler() ->
  820:     {fun(A, {trace_ts, _, call, _, _} ,_,_) -> io:format(A, "ok.~n", []);
  821: 	(_, end_of_trace, _, _) -> ok end, []}.
  822: 
  823: marking_call_handler() ->
  824:     {fun(_, _, _, initial) -> file:write_file("HANDLER_OK", []);
  825: 	(_,_,_,_) -> ok end, initial}.
  826: 
  827: counter_call_handler() ->
  828:     {fun(_, {trace_ts, _, call, _, _} ,_,State) -> State + 1;
  829: 	(A, end_of_trace, _, State) -> io:format(A,"~p.~n", [State]) end, 0}.
  830: 
  831: ret_caller_call_handler() ->
  832:     {fun(A, {trace_ts, _, call, _, _, _} ,_,_) -> io:format(A, "ok.~n", []);
  833: 	(A, {trace_ts, _, return_from, _, _, _}, _, _) -> io:format(A, "ok.~n", []);
  834: 	(_, _, _, _) -> ok end, []}.
  835: 
  836: node_call_handler() ->
  837:     {fun(A, {trace_ts, {_,_,Node}, call, _, _} ,_,_) -> io:format(A, "~p.~n", [Node]);
  838: 	(_, end_of_trace, _, _) -> ok end, []}.
  839: 
  840: otherhandler(_Fd,_,end_of_trace,Relay) ->
  841:     Relay ! end_of_trace,
  842:     Relay;
  843: otherhandler(_Fd,Trace,TI,Relay) ->
  844:     {value,{mytraceinfo,I}} = lists:keysearch(mytraceinfo,1,TI),
  845:     Relay ! {Trace,I},
  846:     Relay.
  847: 
  848: flush() ->
  849:     flush([]).
  850: flush(Acc) ->
  851:     receive
  852: 	X ->
  853: 	    flush(Acc ++ [X])
  854:     after 1000 ->
  855: 	    Acc
  856:     end.
  857: 
  858: foo() ->
  859:     %% Sync between nodes is not always exact, so here is a litle timeout to 
  860:     %% make sure traces come i correct sequence when merging.
  861:     %% In the real world there is no way to avoid this kind of trouble
  862:     timer:sleep(100),
  863:     foo_called.
  864: 
  865: 
  866: seq() ->
  867:     Fun = fun() -> timer:sleep(100),
  868: 		   receive {From,To} -> To ! {self(),From} end 
  869: 	  end,
  870:     P1 = spawn(Fun),
  871:     P2 = spawn(Fun),
  872:     P1 ! {self(),P2},
  873:     receive {P2,P1} -> ok end,
  874:     {P1,P2}.
  875: 
  876: %% Additional test for metatrace which might fail on OtherNode
  877: metatest(Proc,Node,Privdir,Filename) ->
  878:     case Proc of
  879: 	{_,_,Node} -> ok;
  880: 	Pid when is_pid(Pid) ->
  881: 	    io:format("\n\nProcinfo was pid: ~p.\n"
  882: 		      "Should have been {Pid,Name,Node}\n",
  883: 		      [Pid]),
  884: 	    io:format("Trace information file:\n~p\n",
  885: 		      [ttb:dump_ti(
  886: 			 filename:join(Privdir,atom_to_list(Node)++Filename))]),
  887: %	    ?t:fail("metatrace failed on "++atom_to_list(Node))
  888: 	    {error,"metatrace failed on "++atom_to_list(Node)}
  889:     end.
  890: 
  891: check_gone(Dir,File) ->
  892:     ?line {ok,List} = file:list_dir(Dir),
  893:     ?line case lists:member(File,List) of
  894: 	      true -> 
  895: 		  timer:sleep(2000),
  896: 		  {ok,NewList} = file:list_dir(Dir),
  897: 		  case lists:member(File,NewList) of
  898: 		      true ->
  899: 			  io:format("~p: ~p~n",
  900: 				    [Dir,NewList]),
  901: 			  ?t:fail(File ++ " not removed from original place");
  902: 		      false ->
  903: 			  io:format("gone after 2 sec....~n",[]),
  904: 			  ok
  905: 		  end;
  906: 	      false -> 
  907: 		  ok
  908: 	  end.
  909: 
  910: start_client_and_server() ->
  911:     ?line {ok,ClientNode} = ?t:start_node(client,slave,[]),
  912:     ?line ok = ttb_helper:c(code, add_paths, [code:get_path()]),
  913:     ?line {ok,ServerNode} = ?t:start_node(server,slave,[]),
  914:     ?line ok = ttb_helper:s(code, add_paths, [code:get_path()]),
  915:     ?line ttb_helper:clear(),
  916:     {ServerNode, ClientNode}.
  917: 
  918: stop_client_and_server() ->
  919:     ClientNode = ttb_helper:get_node(client),
  920:     ServerNode = ttb_helper:get_node(server),
  921:     erlang:monitor_node(ClientNode,true),
  922:     erlang:monitor_node(ServerNode,true),
  923:     ?line ?t:stop_node(ClientNode),
  924:     ?line ?t:stop_node(ServerNode),
  925:     wait_for_client_and_server_stop(ClientNode,ServerNode).
  926: 
  927: wait_for_client_and_server_stop(undefined,undefined) ->
  928:     ok;
  929: wait_for_client_and_server_stop(ClientNode,ServerNode) ->
  930:     receive
  931: 	{nodedown,ClientNode} ->
  932: 	    erlang:monitor_node(ClientNode,false),
  933: 	    wait_for_client_and_server_stop(undefined,ServerNode);
  934: 	{nodedown,ServerNode} ->
  935: 	    erlang:monitor_node(ServerNode,false),
  936: 	    wait_for_client_and_server_stop(ClientNode,undefined)
  937:     end.
  938: 
  939: begin_trace(ServerNode, ClientNode, Dest) ->
  940:     ?line {ok, _} =
  941: 	ttb:tracer([ServerNode,ClientNode],[{file, Dest}]),
  942:     ?line ttb:p(all, call),
  943:     ?line ttb:tp(server, received, []),
  944:     ?line ttb:tp(client, put, []),
  945:     ?line ttb:tp(client, get, []).
  946: 
  947: begin_trace_local(ServerNode, ClientNode, Dest) ->
  948:     ?line {ok, _} =
  949: 	ttb:tracer([ServerNode,ClientNode],[{file, Dest}]),
  950:     ?line ttb:p(all, call),
  951:     ?line ttb:tpl(server, received, []),
  952:     ?line ttb:tpl(client, put, []),
  953:     ?line ttb:tpl(client, get, []).
  954: 
  955: check_size(N, Dest, Output, ServerNode, ClientNode) ->
  956:     ?line begin_trace(ServerNode, ClientNode, Dest),
  957:     ?line case Dest of
  958:         {local, _} ->
  959:             ?line ttb_helper:msgs_ip(N);
  960:         _ ->
  961:             ?line ttb_helper:msgs(N)
  962:     end,
  963:     ?line {_, D} = ttb:stop([fetch, return_fetch_dir]),
  964:     ?line ttb:format(D, [{out, Output}, {handler, simple_call_handler()}]),
  965:     ?line {ok, Ret} = file:consult(Output),
  966:     ?line true = (N + 1 == length(Ret)).
  967: 
  968: fetch_when_no_option_given(suite) ->
  969:     [];
  970: fetch_when_no_option_given(doc) ->
  971:     ["Fetch when no option given"];
  972: fetch_when_no_option_given(Config) when is_list(Config) ->
  973:     ?line {ServerNode, ClientNode} = start_client_and_server(),
  974:     ?line {ok, Privdir} = file:get_cwd(),
  975:     ?line [] = filelib:wildcard(filename:join(Privdir,"ttb_upload_temptest*")),
  976:     begin_trace(ServerNode, ClientNode, ?FNAME),
  977:     ?line ttb_helper:msgs(4),
  978:     ?line stopped = ttb:stop(),
  979:     ?line [_] = filelib:wildcard(filename:join(Privdir,"ttb_upload_temptest*")).
  980: 
  981: fetch_when_no_option_given(cleanup,_Config) ->
  982:     ?line stop_client_and_server().
  983: 
  984: 
  985: basic_ttb_run_ip_port(suite) ->
  986:     [];
  987: basic_ttb_run_ip_port(doc) ->
  988:     ["Basic ttb run ip port"];
  989: basic_ttb_run_ip_port(Config) when is_list(Config) ->
  990:     ?line {ServerNode, ClientNode} = start_client_and_server(),
  991:     ?line check_size(1, {local, ?FNAME}, ?OUTPUT, ServerNode, ClientNode),
  992:     ?line check_size(2, {local, ?FNAME}, ?OUTPUT, ServerNode, ClientNode),
  993:     ?line check_size(10, {local, ?FNAME}, ?OUTPUT, ServerNode, ClientNode).
  994: basic_ttb_run_ip_port(cleanup,_Config) ->
  995:     ?line stop_client_and_server().
  996: 
  997: basic_ttb_run_file_port(suite) ->
  998:     [];
  999: basic_ttb_run_file_port(doc) ->
 1000:     ["Basic ttb run file port"];
 1001: basic_ttb_run_file_port(Config) when is_list(Config) ->
 1002:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1003:     ?line check_size(1, ?FNAME, ?OUTPUT, ServerNode, ClientNode),
 1004:     ?line check_size(2, ?FNAME, ?OUTPUT, ServerNode, ClientNode),
 1005:     ?line check_size(10, ?FNAME, ?OUTPUT, ServerNode, ClientNode).
 1006: basic_ttb_run_file_port(cleanup,_Config) ->
 1007:     ?line stop_client_and_server().
 1008: 
 1009: return_fetch_dir_implies_fetch(suite) ->
 1010:     [];
 1011: return_fetch_dir_implies_fetch(doc) ->
 1012:     ["Return_fetch_dir implies fetch"];
 1013: return_fetch_dir_implies_fetch(Config) when is_list(Config) ->
 1014:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1015:     ?line begin_trace(ServerNode, ClientNode, ?FNAME),
 1016:     ?line ttb_helper:msgs(2),
 1017:     ?line {_,_} = ttb:stop([return_fetch_dir]).
 1018: return_fetch_dir_implies_fetch(cleanup,_Config) ->
 1019:     ?line stop_client_and_server().
 1020: 
 1021: logfile_name_in_fetch_dir(suite) ->
 1022:     [];
 1023: logfile_name_in_fetch_dir(doc) ->
 1024:     ["Logfile name in fetch dir"];
 1025: logfile_name_in_fetch_dir(Config) when is_list(Config) ->
 1026:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1027:     ?line begin_trace(ServerNode, ClientNode, {local, ?FNAME}),
 1028:     ?line {_,Dir} = ttb:stop([return_fetch_dir]),
 1029:     ?line P1 = lists:nth(3, string:tokens(filename:basename(Dir), "_")),
 1030:     ?line P2 = hd(string:tokens(P1, "-")),
 1031:     ?line _File = P2.
 1032: logfile_name_in_fetch_dir(cleanup,_Config) ->
 1033:     ?line stop_client_and_server().
 1034: 
 1035: upload_to_my_logdir(suite) ->
 1036:     [];
 1037: upload_to_my_logdir(doc) ->
 1038:     ["Upload to my logdir"];
 1039: upload_to_my_logdir(Config) when is_list(Config) ->
 1040:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1041:     ?line {ok, _} =
 1042: 	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}]),
 1043:     ?line {stopped,_} = ttb:stop([return_fetch_dir, {fetch_dir, ?DIRNAME}]),
 1044:     ?line true = filelib:is_file(?DIRNAME),
 1045:     ?line [] = filelib:wildcard("ttb_upload_"++?FNAME).
 1046: upload_to_my_logdir(cleanup,_Config) ->
 1047:     ?line stop_client_and_server().
 1048: 
 1049: upload_to_my_existing_logdir(suite) ->
 1050:     [];
 1051: upload_to_my_existing_logdir(doc) ->
 1052:     ["Upload to my existing logdir"];
 1053: upload_to_my_existing_logdir(Config) when is_list(Config) ->
 1054:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1055:     ?line ok = file:make_dir(?DIRNAME),
 1056:     ?line {ok, _} =
 1057: 	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}]),
 1058:     ?line {error,_,_} = (catch ttb:stop([return_fetch_dir, {fetch_dir, ?DIRNAME}])),
 1059:     ?line {stopped,_} = ttb:stop(return_fetch_dir).
 1060: upload_to_my_existing_logdir(cleanup,_Config) ->
 1061:     ?line stop_client_and_server().
 1062: 
 1063: fetch_with_options_not_as_list(suite) ->
 1064:     [];
 1065: fetch_with_options_not_as_list(doc) ->
 1066:     ["Fetch with options not as list"];
 1067: fetch_with_options_not_as_list(Config) when is_list(Config) ->
 1068:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1069:     ?line {ok, _} =
 1070: 	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}]),
 1071:     ?line {stopped, D} = ttb:stop(return_fetch_dir),
 1072:     ?line false = filelib:is_file(?OUTPUT),
 1073:     ?line ttb:format(D, {out, ?OUTPUT}),
 1074:     ?line true = filelib:is_file(?OUTPUT).
 1075: fetch_with_options_not_as_list(cleanup,_Config) ->
 1076:     ?line stop_client_and_server().
 1077: 
 1078: error_when_formatting_multiple_files_4393(suite) ->
 1079:     [];
 1080: error_when_formatting_multiple_files_4393(doc) ->
 1081:     ["Error when formatting multiple files"];
 1082: error_when_formatting_multiple_files_4393(Config) when is_list(Config) ->
 1083:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1084:     ?line begin_trace(ServerNode, ClientNode, ?FNAME),
 1085:     ?line ttb_helper:msgs(2),
 1086:     ?line {_, Dir} = ttb:stop(return_fetch_dir),
 1087:     ?line Files = [filename:join(Dir, atom_to_list(ServerNode) ++ "-" ++ ?FNAME),
 1088: 		   filename:join(Dir, atom_to_list(ClientNode) ++ "-" ++ ?FNAME)],
 1089:     ?line ok = ttb:format(Files).
 1090: error_when_formatting_multiple_files_4393(cleanup,_Config) ->
 1091:     ?line stop_client_and_server().
 1092: 
 1093: format_on_trace_stop(suite) ->
 1094:     [];
 1095: format_on_trace_stop(doc) ->
 1096:     ["Format on trace stop"];
 1097: format_on_trace_stop(Config) when is_list(Config) ->
 1098:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1099:     ?line begin_trace(ServerNode, ClientNode, {local, ?FNAME}),
 1100:     ?line ttb_helper:msgs_ip(2),
 1101:     ?line file:delete("HANDLER_OK"),
 1102:     ?line {_,_} = ttb:stop([fetch, return_fetch_dir, {format, {handler, marking_call_handler()}}]),
 1103:     ?line true = filelib:is_file("HANDLER_OK"),
 1104:     ?line ok = file:delete("HANDLER_OK").
 1105: format_on_trace_stop(cleanup,_Config) ->
 1106:     ?line stop_client_and_server().
 1107: 
 1108: %% The following three tests are for the issue "fixes fetch fail when nodes on the same host
 1109: %% have different cwd"
 1110: trace_to_remote_files_on_localhost_with_different_pwd(suite) ->
 1111:     [];
 1112: trace_to_remote_files_on_localhost_with_different_pwd(doc) ->
 1113:     ["Trace to remote files on localhost with different pwd"];
 1114: trace_to_remote_files_on_localhost_with_different_pwd(Config) when is_list(Config) ->
 1115:     ?line {ok, OldDir} = file:get_cwd(),
 1116:     ?line ok = file:set_cwd(".."),
 1117:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1118:     ?line check_size(2, ?FNAME, ?OUTPUT, ServerNode, ClientNode),
 1119:     ?line ok = file:set_cwd(OldDir).
 1120: trace_to_remote_files_on_localhost_with_different_pwd(cleanup,_Config) ->
 1121:     ?line stop_client_and_server().
 1122: 
 1123: trace_to_local_files_on_localhost_with_different_pwd(suite) ->
 1124:     [];
 1125: trace_to_local_files_on_localhost_with_different_pwd(doc) ->
 1126:     ["Trace to local files on localhost with different pwd"];
 1127: trace_to_local_files_on_localhost_with_different_pwd(Config) when is_list(Config) ->
 1128:     ?line {ok, OldDir} = file:get_cwd(),
 1129:     ?line ok = file:set_cwd(".."),
 1130:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1131:     ?line check_size(2, {local, ?FNAME}, ?OUTPUT, ServerNode, ClientNode),
 1132:     ?line ok = file:set_cwd(OldDir).
 1133: trace_to_local_files_on_localhost_with_different_pwd(cleanup,_Config) ->
 1134:     ?line stop_client_and_server().
 1135: 
 1136: trace_to_remote_files_on_localhost_with_different_pwd_abs(suite) ->
 1137:     [];
 1138: trace_to_remote_files_on_localhost_with_different_pwd_abs(doc) ->
 1139:     ["Trace to remote files on localhost with different pwd abs"];
 1140: trace_to_remote_files_on_localhost_with_different_pwd_abs(Config) when is_list(Config) ->
 1141:     ?line {ok, OldDir} = file:get_cwd(),
 1142:     ?line ok = file:set_cwd(".."),
 1143:     ?line {ok, Path} = file:get_cwd(),
 1144:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1145:     ?line File = filename:join(Path, ?FNAME),
 1146:     ?line check_size(2, File, ?OUTPUT, ServerNode, ClientNode),
 1147:     ?line ok = file:set_cwd(OldDir).
 1148: trace_to_remote_files_on_localhost_with_different_pwd_abs(cleanup,_Config) ->
 1149:     ?line stop_client_and_server().
 1150: 
 1151: %% Trace is not affected by changes of cwd on control node or remote nodes during tracing
 1152: %% (three tests)
 1153: changing_cwd_on_control_node(suite) ->
 1154:     [];
 1155: changing_cwd_on_control_node(doc) ->
 1156:     ["Changing cwd on control node during tracing is safe"];
 1157: changing_cwd_on_control_node(Config) when is_list(Config) ->
 1158:     ?line {ok, OldDir} = file:get_cwd(),
 1159:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1160:     ?line begin_trace(ServerNode, ClientNode, ?FNAME),
 1161:     ?line NumMsgs = 3,
 1162:     ?line ttb_helper:msgs(NumMsgs),
 1163:     ?line ok = file:set_cwd(".."),
 1164:     ?line ttb_helper:msgs(NumMsgs),
 1165:     ?line {_, D} = ttb:stop([fetch, return_fetch_dir]),
 1166:     ?line ttb:format(D, [{out, ?OUTPUT}, {handler, simple_call_handler()}]),
 1167:     ?line {ok, Ret} = file:consult(?OUTPUT),
 1168:     ?line true = (2*(NumMsgs + 1) == length(Ret)),
 1169:     ?line ok = file:set_cwd(OldDir).
 1170: changing_cwd_on_control_node(cleanup,_Config) ->
 1171:     ?line stop_client_and_server().
 1172: 
 1173: changing_cwd_on_control_node_with_local_trace(suite) ->
 1174:     [];
 1175: changing_cwd_on_control_node_with_local_trace(doc) ->
 1176:     ["Changing cwd on control node during local tracing is safe"];
 1177: changing_cwd_on_control_node_with_local_trace(Config) when is_list(Config) ->
 1178:     ?line {ok, OldDir} = file:get_cwd(),
 1179:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1180:     ?line begin_trace(ServerNode, ClientNode, {local, ?FNAME}),
 1181:     ?line NumMsgs = 3,
 1182:     ?line ttb_helper:msgs_ip(NumMsgs),
 1183:     ?line ok = file:set_cwd(".."),
 1184:     ?line ttb_helper:msgs_ip(NumMsgs),
 1185:     ?line {_, D} = ttb:stop([fetch, return_fetch_dir]),
 1186:     ?line ttb:format(D, [{out, ?OUTPUT}, {handler, simple_call_handler()}]),
 1187:     ?line {ok, Ret} = file:consult(?OUTPUT),
 1188:     ?line true = (2*(NumMsgs + 1) == length(Ret)),
 1189:     ?line ok = file:set_cwd(OldDir).
 1190: changing_cwd_on_control_node_with_local_trace(cleanup,_Config) ->
 1191:     ?line stop_client_and_server().
 1192: 
 1193: changing_cwd_on_remote_node(suite) ->
 1194:     [];
 1195: changing_cwd_on_remote_node(doc) ->
 1196:     ["Changing cwd on remote node during tracing is safe"];
 1197: changing_cwd_on_remote_node(Config) when is_list(Config) ->
 1198:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1199:     ?line begin_trace(ServerNode, ClientNode, ?FNAME),
 1200:     ?line NumMsgs = 2,
 1201:     ?line ttb_helper:msgs(NumMsgs),
 1202:     ?line ok = rpc:call(ClientNode, file, set_cwd, [".."]),
 1203:     ?line ttb_helper:msgs(NumMsgs),
 1204:     ?line {_, D} = ttb:stop([fetch, return_fetch_dir]),
 1205:     ?line ttb:format(D, [{out, ?OUTPUT}, {handler, simple_call_handler()}]),
 1206:     ?line {ok, Ret} = file:consult(?OUTPUT),
 1207:     ?line true = (2*(NumMsgs + 1) == length(Ret)).
 1208: changing_cwd_on_remote_node(cleanup,_Config) ->
 1209:     ?line stop_client_and_server().
 1210: 
 1211: one_command_trace_setup(suite) ->
 1212:     [];
 1213: one_command_trace_setup(doc) ->
 1214:     ["One command trace setup"];
 1215: one_command_trace_setup(Config) when is_list(Config) ->
 1216:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1217:     ?line ttb:start_trace([ClientNode, ServerNode],
 1218: 			  [{server, received, '_', []},
 1219: 			   {client, put, 1, []},
 1220: 			   {client, get, '_', []}],
 1221: 			  {all, call},
 1222: 			  [{file, ?FNAME}]),
 1223:     ?line ttb_helper:msgs(2),
 1224:     ?line {_, D} = ttb:stop(return_fetch_dir),
 1225:     ?line ttb:format(D, [{out, ?OUTPUT}, {handler, simple_call_handler()}]),
 1226:     ?line {ok, Ret} = file:consult(?OUTPUT),
 1227:     ?line 5 = length(Ret).
 1228: one_command_trace_setup(cleanup,_Config) ->
 1229:     ?line stop_client_and_server().
 1230: 
 1231: dbg_style_fetch(suite) ->
 1232:     [];
 1233: dbg_style_fetch(doc) ->
 1234:     ["Dbg style fetch"];
 1235: dbg_style_fetch(Config) when is_list(Config) ->
 1236:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1237:     ?line DirSize = length(element(2, file:list_dir("."))),
 1238:     ?line ttb:start_trace([ClientNode, ServerNode],
 1239: 			  [{server, received, '_', []},
 1240: 			   {client, put, 1, []},
 1241: 			   {client, get, '_', []}],
 1242: 			  {all, call},
 1243: 			  [{shell, only}]),
 1244:     ?line DirSize = length(element(2, file:list_dir("."))),
 1245:     ?line ttb_helper:msgs(2),
 1246:     ?line DirSize = length(element(2, file:list_dir("."))),
 1247:     ?line stopped, ttb:stop(format),
 1248:     %%+1 -> ttb_last_trace
 1249:     ?line true = (DirSize + 1 == length(element(2, file:list_dir(".")))),
 1250:     ?line {ok,[{all, [{matched,_,_}, {matched,_,_}]}]} =
 1251: 	ttb:start_trace([ClientNode, ServerNode],
 1252: 			[{server, received, '_', []},
 1253: 			 {client, put, 1, []},
 1254: 			 {client, get, '_', []}],
 1255: 			{all, call},
 1256: 			[{shell, only}]),
 1257:     ?line ttb:stop().
 1258: dbg_style_fetch(cleanup,_Config) ->
 1259:     ?line stop_client_and_server().
 1260: 
 1261: shell_tracing_init(suite) ->
 1262:     [];
 1263: shell_tracing_init(doc) ->
 1264:     ["Shell tracing init"];
 1265: shell_tracing_init(Config) when is_list(Config) ->
 1266:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1267:     ?line ttb:tracer([ClientNode, ServerNode], shell),
 1268:     ?line ttb:stop(),
 1269:     ?line ttb:tracer([ClientNode, ServerNode],
 1270: 		     [{file, {local, ?FNAME}}, shell]),
 1271:     ?line ttb:stop(),
 1272:     ?line local_client_required_on_shell_tracing =
 1273: 	try  ttb:tracer([ClientNode, ServerNode],[{file, ?FNAME}, shell])
 1274: 	catch
 1275: 	    exit:local_client_required_on_shell_tracing ->
 1276: 		local_client_required_on_shell_tracing
 1277: 	end.
 1278: shell_tracing_init(cleanup,_Config) ->
 1279:     ?line stop_client_and_server().
 1280: 
 1281: only_one_state_for_format_handler(suite) ->
 1282:     [];
 1283: only_one_state_for_format_handler(doc) ->
 1284:     ["Only one state for format handler"];
 1285: only_one_state_for_format_handler(Config) when is_list(Config) ->
 1286:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1287:     ?line begin_trace_local(ServerNode, ClientNode, ?FNAME),
 1288:     ?line ttb_helper:msgs(2),
 1289:     ?line {_, D} = ttb:stop([return_fetch_dir]),
 1290:     ?line ttb:format(D, [{out, ?OUTPUT}, {handler, counter_call_handler()}]),
 1291:     ?line {ok, Ret} = file:consult(?OUTPUT),
 1292:     ?line [5] = Ret.
 1293: only_one_state_for_format_handler(cleanup,_Config) ->
 1294:     ?line stop_client_and_server().
 1295: 
 1296: only_one_state_with_default_format_handler(suite) ->
 1297:     [];
 1298: only_one_state_with_default_format_handler(doc) ->
 1299:     ["Only one state with default format handler"];
 1300: only_one_state_with_default_format_handler(Config) when is_list(Config) ->
 1301:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1302:     ?line begin_trace_local(ServerNode, ClientNode, ?FNAME),
 1303:     ?line ttb_helper:msgs(2),
 1304:     ?line {_, D} = ttb:stop([return_fetch_dir]),
 1305:     ?line ttb:format(D, [{out, ?OUTPUT}]),
 1306:     ?line true = filelib:is_file(?OUTPUT).
 1307: only_one_state_with_default_format_handler(cleanup,_Config) ->
 1308:     ?line stop_client_and_server().
 1309: 
 1310: only_one_state_with_initial_format_handler(suite) ->
 1311:     [];
 1312: only_one_state_with_initial_format_handler(doc) ->
 1313:     ["Only one state with initial format handler"];
 1314: only_one_state_with_initial_format_handler(Config) when is_list(Config) ->
 1315:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1316:     ?line {ok, _} =
 1317: 	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}, {handler, counter_call_handler()}]),
 1318:     ?line ttb:p(all, call),
 1319:     ?line ttb:tpl(server, received, []),
 1320:     ?line ttb:tpl(client, put, []),
 1321:     ?line ttb:tpl(client, get, []),
 1322:     ?line ttb_helper:msgs(2),
 1323:     ?line {_, D} = ttb:stop([return_fetch_dir]),
 1324:     ?line ttb:format(D, [{out, ?OUTPUT}]),
 1325:     ?line {ok, Ret} = file:consult(?OUTPUT),
 1326:     ?line [5] = Ret.
 1327: only_one_state_with_initial_format_handler(cleanup,_Config) ->
 1328:     ?line stop_client_and_server().
 1329: 
 1330: run_trace_with_shortcut(Shortcut, Ret, F) ->
 1331:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1332:     ?line {ok, _} =
 1333: 	ttb:tracer([ServerNode,ClientNode],[{file, ?FNAME}]),
 1334:     ?line ttb:p(all, call),
 1335:     ?line ttb:F(client, put, Shortcut),
 1336:     ?line ttb_helper:msgs(2),
 1337:     ?line {_, D} = ttb:stop([return_fetch_dir]),
 1338:     ?line ttb:format(D, [{out, ?OUTPUT}, {handler, ret_caller_call_handler()}]),
 1339:     ?line {ok, Ret} =file:consult(?OUTPUT),
 1340:     ?line stop_client_and_server().
 1341: 
 1342: fun_for(return) ->
 1343:     {codestr, "fun(_) -> return_trace() end"};
 1344: fun_for(msg_false) ->
 1345:     {codestr, "fun(_) -> message(false) end"}.
 1346: 
 1347: run_trace_with_shortcut1(suite) ->
 1348:     [];
 1349: run_trace_with_shortcut1(doc) ->
 1350:     ["Run trace with shortcut 1"];
 1351: run_trace_with_shortcut1(Config) when is_list(Config) ->
 1352:     ?line run_trace_with_shortcut(caller, [ok,ok], tp),
 1353:     ?line run_trace_with_shortcut(caller, [ok,ok], tpl).
 1354: run_trace_with_shortcut1(cleanup,_Config) ->
 1355:     ?line stop_client_and_server().
 1356: 
 1357: run_trace_with_shortcut2(suite) ->
 1358:     [];
 1359: run_trace_with_shortcut2(doc) ->
 1360:     ["Run trace with shortcut 2"];
 1361: run_trace_with_shortcut2(Config) when is_list(Config) ->
 1362:     ?line run_trace_with_shortcut(return, [ok,ok], tp),
 1363:     ?line run_trace_with_shortcut(return, [ok,ok], tpl).
 1364: run_trace_with_shortcut2(cleanup,_Config) ->
 1365:     ?line stop_client_and_server().
 1366: 
 1367: run_trace_with_shortcut3(suite) ->
 1368:     [];
 1369: run_trace_with_shortcut3(doc) ->
 1370:     ["Run trace with shortcut 3"];
 1371: run_trace_with_shortcut3(Config) when is_list(Config) ->
 1372:     ?line run_trace_with_shortcut(fun_for(return), [ok,ok], tp),
 1373:     ?line run_trace_with_shortcut(fun_for(return), [ok,ok], tpl).
 1374: run_trace_with_shortcut3(cleanup,_Config) ->
 1375:     ?line stop_client_and_server().
 1376: 
 1377: run_trace_with_shortcut4(suite) ->
 1378:     [];
 1379: run_trace_with_shortcut4(doc) ->
 1380:     ["Run trace with shortcut 4"];
 1381: run_trace_with_shortcut4(Config) when is_list(Config) ->
 1382:     ?line run_trace_with_shortcut(fun_for(msg_false), [], tp),
 1383:     ?line run_trace_with_shortcut(fun_for(msg_false), [], tpl).
 1384: run_trace_with_shortcut4(cleanup,_Config) ->
 1385:     ?line stop_client_and_server().
 1386: 
 1387: cant_specify_local_and_flush(suite) ->
 1388:     [];
 1389: cant_specify_local_and_flush(doc) ->
 1390:     ["Can't specify local and flush"];
 1391: cant_specify_local_and_flush(Config) when is_list(Config) ->
 1392:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1393:     ?line flush_unsupported_with_ip_trace_port =
 1394: 	try ttb:tracer([ServerNode, ClientNode],
 1395: 		       [{flush, 1000}, {file, {local, ?FNAME}}])
 1396: 	catch
 1397: 	    exit:flush_unsupported_with_ip_trace_port ->
 1398: 		flush_unsupported_with_ip_trace_port
 1399: 	end.
 1400: cant_specify_local_and_flush(cleanup,_Config) ->
 1401:     ?line stop_client_and_server().
 1402: 
 1403: trace_sorted_by_default(suite) ->
 1404:     [];
 1405: trace_sorted_by_default(doc) ->
 1406:     ["Trace sorted by default"];
 1407: trace_sorted_by_default(Config) when is_list(Config) ->
 1408:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1409:     ?line begin_trace_local(ServerNode, ClientNode, ?FILE),
 1410:     ?line ttb_helper:msgs(2),
 1411:     ?line {_, D} = ttb:stop([return_fetch_dir]),
 1412:     ?line ttb:format(D, [{out, ?OUTPUT}, {handler, node_call_handler()}, {disable_sort, false}]),
 1413:     {ok, Ret} = file:consult(?OUTPUT),
 1414:     ?line [ClientNode,ServerNode,ClientNode,ServerNode,ServerNode] = Ret.
 1415: trace_sorted_by_default(cleanup,_Config) ->
 1416:     ?line stop_client_and_server().
 1417: 
 1418: disable_sorting(suite) ->
 1419:     [];
 1420: disable_sorting(doc) ->
 1421:     ["Disable sorting"];
 1422: disable_sorting(Config) when is_list(Config) ->
 1423:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1424:     ?line begin_trace_local(ServerNode, ClientNode, ?FILE),
 1425:     ?line ttb_helper:msgs(2),
 1426:     ?line {_, D} = ttb:stop([return_fetch_dir]),
 1427:     ?line ttb:format(D, [{out, ?OUTPUT}, {handler, node_call_handler()}, {disable_sort, true}]),
 1428:     {ok, Ret} = file:consult(?OUTPUT),
 1429:     ?line [ClientNode,ClientNode,ServerNode,ServerNode,ServerNode] = Ret.
 1430: disable_sorting(cleanup,_Config) ->
 1431:     ?line stop_client_and_server().
 1432: 
 1433: %% -----------------------------------------------------------------------------
 1434: %% tests for autoresume of tracing
 1435: %% -----------------------------------------------------------------------------
 1436: 
 1437: trace_resumed_after_node_restart(suite) ->
 1438:     [];
 1439: trace_resumed_after_node_restart(doc) ->
 1440:     ["Test trace resumed after node restart, trace to files on remote node."];
 1441: trace_resumed_after_node_restart(Config) when is_list(Config) ->
 1442:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1443:     ?line begin_trace_with_resume(ServerNode, ClientNode, ?FNAME),
 1444:     ?line logic(2,6,file).
 1445: trace_resumed_after_node_restart(cleanup,_Config) ->
 1446:     ?line stop_client_and_server().
 1447: 
 1448: trace_resumed_after_node_restart_ip(suite) ->
 1449:     [];
 1450: trace_resumed_after_node_restart_ip(doc) ->
 1451:     ["Test trace resumed after node restart, trace via tcp/ip to local node."];
 1452: trace_resumed_after_node_restart_ip(Config) when is_list(Config) ->
 1453:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1454:     ?line begin_trace_with_resume(ServerNode, ClientNode, {local, ?FNAME}),
 1455:     ?line logic(2,6,local).
 1456: trace_resumed_after_node_restart_ip(cleanup,_Config) ->
 1457:     ?line stop_client_and_server().
 1458: 
 1459: trace_resumed_after_node_restart_wrap(suite) ->
 1460:     [];
 1461: trace_resumed_after_node_restart_wrap(doc) ->
 1462:     ["Test trace resumed after node restart, wrap option."];
 1463: trace_resumed_after_node_restart_wrap(Config) when is_list(Config) ->
 1464:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1465:     ?line begin_trace_with_resume(ServerNode, ClientNode, {wrap, ?FNAME, 10, 4}),
 1466:     ?line logic(1,4,file).
 1467: trace_resumed_after_node_restart_wrap(cleanup,_Config) ->
 1468:     ?line stop_client_and_server().
 1469: 
 1470: trace_resumed_after_node_restart_wrap_mult(suite) ->
 1471:     [];
 1472: trace_resumed_after_node_restart_wrap_mult(doc) ->
 1473:     ["Test trace resumed after node restart, wrap option, multiple files."];
 1474: trace_resumed_after_node_restart_wrap_mult(Config) when is_list(Config) ->
 1475:     ?line {ServerNode, ClientNode} = start_client_and_server(),
 1476:     ?line begin_trace_with_resume(ServerNode, ClientNode, {wrap, ?FNAME, 10, 4}),
 1477:     ?line logic(20,8,file).
 1478: trace_resumed_after_node_restart_wrap_mult(cleanup,_Config) ->
 1479:     ?line stop_client_and_server().
 1480: 
 1481: logic(N, M, TracingType) ->
 1482:     helper_msgs(N, TracingType),
 1483:     ?t:stop_node(ttb_helper:get_node(client)),
 1484:     timer:sleep(2500),
 1485:     ?line {ok,_ClientNode} = ?t:start_node(client,slave,[]),
 1486:     ct:log("client started",[]),
 1487:     ?line ok = ttb_helper:c(code, add_paths, [code:get_path()]),
 1488:     ct:log("paths added",[]),
 1489:     ?line ttb_helper:c(client, init, []),
 1490:     ct:log("client initiated",[]),
 1491:     ?line helper_msgs(N, TracingType),
 1492:     ct:log("helper msgs sent and flushed",[]),
 1493:     ?line {_, D} = ttb:stop([return_fetch_dir]),
 1494:     ct:log("stopped ~p",[D]),
 1495:     ?line ttb:format(D, [{out, ?OUTPUT}, {handler, ret_caller_call_handler2()}]),
 1496:     ct:log("formatted ~p",[{D,?OUTPUT}]),
 1497:     ?line {ok, Ret} = file:consult(?OUTPUT),
 1498:     ct:log("consulted: ~p",[Ret]),
 1499:     ?line M = length(Ret).
 1500: 
 1501: begin_trace_with_resume(ServerNode, ClientNode, Dest) ->
 1502:     ?line {ok, _} = ttb:tracer([ServerNode,ClientNode], [{file, Dest}, resume]),
 1503:     ?line ttb:p(all, [call, timestamp]),
 1504:     ?line ttb:tp(server, received, []),
 1505:     ?line ttb:tp(client, put, []),
 1506:     ?line ttb:tp(client, get, []).
 1507: 
 1508: ret_caller_call_handler2() ->
 1509:     {fun(A, {trace_ts, _, call, _, _} ,_,_) -> io:format(A, "ok.~n", []);
 1510: 	(_, _, _, _) -> ok end, []}.
 1511: 
 1512: helper_msgs(N, TracingType) ->
 1513:     case TracingType of
 1514:         local ->
 1515:             ttb_helper:msgs_ip(N);
 1516:         _ ->
 1517:             ttb_helper:msgs(N)
 1518:     end.
 1519: 
 1520: priv_dir(Conf) ->
 1521:     %% Due to problem with long paths on windows => creating a new
 1522:     %% priv_dir under data_dir
 1523:     Dir = filename:absname(filename:join(?config(data_dir, Conf),priv_dir)),
 1524:     filelib:ensure_dir(filename:join(Dir,"*")),
 1525:     Dir.
 1526: 
 1527: clean_priv_dir(Config) ->
 1528:     PrivDir = priv_dir(Config),
 1529:     case filelib:is_dir(PrivDir) of
 1530:         true -> rm(PrivDir);
 1531:         false -> ok
 1532:     end.
 1533: 
 1534: rm(This) ->
 1535:     case filelib:is_dir(This) of
 1536:         true ->
 1537:             {ok,Files} = file:list_dir(This),
 1538:             [rm(filename:join(This,F)) || F <- Files],
 1539: 	    file:del_dir(This);
 1540:         false ->
 1541: 	    file:delete(This)
 1542:     end.