1: %%
    2: %% %CopyrightBegin%
    3: %%
    4: %% Copyright Ericsson AB 1996-2013. All Rights Reserved.
    5: %%
    6: %% The contents of this file are subject to the Erlang Public License,
    7: %% Version 1.1, (the "License"); you may not use this file except in
    8: %% compliance with the License. You should have received a copy of the
    9: %% Erlang Public License along with this software. If not, it can be
   10: %% retrieved online at http://www.erlang.org/.
   11: %%
   12: %% Software distributed under the License is distributed on an "AS IS"
   13: %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
   14: %% the License for the specific language governing rights and limitations
   15: %% under the License.
   16: %%
   17: %% %CopyrightEnd%
   18: %%
   19: 
   20: %%
   21: -module(mnesia_recovery_test).
   22: -author('hakan@erix.ericsson.se').
   23: -compile([export_all]).
   24: 
   25: -include("mnesia_test_lib.hrl").
   26: -include_lib("kernel/include/file.hrl").
   27: 
   28: init_per_testcase(Func, Conf) ->
   29:     mnesia_test_lib:init_per_testcase(Func, Conf).
   30: 
   31: end_per_testcase(Func, Conf) ->
   32:     mnesia_test_lib:end_per_testcase(Func, Conf).
   33: 
   34: -define(receive_messages(Msgs), receive_messages(Msgs, ?FILE, ?LINE)).
   35: 
   36: % First Some debug logging 
   37: -define(dgb, true).
   38: -ifdef(dgb).
   39: -define(dl(X, Y), ?verbose("**TRACING: " ++ X ++ "**~n", Y)).
   40: -else. 
   41: -define(dl(X, Y), ok).
   42: -endif.
   43: 
   44: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
   45: all() -> 
   46:     [{group, mnesia_down}, {group, explicit_stop},
   47:      coord_dies, {group, schema_trans}, {group, async_dirty},
   48:      {group, sync_dirty}, {group, sym_trans},
   49:      {group, asym_trans}, after_full_disc_partition,
   50:      {group, after_corrupt_files}, disc_less, garb_decision,
   51:      system_upgrade].
   52: 
   53: groups() -> 
   54:     [{schema_trans, [],
   55:       [{mnesia_schema_recovery_test, all}]},
   56:      {mnesia_down, [],
   57:       [{group, mnesia_down_during_startup},
   58:        {group, master_node_tests}, {group, read_during_down},
   59:        {group, with_checkpoint}, delete_during_start]},
   60:      {master_node_tests, [],
   61:       [no_master_2, no_master_3, one_master_2, one_master_3,
   62:        two_master_2, two_master_3, all_master_2,
   63:        all_master_3]},
   64:      {read_during_down, [],
   65:       [dirty_read_during_down, trans_read_during_down]},
   66:      {mnesia_down_during_startup, [],
   67:       [mnesia_down_during_startup_disk_ram,
   68:        mnesia_down_during_startup_init_ram,
   69:        mnesia_down_during_startup_init_disc,
   70:        mnesia_down_during_startup_init_disc_only,
   71:        mnesia_down_during_startup_tm_ram,
   72:        mnesia_down_during_startup_tm_disc,
   73:        mnesia_down_during_startup_tm_disc_only]},
   74:      {with_checkpoint, [],
   75:       [with_checkpoint_same, with_checkpoint_other]},
   76:      {explicit_stop, [], [explicit_stop_during_snmp]},
   77:      {sym_trans, [],
   78:       [sym_trans_before_commit_kill_coord_node,
   79:        sym_trans_before_commit_kill_coord_pid,
   80:        sym_trans_before_commit_kill_part_after_ask,
   81:        sym_trans_before_commit_kill_part_before_ask,
   82:        sym_trans_after_commit_kill_coord_node,
   83:        sym_trans_after_commit_kill_coord_pid,
   84:        sym_trans_after_commit_kill_part_after_ask,
   85:        sym_trans_after_commit_kill_part_do_commit_pre,
   86:        sym_trans_after_commit_kill_part_do_commit_post]},
   87:      {sync_dirty, [],
   88:       [sync_dirty_pre_kill_part,
   89:        sync_dirty_pre_kill_coord_node,
   90:        sync_dirty_pre_kill_coord_pid,
   91:        sync_dirty_post_kill_part,
   92:        sync_dirty_post_kill_coord_node,
   93:        sync_dirty_post_kill_coord_pid]},
   94:      {async_dirty, [],
   95:       [async_dirty_pre_kill_part,
   96:        async_dirty_pre_kill_coord_node,
   97:        async_dirty_pre_kill_coord_pid,
   98:        async_dirty_post_kill_part,
   99:        async_dirty_post_kill_coord_node,
  100:        async_dirty_post_kill_coord_pid]},
  101:      {asym_trans, [],
  102:       [asymtrans_part_ask,
  103:        asymtrans_part_commit_vote,
  104:        asymtrans_part_pre_commit,
  105:        asymtrans_part_log_commit,
  106:        asymtrans_part_do_commit,
  107:        asymtrans_coord_got_votes,
  108:        asymtrans_coord_pid_got_votes,
  109:        asymtrans_coord_log_commit_rec,
  110:        asymtrans_coord_pid_log_commit_rec,
  111:        asymtrans_coord_log_commit_dec,
  112:        asymtrans_coord_pid_log_commit_dec,
  113:        asymtrans_coord_rec_acc_pre_commit_log_commit,
  114:        asymtrans_coord_pid_rec_acc_pre_commit_log_commit,
  115:        asymtrans_coord_rec_acc_pre_commit_done_commit,
  116:        asymtrans_coord_pid_rec_acc_pre_commit_done_commit]},
  117:      {after_corrupt_files, [],
  118:       [after_corrupt_files_decision_log_head,
  119:        after_corrupt_files_decision_log_tail,
  120:        after_corrupt_files_latest_log_head,
  121:        after_corrupt_files_latest_log_tail,
  122:        after_corrupt_files_table_dat_head,
  123:        after_corrupt_files_table_dat_tail,
  124:        after_corrupt_files_schema_dat_head,
  125:        after_corrupt_files_schema_dat_tail]}].
  126: 
  127: init_per_group(_GroupName, Config) ->
  128:     Config.
  129: 
  130: end_per_group(_GroupName, Config) ->
  131:     Config.
  132: 
  133: tpcb_config(ReplicaType, _NodeConfig, Nodes) ->
  134:     [{n_branches, 5},
  135:      {n_drivers_per_node, 5},
  136:      {replica_nodes, Nodes},
  137:      {driver_nodes, Nodes},
  138:      {use_running_mnesia, true},
  139:      {report_interval, infinity},
  140:      {n_accounts_per_branch, 20},
  141:      {replica_type, ReplicaType}].
  142: 
  143: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  144: 
  145: 
  146: 
  147: no_master_2(suite) -> [];
  148: no_master_2(Config) when is_list(Config) ->    mnesia_down_2(no, Config).
  149:     
  150: no_master_3(suite) -> [];
  151: no_master_3(Config) when is_list(Config) ->    mnesia_down_3(no, Config).
  152: 
  153: one_master_2(suite) -> [];
  154: one_master_2(Config) when is_list(Config) ->   mnesia_down_2(one, Config).
  155: 
  156: one_master_3(suite) -> [];
  157: one_master_3(Config) when is_list(Config) ->   mnesia_down_3(one, Config).
  158: 
  159: two_master_2(suite) -> [];
  160: two_master_2(Config) when is_list(Config) ->   mnesia_down_2(two, Config).
  161: 
  162: two_master_3(suite) -> [];
  163: two_master_3(Config) when is_list(Config) ->   mnesia_down_3(two, Config).
  164: 
  165: all_master_2(suite) -> [];
  166: all_master_2(Config) when is_list(Config) ->   mnesia_down_2(all, Config).
  167: 
  168: all_master_3(suite) -> [];
  169: all_master_3(Config) when is_list(Config) ->   mnesia_down_3(all, Config).
  170: 
  171: mnesia_down_2(Masters, Config) -> 
  172:     Nodes = [N1, N2] = ?acquire_nodes(2, Config),
  173:     ?match({atomic, ok}, mnesia:create_table(tab1, [{ram_copies, Nodes}])),
  174:     ?match({atomic, ok}, mnesia:create_table(tab2, [{disc_copies, Nodes}])),
  175:     ?match({atomic, ok}, mnesia:create_table(tab3, [{disc_only_copies, Nodes}])),
  176:     ?match({atomic, ok}, mnesia:create_table(tab4, [{ram_copies, [N1]}])),
  177:     ?match({atomic, ok}, mnesia:create_table(tab5, [{ram_copies, [N2]}])),
  178:     ?match({atomic, ok}, mnesia:create_table(tab6, [{disc_copies, [N1]}])),
  179:     ?match({atomic, ok}, mnesia:create_table(tab7, [{disc_copies, [N2]}])),
  180:     ?match({atomic, ok}, mnesia:create_table(tab8, [{disc_only_copies, [N1]}])),
  181:     ?match({atomic, ok}, mnesia:create_table(tab9, [{disc_only_copies, [N2]}])),
  182:     ?match({atomic, ok}, mnesia:create_table(tab10, [{ram_copies, [N1]}, {disc_copies, [N2]}])),
  183:     ?match({atomic, ok}, mnesia:create_table(tab11, [{ram_copies, [N2]}, {disc_copies, [N1]}])),
  184:     ?match({atomic, ok}, mnesia:create_table(tab12, [{ram_copies, [N1]}, {disc_only_copies, [N2]}])),
  185:     ?match({atomic, ok}, mnesia:create_table(tab13, [{ram_copies, [N2]}, {disc_only_copies, [N1]}])),
  186:     ?match({atomic, ok}, mnesia:create_table(tab14, [{disc_only_copies, [N1]}, {disc_copies, [N2]}])),
  187:     ?match({atomic, ok}, mnesia:create_table(tab15, [{disc_only_copies, [N2]}, {disc_copies, [N1]}])),
  188: 
  189:     Tabs = [tab1, tab2, tab3, tab4, tab5, tab6, tab7, tab8, 
  190: 	    tab9, tab10, tab11, tab12, tab13, tab14, tab15],
  191:     [?match(ok, rpc:call(Node, mnesia, wait_for_tables, [Tabs, 10000])) || Node <- Nodes],
  192:     [insert_data(Tab, 20) || Tab <- Tabs],
  193: 
  194:     VTabs =
  195: 	case Masters of
  196: 	    no ->
  197: 		Tabs -- [tab4, tab5]; % ram copies
  198: 	    one ->
  199: 		?match(ok, rpc:call(N1, mnesia, set_master_nodes, [[N1]])),
  200: 		Tabs -- [tab1, tab4, tab5, tab10, tab12]; % ram_copies
  201: 	    two ->
  202: 		?match(ok, rpc:call(N1, mnesia, set_master_nodes, [Nodes])),
  203: 		Tabs -- [tab4, tab5];
  204: 	    all -> 
  205: 		[?match(ok, rpc:call(Node, mnesia, set_master_nodes, [[Node]])) || Node <- Nodes],
  206: 		Tabs -- [tab1, tab4, tab5, tab10, tab11, tab12, tab13]
  207: 	end,
  208: 
  209:     mnesia_test_lib:kill_mnesia([N1]),
  210:     ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
  211: 
  212:     ?match([], mnesia_test_lib:kill_mnesia([N2])),
  213:     ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
  214: 
  215:     [?match(ok, rpc:call(N1, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs],
  216:     [?match(ok, rpc:call(N2, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs],
  217:     ?verify_mnesia(Nodes, []).
  218: 
  219: mnesia_down_3(Masters, Config) -> 
  220:     Nodes = [N1, N2, N3] = ?acquire_nodes(3, Config),
  221:     ?match({atomic, ok}, mnesia:create_table(tab1, [{ram_copies, Nodes}])),
  222:     ?match({atomic, ok}, mnesia:create_table(tab2, [{disc_copies, Nodes}])),
  223:     ?match({atomic, ok}, mnesia:create_table(tab3, [{disc_only_copies, Nodes}])),
  224:     ?match({atomic, ok}, mnesia:create_table(tab4, [{ram_copies, [N1]}])),
  225:     ?match({atomic, ok}, mnesia:create_table(tab5, [{ram_copies, [N2]}])),
  226:     ?match({atomic, ok}, mnesia:create_table(tab16, [{ram_copies, [N3]}])),
  227:     ?match({atomic, ok}, mnesia:create_table(tab6, [{disc_copies, [N1]}])),
  228:     ?match({atomic, ok}, mnesia:create_table(tab7, [{disc_copies, [N2]}])),
  229:     ?match({atomic, ok}, mnesia:create_table(tab17, [{disc_copies, [N3]}])),
  230:     ?match({atomic, ok}, mnesia:create_table(tab8, [{disc_only_copies, [N1]}])),
  231:     ?match({atomic, ok}, mnesia:create_table(tab9, [{disc_only_copies, [N2]}])),
  232:     ?match({atomic, ok}, mnesia:create_table(tab18, [{disc_only_copies, [N3]}])),
  233:     ?match({atomic, ok}, mnesia:create_table(tab10, [{ram_copies, [N1]}, {disc_copies, [N2, N3]}])),
  234:     ?match({atomic, ok}, mnesia:create_table(tab11, [{ram_copies, [N2]}, {disc_copies, [N3, N1]}])),
  235:     ?match({atomic, ok}, mnesia:create_table(tab19, [{ram_copies, [N3]}, {disc_copies, [N1, N2]}])),
  236:     ?match({atomic, ok}, mnesia:create_table(tab12, [{ram_copies, [N1]}, {disc_only_copies, [N2, N3]}])),
  237:     ?match({atomic, ok}, mnesia:create_table(tab13, [{ram_copies, [N2]}, {disc_only_copies, [N3, N1]}])),
  238:     ?match({atomic, ok}, mnesia:create_table(tab20, [{ram_copies, [N3]}, {disc_only_copies, [N1, N2]}])),
  239:     ?match({atomic, ok}, mnesia:create_table(tab14, [{disc_only_copies, [N1]}, {disc_copies, [N2, N3]}])),
  240:     ?match({atomic, ok}, mnesia:create_table(tab15, [{disc_only_copies, [N2]}, {disc_copies, [N3, N1]}])),
  241:     ?match({atomic, ok}, mnesia:create_table(tab21, [{disc_only_copies, [N3]}, {disc_copies, [N1, N2]}])),
  242: 
  243:     Tabs = [tab1, tab2, tab3, tab4, tab5, tab6, tab7, tab8, 
  244: 	    tab9, tab10, tab11, tab12, tab13, tab14, tab15,
  245: 	    tab16, tab17, tab18, tab19, tab20, tab21],
  246:     [?match(ok, rpc:call(Node, mnesia, wait_for_tables, [Tabs, 10000])) || Node <- Nodes],
  247:     [insert_data(Tab, 20) || Tab <- Tabs],
  248: 
  249:     VTabs = 
  250: 	case Masters of
  251: 	    no ->
  252: 		Tabs -- [tab4, tab5, tab16]; % ram copies
  253: 	    one ->
  254: 		?match(ok, rpc:call(N1, mnesia, set_master_nodes, [[N1]])),
  255: 		Tabs -- [tab1, tab4, tab5, tab16, tab10, tab12]; % ram copies
  256: 	    two ->
  257: 		?match(ok, rpc:call(N1, mnesia, set_master_nodes, [Nodes])),
  258: 		Tabs -- [tab4, tab5, tab16]; % ram copies
  259: 	    all -> 
  260: 		[?match(ok, rpc:call(Node, mnesia, set_master_nodes, [[Node]])) || Node <- Nodes],
  261: 		Tabs -- [tab1, tab4, tab5, tab16, tab10, 
  262: 			 tab11, tab19, tab12, tab13, tab20] % ram copies
  263: 	end,
  264:     
  265:     mnesia_test_lib:kill_mnesia([N1]),
  266:     ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
  267:     
  268:     ?match([], mnesia_test_lib:kill_mnesia([N2])),
  269:     ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
  270: 
  271:     ?match([], mnesia_test_lib:kill_mnesia([N3])),
  272:     ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
  273: 
  274:     ?match([], mnesia_test_lib:kill_mnesia([N2, N1])),
  275:     ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
  276: 
  277:     ?match([], mnesia_test_lib:kill_mnesia([N2, N3])),
  278:     ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
  279: 
  280:     ?match([], mnesia_test_lib:kill_mnesia([N1, N3])),
  281:     ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
  282: 
  283:     [?match(ok, rpc:call(N1, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs],
  284:     [?match(ok, rpc:call(N2, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs],
  285:     [?match(ok, rpc:call(N3, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs],
  286:     
  287:     ?verify_mnesia(Nodes, []).
  288: 
  289: 
  290: 
  291: dirty_read_during_down(suite) ->
  292:     [];
  293: dirty_read_during_down(Config) when is_list(Config) ->
  294:     read_during_down(dirty, Config).
  295: 
  296: trans_read_during_down(suite) ->
  297:     [];
  298: trans_read_during_down(Config) when is_list(Config) ->
  299:     read_during_down(trans, Config).
  300: 
  301: 
  302: read_during_down(Op, Config) when is_list(Config) ->
  303:     Ns = [N1|TNs] = ?acquire_nodes(3, Config),
  304:     Tabs = [ram, disc, disco],
  305:     
  306:     ?match({atomic, ok}, mnesia:create_table(ram, [{ram_copies, TNs}])),
  307:     ?match({atomic, ok}, mnesia:create_table(disc, [{disc_copies, TNs}])),
  308:     ?match({atomic, ok}, mnesia:create_table(disco, [{disc_only_copies, TNs}])),
  309: 
  310:     %% Create some work for mnesia_controller when a node goes down
  311:     [{atomic, ok} = mnesia:create_table(list_to_atom("temp" ++ integer_to_list(N)),
  312: 					[{ram_copies, Ns}]) || N <- lists:seq(1, 50)],    
  313:     
  314:     Write = fun(Tab) -> mnesia:write({Tab, key, val}) end,
  315:     ?match([ok,ok,ok],
  316: 	   [mnesia:sync_dirty(Write, [Tab]) || Tab <- Tabs]),
  317:     
  318:     Readers = [spawn_link(N1, ?MODULE, reader, [Tab, Op]) || Tab <- Tabs],
  319:     [_|_] = W2R= [mnesia:table_info(Tab, where_to_read) || Tab <- Tabs],
  320:     ?log("W2R ~p~n", [W2R]),
  321:     loop_and_kill_mnesia(10, hd(W2R), Tabs),
  322:     [Pid ! self() || Pid <- Readers],
  323:     ?match([ok, ok, ok], [receive ok -> ok after 1000 -> {Pid, mnesia_lib:dist_coredump()} end || Pid <- Readers]),
  324:     ?verify_mnesia(Ns, []).
  325: 
  326: reader(Tab, OP) ->
  327:     Res = case OP of 
  328: 	      dirty ->
  329: 		  catch mnesia:dirty_read({Tab, key});
  330: 	      trans ->
  331: 		  Read = fun() -> mnesia:read({Tab, key}) end,
  332: 		  {_, Temp} = mnesia:transaction(Read),
  333: 		  Temp
  334: 	  end,
  335:     case Res of 
  336: 	[{Tab, key, val}] -> ok;
  337: 	Else -> 
  338: 	    ?error("Expected ~p Got ~p ~n", [[{Tab, key, val}], Else]),
  339: 	    erlang:error(test_failed)
  340:     end,
  341:     receive Pid ->
  342: 	    Pid ! ok
  343:     after 50 ->
  344: 	    reader(Tab, OP)
  345:     end.
  346: 
  347: loop_and_kill_mnesia(0, _Node, _Tabs) -> ok;
  348: loop_and_kill_mnesia(N, Node, Tabs) ->
  349:     mnesia_test_lib:kill_mnesia([Node]),
  350:     timer:sleep(100), 
  351:     ?match([], mnesia_test_lib:start_mnesia([Node], Tabs)),
  352:     [KN | _] = W2R= [mnesia:table_info(Tab, where_to_read) || Tab <- Tabs],
  353:     ?match([KN, KN,KN], W2R),
  354:     timer:sleep(100), 
  355:     loop_and_kill_mnesia(N-1, KN, Tabs).
  356: 
  357: 
  358: mnesia_down_during_startup_disk_ram(suite) -> [];
  359: mnesia_down_during_startup_disk_ram(Config) when is_list(Config)->
  360:     [Node1, Node2] = ?acquire_nodes(2, Config ++ 
  361: 				    [{tc_timeout, timer:minutes(2)}]),
  362:     Tab = down_during_startup,
  363:     Def = [{ram_copies, [Node2]}, {disc_copies, [Node1]}],
  364:     
  365:     ?match({atomic, ok}, mnesia:create_table(Tab, Def)),
  366:     ?match(ok, mnesia:dirty_write({Tab, 876234, test_ok})),
  367:     timer:sleep(500),
  368:     mnesia_test_lib:kill_mnesia([Node1, Node2]),
  369:     timer:sleep(500),
  370:     mnesia_test_lib:start_mnesia([Node1, Node2], [Tab]),
  371:     mnesia_test_lib:kill_mnesia([Node1]),
  372:     timer:sleep(500),
  373:     ?match([], mnesia_test_lib:start_mnesia([Node1], [Tab])),
  374:     ?match([{Tab, 876234, test_ok}], mnesia:dirty_read({Tab,876234})),
  375:     ?verify_mnesia([Node1, Node2], []).
  376:     
  377: mnesia_down_during_startup_init_ram(suite) -> [];
  378: mnesia_down_during_startup_init_ram(Config) when is_list(Config) ->
  379:     ?is_debug_compiled,
  380:     DP = {mnesia_loader, do_get_network_copy},
  381:     Type = ram_copies,
  382:     mnesia_down_during_startup2(Config, Type, DP, self()).
  383: 
  384: mnesia_down_during_startup_init_disc(suite)  -> [];
  385: mnesia_down_during_startup_init_disc(Config) when is_list(Config) ->
  386:     ?is_debug_compiled,
  387:     DP = {mnesia_loader, do_get_network_copy},
  388:     Type = disc_copies,
  389:     mnesia_down_during_startup2(Config, Type, DP, self()).
  390: 
  391: mnesia_down_during_startup_init_disc_only(suite) -> [];
  392: mnesia_down_during_startup_init_disc_only(Config) when is_list(Config) ->
  393:     ?is_debug_compiled,
  394:     DP = {mnesia_loader, do_get_network_copy},
  395:     Type = disc_only_copies,
  396:     mnesia_down_during_startup2(Config, Type, DP, self()).
  397: 
  398: mnesia_down_during_startup_tm_ram(suite) -> [];
  399: mnesia_down_during_startup_tm_ram(Config) when is_list(Config) ->
  400:     ?is_debug_compiled,
  401:     DP = {mnesia_tm, init},
  402:     Type = ram_copies,
  403:     mnesia_down_during_startup2(Config, Type, DP, self()).
  404: 
  405: mnesia_down_during_startup_tm_disc(suite) -> [];
  406: mnesia_down_during_startup_tm_disc(Config) when is_list(Config) ->
  407:     ?is_debug_compiled,
  408:     DP = {mnesia_tm, init},
  409:     Type = disc_copies,
  410:     mnesia_down_during_startup2(Config, Type, DP, self()).
  411: 
  412: mnesia_down_during_startup_tm_disc_only(suite) -> [];
  413: mnesia_down_during_startup_tm_disc_only(Config) when is_list(Config) ->
  414:     ?is_debug_compiled,
  415:     DP = {mnesia_tm, init},
  416:     Type = disc_only_copies,
  417:     mnesia_down_during_startup2(Config, Type, DP, self()).
  418: 
  419: mnesia_down_during_startup2(Config, ReplicaType, Debug_Point, _Father) -> 
  420:     ?log("TC~n mnesia_down_during_startup with type ~w and stops at ~w~n",
  421: 	 [ReplicaType, Debug_Point]),
  422:     Tpcb_tabs = [history,teller,account,branch],
  423:     Nodes = ?acquire_nodes(2, Config),
  424:     Node1 = hd(Nodes),
  425:     {success, [A]} = ?start_activities([Node1]),
  426:     TpcbConfig = tpcb_config(ReplicaType, 2, Nodes),
  427:     mnesia_tpcb:init(TpcbConfig),
  428:     A ! fun () -> mnesia_tpcb:run(TpcbConfig) end,
  429:     ?match_receive(timeout),
  430:     timer:sleep(timer:seconds(10)),  % Let tpcb run for a while
  431:     mnesia_tpcb:stop(),
  432:     ?match(ok, mnesia_tpcb:verify_tabs()), 
  433:     mnesia_test_lib:kill_mnesia([Node1]),
  434:     timer:sleep(timer:seconds(2)),
  435:     Self = self(),
  436:     TestFun = fun(_MnesiaEnv, _EvalEnv) ->
  437: 		      ?deactivate_debug_fun(Debug_Point),
  438: 		      Self ! fun_done, 
  439: 		      spawn(mnesia_test_lib, kill_mnesia, [[Node1]])
  440: 	      end,    
  441:     ?activate_debug_fun(Debug_Point, TestFun, []),      % Kill when debug has been reached
  442:     mnesia:start(),
  443:     Res = receive fun_done -> ok after timer:minutes(3) -> timeout end, % Wait till it's killed
  444:     ?match(ok, Res),
  445:     ?match(ok, timer:sleep(timer:seconds(2))), % Wait a while, at least till it dies;
  446:     ?match([], mnesia_test_lib:start_mnesia([Node1], Tpcb_tabs)),  
  447:     ?match(ok, mnesia_tpcb:verify_tabs()), % Verify it
  448:     ?verify_mnesia(Nodes, []).    	    
  449: 
  450: 
  451: 
  452: with_checkpoint_same(suite) -> [];
  453: with_checkpoint_same(Config) when is_list(Config) -> 
  454:     with_checkpoint(Config, same).
  455: 
  456: with_checkpoint_other(suite) -> [];
  457: with_checkpoint_other(Config) when is_list(Config) -> 
  458:     with_checkpoint(Config, other).
  459: 
  460: with_checkpoint(Config, Type) when is_list(Config) ->
  461:     Nodes = [Node1, Node2] = ?acquire_nodes(2, Config),
  462:     Kill = case Type of
  463: 	       same ->    %% Node1 is the one used for creating the checkpoint
  464: 		   Node1; %% and which we bring down
  465: 	       other ->
  466: 		   Node2  %% Here we bring node2 down..
  467: 	   end,
  468:     
  469:     ?match({atomic, ok}, mnesia:create_table(ram, [{ram_copies, Nodes}])),
  470:     ?match({atomic, ok}, mnesia:create_table(disc, [{disc_copies, Nodes}])),
  471:     ?match({atomic, ok}, mnesia:create_table(disco, [{disc_only_copies, Nodes}])),
  472:     Tabs = [ram, disc, disco],
  473:     
  474:     ?match({ok, sune, _}, mnesia:activate_checkpoint([{name, sune}, 
  475: 						      {max, mnesia:system_info(tables)}, 
  476: 						      {ram_overrides_dump, true}])),
  477: 
  478:     ?match([], check_retainers(sune, Nodes)),
  479:     
  480:     ?match(ok, mnesia:deactivate_checkpoint(sune)),
  481:     ?match([], check_chkp(Nodes)),
  482:     
  483:     timer:sleep(500),  %% Just to help debugging the io:formats now comes in the
  484:     %% correct order... :-)    
  485: 
  486:     ?match({ok, sune, _}, mnesia:activate_checkpoint([{name, sune}, 
  487: 	{max, mnesia:system_info(tables)}, 
  488: 	{ram_overrides_dump, true}])),
  489: 
  490:     [[mnesia:dirty_write({Tab,Key,Key}) || Key <- lists:seq(1,10)] || Tab <- Tabs],
  491: 
  492:     mnesia_test_lib:kill_mnesia([Kill]),
  493:     timer:sleep(100),     
  494:     mnesia_test_lib:start_mnesia([Kill], Tabs),
  495:     io:format("Mnesia on ~p started~n", [Kill]),
  496:     ?match([], check_retainers(sune, Nodes)),
  497:     ?match(ok, mnesia:deactivate_checkpoint(sune)),
  498:     ?match([], check_chkp(Nodes)),
  499: 
  500:     case Kill of
  501: 	Node1 -> 
  502: 	    ignore;
  503: 	Node2 ->
  504: 	    mnesia_test_lib:kill_mnesia([Kill]),
  505: 	    timer:sleep(500),  %% Just to help debugging
  506: 	    ?match({ok, sune, _}, mnesia:activate_checkpoint([{name, sune}, 
  507: 		{max, mnesia:system_info(tables)}, 
  508: 		{ram_overrides_dump, true}])),
  509: 
  510: 	    [[mnesia:dirty_write({Tab,Key,Key+2}) || Key <- lists:seq(1,10)] ||
  511: 		Tab <- Tabs],
  512: 
  513: 	    mnesia_test_lib:start_mnesia([Kill], Tabs),
  514: 	    io:format("Mnesia on ~p started ~n", [Kill]),
  515: 	    ?match([], check_retainers(sune, Nodes)),
  516: 	    ?match(ok, mnesia:deactivate_checkpoint(sune)),
  517: 	    ?match([], check_chkp(Nodes)),	    
  518: 	    ok
  519:     end,
  520:     ?verify_mnesia(Nodes, []).
  521: 
  522: check_chkp(Nodes) ->
  523:     {Good, Bad} = rpc:multicall(Nodes, ?MODULE, check, []),
  524:     lists:flatten(Good ++ Bad).
  525: 
  526: check() ->
  527:     [PCP] = ets:match_object(mnesia_gvar, {pending_checkpoint_pids, '_'}),
  528:     [PC]  = ets:match_object(mnesia_gvar, {pending_checkpoints, '_'}),
  529:     [CPN] = ets:match_object(mnesia_gvar, {checkpoints, '_'}),    
  530:     F = lists:filter(fun({_, []}) -> false; (_W) -> true end, 
  531: 		     [PCP,PC,CPN]),    
  532:     CPP = ets:match_object(mnesia_gvar, {{checkpoint, '_'}, '_'}),
  533:     Rt  = ets:match_object(mnesia_gvar, {{'_', {retainer, '_'}}, '_'}),
  534:     F ++ CPP ++ Rt.
  535: 
  536: 
  537: check_retainers(CHP, Nodes) ->
  538:     {[R1,R2], []} = rpc:multicall(Nodes, ?MODULE, get_all_retainers, [CHP]),
  539:     (R1 -- R2) ++ (R2 -- R1).
  540:         
  541: get_all_retainers(CHP) ->
  542:     Tabs = mnesia:system_info(local_tables),
  543:     Iter = fun(Tab) ->
  544: 		   {ok, Res} = 
  545: 		       mnesia_checkpoint:iterate(CHP, Tab, fun(R, A) -> [R|A] end, [],
  546: 						 retainer, checkpoint),
  547: %%		   io:format("Retainer content ~w ~n", [Res]),
  548: 		   Res
  549: 	   end,
  550:     Elements = [Iter(Tab) || Tab <- Tabs],
  551:     lists:sort(lists:flatten(Elements)).
  552:     
  553: delete_during_start(doc) ->
  554:     ["Test that tables can be delete during start, hopefully with tables"
  555:      " in the loader queue or soon to be"];
  556: delete_during_start(suite) -> [];
  557: delete_during_start(Config) when is_list(Config) ->
  558:     [N1, N2, N3] = Nodes = ?acquire_nodes(3, Config),
  559:     Tabs = [list_to_atom("tab" ++ integer_to_list(I)) || I <- lists:seq(1, 30)],
  560:     ?match({atomic, ok}, mnesia:change_table_copy_type(schema, N2, ram_copies)),
  561:     ?match({atomic, ok}, mnesia:change_table_copy_type(schema, N3, ram_copies)),
  562: 
  563:     [?match({atomic, ok},mnesia:create_table(Tab, [{ram_copies,Nodes}])) || Tab <- Tabs],
  564:     lists:foldl(fun(Tab, I) ->
  565: 			?match({atomic, ok},
  566: 			       mnesia:change_table_load_order(Tab,I)),
  567: 			I+1
  568: 		end, 1, Tabs),
  569:     mnesia_test_lib:kill_mnesia([N2,N3]),
  570: %%    timer:sleep(500),
  571:     ?match({[ok,ok],[]}, rpc:multicall([N2,N3], mnesia,start, 
  572: 				       [[{extra_db_nodes,[N1]}]])),
  573:     [Tab1,Tab2,Tab3|_] = Tabs,
  574:     ?match({atomic, ok}, mnesia:delete_table(Tab1)),
  575:     ?match({atomic, ok}, mnesia:delete_table(Tab2)),
  576:     
  577:     ?log("W4T ~p~n", [rpc:multicall([N2,N3], mnesia, wait_for_tables, [[Tab1,Tab2,Tab3],1])]),
  578:     
  579:     Remain = Tabs--[Tab1,Tab2],
  580:     ?match(ok, rpc:call(N2, mnesia, wait_for_tables, [Remain,10000])),
  581:     ?match(ok, rpc:call(N3, mnesia, wait_for_tables, [Remain,10000])),
  582: 
  583:     ?match(ok, rpc:call(N2, ?MODULE, verify_where2read, [Remain])),
  584:     ?match(ok, rpc:call(N3, ?MODULE, verify_where2read, [Remain])),
  585: 
  586:     ?verify_mnesia(Nodes, []).
  587:     
  588: verify_where2read([Tab|Tabs]) ->
  589:     true = (node() == mnesia:table_info(Tab,where_to_read)),
  590:     verify_where2read(Tabs);
  591: verify_where2read([]) -> ok.
  592: 
  593: 
  594: %%-------------------------------------------------------------------------------------------
  595: %% This is a bad implementation, but at least gives a indication if something is wrong
  596: explicit_stop_during_snmp(suite) -> [];
  597: explicit_stop_during_snmp(Config) when is_list(Config) ->
  598:     Nodes = ?acquire_nodes(2, Config),
  599:     [Node1, Node2] = Nodes,
  600:     Tab = snmp_tab,
  601:     Def = [{attributes, [key, value]},
  602: 	   {snmp, [{key, integer}]},
  603: 	   {mnesia_test_lib:storage_type(disc_copies, Config), 
  604: 	    [Node1, Node2]}],
  605:     ?match({atomic, ok}, mnesia:create_table(Tab, Def)),
  606:     ?match({atomic, ok}, mnesia:transaction(fun() -> mnesia:write({Tab, 1, 1}) end)),
  607: 
  608:     Do_trans_Pid1 = spawn_link(Node2, ?MODULE, do_trans_loop, [Tab, self()]),
  609:     Do_trans_Pid2 = spawn_link(?MODULE, do_trans_loop, [Tab, self()]),
  610:     Start_stop_Pid = spawn_link(?MODULE, start_stop, [Node1, 5, self()]),
  611:     receive 
  612: 	test_done -> 
  613: 	    ok
  614:     after timer:minutes(5) ->
  615: 	    ?error("test case time out~n", [])
  616:     end,
  617:     ?verify_mnesia(Nodes, []),
  618:     exit(Do_trans_Pid1, kill),
  619:     exit(Do_trans_Pid2, kill),
  620:     exit(Start_stop_Pid, kill),
  621:     ok.
  622: 
  623: do_trans_loop(Tab, Father) ->
  624:     %% Do not trap exit
  625:     do_trans_loop2(Tab, Father).
  626: do_trans_loop2(Tab, Father) ->
  627:     Trans =
  628: 	fun() ->
  629: 		[{Tab, 1, Val}] = mnesia:read({Tab, 1}),
  630: 		mnesia:write({Tab, 1, Val + 1})
  631: 	end,
  632:     case mnesia:transaction(Trans) of
  633: 	{atomic, ok} ->
  634: 	    timer:sleep(100),
  635: 	    do_trans_loop2(Tab, Father);
  636: 	{aborted, {node_not_running, N}} when N == node() ->
  637: 	    timer:sleep(100),
  638: 	    do_trans_loop2(Tab, Father);
  639: 	{aborted, {no_exists, Tab}} ->
  640: 	    timer:sleep(100),
  641: 	    do_trans_loop2(Tab, Father);
  642: 	Else ->
  643: 	    ?error("Transaction failed: ~p ~n", [Else]),
  644: 	    Father ! test_done,
  645: 	    exit(shutdown)
  646:     end.
  647: 
  648: start_stop(_Node1, 0, Father) ->
  649:     Father ! test_done,
  650:     exit(shutdown);
  651: start_stop(Node1, N, Father) when N > 0->
  652:     timer:sleep(timer:seconds(2)),
  653:     ?match(stopped, rpc:call(Node1, mnesia, stop, [])),
  654:     timer:sleep(timer:seconds(1)),
  655:     ?match([], mnesia_test_lib:start_mnesia([Node1])),
  656:     start_stop(Node1, N-1, Father).    
  657: 
  658: coord_dies(suite) -> [];
  659: coord_dies(doc) -> [""];
  660: coord_dies(Config) when is_list(Config) ->
  661:     Nodes = [N1, N2] = ?acquire_nodes(2, Config),
  662:     ?match({atomic, ok}, mnesia:create_table(tab1, [{ram_copies, Nodes}])),
  663:     ?match({atomic, ok}, mnesia:create_table(tab2, [{ram_copies, [N1]}])),
  664:     ?match({atomic, ok}, mnesia:create_table(tab3, [{ram_copies, [N2]}])),
  665:     Tester = self(),
  666: 
  667:     U1 = fun(Tab) -> 
  668: 		 [{Tab,key,Val}] = mnesia:read(Tab,key,write),
  669: 		 mnesia:write({Tab,key, Val+1}),
  670: 		 Tester ! {self(),continue},
  671: 		 receive 
  672: 		     continue -> exit(crash)
  673: 		 end
  674: 	 end,
  675:     U2 = fun(Tab) -> 
  676: 		 [{Tab,key,Val}] = mnesia:read(Tab,key,write),
  677: 		 mnesia:write({Tab,key, Val+1}),
  678: 		 mnesia:transaction(U1, [Tab])
  679: 	 end,
  680:     [mnesia:dirty_write(Tab,{Tab,key,0}) || Tab <- [tab1,tab2,tab3]],
  681:     Pid1 = spawn(fun() -> mnesia:transaction(U2, [tab1]) end), 
  682:     Pid2 = spawn(fun() -> mnesia:transaction(U2, [tab2]) end), 
  683:     Pid3 = spawn(fun() -> mnesia:transaction(U2, [tab3]) end), 
  684:     [receive {Pid,continue} -> ok end || Pid <- [Pid1,Pid2,Pid3]],
  685:     Pid1 ! continue,    Pid2 ! continue,    Pid3 ! continue,
  686:     ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab1,key}) end)),
  687:     ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab2,key}) end)),
  688:     ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab3,key}) end)),
  689: 
  690:     Pid4 = spawn(fun() -> mnesia:transaction(U2, [tab1]) end), 
  691:     Pid5 = spawn(fun() -> mnesia:transaction(U2, [tab2]) end), 
  692:     Pid6 = spawn(fun() -> mnesia:transaction(U2, [tab3]) end), 
  693:     erlang:monitor(process, Pid4),erlang:monitor(process, Pid5),erlang:monitor(process, Pid6),
  694:     
  695:     [receive {Pid,continue} -> ok end || Pid <- [Pid4,Pid5,Pid6]],
  696:     exit(Pid4,crash),    
  697:     ?match_receive({'DOWN',_,_,Pid4, _}),
  698:     ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab1,key}) end)),
  699:     exit(Pid5,crash),
  700:     ?match_receive({'DOWN',_,_,Pid5, _}),
  701:     ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab2,key}) end)),
  702:     exit(Pid6,crash),
  703:     ?match_receive({'DOWN',_,_,Pid6, _}),
  704:     ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab3,key}) end)),
  705:     
  706:     ?verify_mnesia(Nodes, []).
  707: 
  708: 
  709: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  710:  
  711: 
  712: %kill_after_debug_point(Config, TestCase, {Debug_node, Debug_Point}, TransFun, Tab)
  713: 
  714: sym_trans_before_commit_kill_coord_node(suite) -> [];
  715: sym_trans_before_commit_kill_coord_node(Config) when is_list(Config) ->
  716:     ?is_debug_compiled,
  717:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  718:     [Coord, Part1, Part2] = Nodes,
  719:     Tab = sym_trans_before_commit_kill_coord,
  720:     Def = [{attributes, [key, value]}, {ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  721:     kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_sym}},
  722: 			   do_sym_trans, [{Tab, Def}], Nodes).
  723: 
  724: sym_trans_before_commit_kill_coord_pid(suite) -> [];
  725: sym_trans_before_commit_kill_coord_pid(Config) when is_list(Config) ->
  726:     ?is_debug_compiled,
  727:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  728:     [Coord, Part1, Part2] = Nodes,
  729:     Tab = sym_trans_before_commit_kill_coord,
  730:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  731:     kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_sym}},
  732: 			   do_sym_trans, [{Tab, Def}], Nodes).
  733: 
  734: sym_trans_before_commit_kill_part_after_ask(suite) -> [];
  735: sym_trans_before_commit_kill_part_after_ask(Config) when is_list(Config) ->
  736:     ?is_debug_compiled,
  737:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  738:     [Coord, Part1, Part2] = Nodes,
  739:     Tab = sym_trans_before_commit_kill_part_after_ask,
  740:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  741:     kill_after_debug_point(Part1, {Coord, {mnesia_tm, multi_commit_sym}},
  742: 			   do_sym_trans, [{Tab, Def}], Nodes).
  743: 
  744: sym_trans_before_commit_kill_part_before_ask(suite) -> [];
  745: sym_trans_before_commit_kill_part_before_ask(Config) when is_list(Config) ->
  746:     ?is_debug_compiled,
  747:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  748:     [Coord, Part1, Part2] = Nodes,
  749:     Tab = sym_trans_before_commit_kill_part_before_ask,
  750:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  751:     kill_after_debug_point(Part1, {Part1, {mnesia_tm, doit_ask_commit}},
  752: 			   do_sym_trans, [{Tab, Def}], Nodes).
  753: 
  754: sym_trans_after_commit_kill_coord_node(suite) -> [];
  755: sym_trans_after_commit_kill_coord_node(Config) when is_list(Config) ->
  756:     ?is_debug_compiled,
  757:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  758:     [Coord, Part1, Part2] = Nodes,
  759:     Tab = sym_trans_after_commit_kill_coord,
  760:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  761:     kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_sym, post}},
  762: 			  do_sym_trans, [{Tab, Def}], Nodes).
  763: 
  764: sym_trans_after_commit_kill_coord_pid(suite) -> [];
  765: sym_trans_after_commit_kill_coord_pid(Config) when is_list(Config) ->
  766:     ?is_debug_compiled,
  767:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  768:     [Coord, Part1, Part2] = Nodes,
  769:     Tab = sym_trans_after_commit_kill_coord,
  770:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  771:     kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_sym, post}},
  772: 			  do_sym_trans, [{Tab,Def}], Nodes).
  773: 
  774: sym_trans_after_commit_kill_part_after_ask(suite) -> [];
  775: sym_trans_after_commit_kill_part_after_ask(Config) when is_list(Config) ->
  776:     ?is_debug_compiled,
  777:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  778:     [Coord, Part1, Part2] = Nodes,
  779:     Tab =  sym_trans_after_commit_kill_part_after_ask,
  780:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  781:     kill_after_debug_point(Part1, {Coord, {mnesia_tm, multi_commit_sym, post}},
  782: 			   do_sym_trans, [{Tab, Def}], Nodes).
  783: 
  784: sym_trans_after_commit_kill_part_do_commit_pre(suite) -> [];
  785: sym_trans_after_commit_kill_part_do_commit_pre(Config) when is_list(Config) ->
  786:     ?is_debug_compiled,
  787:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  788:     [Coord, Part1, Part2] = Nodes,
  789:     Tab = sym_trans_after_commit_kill_part_do_commit_pre,
  790:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  791:     TransFun = do_sym_trans,
  792:     kill_after_debug_point(Part1, {Part1, {mnesia_tm, do_commit, pre}},
  793: 			   TransFun, [{Tab, Def}], Nodes).
  794: 
  795: sym_trans_after_commit_kill_part_do_commit_post(suite) -> [];
  796: sym_trans_after_commit_kill_part_do_commit_post(Config) when is_list(Config) ->
  797:     ?is_debug_compiled,
  798:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  799:     [Coord, Part1, Part2] = Nodes,
  800:     Tab = sym_trans_after_commit_kill_part_do_commit_post,
  801:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  802:     TransFun = do_sym_trans,
  803:     kill_after_debug_point(Part1, {Part1, {mnesia_tm, do_commit, post}}, 
  804: 			   TransFun, [{Tab, Def}], Nodes).
  805: 
  806: do_sym_trans([Tab], _Fahter) ->
  807:     ?dl("Starting SYM_TRANS with active debug fun ", []),
  808:     Trans = fun() -> 
  809: 		    [{_,_,Val}] = mnesia:read({Tab, 1}),
  810: 		    mnesia:write({Tab, 1, Val+1})
  811: 	    end,
  812:     Res = mnesia:transaction(Trans),
  813:     case Res of
  814: 	{atomic, ok} -> ok;
  815: 	{aborted, _Reason} -> ok;
  816: 	Else -> ?error("Wrong output from mensia:transaction(FUN):~n ~p~n",
  817: 		       [Else])
  818:     end,
  819:     ?dl("SYM_TRANSACTION done: ~p  (deactiv dbgfun) ", [Res]),
  820:     ok.
  821: 
  822: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  823: 
  824: 
  825: sync_dirty_pre_kill_part(suite) -> [];
  826: sync_dirty_pre_kill_part(Config) when is_list(Config) ->
  827:     ?is_debug_compiled,
  828:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  829:     [Coord, Part1, Part2] = Nodes,
  830:     Tab = sync_dirty_pre,
  831:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  832:     TransFun = do_sync_dirty,
  833:     kill_after_debug_point(Part1, {Part1, {mnesia_tm, sync_dirty, pre}}, 
  834: 			   TransFun, [{Tab, Def}], Nodes).
  835: 
  836: sync_dirty_pre_kill_coord_node(suite) -> [];
  837: sync_dirty_pre_kill_coord_node(Config) when is_list(Config) ->
  838:     ?is_debug_compiled,
  839:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  840:     [Coord, Part1, Part2] = Nodes,
  841:     Tab = sync_dirty_pre,
  842:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  843:     TransFun = do_sync_dirty,
  844:     kill_after_debug_point(Coord, {Part1, {mnesia_tm, sync_dirty, pre}}, 
  845: 			   TransFun, [{Tab, Def}], Nodes).
  846: 
  847: sync_dirty_pre_kill_coord_pid(suite) -> [];
  848: sync_dirty_pre_kill_coord_pid(Config) when is_list(Config) ->
  849:     ?is_debug_compiled,
  850:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  851:     [Coord, Part1, Part2] = Nodes,
  852:     Tab = sync_dirty_pre,
  853:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  854:     TransFun = do_sync_dirty,
  855:     kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, sync_dirty, pre}}, 
  856: 			   TransFun, [{Tab, Def}], Nodes).
  857: 
  858: sync_dirty_post_kill_part(suite) -> [];
  859: sync_dirty_post_kill_part(Config) when is_list(Config) ->
  860:     ?is_debug_compiled,
  861:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  862:     [Coord, Part1, Part2] = Nodes,
  863:     Tab = sync_dirty_post,
  864:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  865:     TransFun = do_sync_dirty,
  866:     kill_after_debug_point(Part1, {Part1, {mnesia_tm, sync_dirty, post}}, 
  867: 			   TransFun, [{Tab, Def}], Nodes).
  868: 
  869: sync_dirty_post_kill_coord_node(suite) -> [];
  870: sync_dirty_post_kill_coord_node(Config) when is_list(Config) ->
  871:     ?is_debug_compiled,
  872:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  873:     [Coord, Part1, Part2] = Nodes,
  874:     Tab = sync_dirty_post,
  875:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  876:     TransFun = do_sync_dirty,
  877:     kill_after_debug_point(Coord, {Part1, {mnesia_tm, sync_dirty, post}}, 
  878: 			   TransFun, [{Tab, Def}], Nodes).
  879: 
  880: sync_dirty_post_kill_coord_pid(suite) -> [];
  881: sync_dirty_post_kill_coord_pid(Config) when is_list(Config) ->
  882:     ?is_debug_compiled,
  883:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  884:     [Coord, Part1, Part2] = Nodes,
  885:     Tab = sync_dirty_post,
  886:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  887:     TransFun = do_sync_dirty,
  888:     kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, sync_dirty, post}}, 
  889: 			   TransFun, [{Tab, Def}], Nodes).
  890: 
  891: do_sync_dirty([Tab], _Father) ->
  892:     ?dl("Starting SYNC_DIRTY", []),
  893:     SYNC = fun() -> 
  894: 		   [{_,_,Val}] = mnesia:read({Tab, 1}),
  895: 		   mnesia:write({Tab, 1, Val+1})
  896: 	   end,
  897:     {_, Res} = ?match(ok, mnesia:sync_dirty(SYNC)),
  898:     ?dl("SYNC_DIRTY done: ~p ", [Res]),
  899:     ok.
  900: 
  901: 
  902: async_dirty_pre_kill_part(suite) -> [];
  903: async_dirty_pre_kill_part(Config) when is_list(Config) ->
  904:     ?is_debug_compiled,
  905:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  906:     [Coord, Part1, Part2] = Nodes,
  907:     Tab = async_dirty_pre,
  908:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  909:     TransFun = do_async_dirty,
  910:     kill_after_debug_point(Part1, {Part1, {mnesia_tm, async_dirty, pre}}, 
  911: 			   TransFun, [{Tab, Def}], Nodes).
  912: 
  913: async_dirty_pre_kill_coord_node(suite) -> [];
  914: async_dirty_pre_kill_coord_node(Config) when is_list(Config) ->
  915:     ?is_debug_compiled,
  916:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  917:     [Coord, Part1, Part2] = Nodes,
  918:     Tab = async_dirty_pre,
  919:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  920:     TransFun = do_async_dirty,
  921:     kill_after_debug_point(Coord, {Part1, {mnesia_tm, async_dirty, pre}}, 
  922: 			   TransFun, [{Tab, Def}], Nodes).
  923: 
  924: async_dirty_pre_kill_coord_pid(suite) -> [];
  925: async_dirty_pre_kill_coord_pid(Config) when is_list(Config) ->
  926:     ?is_debug_compiled,
  927:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  928:     [Coord, Part1, Part2] = Nodes,
  929:     Tab = async_dirty_pre,
  930:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  931:     TransFun = do_async_dirty,
  932:     kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, async_dirty, pre}}, 
  933: 			   TransFun, [{Tab, Def}], Nodes).
  934: 
  935: async_dirty_post_kill_part(suite) -> [];
  936: async_dirty_post_kill_part(Config) when is_list(Config) ->
  937:     ?is_debug_compiled,
  938:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  939:     [Coord, Part1, Part2] = Nodes,
  940:     Tab = async_dirty_post,
  941:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  942:     TransFun = do_async_dirty,
  943:     kill_after_debug_point(Part1, {Part1, {mnesia_tm, async_dirty, post}}, 
  944: 			   TransFun, [{Tab, Def}], Nodes).
  945: 
  946: async_dirty_post_kill_coord_node(suite) -> [];
  947: async_dirty_post_kill_coord_node(Config) when is_list(Config) ->
  948:     ?is_debug_compiled,
  949:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  950:     [Coord, Part1, Part2] = Nodes,
  951:     Tab = async_dirty_post,
  952:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  953:     TransFun = do_async_dirty,
  954:     kill_after_debug_point(Coord, {Part1, {mnesia_tm, async_dirty, post}}, 
  955: 			   TransFun, [{Tab, Def}], Nodes).
  956: 
  957: async_dirty_post_kill_coord_pid(suite) -> [];
  958: async_dirty_post_kill_coord_pid(Config) when is_list(Config) ->
  959:     ?is_debug_compiled,
  960:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  961:     [Coord, Part1, Part2] = Nodes,
  962:     Tab = async_dirty_post,
  963:     Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
  964:     TransFun = do_async_dirty,
  965:     kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, async_dirty, post}}, 
  966: 			   TransFun, [{Tab, Def}], Nodes).
  967: 
  968: do_async_dirty([Tab], _Fahter) ->
  969:     ?dl("Starting ASYNC", []),
  970:     ASYNC = fun() -> 
  971: 		    [{_,_,Val}] = mnesia:read({Tab, 1}),
  972: 		    mnesia:write({Tab, 1, Val+1})
  973: 	    end,
  974:     {_, Res} = ?match(ok, mnesia:async_dirty(ASYNC)),
  975:     ?dl("ASYNC done: ~p ", [Res]),
  976:     ok.
  977: 
  978: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  979: 
  980: 
  981: asymtrans_part_ask(suite) -> [];
  982: asymtrans_part_ask(Config) when is_list(Config) -> 
  983:     ?is_debug_compiled,
  984:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  985:     [Coord, Part1, Part2] = Nodes,
  986:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
  987:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
  988:     TransFun = do_asym_trans,
  989:     kill_after_debug_point(Part1, {Part1, {mnesia_tm, doit_ask_commit}}, 
  990: 			   TransFun, [Tab1, Tab2], Nodes).
  991: 
  992: asymtrans_part_commit_vote(suite) -> [];
  993: asymtrans_part_commit_vote(Config) when is_list(Config) -> 
  994:     ?is_debug_compiled,
  995:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
  996:     [Coord, Part1, Part2] = Nodes,
  997:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
  998:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
  999:     TransFun = do_asym_trans,
 1000:     kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, vote_yes}}, 
 1001: 			   TransFun, [Tab1, Tab2], Nodes).
 1002: 
 1003: asymtrans_part_pre_commit(suite) -> [];
 1004: asymtrans_part_pre_commit(Config) when is_list(Config) -> 
 1005:     ?is_debug_compiled,
 1006:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1007:     [Coord, Part1, Part2] = Nodes,
 1008:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
 1009:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
 1010:     TransFun = do_asym_trans,
 1011:     kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, pre_commit}}, 
 1012: 			   TransFun, [Tab1, Tab2], Nodes).
 1013: 
 1014: asymtrans_part_log_commit(suite) -> [];
 1015: asymtrans_part_log_commit(Config) when is_list(Config) -> 
 1016:     ?is_debug_compiled,
 1017:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1018:     [Coord, Part1, Part2] = Nodes,
 1019:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
 1020:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
 1021:     TransFun = do_asym_trans,
 1022:     kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, log_commit}}, 
 1023: 			   TransFun, [Tab1, Tab2], Nodes).
 1024: 
 1025: asymtrans_part_do_commit(suite) -> [];
 1026: asymtrans_part_do_commit(Config) when is_list(Config) -> 
 1027:     ?is_debug_compiled,
 1028:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1029:     [Coord, Part1, Part2] = Nodes,
 1030:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
 1031:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
 1032:     TransFun = do_asym_trans,
 1033:     kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, do_commit}}, 
 1034: 			   TransFun, [Tab1, Tab2], Nodes).
 1035: 
 1036: asymtrans_coord_got_votes(suite) -> [];
 1037: asymtrans_coord_got_votes(Config) when is_list(Config) -> 
 1038:     ?is_debug_compiled,
 1039:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1040:     [Coord, Part1, Part2] = Nodes,
 1041:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
 1042:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
 1043:     TransFun = do_asym_trans,
 1044:     kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_asym_got_votes}}, 
 1045: 			   TransFun, [Tab1, Tab2], Nodes).
 1046: 
 1047: asymtrans_coord_pid_got_votes(suite) -> [];
 1048: asymtrans_coord_pid_got_votes(Config) when is_list(Config) -> 
 1049:     ?is_debug_compiled,
 1050:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1051:     [Coord, Part1, Part2] = Nodes,
 1052:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
 1053:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
 1054:     TransFun = do_asym_trans,
 1055:     kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_asym_got_votes}}, 
 1056: 			   TransFun, [Tab1, Tab2], Nodes).
 1057: 
 1058: asymtrans_coord_log_commit_rec(suite) -> [];
 1059: asymtrans_coord_log_commit_rec(Config) when is_list(Config) -> 
 1060:     ?is_debug_compiled,
 1061:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1062:     [Coord, Part1, Part2] = Nodes,
 1063:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
 1064:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
 1065:     TransFun = do_asym_trans,
 1066:     kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_asym_log_commit_rec}}, 
 1067: 			   TransFun, [Tab1, Tab2], Nodes).
 1068: 
 1069: asymtrans_coord_pid_log_commit_rec(suite) -> [];
 1070: asymtrans_coord_pid_log_commit_rec(Config) when is_list(Config) -> 
 1071:     ?is_debug_compiled,
 1072:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1073:     [Coord, Part1, Part2] = Nodes,
 1074:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
 1075:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
 1076:     TransFun = do_asym_trans,
 1077:     kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_asym_log_commit_rec}}, 
 1078: 			   TransFun, [Tab1, Tab2], Nodes).
 1079: 
 1080: asymtrans_coord_log_commit_dec(suite) -> [];
 1081: asymtrans_coord_log_commit_dec(Config) when is_list(Config) -> 
 1082:     ?is_debug_compiled,
 1083:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1084:     [Coord, Part1, Part2] = Nodes,
 1085:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
 1086:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
 1087:     TransFun = do_asym_trans,
 1088:     kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_asym_log_commit_dec}}, 
 1089: 			   TransFun, [Tab1, Tab2], Nodes).
 1090: 
 1091: asymtrans_coord_pid_log_commit_dec(suite) -> [];
 1092: asymtrans_coord_pid_log_commit_dec(Config) when is_list(Config) -> 
 1093:     ?is_debug_compiled,
 1094:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1095:     [Coord, Part1, Part2] = Nodes,
 1096:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
 1097:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
 1098:     TransFun = do_asym_trans,
 1099:     kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_asym_log_commit_dec}}, 
 1100: 			   TransFun, [Tab1, Tab2], Nodes).
 1101: 
 1102: asymtrans_coord_rec_acc_pre_commit_log_commit(suite) -> [];
 1103: asymtrans_coord_rec_acc_pre_commit_log_commit(Config) when is_list(Config) -> 
 1104:     ?is_debug_compiled,
 1105:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1106:     [Coord, Part1, Part2] = Nodes,
 1107:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
 1108:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
 1109:     TransFun = do_asym_trans,
 1110:     kill_after_debug_point(Coord, {Coord, {mnesia_tm, rec_acc_pre_commit_log_commit}}, 
 1111: 			   TransFun, [Tab1, Tab2], Nodes).
 1112: 
 1113: asymtrans_coord_pid_rec_acc_pre_commit_log_commit(suite) -> [];
 1114: asymtrans_coord_pid_rec_acc_pre_commit_log_commit(Config) when is_list(Config) -> 
 1115:     ?is_debug_compiled,
 1116:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1117:     [Coord, Part1, Part2] = Nodes,
 1118:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
 1119:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
 1120:     TransFun = do_asym_trans,
 1121:     kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, rec_acc_pre_commit_log_commit}}, 
 1122: 			   TransFun, [Tab1, Tab2], Nodes).
 1123: 
 1124: asymtrans_coord_rec_acc_pre_commit_done_commit(suite) -> [];
 1125: asymtrans_coord_rec_acc_pre_commit_done_commit(Config) when is_list(Config) -> 
 1126:     ?is_debug_compiled,
 1127:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1128:     [Coord, Part1, Part2] = Nodes,
 1129:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
 1130:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
 1131:     TransFun = do_asym_trans,
 1132:     kill_after_debug_point(Coord, {Coord, {mnesia_tm, rec_acc_pre_commit_done_commit}}, 
 1133: 			   TransFun, [Tab1, Tab2], Nodes).
 1134: 
 1135: asymtrans_coord_pid_rec_acc_pre_commit_done_commit(suite) -> [];
 1136: asymtrans_coord_pid_rec_acc_pre_commit_done_commit(Config) when is_list(Config) -> 
 1137:     ?is_debug_compiled,
 1138:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1139:     [Coord, Part1, Part2] = Nodes,
 1140:     Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
 1141:     Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
 1142:     TransFun = do_asym_trans,
 1143:     kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, rec_acc_pre_commit_done_commit}}, 
 1144: 			   TransFun, [Tab1, Tab2], Nodes).
 1145: 
 1146: do_asym_trans([Tab1, Tab2 | _R], Garbhandler) ->
 1147:     ?dl("Starting asym trans ", []),
 1148:     ASym_Trans = fun() ->
 1149: 			 TidTs = {_Mod, Tid, _Store} =
 1150: 			     mnesia:get_activity_id(),
 1151: 			 ?verbose("===> asym_trans: ~w~n", [TidTs]),
 1152: 			 Garbhandler ! {trans_id, Tid},
 1153: 			 [{_, _, Val1}] = mnesia:read({Tab1, 1}),
 1154: 			 [{_, _, Val2}] = mnesia:read({Tab2, 1}),
 1155: 			 mnesia:write({Tab1, 1, Val1+1}),
 1156: 			 mnesia:write({Tab2, 1, Val2+1})
 1157: 		 end,
 1158:     Res = mnesia:transaction(ASym_Trans),
 1159:     case Res of
 1160: 	{atomic, ok} -> ok;
 1161: 	{aborted, _Reason} -> ok;
 1162: 	_Else -> ?error("Wrong output from mensia:transaction(FUN):~n ~p~n", [Res])
 1163:     end,			
 1164:     ?dl("Asym trans finished with: ~p ", [Res]).
 1165: 
 1166: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1167: 
 1168: kill_after_debug_point(Kill, {DebugNode, Debug_Point}, TransFun, TabsAndDefs, Nodes) ->
 1169:     [Coord | _rest] = Nodes,
 1170: 
 1171:     Create = fun({Tab, Def}) -> ?match({atomic, ok}, mnesia:create_table(Tab, Def)) end,
 1172:     lists:foreach(Create, TabsAndDefs),
 1173:     Tabs = [T || {T, _} <- TabsAndDefs],
 1174:     Write = fun(Tab) -> ?match(ok, mnesia:dirty_write({Tab, 1, 100})) end,
 1175:     lists:foreach(Write, Tabs),
 1176: 
 1177:     Self = self(),    
 1178:     SyncFun = fun(_Env1, _Env2) ->  % Just Sync with test prog
 1179: 		      Self ! {self(), fun_in_position}, 
 1180: 		      ?dl("SyncFun, sending fun_in_position ", []),
 1181: 		      receive continue -> 
 1182: 			      ?dl("SyncFun received continue ",[]),
 1183: 			      ok
 1184: 		      after timer:seconds(60) -> 
 1185: 			      ?error("Timeout in sync_fun on ~p~n", [node()])
 1186: 		      end
 1187: 	      end,
 1188: 
 1189:     Garb_handler = spawn_link(?MODULE, garb_handler, [[]]),
 1190: 
 1191:     ?remote_activate_debug_fun(DebugNode, Debug_Point, SyncFun, []),
 1192:     ?dl("fun_in_position activated at ~p with ~p", [DebugNode, Debug_Point]),
 1193:     %% Spawn and do the transaction
 1194:     Pid = spawn(Coord, ?MODULE, TransFun, [Tabs, Garb_handler]),
 1195:     %% Wait till all the Nodes are in correct position
 1196:     [{StoppedPid,_}] = ?receive_messages([fun_in_position]),
 1197:     ?dl("Received fun_in_position; Removing the debug funs ~p", [DebugNode]),
 1198:     ?remote_deactivate_debug_fun(DebugNode, Debug_Point),    
 1199: 
 1200:     case Kill of
 1201: 	coord_pid -> 
 1202: 	    ?dl("Intentionally killing pid ~p ", [Pid]),
 1203: 	    exit(Pid, normal);
 1204: 	Node -> 		
 1205: 	    mnesia_test_lib:kill_mnesia([Node])
 1206:     end,
 1207: 
 1208:     StoppedPid ! continue,    %% Send continue, it may still be alive
 1209: 
 1210:     %% Start and check that the databases are consistent
 1211:     ?dl("Done, Restarting and verifying result ",[]),
 1212:     case Kill of
 1213: 	coord_pid -> ok;
 1214: 	_ ->  % Ok, mnesia on some node was killed restart it
 1215: 	    timer:sleep(timer:seconds(3)), %% Just let it have the time to die
 1216: 	    ?match(ok, rpc:call(Kill, mnesia, start, [[]])),
 1217: 	    ?match(ok, rpc:call(Kill, mnesia, wait_for_tables, [Tabs, 60000]))
 1218:     end,
 1219:     Trans_res = verify_tabs(Tabs, Nodes),
 1220:     case TransFun of
 1221: 	do_asym_trans ->
 1222: 	    %% Verifies that decisions are garbed, only valid for asym_tran
 1223: 	    Garb_handler ! {get_tids, self()},
 1224: 	    Tid_list = receive 
 1225: 			   {tids, List} -> 
 1226: 			       ?dl("Fun rec ~w", [List]),
 1227: 			       List
 1228: 		       end,
 1229: 	    garb_of_decisions(Kill, Nodes, Tid_list, Trans_res);
 1230: 	_ ->
 1231: 	    ignore
 1232:     end,
 1233:     ?verify_mnesia(Nodes, []).
 1234: 
 1235: garb_of_decisions(Kill, Nodes, Tid_list, Trans_res) ->
 1236:     [Coord, Part1, Part2]  = Nodes,
 1237:     %% Check that decision log is empty on all nodes after the trans is finished
 1238:     verify_garb_decision_log(Nodes, Tid_list),  
 1239:     case Trans_res of
 1240: 	aborted ->
 1241: 	    %% Check that aborted trans have not been restarted!!
 1242: 	    ?match(1, length(Tid_list)),
 1243: 	    %% Check the transient decision logs
 1244: 	    %% A transaction should only be aborted in an early stage of 
 1245: 	    %% the trans before the any Node have logged anything
 1246: 	    verify_garb_transient_logs(Nodes, Tid_list, aborted),
 1247: 	    %% And only when the coordinator are have died
 1248: 	    %% Else he would have restarted the transaction
 1249: 	    ?match(Kill, Coord);
 1250: 	updated ->
 1251: 	    case length(Tid_list) of
 1252: 		1 -> 
 1253: 		    %% If there was only one transaction, it should be logged as
 1254: 		    %% comitted on every node!
 1255: 		    [Tid1] = Tid_list,
 1256: 		    verify_garb_transient_logs(Nodes, [Tid1], committed);
 1257: 		2 -> 
 1258: 		    %% If there is two transaction id, then the first
 1259: 		    %% TID should have been aborted and the transaction
 1260: 		    %% restarted with a new TID
 1261: 		    [Tid1, Tid2] = Tid_list,
 1262: 		    verify_garb_transient_logs(Nodes, [Tid1], aborted),
 1263: 		    %% If mnesia is killed on a node i.e Coord and Part1 than they
 1264: 		    %% won't know about the restarted trans! The rest of the nodes
 1265: 		    %% should know that the trans was committed
 1266: 		    case Kill of 
 1267: 			coord_pid -> 
 1268: 			    verify_garb_transient_logs(Nodes, [Tid2], committed);
 1269: 			Coord -> 
 1270: 			    verify_garb_transient_logs([Part1, Part2], [Tid2], committed),
 1271: 			    verify_garb_transient_logs([Coord], [Tid2], not_found);
 1272: 			Part1 ->
 1273: 			    verify_garb_transient_logs([Coord, Part2], [Tid2], committed),
 1274: 			    verify_garb_transient_logs([Part1], [Tid2], not_found)
 1275: 		    end
 1276: 	    end
 1277:     end.
 1278: 
 1279: verify_garb_decision_log([], _Tids) -> ok;
 1280: verify_garb_decision_log([Node|R], Tids) -> 
 1281:     Check = fun(Tid) ->   %% Node, Tid used in debugging!
 1282: 		    ?match({{not_found, _}, Node, Tid}, 
 1283: 			   {outcome(Tid, [mnesia_decision]), Node, Tid})
 1284: 	    end,
 1285:     rpc:call(Node, lists, foreach, [Check, Tids]),
 1286:     verify_garb_decision_log(R, Tids).
 1287: 
 1288: verify_garb_transient_logs([], _Tids, _) -> ok;
 1289: verify_garb_transient_logs([Node|R], Tids, Exp_Res) -> 
 1290:     Check = fun(Tid) -> 
 1291: 		    LatestTab = mnesia_lib:val(latest_transient_decision),
 1292: 		    PrevTabs = mnesia_lib:val(previous_transient_decisions),
 1293: 		    case outcome(Tid, [LatestTab | PrevTabs]) of
 1294: 			{found, {_, [{_,_Tid, Exp_Res}]}} -> ok;
 1295: 			{not_found, _} when Exp_Res == not_found -> ok;
 1296: 			{not_found, _} when Exp_Res == aborted -> ok;
 1297: 			Else  -> ?error("Expected ~p in trans ~p on ~p got ~p~n",
 1298: 					[Exp_Res, Tid, Node, Else])
 1299: 		    end
 1300: 	    end,    
 1301:     rpc:call(Node, lists, foreach, [Check, Tids]),
 1302:     verify_garb_transient_logs(R, Tids, Exp_Res). 
 1303:     
 1304: outcome(Tid, Tabs) ->
 1305:     outcome(Tid, Tabs, Tabs).
 1306: 
 1307: outcome(Tid, [Tab | Tabs], AllTabs) ->
 1308:     case catch ets:lookup(Tab, Tid) of
 1309: 	{'EXIT', _} ->
 1310: 	    outcome(Tid, Tabs, AllTabs);
 1311: 	[] ->
 1312: 	    outcome(Tid, Tabs, AllTabs);
 1313: 	Val ->
 1314: 	    {found, {Tab, Val}}
 1315:     end;
 1316: outcome(_Tid, [], AllTabs) ->
 1317:     {not_found, AllTabs}.
 1318: 
 1319: 
 1320: verify_tabs([Tab|R], Nodes) -> 
 1321:     [_Coord, Part1, Part2 | _rest] = Nodes, 
 1322:     Read = fun() -> mnesia:read({Tab, 1}) end,
 1323:     {success, A} = ?match({atomic, _}, mnesia:transaction(Read)),
 1324:     ?match(A, rpc:call(Part1, mnesia, transaction, [Read])),
 1325:     ?match(A, rpc:call(Part2, mnesia, transaction, [Read])),
 1326:     {atomic, [{Tab, 1, Res}]} = A,
 1327:     verify_tabs(R, Nodes, Res).
 1328: 
 1329: verify_tabs([], _Nodes, Res) -> 
 1330:     case Res of 
 1331: 	100 -> aborted;
 1332: 	101 -> updated
 1333:     end;
 1334: 
 1335: verify_tabs([Tab | Rest], Nodes, Res) ->
 1336:     [Coord, Part1, Part2 | _rest] = Nodes, 
 1337:     Read = fun() -> mnesia:read({Tab, 1}) end,
 1338:     Exp = {atomic, [{Tab, 1, Res}]},
 1339:     ?match(Exp, rpc:call(Coord, mnesia, transaction, [Read])),
 1340:     ?match(Exp, rpc:call(Part1, mnesia, transaction, [Read])),
 1341:     ?match(Exp, rpc:call(Part2, mnesia, transaction, [Read])),
 1342:     verify_tabs(Rest, Nodes, Res).
 1343: 
 1344: %% Gather TIDS and send them to requesting process and exit!
 1345: garb_handler(List) -> 
 1346:     receive 
 1347: 	{trans_id, ID} -> garb_handler([ID|List]);
 1348: 	{get_tids, Pid} -> Pid ! {tids, lists:reverse(List)}
 1349:     end.
 1350: 
 1351: %%%%%%%%%%%%%%%%%%%%%%%
 1352: receive_messages([], _File, _Line) -> [];
 1353: receive_messages(ListOfMsgs, File, Line) ->
 1354:     receive 
 1355: 	{Pid, Msg} ->     
 1356: 	    case lists:member(Msg, ListOfMsgs) of
 1357: 		false -> 
 1358: 		    mnesia_test_lib:log("<>WARNING<>~n"
 1359: 					"Received unexpected msg~n ~p ~n"
 1360: 					"While waiting for ~p~n",  
 1361: 					[{Pid, Msg}, ListOfMsgs], File, Line),
 1362: 		    receive_messages(ListOfMsgs, File, Line);
 1363: 		true -> 
 1364: 		    ?dl("Got msg ~p from ~p ", [Msg, node(Pid)]),
 1365: 		    [{Pid, Msg} | receive_messages(ListOfMsgs -- [Msg], File, Line)]
 1366: 	    end;
 1367: 	Else -> mnesia_test_lib:log("<>WARNING<>~n"
 1368: 				    "Recevied unexpected or bad formatted msg~n ~p ~n"
 1369: 				    "While waiting for ~p~n", 
 1370: 	 			    [Else, ListOfMsgs], File, Line),
 1371: 		receive_messages(ListOfMsgs, File, Line)
 1372:     after timer:minutes(2) -> 
 1373: 	    ?error("Timeout in receive msgs while waiting for ~p~n", 
 1374: 		   [ListOfMsgs])
 1375:     end.    
 1376: 
 1377: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1378: 
 1379: after_full_disc_partition(doc) ->
 1380:     ["Verify that the database does not get corrupt",
 1381:      "when Mnesia encounters a full disc partition"].
 1382: 
 1383: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1384: %% interrupted_fallback_start 
 1385: %% is implemented in consistency interupted_install_fallback!
 1386: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1387: 
 1388: after_corrupt_files_decision_log_head(suite) -> [];
 1389: after_corrupt_files_decision_log_head(Config) when is_list(Config) ->
 1390:     after_corrupt_files(Config, "DECISION.LOG", head, repair).
 1391: 
 1392: after_corrupt_files_decision_log_tail(suite) -> [];
 1393: after_corrupt_files_decision_log_tail(Config) when is_list(Config) ->
 1394:     after_corrupt_files(Config, "DECISION.LOG", tail, repair).
 1395: 
 1396: after_corrupt_files_latest_log_head(suite) -> [];
 1397: after_corrupt_files_latest_log_head(Config) when is_list(Config) ->
 1398:     after_corrupt_files(Config, "LATEST.LOG", head, repair).
 1399: 
 1400: after_corrupt_files_latest_log_tail(suite) -> [];
 1401: after_corrupt_files_latest_log_tail(Config) when is_list(Config) ->
 1402:     after_corrupt_files(Config, "LATEST.LOG", tail, repair).
 1403: 
 1404: after_corrupt_files_table_dat_head(suite) -> [];
 1405: after_corrupt_files_table_dat_head(Config) when is_list(Config) ->
 1406:     after_corrupt_files(Config, "rec_files.DAT", head, crash).
 1407: 
 1408: after_corrupt_files_table_dat_tail(suite) -> [];
 1409: after_corrupt_files_table_dat_tail(Config) when is_list(Config) ->
 1410:     after_corrupt_files(Config, "rec_files.DAT", tail, repair).
 1411: 
 1412: after_corrupt_files_schema_dat_head(suite) -> [];
 1413: after_corrupt_files_schema_dat_head(Config) when is_list(Config) ->
 1414:     after_corrupt_files(Config, "schema.DAT", head, crash).
 1415: 
 1416: after_corrupt_files_schema_dat_tail(suite) -> [];
 1417: after_corrupt_files_schema_dat_tail(Config) when is_list(Config) ->
 1418:     after_corrupt_files(Config, "schema.DAT", tail, crash).
 1419: 
 1420: 
 1421: 
 1422: %%% BUGBUG: We should also write testcase's for autorepair=false i.e. 
 1423: %%% not the standard case!
 1424: after_corrupt_files(Config, File, Where, Behaviour) ->
 1425:     [Node] = ?acquire_nodes(1, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1426:     Tab = rec_files,
 1427:     Def = [{disc_only_copies, [Node]}],
 1428:     ?match({atomic, ok}, mnesia:create_table(Tab, Def)),
 1429:     insert_data(Tab, 100),
 1430:     Dir = mnesia:system_info(directory),
 1431:     mnesia_test_lib:kill_mnesia([Node]),
 1432:     timer:sleep(timer:seconds(10)),   % Let dets finish whatever it does
 1433: 
 1434:     DirFile = Dir ++ "/" ++ File,
 1435: 
 1436:     {ok, Fd} = file:open(DirFile, read_write),
 1437:     {ok, FileInfo} = file:read_file_info(DirFile),
 1438:     case Where of
 1439: 	head -> 
 1440: 	    ?match({ok, _NewP}, file:position(Fd, {bof, 1})),
 1441: 	    ?match(ok, file:write(Fd, [255, 255, 255, 255, 255, 255, 255, 255, 254])),
 1442: 	    ok;
 1443: 	tail ->
 1444: 	    Size = FileInfo#file_info.size,
 1445: 	    Half = Size div 2,
 1446: 
 1447: 	    ?dl(" Size = ~p Half = ~p ", [Size, Half]),
 1448: 	    ?match({ok, _NewP}, file:position(Fd, {bof, Half})),
 1449: 	    ?match(ok, file:truncate(Fd)),
 1450: 	    ok
 1451:     end,
 1452:     ?match(ok, file:close(Fd)),
 1453: 
 1454:     ?warning("++++++SOME OF THE after_corrupt* TEST CASES WILL INTENTIONALLY CRASH MNESIA+++++++~n", []),
 1455:     Pid = spawn_link(?MODULE, mymnesia_start, [self()]),
 1456:     receive
 1457: 	{Pid, ok} -> 
 1458: 	    ?match(ok, mnesia:wait_for_tables([schema, Tab], 10000)),
 1459: 	    ?match(ok, verify_data(Tab, 100)),
 1460: 	    case mnesia_monitor:get_env(auto_repair) of
 1461: 		false ->
 1462: 		    ?error("Mnesia should have crashed in ~p ~p ~n",
 1463: 			   [File, Where]);
 1464: 		true ->
 1465: 		    ok
 1466: 	    end,
 1467: 	    ?verify_mnesia([Node], []);
 1468: 	{Pid, {error, ED}} ->
 1469: 	    case {mnesia_monitor:get_env(auto_repair), Behaviour} of
 1470: 		{true, repair} ->
 1471: 		    ?error("Mnesia crashed with ~p: in ~p ~p ~n",
 1472: 			   [ED, File, Where]);
 1473: 		_ ->  %% Every other can crash!
 1474: 		    ok
 1475: 	    end,
 1476: 	    ?verify_mnesia([], [Node]);
 1477: 	Msg ->
 1478: 	    ?error("~p ~p: Got ~p during start of Mnesia~n",
 1479: 		   [File, Where, Msg])
 1480:     end.
 1481: 
 1482: mymnesia_start(Tester) ->
 1483:     Res = mnesia:start(),
 1484:     unlink(Tester),
 1485:     Tester ! {self(), Res}.
 1486: 
 1487: verify_data(_, 0) -> ok;
 1488: verify_data(Tab, N) -> 
 1489:     Actual = mnesia:dirty_read({Tab, N}),
 1490:     Expected = [{Tab, N, N}],
 1491:     if
 1492: 	Expected == Actual ->
 1493: 	    verify_data(Tab, N - 1);
 1494: 	true  ->
 1495: 	    mnesia:schema(Tab),
 1496: 	    {not_equal, node(), Expected, Actual}
 1497:     end.
 1498: 
 1499: insert_data(_Tab, 0) -> ok;
 1500: insert_data(Tab, N) ->
 1501:     ok = mnesia:sync_dirty(fun() -> mnesia:write({Tab, N, N}) end),
 1502:     insert_data(Tab, N-1).
 1503: 
 1504: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1505: 
 1506: disc_less(doc) -> 
 1507:     ["Here is a simple test case of a simple recovery of a disc less node. "
 1508:      "However a lot more test cases involving disc less nodes should "
 1509:      "be written"];
 1510: disc_less(suite) -> [];
 1511: disc_less(Config) when is_list(Config) ->
 1512:     [Node1, Node2, Node3] = Nodes = ?acquire_nodes(3, Config),
 1513:     case mnesia_test_lib:diskless(Config) of
 1514: 	true -> skip;
 1515: 	false ->    
 1516: 	    ?match({atomic, ok}, mnesia:change_table_copy_type(schema, Node3, ram_copies))
 1517:     end,
 1518:     Tab1 = disc_less1,
 1519:     Tab2 = disc_less2,
 1520:     Tab3 = disc_less3,
 1521:     Def1 = [{ram_copies, [Node3]}, {disc_copies, [Node1, Node2]}],
 1522:     Def2 = [{ram_copies, [Node3]}, {disc_copies, [Node1]}],
 1523:     Def3 = [{ram_copies, [Node3]}, {disc_copies, [Node2]}],
 1524:     ?match({atomic, ok}, mnesia:create_table(Tab1, Def1)),
 1525:     ?match({atomic, ok}, mnesia:create_table(Tab2, Def2)),
 1526:     ?match({atomic, ok}, mnesia:create_table(Tab3, Def3)),   
 1527:     insert_data(Tab1, 100),
 1528:     insert_data(Tab2, 100),
 1529:     insert_data(Tab3, 100),
 1530: 
 1531:     mnesia_test_lib:kill_mnesia([Node1, Node2]),
 1532:     timer:sleep(500),
 1533:     mnesia_test_lib:kill_mnesia([Node3]),
 1534:     ?match(ok, rpc:call(Node1, mnesia, start, [])),
 1535:     ?match(ok, rpc:call(Node2, mnesia, start, [])),
 1536: 
 1537:     timer:sleep(500),
 1538:     ?match(ok, rpc:call(Node3, mnesia, start, [[{extra_db_nodes, [Node1, Node2]}]])),
 1539:     ?match(ok, rpc:call(Node3, mnesia, wait_for_tables, [[Tab1, Tab2, Tab3], 20000])),
 1540: 
 1541:     ?match(ok, rpc:call(Node3, ?MODULE, verify_data, [Tab1, 100])),
 1542:     ?match(ok, rpc:call(Node3, ?MODULE, verify_data, [Tab2, 100])),
 1543:     ?match(ok, rpc:call(Node3, ?MODULE, verify_data, [Tab3, 100])),
 1544: 
 1545: 
 1546:     ?match(ok, rpc:call(Node2, ?MODULE, verify_data, [Tab1, 100])),
 1547:     ?match(ok, rpc:call(Node2, ?MODULE, verify_data, [Tab2, 100])),
 1548:     ?match(ok, rpc:call(Node2, ?MODULE, verify_data, [Tab3, 100])),
 1549: 
 1550:     ?match(ok, rpc:call(Node1, ?MODULE, verify_data, [Tab1, 100])),
 1551:     ?match(ok, rpc:call(Node1, ?MODULE, verify_data, [Tab2, 100])),
 1552:     ?match(ok, rpc:call(Node1, ?MODULE, verify_data, [Tab3, 100])),
 1553:     ?verify_mnesia(Nodes, []).
 1554: 
 1555: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1556: 
 1557: system_upgrade(doc) ->
 1558:     ["Test on-line and off-line upgrade of the Mnesia application"].
 1559: 
 1560: garb_decision(doc) ->
 1561:     ["Test that decisions are garbed correctly."];
 1562: garb_decision(suite) -> [];
 1563: garb_decision(Config) when is_list(Config) ->
 1564:     [Node1, Node2, Node3] = Nodes = ?acquire_nodes(3, Config),
 1565:     check_garb(Nodes),
 1566:     ?match({atomic, ok},mnesia:create_table(a, [{disc_copies, Nodes}])),
 1567:     check_garb(Nodes),
 1568:     ?match({atomic, ok},mnesia:create_table(b, [{ram_copies, Nodes}])),
 1569:     check_garb(Nodes),
 1570:     ?match({atomic, ok},mnesia:create_table(c, [{ram_copies, [Node1, Node3]},
 1571: 						{disc_copies, [Node2]}])),
 1572:     check_garb(Nodes),
 1573:     ?match({atomic, ok},mnesia:create_table(d, [{disc_copies, [Node1, Node3]},
 1574: 						{ram_copies, [Node2]}])),
 1575:     check_garb(Nodes),
 1576: 
 1577:     W = fun(Tab) ->  mnesia:write({Tab,1,1}) end,
 1578:     A = fun(Tab) ->  mnesia:write({Tab,1,1}), exit(1) end,
 1579: 
 1580:     ?match({atomic, ok}, mnesia:transaction(W,[a])),
 1581:     check_garb(Nodes),
 1582:     ?match({atomic, ok}, mnesia:transaction(W,[b])),
 1583:     check_garb(Nodes),
 1584:     ?match({atomic, ok}, mnesia:transaction(W,[c])),
 1585:     check_garb(Nodes),
 1586:     ?match({atomic, ok}, mnesia:transaction(W,[d])),
 1587:     check_garb(Nodes),
 1588:     ?match({aborted,1}, mnesia:transaction(A,[a])),
 1589:     check_garb(Nodes),
 1590:     ?match({aborted,1}, mnesia:transaction(A,[b])),
 1591:     check_garb(Nodes),
 1592:     ?match({aborted,1}, mnesia:transaction(A,[c])),
 1593:     check_garb(Nodes),
 1594:     ?match({aborted,1}, mnesia:transaction(A,[d])),
 1595:     check_garb(Nodes),
 1596: 
 1597:     rpc:call(Node2, mnesia, lkill, []),
 1598:     ?match({atomic, ok}, mnesia:transaction(W,[a])),
 1599:     ?match({atomic, ok}, mnesia:transaction(W,[b])),
 1600:     ?match({atomic, ok}, mnesia:transaction(W,[c])),
 1601:     ?match({atomic, ok}, mnesia:transaction(W,[d])),
 1602:     check_garb(Nodes),
 1603:     ?match([], mnesia_test_lib:start_mnesia([Node2])),    
 1604:     check_garb(Nodes),
 1605:     timer:sleep(2000),
 1606:     check_garb(Nodes),
 1607:     %%%%%% Check transient_decision logs %%%%%
 1608: 
 1609:     ?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync   
 1610:     [{atomic, ok} = mnesia:transaction(W,[a]) || _ <- lists:seq(1,30)],
 1611:     ?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync   
 1612:     TD0 = mnesia_lib:val(latest_transient_decision),
 1613:     ?match(0, ets:info(TD0, size)),
 1614:     {atomic, ok} = mnesia:transaction(W,[a]),
 1615:     ?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync   
 1616:     ?match(TD0, mnesia_lib:val(latest_transient_decision)),
 1617:     [{atomic, ok} = mnesia:transaction(W,[a]) || _ <- lists:seq(1,30)],
 1618:     ?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync   
 1619:     ?match(false, TD0 =:= mnesia_lib:val(latest_transient_decision)),
 1620:     ?match(true, lists:member(TD0, mnesia_lib:val(previous_transient_decisions))),
 1621:     ?verify_mnesia(Nodes, []).
 1622: 
 1623: check_garb(Nodes) ->
 1624:     rpc:multicall(Nodes, sys, get_status, [mnesia_recover]),
 1625:     ?match({_, []},rpc:multicall(Nodes, erlang, apply, [fun check_garb/0, []])).
 1626: 
 1627: check_garb() ->
 1628:     try 
 1629: 	Ds = ets:tab2list(mnesia_decision),
 1630: 	Check = fun({trans_tid,serial, _}) -> false;
 1631: 		   ({mnesia_down,_,_,_}) -> false;
 1632: 		   (_Else) -> true
 1633: 		end,
 1634: 	Node = node(),
 1635: 	?match({Node, []}, {node(), lists:filter(Check, Ds)})
 1636:     catch _:_ -> ok  
 1637:     end,
 1638:     ok.