1: %% 2: %% %CopyrightBegin% 3: %% 4: %% Copyright Ericsson AB 1996-2013. All Rights Reserved. 5: %% 6: %% The contents of this file are subject to the Erlang Public License, 7: %% Version 1.1, (the "License"); you may not use this file except in 8: %% compliance with the License. You should have received a copy of the 9: %% Erlang Public License along with this software. If not, it can be 10: %% retrieved online at http://www.erlang.org/. 11: %% 12: %% Software distributed under the License is distributed on an "AS IS" 13: %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 14: %% the License for the specific language governing rights and limitations 15: %% under the License. 16: %% 17: %% %CopyrightEnd% 18: %% 19: 20: %% 21: -module(mnesia_recovery_test). 22: -author('hakan@erix.ericsson.se'). 23: -compile([export_all]). 24: 25: -include("mnesia_test_lib.hrl"). 26: -include_lib("kernel/include/file.hrl"). 27: 28: init_per_testcase(Func, Conf) -> 29: mnesia_test_lib:init_per_testcase(Func, Conf). 30: 31: end_per_testcase(Func, Conf) -> 32: mnesia_test_lib:end_per_testcase(Func, Conf). 33: 34: -define(receive_messages(Msgs), receive_messages(Msgs, ?FILE, ?LINE)). 35: 36: % First Some debug logging 37: -define(dgb, true). 38: -ifdef(dgb). 39: -define(dl(X, Y), ?verbose("**TRACING: " ++ X ++ "**~n", Y)). 40: -else. 41: -define(dl(X, Y), ok). 42: -endif. 43: 44: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 45: all() -> 46: [{group, mnesia_down}, {group, explicit_stop}, 47: coord_dies, {group, schema_trans}, {group, async_dirty}, 48: {group, sync_dirty}, {group, sym_trans}, 49: {group, asym_trans}, after_full_disc_partition, 50: {group, after_corrupt_files}, disc_less, garb_decision, 51: system_upgrade]. 52: 53: groups() -> 54: [{schema_trans, [], 55: [{mnesia_schema_recovery_test, all}]}, 56: {mnesia_down, [], 57: [{group, mnesia_down_during_startup}, 58: {group, master_node_tests}, {group, read_during_down}, 59: {group, with_checkpoint}, delete_during_start]}, 60: {master_node_tests, [], 61: [no_master_2, no_master_3, one_master_2, one_master_3, 62: two_master_2, two_master_3, all_master_2, 63: all_master_3]}, 64: {read_during_down, [], 65: [dirty_read_during_down, trans_read_during_down]}, 66: {mnesia_down_during_startup, [], 67: [mnesia_down_during_startup_disk_ram, 68: mnesia_down_during_startup_init_ram, 69: mnesia_down_during_startup_init_disc, 70: mnesia_down_during_startup_init_disc_only, 71: mnesia_down_during_startup_tm_ram, 72: mnesia_down_during_startup_tm_disc, 73: mnesia_down_during_startup_tm_disc_only]}, 74: {with_checkpoint, [], 75: [with_checkpoint_same, with_checkpoint_other]}, 76: {explicit_stop, [], [explicit_stop_during_snmp]}, 77: {sym_trans, [], 78: [sym_trans_before_commit_kill_coord_node, 79: sym_trans_before_commit_kill_coord_pid, 80: sym_trans_before_commit_kill_part_after_ask, 81: sym_trans_before_commit_kill_part_before_ask, 82: sym_trans_after_commit_kill_coord_node, 83: sym_trans_after_commit_kill_coord_pid, 84: sym_trans_after_commit_kill_part_after_ask, 85: sym_trans_after_commit_kill_part_do_commit_pre, 86: sym_trans_after_commit_kill_part_do_commit_post]}, 87: {sync_dirty, [], 88: [sync_dirty_pre_kill_part, 89: sync_dirty_pre_kill_coord_node, 90: sync_dirty_pre_kill_coord_pid, 91: sync_dirty_post_kill_part, 92: sync_dirty_post_kill_coord_node, 93: sync_dirty_post_kill_coord_pid]}, 94: {async_dirty, [], 95: [async_dirty_pre_kill_part, 96: async_dirty_pre_kill_coord_node, 97: async_dirty_pre_kill_coord_pid, 98: async_dirty_post_kill_part, 99: async_dirty_post_kill_coord_node, 100: async_dirty_post_kill_coord_pid]}, 101: {asym_trans, [], 102: [asymtrans_part_ask, 103: asymtrans_part_commit_vote, 104: asymtrans_part_pre_commit, 105: asymtrans_part_log_commit, 106: asymtrans_part_do_commit, 107: asymtrans_coord_got_votes, 108: asymtrans_coord_pid_got_votes, 109: asymtrans_coord_log_commit_rec, 110: asymtrans_coord_pid_log_commit_rec, 111: asymtrans_coord_log_commit_dec, 112: asymtrans_coord_pid_log_commit_dec, 113: asymtrans_coord_rec_acc_pre_commit_log_commit, 114: asymtrans_coord_pid_rec_acc_pre_commit_log_commit, 115: asymtrans_coord_rec_acc_pre_commit_done_commit, 116: asymtrans_coord_pid_rec_acc_pre_commit_done_commit]}, 117: {after_corrupt_files, [], 118: [after_corrupt_files_decision_log_head, 119: after_corrupt_files_decision_log_tail, 120: after_corrupt_files_latest_log_head, 121: after_corrupt_files_latest_log_tail, 122: after_corrupt_files_table_dat_head, 123: after_corrupt_files_table_dat_tail, 124: after_corrupt_files_schema_dat_head, 125: after_corrupt_files_schema_dat_tail]}]. 126: 127: init_per_group(_GroupName, Config) -> 128: Config. 129: 130: end_per_group(_GroupName, Config) -> 131: Config. 132: 133: tpcb_config(ReplicaType, _NodeConfig, Nodes) -> 134: [{n_branches, 5}, 135: {n_drivers_per_node, 5}, 136: {replica_nodes, Nodes}, 137: {driver_nodes, Nodes}, 138: {use_running_mnesia, true}, 139: {report_interval, infinity}, 140: {n_accounts_per_branch, 20}, 141: {replica_type, ReplicaType}]. 142: 143: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 144: 145: 146: 147: no_master_2(suite) -> []; 148: no_master_2(Config) when is_list(Config) -> mnesia_down_2(no, Config). 149: 150: no_master_3(suite) -> []; 151: no_master_3(Config) when is_list(Config) -> mnesia_down_3(no, Config). 152: 153: one_master_2(suite) -> []; 154: one_master_2(Config) when is_list(Config) -> mnesia_down_2(one, Config). 155: 156: one_master_3(suite) -> []; 157: one_master_3(Config) when is_list(Config) -> mnesia_down_3(one, Config). 158: 159: two_master_2(suite) -> []; 160: two_master_2(Config) when is_list(Config) -> mnesia_down_2(two, Config). 161: 162: two_master_3(suite) -> []; 163: two_master_3(Config) when is_list(Config) -> mnesia_down_3(two, Config). 164: 165: all_master_2(suite) -> []; 166: all_master_2(Config) when is_list(Config) -> mnesia_down_2(all, Config). 167: 168: all_master_3(suite) -> []; 169: all_master_3(Config) when is_list(Config) -> mnesia_down_3(all, Config). 170: 171: mnesia_down_2(Masters, Config) -> 172: Nodes = [N1, N2] = ?acquire_nodes(2, Config), 173: ?match({atomic, ok}, mnesia:create_table(tab1, [{ram_copies, Nodes}])), 174: ?match({atomic, ok}, mnesia:create_table(tab2, [{disc_copies, Nodes}])), 175: ?match({atomic, ok}, mnesia:create_table(tab3, [{disc_only_copies, Nodes}])), 176: ?match({atomic, ok}, mnesia:create_table(tab4, [{ram_copies, [N1]}])), 177: ?match({atomic, ok}, mnesia:create_table(tab5, [{ram_copies, [N2]}])), 178: ?match({atomic, ok}, mnesia:create_table(tab6, [{disc_copies, [N1]}])), 179: ?match({atomic, ok}, mnesia:create_table(tab7, [{disc_copies, [N2]}])), 180: ?match({atomic, ok}, mnesia:create_table(tab8, [{disc_only_copies, [N1]}])), 181: ?match({atomic, ok}, mnesia:create_table(tab9, [{disc_only_copies, [N2]}])), 182: ?match({atomic, ok}, mnesia:create_table(tab10, [{ram_copies, [N1]}, {disc_copies, [N2]}])), 183: ?match({atomic, ok}, mnesia:create_table(tab11, [{ram_copies, [N2]}, {disc_copies, [N1]}])), 184: ?match({atomic, ok}, mnesia:create_table(tab12, [{ram_copies, [N1]}, {disc_only_copies, [N2]}])), 185: ?match({atomic, ok}, mnesia:create_table(tab13, [{ram_copies, [N2]}, {disc_only_copies, [N1]}])), 186: ?match({atomic, ok}, mnesia:create_table(tab14, [{disc_only_copies, [N1]}, {disc_copies, [N2]}])), 187: ?match({atomic, ok}, mnesia:create_table(tab15, [{disc_only_copies, [N2]}, {disc_copies, [N1]}])), 188: 189: Tabs = [tab1, tab2, tab3, tab4, tab5, tab6, tab7, tab8, 190: tab9, tab10, tab11, tab12, tab13, tab14, tab15], 191: [?match(ok, rpc:call(Node, mnesia, wait_for_tables, [Tabs, 10000])) || Node <- Nodes], 192: [insert_data(Tab, 20) || Tab <- Tabs], 193: 194: VTabs = 195: case Masters of 196: no -> 197: Tabs -- [tab4, tab5]; % ram copies 198: one -> 199: ?match(ok, rpc:call(N1, mnesia, set_master_nodes, [[N1]])), 200: Tabs -- [tab1, tab4, tab5, tab10, tab12]; % ram_copies 201: two -> 202: ?match(ok, rpc:call(N1, mnesia, set_master_nodes, [Nodes])), 203: Tabs -- [tab4, tab5]; 204: all -> 205: [?match(ok, rpc:call(Node, mnesia, set_master_nodes, [[Node]])) || Node <- Nodes], 206: Tabs -- [tab1, tab4, tab5, tab10, tab11, tab12, tab13] 207: end, 208: 209: mnesia_test_lib:kill_mnesia([N1]), 210: ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 211: 212: ?match([], mnesia_test_lib:kill_mnesia([N2])), 213: ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 214: 215: [?match(ok, rpc:call(N1, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs], 216: [?match(ok, rpc:call(N2, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs], 217: ?verify_mnesia(Nodes, []). 218: 219: mnesia_down_3(Masters, Config) -> 220: Nodes = [N1, N2, N3] = ?acquire_nodes(3, Config), 221: ?match({atomic, ok}, mnesia:create_table(tab1, [{ram_copies, Nodes}])), 222: ?match({atomic, ok}, mnesia:create_table(tab2, [{disc_copies, Nodes}])), 223: ?match({atomic, ok}, mnesia:create_table(tab3, [{disc_only_copies, Nodes}])), 224: ?match({atomic, ok}, mnesia:create_table(tab4, [{ram_copies, [N1]}])), 225: ?match({atomic, ok}, mnesia:create_table(tab5, [{ram_copies, [N2]}])), 226: ?match({atomic, ok}, mnesia:create_table(tab16, [{ram_copies, [N3]}])), 227: ?match({atomic, ok}, mnesia:create_table(tab6, [{disc_copies, [N1]}])), 228: ?match({atomic, ok}, mnesia:create_table(tab7, [{disc_copies, [N2]}])), 229: ?match({atomic, ok}, mnesia:create_table(tab17, [{disc_copies, [N3]}])), 230: ?match({atomic, ok}, mnesia:create_table(tab8, [{disc_only_copies, [N1]}])), 231: ?match({atomic, ok}, mnesia:create_table(tab9, [{disc_only_copies, [N2]}])), 232: ?match({atomic, ok}, mnesia:create_table(tab18, [{disc_only_copies, [N3]}])), 233: ?match({atomic, ok}, mnesia:create_table(tab10, [{ram_copies, [N1]}, {disc_copies, [N2, N3]}])), 234: ?match({atomic, ok}, mnesia:create_table(tab11, [{ram_copies, [N2]}, {disc_copies, [N3, N1]}])), 235: ?match({atomic, ok}, mnesia:create_table(tab19, [{ram_copies, [N3]}, {disc_copies, [N1, N2]}])), 236: ?match({atomic, ok}, mnesia:create_table(tab12, [{ram_copies, [N1]}, {disc_only_copies, [N2, N3]}])), 237: ?match({atomic, ok}, mnesia:create_table(tab13, [{ram_copies, [N2]}, {disc_only_copies, [N3, N1]}])), 238: ?match({atomic, ok}, mnesia:create_table(tab20, [{ram_copies, [N3]}, {disc_only_copies, [N1, N2]}])), 239: ?match({atomic, ok}, mnesia:create_table(tab14, [{disc_only_copies, [N1]}, {disc_copies, [N2, N3]}])), 240: ?match({atomic, ok}, mnesia:create_table(tab15, [{disc_only_copies, [N2]}, {disc_copies, [N3, N1]}])), 241: ?match({atomic, ok}, mnesia:create_table(tab21, [{disc_only_copies, [N3]}, {disc_copies, [N1, N2]}])), 242: 243: Tabs = [tab1, tab2, tab3, tab4, tab5, tab6, tab7, tab8, 244: tab9, tab10, tab11, tab12, tab13, tab14, tab15, 245: tab16, tab17, tab18, tab19, tab20, tab21], 246: [?match(ok, rpc:call(Node, mnesia, wait_for_tables, [Tabs, 10000])) || Node <- Nodes], 247: [insert_data(Tab, 20) || Tab <- Tabs], 248: 249: VTabs = 250: case Masters of 251: no -> 252: Tabs -- [tab4, tab5, tab16]; % ram copies 253: one -> 254: ?match(ok, rpc:call(N1, mnesia, set_master_nodes, [[N1]])), 255: Tabs -- [tab1, tab4, tab5, tab16, tab10, tab12]; % ram copies 256: two -> 257: ?match(ok, rpc:call(N1, mnesia, set_master_nodes, [Nodes])), 258: Tabs -- [tab4, tab5, tab16]; % ram copies 259: all -> 260: [?match(ok, rpc:call(Node, mnesia, set_master_nodes, [[Node]])) || Node <- Nodes], 261: Tabs -- [tab1, tab4, tab5, tab16, tab10, 262: tab11, tab19, tab12, tab13, tab20] % ram copies 263: end, 264: 265: mnesia_test_lib:kill_mnesia([N1]), 266: ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 267: 268: ?match([], mnesia_test_lib:kill_mnesia([N2])), 269: ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 270: 271: ?match([], mnesia_test_lib:kill_mnesia([N3])), 272: ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 273: 274: ?match([], mnesia_test_lib:kill_mnesia([N2, N1])), 275: ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 276: 277: ?match([], mnesia_test_lib:kill_mnesia([N2, N3])), 278: ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 279: 280: ?match([], mnesia_test_lib:kill_mnesia([N1, N3])), 281: ?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)), 282: 283: [?match(ok, rpc:call(N1, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs], 284: [?match(ok, rpc:call(N2, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs], 285: [?match(ok, rpc:call(N3, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs], 286: 287: ?verify_mnesia(Nodes, []). 288: 289: 290: 291: dirty_read_during_down(suite) -> 292: []; 293: dirty_read_during_down(Config) when is_list(Config) -> 294: read_during_down(dirty, Config). 295: 296: trans_read_during_down(suite) -> 297: []; 298: trans_read_during_down(Config) when is_list(Config) -> 299: read_during_down(trans, Config). 300: 301: 302: read_during_down(Op, Config) when is_list(Config) -> 303: Ns = [N1|TNs] = ?acquire_nodes(3, Config), 304: Tabs = [ram, disc, disco], 305: 306: ?match({atomic, ok}, mnesia:create_table(ram, [{ram_copies, TNs}])), 307: ?match({atomic, ok}, mnesia:create_table(disc, [{disc_copies, TNs}])), 308: ?match({atomic, ok}, mnesia:create_table(disco, [{disc_only_copies, TNs}])), 309: 310: %% Create some work for mnesia_controller when a node goes down 311: [{atomic, ok} = mnesia:create_table(list_to_atom("temp" ++ integer_to_list(N)), 312: [{ram_copies, Ns}]) || N <- lists:seq(1, 50)], 313: 314: Write = fun(Tab) -> mnesia:write({Tab, key, val}) end, 315: ?match([ok,ok,ok], 316: [mnesia:sync_dirty(Write, [Tab]) || Tab <- Tabs]), 317: 318: Readers = [spawn_link(N1, ?MODULE, reader, [Tab, Op]) || Tab <- Tabs], 319: [_|_] = W2R= [mnesia:table_info(Tab, where_to_read) || Tab <- Tabs], 320: ?log("W2R ~p~n", [W2R]), 321: loop_and_kill_mnesia(10, hd(W2R), Tabs), 322: [Pid ! self() || Pid <- Readers], 323: ?match([ok, ok, ok], [receive ok -> ok after 1000 -> {Pid, mnesia_lib:dist_coredump()} end || Pid <- Readers]), 324: ?verify_mnesia(Ns, []). 325: 326: reader(Tab, OP) -> 327: Res = case OP of 328: dirty -> 329: catch mnesia:dirty_read({Tab, key}); 330: trans -> 331: Read = fun() -> mnesia:read({Tab, key}) end, 332: {_, Temp} = mnesia:transaction(Read), 333: Temp 334: end, 335: case Res of 336: [{Tab, key, val}] -> ok; 337: Else -> 338: ?error("Expected ~p Got ~p ~n", [[{Tab, key, val}], Else]), 339: erlang:error(test_failed) 340: end, 341: receive Pid -> 342: Pid ! ok 343: after 50 -> 344: reader(Tab, OP) 345: end. 346: 347: loop_and_kill_mnesia(0, _Node, _Tabs) -> ok; 348: loop_and_kill_mnesia(N, Node, Tabs) -> 349: mnesia_test_lib:kill_mnesia([Node]), 350: timer:sleep(100), 351: ?match([], mnesia_test_lib:start_mnesia([Node], Tabs)), 352: [KN | _] = W2R= [mnesia:table_info(Tab, where_to_read) || Tab <- Tabs], 353: ?match([KN, KN,KN], W2R), 354: timer:sleep(100), 355: loop_and_kill_mnesia(N-1, KN, Tabs). 356: 357: 358: mnesia_down_during_startup_disk_ram(suite) -> []; 359: mnesia_down_during_startup_disk_ram(Config) when is_list(Config)-> 360: [Node1, Node2] = ?acquire_nodes(2, Config ++ 361: [{tc_timeout, timer:minutes(2)}]), 362: Tab = down_during_startup, 363: Def = [{ram_copies, [Node2]}, {disc_copies, [Node1]}], 364: 365: ?match({atomic, ok}, mnesia:create_table(Tab, Def)), 366: ?match(ok, mnesia:dirty_write({Tab, 876234, test_ok})), 367: timer:sleep(500), 368: mnesia_test_lib:kill_mnesia([Node1, Node2]), 369: timer:sleep(500), 370: mnesia_test_lib:start_mnesia([Node1, Node2], [Tab]), 371: mnesia_test_lib:kill_mnesia([Node1]), 372: timer:sleep(500), 373: ?match([], mnesia_test_lib:start_mnesia([Node1], [Tab])), 374: ?match([{Tab, 876234, test_ok}], mnesia:dirty_read({Tab,876234})), 375: ?verify_mnesia([Node1, Node2], []). 376: 377: mnesia_down_during_startup_init_ram(suite) -> []; 378: mnesia_down_during_startup_init_ram(Config) when is_list(Config) -> 379: ?is_debug_compiled, 380: DP = {mnesia_loader, do_get_network_copy}, 381: Type = ram_copies, 382: mnesia_down_during_startup2(Config, Type, DP, self()). 383: 384: mnesia_down_during_startup_init_disc(suite) -> []; 385: mnesia_down_during_startup_init_disc(Config) when is_list(Config) -> 386: ?is_debug_compiled, 387: DP = {mnesia_loader, do_get_network_copy}, 388: Type = disc_copies, 389: mnesia_down_during_startup2(Config, Type, DP, self()). 390: 391: mnesia_down_during_startup_init_disc_only(suite) -> []; 392: mnesia_down_during_startup_init_disc_only(Config) when is_list(Config) -> 393: ?is_debug_compiled, 394: DP = {mnesia_loader, do_get_network_copy}, 395: Type = disc_only_copies, 396: mnesia_down_during_startup2(Config, Type, DP, self()). 397: 398: mnesia_down_during_startup_tm_ram(suite) -> []; 399: mnesia_down_during_startup_tm_ram(Config) when is_list(Config) -> 400: ?is_debug_compiled, 401: DP = {mnesia_tm, init}, 402: Type = ram_copies, 403: mnesia_down_during_startup2(Config, Type, DP, self()). 404: 405: mnesia_down_during_startup_tm_disc(suite) -> []; 406: mnesia_down_during_startup_tm_disc(Config) when is_list(Config) -> 407: ?is_debug_compiled, 408: DP = {mnesia_tm, init}, 409: Type = disc_copies, 410: mnesia_down_during_startup2(Config, Type, DP, self()). 411: 412: mnesia_down_during_startup_tm_disc_only(suite) -> []; 413: mnesia_down_during_startup_tm_disc_only(Config) when is_list(Config) -> 414: ?is_debug_compiled, 415: DP = {mnesia_tm, init}, 416: Type = disc_only_copies, 417: mnesia_down_during_startup2(Config, Type, DP, self()). 418: 419: mnesia_down_during_startup2(Config, ReplicaType, Debug_Point, _Father) -> 420: ?log("TC~n mnesia_down_during_startup with type ~w and stops at ~w~n", 421: [ReplicaType, Debug_Point]), 422: Tpcb_tabs = [history,teller,account,branch], 423: Nodes = ?acquire_nodes(2, Config), 424: Node1 = hd(Nodes), 425: {success, [A]} = ?start_activities([Node1]), 426: TpcbConfig = tpcb_config(ReplicaType, 2, Nodes), 427: mnesia_tpcb:init(TpcbConfig), 428: A ! fun () -> mnesia_tpcb:run(TpcbConfig) end, 429: ?match_receive(timeout), 430: timer:sleep(timer:seconds(10)), % Let tpcb run for a while 431: mnesia_tpcb:stop(), 432: ?match(ok, mnesia_tpcb:verify_tabs()), 433: mnesia_test_lib:kill_mnesia([Node1]), 434: timer:sleep(timer:seconds(2)), 435: Self = self(), 436: TestFun = fun(_MnesiaEnv, _EvalEnv) -> 437: ?deactivate_debug_fun(Debug_Point), 438: Self ! fun_done, 439: spawn(mnesia_test_lib, kill_mnesia, [[Node1]]) 440: end, 441: ?activate_debug_fun(Debug_Point, TestFun, []), % Kill when debug has been reached 442: mnesia:start(), 443: Res = receive fun_done -> ok after timer:minutes(3) -> timeout end, % Wait till it's killed 444: ?match(ok, Res), 445: ?match(ok, timer:sleep(timer:seconds(2))), % Wait a while, at least till it dies; 446: ?match([], mnesia_test_lib:start_mnesia([Node1], Tpcb_tabs)), 447: ?match(ok, mnesia_tpcb:verify_tabs()), % Verify it 448: ?verify_mnesia(Nodes, []). 449: 450: 451: 452: with_checkpoint_same(suite) -> []; 453: with_checkpoint_same(Config) when is_list(Config) -> 454: with_checkpoint(Config, same). 455: 456: with_checkpoint_other(suite) -> []; 457: with_checkpoint_other(Config) when is_list(Config) -> 458: with_checkpoint(Config, other). 459: 460: with_checkpoint(Config, Type) when is_list(Config) -> 461: Nodes = [Node1, Node2] = ?acquire_nodes(2, Config), 462: Kill = case Type of 463: same -> %% Node1 is the one used for creating the checkpoint 464: Node1; %% and which we bring down 465: other -> 466: Node2 %% Here we bring node2 down.. 467: end, 468: 469: ?match({atomic, ok}, mnesia:create_table(ram, [{ram_copies, Nodes}])), 470: ?match({atomic, ok}, mnesia:create_table(disc, [{disc_copies, Nodes}])), 471: ?match({atomic, ok}, mnesia:create_table(disco, [{disc_only_copies, Nodes}])), 472: Tabs = [ram, disc, disco], 473: 474: ?match({ok, sune, _}, mnesia:activate_checkpoint([{name, sune}, 475: {max, mnesia:system_info(tables)}, 476: {ram_overrides_dump, true}])), 477: 478: ?match([], check_retainers(sune, Nodes)), 479: 480: ?match(ok, mnesia:deactivate_checkpoint(sune)), 481: ?match([], check_chkp(Nodes)), 482: 483: timer:sleep(500), %% Just to help debugging the io:formats now comes in the 484: %% correct order... :-) 485: 486: ?match({ok, sune, _}, mnesia:activate_checkpoint([{name, sune}, 487: {max, mnesia:system_info(tables)}, 488: {ram_overrides_dump, true}])), 489: 490: [[mnesia:dirty_write({Tab,Key,Key}) || Key <- lists:seq(1,10)] || Tab <- Tabs], 491: 492: mnesia_test_lib:kill_mnesia([Kill]), 493: timer:sleep(100), 494: mnesia_test_lib:start_mnesia([Kill], Tabs), 495: io:format("Mnesia on ~p started~n", [Kill]), 496: ?match([], check_retainers(sune, Nodes)), 497: ?match(ok, mnesia:deactivate_checkpoint(sune)), 498: ?match([], check_chkp(Nodes)), 499: 500: case Kill of 501: Node1 -> 502: ignore; 503: Node2 -> 504: mnesia_test_lib:kill_mnesia([Kill]), 505: timer:sleep(500), %% Just to help debugging 506: ?match({ok, sune, _}, mnesia:activate_checkpoint([{name, sune}, 507: {max, mnesia:system_info(tables)}, 508: {ram_overrides_dump, true}])), 509: 510: [[mnesia:dirty_write({Tab,Key,Key+2}) || Key <- lists:seq(1,10)] || 511: Tab <- Tabs], 512: 513: mnesia_test_lib:start_mnesia([Kill], Tabs), 514: io:format("Mnesia on ~p started ~n", [Kill]), 515: ?match([], check_retainers(sune, Nodes)), 516: ?match(ok, mnesia:deactivate_checkpoint(sune)), 517: ?match([], check_chkp(Nodes)), 518: ok 519: end, 520: ?verify_mnesia(Nodes, []). 521: 522: check_chkp(Nodes) -> 523: {Good, Bad} = rpc:multicall(Nodes, ?MODULE, check, []), 524: lists:flatten(Good ++ Bad). 525: 526: check() -> 527: [PCP] = ets:match_object(mnesia_gvar, {pending_checkpoint_pids, '_'}), 528: [PC] = ets:match_object(mnesia_gvar, {pending_checkpoints, '_'}), 529: [CPN] = ets:match_object(mnesia_gvar, {checkpoints, '_'}), 530: F = lists:filter(fun({_, []}) -> false; (_W) -> true end, 531: [PCP,PC,CPN]), 532: CPP = ets:match_object(mnesia_gvar, {{checkpoint, '_'}, '_'}), 533: Rt = ets:match_object(mnesia_gvar, {{'_', {retainer, '_'}}, '_'}), 534: F ++ CPP ++ Rt. 535: 536: 537: check_retainers(CHP, Nodes) -> 538: {[R1,R2], []} = rpc:multicall(Nodes, ?MODULE, get_all_retainers, [CHP]), 539: (R1 -- R2) ++ (R2 -- R1). 540: 541: get_all_retainers(CHP) -> 542: Tabs = mnesia:system_info(local_tables), 543: Iter = fun(Tab) -> 544: {ok, Res} = 545: mnesia_checkpoint:iterate(CHP, Tab, fun(R, A) -> [R|A] end, [], 546: retainer, checkpoint), 547: %% io:format("Retainer content ~w ~n", [Res]), 548: Res 549: end, 550: Elements = [Iter(Tab) || Tab <- Tabs], 551: lists:sort(lists:flatten(Elements)). 552: 553: delete_during_start(doc) -> 554: ["Test that tables can be delete during start, hopefully with tables" 555: " in the loader queue or soon to be"]; 556: delete_during_start(suite) -> []; 557: delete_during_start(Config) when is_list(Config) -> 558: [N1, N2, N3] = Nodes = ?acquire_nodes(3, Config), 559: Tabs = [list_to_atom("tab" ++ integer_to_list(I)) || I <- lists:seq(1, 30)], 560: ?match({atomic, ok}, mnesia:change_table_copy_type(schema, N2, ram_copies)), 561: ?match({atomic, ok}, mnesia:change_table_copy_type(schema, N3, ram_copies)), 562: 563: [?match({atomic, ok},mnesia:create_table(Tab, [{ram_copies,Nodes}])) || Tab <- Tabs], 564: lists:foldl(fun(Tab, I) -> 565: ?match({atomic, ok}, 566: mnesia:change_table_load_order(Tab,I)), 567: I+1 568: end, 1, Tabs), 569: mnesia_test_lib:kill_mnesia([N2,N3]), 570: %% timer:sleep(500), 571: ?match({[ok,ok],[]}, rpc:multicall([N2,N3], mnesia,start, 572: [[{extra_db_nodes,[N1]}]])), 573: [Tab1,Tab2,Tab3|_] = Tabs, 574: ?match({atomic, ok}, mnesia:delete_table(Tab1)), 575: ?match({atomic, ok}, mnesia:delete_table(Tab2)), 576: 577: ?log("W4T ~p~n", [rpc:multicall([N2,N3], mnesia, wait_for_tables, [[Tab1,Tab2,Tab3],1])]), 578: 579: Remain = Tabs--[Tab1,Tab2], 580: ?match(ok, rpc:call(N2, mnesia, wait_for_tables, [Remain,10000])), 581: ?match(ok, rpc:call(N3, mnesia, wait_for_tables, [Remain,10000])), 582: 583: ?match(ok, rpc:call(N2, ?MODULE, verify_where2read, [Remain])), 584: ?match(ok, rpc:call(N3, ?MODULE, verify_where2read, [Remain])), 585: 586: ?verify_mnesia(Nodes, []). 587: 588: verify_where2read([Tab|Tabs]) -> 589: true = (node() == mnesia:table_info(Tab,where_to_read)), 590: verify_where2read(Tabs); 591: verify_where2read([]) -> ok. 592: 593: 594: %%------------------------------------------------------------------------------------------- 595: %% This is a bad implementation, but at least gives a indication if something is wrong 596: explicit_stop_during_snmp(suite) -> []; 597: explicit_stop_during_snmp(Config) when is_list(Config) -> 598: Nodes = ?acquire_nodes(2, Config), 599: [Node1, Node2] = Nodes, 600: Tab = snmp_tab, 601: Def = [{attributes, [key, value]}, 602: {snmp, [{key, integer}]}, 603: {mnesia_test_lib:storage_type(disc_copies, Config), 604: [Node1, Node2]}], 605: ?match({atomic, ok}, mnesia:create_table(Tab, Def)), 606: ?match({atomic, ok}, mnesia:transaction(fun() -> mnesia:write({Tab, 1, 1}) end)), 607: 608: Do_trans_Pid1 = spawn_link(Node2, ?MODULE, do_trans_loop, [Tab, self()]), 609: Do_trans_Pid2 = spawn_link(?MODULE, do_trans_loop, [Tab, self()]), 610: Start_stop_Pid = spawn_link(?MODULE, start_stop, [Node1, 5, self()]), 611: receive 612: test_done -> 613: ok 614: after timer:minutes(5) -> 615: ?error("test case time out~n", []) 616: end, 617: ?verify_mnesia(Nodes, []), 618: exit(Do_trans_Pid1, kill), 619: exit(Do_trans_Pid2, kill), 620: exit(Start_stop_Pid, kill), 621: ok. 622: 623: do_trans_loop(Tab, Father) -> 624: %% Do not trap exit 625: do_trans_loop2(Tab, Father). 626: do_trans_loop2(Tab, Father) -> 627: Trans = 628: fun() -> 629: [{Tab, 1, Val}] = mnesia:read({Tab, 1}), 630: mnesia:write({Tab, 1, Val + 1}) 631: end, 632: case mnesia:transaction(Trans) of 633: {atomic, ok} -> 634: timer:sleep(100), 635: do_trans_loop2(Tab, Father); 636: {aborted, {node_not_running, N}} when N == node() -> 637: timer:sleep(100), 638: do_trans_loop2(Tab, Father); 639: {aborted, {no_exists, Tab}} -> 640: timer:sleep(100), 641: do_trans_loop2(Tab, Father); 642: Else -> 643: ?error("Transaction failed: ~p ~n", [Else]), 644: Father ! test_done, 645: exit(shutdown) 646: end. 647: 648: start_stop(_Node1, 0, Father) -> 649: Father ! test_done, 650: exit(shutdown); 651: start_stop(Node1, N, Father) when N > 0-> 652: timer:sleep(timer:seconds(2)), 653: ?match(stopped, rpc:call(Node1, mnesia, stop, [])), 654: timer:sleep(timer:seconds(1)), 655: ?match([], mnesia_test_lib:start_mnesia([Node1])), 656: start_stop(Node1, N-1, Father). 657: 658: coord_dies(suite) -> []; 659: coord_dies(doc) -> [""]; 660: coord_dies(Config) when is_list(Config) -> 661: Nodes = [N1, N2] = ?acquire_nodes(2, Config), 662: ?match({atomic, ok}, mnesia:create_table(tab1, [{ram_copies, Nodes}])), 663: ?match({atomic, ok}, mnesia:create_table(tab2, [{ram_copies, [N1]}])), 664: ?match({atomic, ok}, mnesia:create_table(tab3, [{ram_copies, [N2]}])), 665: Tester = self(), 666: 667: U1 = fun(Tab) -> 668: [{Tab,key,Val}] = mnesia:read(Tab,key,write), 669: mnesia:write({Tab,key, Val+1}), 670: Tester ! {self(),continue}, 671: receive 672: continue -> exit(crash) 673: end 674: end, 675: U2 = fun(Tab) -> 676: [{Tab,key,Val}] = mnesia:read(Tab,key,write), 677: mnesia:write({Tab,key, Val+1}), 678: mnesia:transaction(U1, [Tab]) 679: end, 680: [mnesia:dirty_write(Tab,{Tab,key,0}) || Tab <- [tab1,tab2,tab3]], 681: Pid1 = spawn(fun() -> mnesia:transaction(U2, [tab1]) end), 682: Pid2 = spawn(fun() -> mnesia:transaction(U2, [tab2]) end), 683: Pid3 = spawn(fun() -> mnesia:transaction(U2, [tab3]) end), 684: [receive {Pid,continue} -> ok end || Pid <- [Pid1,Pid2,Pid3]], 685: Pid1 ! continue, Pid2 ! continue, Pid3 ! continue, 686: ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab1,key}) end)), 687: ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab2,key}) end)), 688: ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab3,key}) end)), 689: 690: Pid4 = spawn(fun() -> mnesia:transaction(U2, [tab1]) end), 691: Pid5 = spawn(fun() -> mnesia:transaction(U2, [tab2]) end), 692: Pid6 = spawn(fun() -> mnesia:transaction(U2, [tab3]) end), 693: erlang:monitor(process, Pid4),erlang:monitor(process, Pid5),erlang:monitor(process, Pid6), 694: 695: [receive {Pid,continue} -> ok end || Pid <- [Pid4,Pid5,Pid6]], 696: exit(Pid4,crash), 697: ?match_receive({'DOWN',_,_,Pid4, _}), 698: ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab1,key}) end)), 699: exit(Pid5,crash), 700: ?match_receive({'DOWN',_,_,Pid5, _}), 701: ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab2,key}) end)), 702: exit(Pid6,crash), 703: ?match_receive({'DOWN',_,_,Pid6, _}), 704: ?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab3,key}) end)), 705: 706: ?verify_mnesia(Nodes, []). 707: 708: 709: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 710: 711: 712: %kill_after_debug_point(Config, TestCase, {Debug_node, Debug_Point}, TransFun, Tab) 713: 714: sym_trans_before_commit_kill_coord_node(suite) -> []; 715: sym_trans_before_commit_kill_coord_node(Config) when is_list(Config) -> 716: ?is_debug_compiled, 717: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 718: [Coord, Part1, Part2] = Nodes, 719: Tab = sym_trans_before_commit_kill_coord, 720: Def = [{attributes, [key, value]}, {ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 721: kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_sym}}, 722: do_sym_trans, [{Tab, Def}], Nodes). 723: 724: sym_trans_before_commit_kill_coord_pid(suite) -> []; 725: sym_trans_before_commit_kill_coord_pid(Config) when is_list(Config) -> 726: ?is_debug_compiled, 727: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 728: [Coord, Part1, Part2] = Nodes, 729: Tab = sym_trans_before_commit_kill_coord, 730: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 731: kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_sym}}, 732: do_sym_trans, [{Tab, Def}], Nodes). 733: 734: sym_trans_before_commit_kill_part_after_ask(suite) -> []; 735: sym_trans_before_commit_kill_part_after_ask(Config) when is_list(Config) -> 736: ?is_debug_compiled, 737: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 738: [Coord, Part1, Part2] = Nodes, 739: Tab = sym_trans_before_commit_kill_part_after_ask, 740: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 741: kill_after_debug_point(Part1, {Coord, {mnesia_tm, multi_commit_sym}}, 742: do_sym_trans, [{Tab, Def}], Nodes). 743: 744: sym_trans_before_commit_kill_part_before_ask(suite) -> []; 745: sym_trans_before_commit_kill_part_before_ask(Config) when is_list(Config) -> 746: ?is_debug_compiled, 747: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 748: [Coord, Part1, Part2] = Nodes, 749: Tab = sym_trans_before_commit_kill_part_before_ask, 750: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 751: kill_after_debug_point(Part1, {Part1, {mnesia_tm, doit_ask_commit}}, 752: do_sym_trans, [{Tab, Def}], Nodes). 753: 754: sym_trans_after_commit_kill_coord_node(suite) -> []; 755: sym_trans_after_commit_kill_coord_node(Config) when is_list(Config) -> 756: ?is_debug_compiled, 757: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 758: [Coord, Part1, Part2] = Nodes, 759: Tab = sym_trans_after_commit_kill_coord, 760: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 761: kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_sym, post}}, 762: do_sym_trans, [{Tab, Def}], Nodes). 763: 764: sym_trans_after_commit_kill_coord_pid(suite) -> []; 765: sym_trans_after_commit_kill_coord_pid(Config) when is_list(Config) -> 766: ?is_debug_compiled, 767: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 768: [Coord, Part1, Part2] = Nodes, 769: Tab = sym_trans_after_commit_kill_coord, 770: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 771: kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_sym, post}}, 772: do_sym_trans, [{Tab,Def}], Nodes). 773: 774: sym_trans_after_commit_kill_part_after_ask(suite) -> []; 775: sym_trans_after_commit_kill_part_after_ask(Config) when is_list(Config) -> 776: ?is_debug_compiled, 777: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 778: [Coord, Part1, Part2] = Nodes, 779: Tab = sym_trans_after_commit_kill_part_after_ask, 780: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 781: kill_after_debug_point(Part1, {Coord, {mnesia_tm, multi_commit_sym, post}}, 782: do_sym_trans, [{Tab, Def}], Nodes). 783: 784: sym_trans_after_commit_kill_part_do_commit_pre(suite) -> []; 785: sym_trans_after_commit_kill_part_do_commit_pre(Config) when is_list(Config) -> 786: ?is_debug_compiled, 787: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 788: [Coord, Part1, Part2] = Nodes, 789: Tab = sym_trans_after_commit_kill_part_do_commit_pre, 790: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 791: TransFun = do_sym_trans, 792: kill_after_debug_point(Part1, {Part1, {mnesia_tm, do_commit, pre}}, 793: TransFun, [{Tab, Def}], Nodes). 794: 795: sym_trans_after_commit_kill_part_do_commit_post(suite) -> []; 796: sym_trans_after_commit_kill_part_do_commit_post(Config) when is_list(Config) -> 797: ?is_debug_compiled, 798: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 799: [Coord, Part1, Part2] = Nodes, 800: Tab = sym_trans_after_commit_kill_part_do_commit_post, 801: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 802: TransFun = do_sym_trans, 803: kill_after_debug_point(Part1, {Part1, {mnesia_tm, do_commit, post}}, 804: TransFun, [{Tab, Def}], Nodes). 805: 806: do_sym_trans([Tab], _Fahter) -> 807: ?dl("Starting SYM_TRANS with active debug fun ", []), 808: Trans = fun() -> 809: [{_,_,Val}] = mnesia:read({Tab, 1}), 810: mnesia:write({Tab, 1, Val+1}) 811: end, 812: Res = mnesia:transaction(Trans), 813: case Res of 814: {atomic, ok} -> ok; 815: {aborted, _Reason} -> ok; 816: Else -> ?error("Wrong output from mensia:transaction(FUN):~n ~p~n", 817: [Else]) 818: end, 819: ?dl("SYM_TRANSACTION done: ~p (deactiv dbgfun) ", [Res]), 820: ok. 821: 822: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 823: 824: 825: sync_dirty_pre_kill_part(suite) -> []; 826: sync_dirty_pre_kill_part(Config) when is_list(Config) -> 827: ?is_debug_compiled, 828: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 829: [Coord, Part1, Part2] = Nodes, 830: Tab = sync_dirty_pre, 831: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 832: TransFun = do_sync_dirty, 833: kill_after_debug_point(Part1, {Part1, {mnesia_tm, sync_dirty, pre}}, 834: TransFun, [{Tab, Def}], Nodes). 835: 836: sync_dirty_pre_kill_coord_node(suite) -> []; 837: sync_dirty_pre_kill_coord_node(Config) when is_list(Config) -> 838: ?is_debug_compiled, 839: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 840: [Coord, Part1, Part2] = Nodes, 841: Tab = sync_dirty_pre, 842: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 843: TransFun = do_sync_dirty, 844: kill_after_debug_point(Coord, {Part1, {mnesia_tm, sync_dirty, pre}}, 845: TransFun, [{Tab, Def}], Nodes). 846: 847: sync_dirty_pre_kill_coord_pid(suite) -> []; 848: sync_dirty_pre_kill_coord_pid(Config) when is_list(Config) -> 849: ?is_debug_compiled, 850: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 851: [Coord, Part1, Part2] = Nodes, 852: Tab = sync_dirty_pre, 853: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 854: TransFun = do_sync_dirty, 855: kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, sync_dirty, pre}}, 856: TransFun, [{Tab, Def}], Nodes). 857: 858: sync_dirty_post_kill_part(suite) -> []; 859: sync_dirty_post_kill_part(Config) when is_list(Config) -> 860: ?is_debug_compiled, 861: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 862: [Coord, Part1, Part2] = Nodes, 863: Tab = sync_dirty_post, 864: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 865: TransFun = do_sync_dirty, 866: kill_after_debug_point(Part1, {Part1, {mnesia_tm, sync_dirty, post}}, 867: TransFun, [{Tab, Def}], Nodes). 868: 869: sync_dirty_post_kill_coord_node(suite) -> []; 870: sync_dirty_post_kill_coord_node(Config) when is_list(Config) -> 871: ?is_debug_compiled, 872: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 873: [Coord, Part1, Part2] = Nodes, 874: Tab = sync_dirty_post, 875: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 876: TransFun = do_sync_dirty, 877: kill_after_debug_point(Coord, {Part1, {mnesia_tm, sync_dirty, post}}, 878: TransFun, [{Tab, Def}], Nodes). 879: 880: sync_dirty_post_kill_coord_pid(suite) -> []; 881: sync_dirty_post_kill_coord_pid(Config) when is_list(Config) -> 882: ?is_debug_compiled, 883: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 884: [Coord, Part1, Part2] = Nodes, 885: Tab = sync_dirty_post, 886: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 887: TransFun = do_sync_dirty, 888: kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, sync_dirty, post}}, 889: TransFun, [{Tab, Def}], Nodes). 890: 891: do_sync_dirty([Tab], _Father) -> 892: ?dl("Starting SYNC_DIRTY", []), 893: SYNC = fun() -> 894: [{_,_,Val}] = mnesia:read({Tab, 1}), 895: mnesia:write({Tab, 1, Val+1}) 896: end, 897: {_, Res} = ?match(ok, mnesia:sync_dirty(SYNC)), 898: ?dl("SYNC_DIRTY done: ~p ", [Res]), 899: ok. 900: 901: 902: async_dirty_pre_kill_part(suite) -> []; 903: async_dirty_pre_kill_part(Config) when is_list(Config) -> 904: ?is_debug_compiled, 905: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 906: [Coord, Part1, Part2] = Nodes, 907: Tab = async_dirty_pre, 908: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 909: TransFun = do_async_dirty, 910: kill_after_debug_point(Part1, {Part1, {mnesia_tm, async_dirty, pre}}, 911: TransFun, [{Tab, Def}], Nodes). 912: 913: async_dirty_pre_kill_coord_node(suite) -> []; 914: async_dirty_pre_kill_coord_node(Config) when is_list(Config) -> 915: ?is_debug_compiled, 916: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 917: [Coord, Part1, Part2] = Nodes, 918: Tab = async_dirty_pre, 919: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 920: TransFun = do_async_dirty, 921: kill_after_debug_point(Coord, {Part1, {mnesia_tm, async_dirty, pre}}, 922: TransFun, [{Tab, Def}], Nodes). 923: 924: async_dirty_pre_kill_coord_pid(suite) -> []; 925: async_dirty_pre_kill_coord_pid(Config) when is_list(Config) -> 926: ?is_debug_compiled, 927: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 928: [Coord, Part1, Part2] = Nodes, 929: Tab = async_dirty_pre, 930: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 931: TransFun = do_async_dirty, 932: kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, async_dirty, pre}}, 933: TransFun, [{Tab, Def}], Nodes). 934: 935: async_dirty_post_kill_part(suite) -> []; 936: async_dirty_post_kill_part(Config) when is_list(Config) -> 937: ?is_debug_compiled, 938: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 939: [Coord, Part1, Part2] = Nodes, 940: Tab = async_dirty_post, 941: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 942: TransFun = do_async_dirty, 943: kill_after_debug_point(Part1, {Part1, {mnesia_tm, async_dirty, post}}, 944: TransFun, [{Tab, Def}], Nodes). 945: 946: async_dirty_post_kill_coord_node(suite) -> []; 947: async_dirty_post_kill_coord_node(Config) when is_list(Config) -> 948: ?is_debug_compiled, 949: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 950: [Coord, Part1, Part2] = Nodes, 951: Tab = async_dirty_post, 952: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 953: TransFun = do_async_dirty, 954: kill_after_debug_point(Coord, {Part1, {mnesia_tm, async_dirty, post}}, 955: TransFun, [{Tab, Def}], Nodes). 956: 957: async_dirty_post_kill_coord_pid(suite) -> []; 958: async_dirty_post_kill_coord_pid(Config) when is_list(Config) -> 959: ?is_debug_compiled, 960: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 961: [Coord, Part1, Part2] = Nodes, 962: Tab = async_dirty_post, 963: Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}], 964: TransFun = do_async_dirty, 965: kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, async_dirty, post}}, 966: TransFun, [{Tab, Def}], Nodes). 967: 968: do_async_dirty([Tab], _Fahter) -> 969: ?dl("Starting ASYNC", []), 970: ASYNC = fun() -> 971: [{_,_,Val}] = mnesia:read({Tab, 1}), 972: mnesia:write({Tab, 1, Val+1}) 973: end, 974: {_, Res} = ?match(ok, mnesia:async_dirty(ASYNC)), 975: ?dl("ASYNC done: ~p ", [Res]), 976: ok. 977: 978: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 979: 980: 981: asymtrans_part_ask(suite) -> []; 982: asymtrans_part_ask(Config) when is_list(Config) -> 983: ?is_debug_compiled, 984: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 985: [Coord, Part1, Part2] = Nodes, 986: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 987: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 988: TransFun = do_asym_trans, 989: kill_after_debug_point(Part1, {Part1, {mnesia_tm, doit_ask_commit}}, 990: TransFun, [Tab1, Tab2], Nodes). 991: 992: asymtrans_part_commit_vote(suite) -> []; 993: asymtrans_part_commit_vote(Config) when is_list(Config) -> 994: ?is_debug_compiled, 995: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 996: [Coord, Part1, Part2] = Nodes, 997: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 998: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 999: TransFun = do_asym_trans, 1000: kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, vote_yes}}, 1001: TransFun, [Tab1, Tab2], Nodes). 1002: 1003: asymtrans_part_pre_commit(suite) -> []; 1004: asymtrans_part_pre_commit(Config) when is_list(Config) -> 1005: ?is_debug_compiled, 1006: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1007: [Coord, Part1, Part2] = Nodes, 1008: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1009: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1010: TransFun = do_asym_trans, 1011: kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, pre_commit}}, 1012: TransFun, [Tab1, Tab2], Nodes). 1013: 1014: asymtrans_part_log_commit(suite) -> []; 1015: asymtrans_part_log_commit(Config) when is_list(Config) -> 1016: ?is_debug_compiled, 1017: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1018: [Coord, Part1, Part2] = Nodes, 1019: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1020: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1021: TransFun = do_asym_trans, 1022: kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, log_commit}}, 1023: TransFun, [Tab1, Tab2], Nodes). 1024: 1025: asymtrans_part_do_commit(suite) -> []; 1026: asymtrans_part_do_commit(Config) when is_list(Config) -> 1027: ?is_debug_compiled, 1028: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1029: [Coord, Part1, Part2] = Nodes, 1030: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1031: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1032: TransFun = do_asym_trans, 1033: kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, do_commit}}, 1034: TransFun, [Tab1, Tab2], Nodes). 1035: 1036: asymtrans_coord_got_votes(suite) -> []; 1037: asymtrans_coord_got_votes(Config) when is_list(Config) -> 1038: ?is_debug_compiled, 1039: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1040: [Coord, Part1, Part2] = Nodes, 1041: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1042: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1043: TransFun = do_asym_trans, 1044: kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_asym_got_votes}}, 1045: TransFun, [Tab1, Tab2], Nodes). 1046: 1047: asymtrans_coord_pid_got_votes(suite) -> []; 1048: asymtrans_coord_pid_got_votes(Config) when is_list(Config) -> 1049: ?is_debug_compiled, 1050: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1051: [Coord, Part1, Part2] = Nodes, 1052: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1053: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1054: TransFun = do_asym_trans, 1055: kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_asym_got_votes}}, 1056: TransFun, [Tab1, Tab2], Nodes). 1057: 1058: asymtrans_coord_log_commit_rec(suite) -> []; 1059: asymtrans_coord_log_commit_rec(Config) when is_list(Config) -> 1060: ?is_debug_compiled, 1061: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1062: [Coord, Part1, Part2] = Nodes, 1063: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1064: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1065: TransFun = do_asym_trans, 1066: kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_asym_log_commit_rec}}, 1067: TransFun, [Tab1, Tab2], Nodes). 1068: 1069: asymtrans_coord_pid_log_commit_rec(suite) -> []; 1070: asymtrans_coord_pid_log_commit_rec(Config) when is_list(Config) -> 1071: ?is_debug_compiled, 1072: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1073: [Coord, Part1, Part2] = Nodes, 1074: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1075: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1076: TransFun = do_asym_trans, 1077: kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_asym_log_commit_rec}}, 1078: TransFun, [Tab1, Tab2], Nodes). 1079: 1080: asymtrans_coord_log_commit_dec(suite) -> []; 1081: asymtrans_coord_log_commit_dec(Config) when is_list(Config) -> 1082: ?is_debug_compiled, 1083: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1084: [Coord, Part1, Part2] = Nodes, 1085: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1086: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1087: TransFun = do_asym_trans, 1088: kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_asym_log_commit_dec}}, 1089: TransFun, [Tab1, Tab2], Nodes). 1090: 1091: asymtrans_coord_pid_log_commit_dec(suite) -> []; 1092: asymtrans_coord_pid_log_commit_dec(Config) when is_list(Config) -> 1093: ?is_debug_compiled, 1094: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1095: [Coord, Part1, Part2] = Nodes, 1096: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1097: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1098: TransFun = do_asym_trans, 1099: kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_asym_log_commit_dec}}, 1100: TransFun, [Tab1, Tab2], Nodes). 1101: 1102: asymtrans_coord_rec_acc_pre_commit_log_commit(suite) -> []; 1103: asymtrans_coord_rec_acc_pre_commit_log_commit(Config) when is_list(Config) -> 1104: ?is_debug_compiled, 1105: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1106: [Coord, Part1, Part2] = Nodes, 1107: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1108: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1109: TransFun = do_asym_trans, 1110: kill_after_debug_point(Coord, {Coord, {mnesia_tm, rec_acc_pre_commit_log_commit}}, 1111: TransFun, [Tab1, Tab2], Nodes). 1112: 1113: asymtrans_coord_pid_rec_acc_pre_commit_log_commit(suite) -> []; 1114: asymtrans_coord_pid_rec_acc_pre_commit_log_commit(Config) when is_list(Config) -> 1115: ?is_debug_compiled, 1116: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1117: [Coord, Part1, Part2] = Nodes, 1118: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1119: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1120: TransFun = do_asym_trans, 1121: kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, rec_acc_pre_commit_log_commit}}, 1122: TransFun, [Tab1, Tab2], Nodes). 1123: 1124: asymtrans_coord_rec_acc_pre_commit_done_commit(suite) -> []; 1125: asymtrans_coord_rec_acc_pre_commit_done_commit(Config) when is_list(Config) -> 1126: ?is_debug_compiled, 1127: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1128: [Coord, Part1, Part2] = Nodes, 1129: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1130: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1131: TransFun = do_asym_trans, 1132: kill_after_debug_point(Coord, {Coord, {mnesia_tm, rec_acc_pre_commit_done_commit}}, 1133: TransFun, [Tab1, Tab2], Nodes). 1134: 1135: asymtrans_coord_pid_rec_acc_pre_commit_done_commit(suite) -> []; 1136: asymtrans_coord_pid_rec_acc_pre_commit_done_commit(Config) when is_list(Config) -> 1137: ?is_debug_compiled, 1138: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1139: [Coord, Part1, Part2] = Nodes, 1140: Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]}, 1141: Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]}, 1142: TransFun = do_asym_trans, 1143: kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, rec_acc_pre_commit_done_commit}}, 1144: TransFun, [Tab1, Tab2], Nodes). 1145: 1146: do_asym_trans([Tab1, Tab2 | _R], Garbhandler) -> 1147: ?dl("Starting asym trans ", []), 1148: ASym_Trans = fun() -> 1149: TidTs = {_Mod, Tid, _Store} = 1150: mnesia:get_activity_id(), 1151: ?verbose("===> asym_trans: ~w~n", [TidTs]), 1152: Garbhandler ! {trans_id, Tid}, 1153: [{_, _, Val1}] = mnesia:read({Tab1, 1}), 1154: [{_, _, Val2}] = mnesia:read({Tab2, 1}), 1155: mnesia:write({Tab1, 1, Val1+1}), 1156: mnesia:write({Tab2, 1, Val2+1}) 1157: end, 1158: Res = mnesia:transaction(ASym_Trans), 1159: case Res of 1160: {atomic, ok} -> ok; 1161: {aborted, _Reason} -> ok; 1162: _Else -> ?error("Wrong output from mensia:transaction(FUN):~n ~p~n", [Res]) 1163: end, 1164: ?dl("Asym trans finished with: ~p ", [Res]). 1165: 1166: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1167: 1168: kill_after_debug_point(Kill, {DebugNode, Debug_Point}, TransFun, TabsAndDefs, Nodes) -> 1169: [Coord | _rest] = Nodes, 1170: 1171: Create = fun({Tab, Def}) -> ?match({atomic, ok}, mnesia:create_table(Tab, Def)) end, 1172: lists:foreach(Create, TabsAndDefs), 1173: Tabs = [T || {T, _} <- TabsAndDefs], 1174: Write = fun(Tab) -> ?match(ok, mnesia:dirty_write({Tab, 1, 100})) end, 1175: lists:foreach(Write, Tabs), 1176: 1177: Self = self(), 1178: SyncFun = fun(_Env1, _Env2) -> % Just Sync with test prog 1179: Self ! {self(), fun_in_position}, 1180: ?dl("SyncFun, sending fun_in_position ", []), 1181: receive continue -> 1182: ?dl("SyncFun received continue ",[]), 1183: ok 1184: after timer:seconds(60) -> 1185: ?error("Timeout in sync_fun on ~p~n", [node()]) 1186: end 1187: end, 1188: 1189: Garb_handler = spawn_link(?MODULE, garb_handler, [[]]), 1190: 1191: ?remote_activate_debug_fun(DebugNode, Debug_Point, SyncFun, []), 1192: ?dl("fun_in_position activated at ~p with ~p", [DebugNode, Debug_Point]), 1193: %% Spawn and do the transaction 1194: Pid = spawn(Coord, ?MODULE, TransFun, [Tabs, Garb_handler]), 1195: %% Wait till all the Nodes are in correct position 1196: [{StoppedPid,_}] = ?receive_messages([fun_in_position]), 1197: ?dl("Received fun_in_position; Removing the debug funs ~p", [DebugNode]), 1198: ?remote_deactivate_debug_fun(DebugNode, Debug_Point), 1199: 1200: case Kill of 1201: coord_pid -> 1202: ?dl("Intentionally killing pid ~p ", [Pid]), 1203: exit(Pid, normal); 1204: Node -> 1205: mnesia_test_lib:kill_mnesia([Node]) 1206: end, 1207: 1208: StoppedPid ! continue, %% Send continue, it may still be alive 1209: 1210: %% Start and check that the databases are consistent 1211: ?dl("Done, Restarting and verifying result ",[]), 1212: case Kill of 1213: coord_pid -> ok; 1214: _ -> % Ok, mnesia on some node was killed restart it 1215: timer:sleep(timer:seconds(3)), %% Just let it have the time to die 1216: ?match(ok, rpc:call(Kill, mnesia, start, [[]])), 1217: ?match(ok, rpc:call(Kill, mnesia, wait_for_tables, [Tabs, 60000])) 1218: end, 1219: Trans_res = verify_tabs(Tabs, Nodes), 1220: case TransFun of 1221: do_asym_trans -> 1222: %% Verifies that decisions are garbed, only valid for asym_tran 1223: Garb_handler ! {get_tids, self()}, 1224: Tid_list = receive 1225: {tids, List} -> 1226: ?dl("Fun rec ~w", [List]), 1227: List 1228: end, 1229: garb_of_decisions(Kill, Nodes, Tid_list, Trans_res); 1230: _ -> 1231: ignore 1232: end, 1233: ?verify_mnesia(Nodes, []). 1234: 1235: garb_of_decisions(Kill, Nodes, Tid_list, Trans_res) -> 1236: [Coord, Part1, Part2] = Nodes, 1237: %% Check that decision log is empty on all nodes after the trans is finished 1238: verify_garb_decision_log(Nodes, Tid_list), 1239: case Trans_res of 1240: aborted -> 1241: %% Check that aborted trans have not been restarted!! 1242: ?match(1, length(Tid_list)), 1243: %% Check the transient decision logs 1244: %% A transaction should only be aborted in an early stage of 1245: %% the trans before the any Node have logged anything 1246: verify_garb_transient_logs(Nodes, Tid_list, aborted), 1247: %% And only when the coordinator are have died 1248: %% Else he would have restarted the transaction 1249: ?match(Kill, Coord); 1250: updated -> 1251: case length(Tid_list) of 1252: 1 -> 1253: %% If there was only one transaction, it should be logged as 1254: %% comitted on every node! 1255: [Tid1] = Tid_list, 1256: verify_garb_transient_logs(Nodes, [Tid1], committed); 1257: 2 -> 1258: %% If there is two transaction id, then the first 1259: %% TID should have been aborted and the transaction 1260: %% restarted with a new TID 1261: [Tid1, Tid2] = Tid_list, 1262: verify_garb_transient_logs(Nodes, [Tid1], aborted), 1263: %% If mnesia is killed on a node i.e Coord and Part1 than they 1264: %% won't know about the restarted trans! The rest of the nodes 1265: %% should know that the trans was committed 1266: case Kill of 1267: coord_pid -> 1268: verify_garb_transient_logs(Nodes, [Tid2], committed); 1269: Coord -> 1270: verify_garb_transient_logs([Part1, Part2], [Tid2], committed), 1271: verify_garb_transient_logs([Coord], [Tid2], not_found); 1272: Part1 -> 1273: verify_garb_transient_logs([Coord, Part2], [Tid2], committed), 1274: verify_garb_transient_logs([Part1], [Tid2], not_found) 1275: end 1276: end 1277: end. 1278: 1279: verify_garb_decision_log([], _Tids) -> ok; 1280: verify_garb_decision_log([Node|R], Tids) -> 1281: Check = fun(Tid) -> %% Node, Tid used in debugging! 1282: ?match({{not_found, _}, Node, Tid}, 1283: {outcome(Tid, [mnesia_decision]), Node, Tid}) 1284: end, 1285: rpc:call(Node, lists, foreach, [Check, Tids]), 1286: verify_garb_decision_log(R, Tids). 1287: 1288: verify_garb_transient_logs([], _Tids, _) -> ok; 1289: verify_garb_transient_logs([Node|R], Tids, Exp_Res) -> 1290: Check = fun(Tid) -> 1291: LatestTab = mnesia_lib:val(latest_transient_decision), 1292: PrevTabs = mnesia_lib:val(previous_transient_decisions), 1293: case outcome(Tid, [LatestTab | PrevTabs]) of 1294: {found, {_, [{_,_Tid, Exp_Res}]}} -> ok; 1295: {not_found, _} when Exp_Res == not_found -> ok; 1296: {not_found, _} when Exp_Res == aborted -> ok; 1297: Else -> ?error("Expected ~p in trans ~p on ~p got ~p~n", 1298: [Exp_Res, Tid, Node, Else]) 1299: end 1300: end, 1301: rpc:call(Node, lists, foreach, [Check, Tids]), 1302: verify_garb_transient_logs(R, Tids, Exp_Res). 1303: 1304: outcome(Tid, Tabs) -> 1305: outcome(Tid, Tabs, Tabs). 1306: 1307: outcome(Tid, [Tab | Tabs], AllTabs) -> 1308: case catch ets:lookup(Tab, Tid) of 1309: {'EXIT', _} -> 1310: outcome(Tid, Tabs, AllTabs); 1311: [] -> 1312: outcome(Tid, Tabs, AllTabs); 1313: Val -> 1314: {found, {Tab, Val}} 1315: end; 1316: outcome(_Tid, [], AllTabs) -> 1317: {not_found, AllTabs}. 1318: 1319: 1320: verify_tabs([Tab|R], Nodes) -> 1321: [_Coord, Part1, Part2 | _rest] = Nodes, 1322: Read = fun() -> mnesia:read({Tab, 1}) end, 1323: {success, A} = ?match({atomic, _}, mnesia:transaction(Read)), 1324: ?match(A, rpc:call(Part1, mnesia, transaction, [Read])), 1325: ?match(A, rpc:call(Part2, mnesia, transaction, [Read])), 1326: {atomic, [{Tab, 1, Res}]} = A, 1327: verify_tabs(R, Nodes, Res). 1328: 1329: verify_tabs([], _Nodes, Res) -> 1330: case Res of 1331: 100 -> aborted; 1332: 101 -> updated 1333: end; 1334: 1335: verify_tabs([Tab | Rest], Nodes, Res) -> 1336: [Coord, Part1, Part2 | _rest] = Nodes, 1337: Read = fun() -> mnesia:read({Tab, 1}) end, 1338: Exp = {atomic, [{Tab, 1, Res}]}, 1339: ?match(Exp, rpc:call(Coord, mnesia, transaction, [Read])), 1340: ?match(Exp, rpc:call(Part1, mnesia, transaction, [Read])), 1341: ?match(Exp, rpc:call(Part2, mnesia, transaction, [Read])), 1342: verify_tabs(Rest, Nodes, Res). 1343: 1344: %% Gather TIDS and send them to requesting process and exit! 1345: garb_handler(List) -> 1346: receive 1347: {trans_id, ID} -> garb_handler([ID|List]); 1348: {get_tids, Pid} -> Pid ! {tids, lists:reverse(List)} 1349: end. 1350: 1351: %%%%%%%%%%%%%%%%%%%%%%% 1352: receive_messages([], _File, _Line) -> []; 1353: receive_messages(ListOfMsgs, File, Line) -> 1354: receive 1355: {Pid, Msg} -> 1356: case lists:member(Msg, ListOfMsgs) of 1357: false -> 1358: mnesia_test_lib:log("<>WARNING<>~n" 1359: "Received unexpected msg~n ~p ~n" 1360: "While waiting for ~p~n", 1361: [{Pid, Msg}, ListOfMsgs], File, Line), 1362: receive_messages(ListOfMsgs, File, Line); 1363: true -> 1364: ?dl("Got msg ~p from ~p ", [Msg, node(Pid)]), 1365: [{Pid, Msg} | receive_messages(ListOfMsgs -- [Msg], File, Line)] 1366: end; 1367: Else -> mnesia_test_lib:log("<>WARNING<>~n" 1368: "Recevied unexpected or bad formatted msg~n ~p ~n" 1369: "While waiting for ~p~n", 1370: [Else, ListOfMsgs], File, Line), 1371: receive_messages(ListOfMsgs, File, Line) 1372: after timer:minutes(2) -> 1373: ?error("Timeout in receive msgs while waiting for ~p~n", 1374: [ListOfMsgs]) 1375: end. 1376: 1377: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1378: 1379: after_full_disc_partition(doc) -> 1380: ["Verify that the database does not get corrupt", 1381: "when Mnesia encounters a full disc partition"]. 1382: 1383: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1384: %% interrupted_fallback_start 1385: %% is implemented in consistency interupted_install_fallback! 1386: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1387: 1388: after_corrupt_files_decision_log_head(suite) -> []; 1389: after_corrupt_files_decision_log_head(Config) when is_list(Config) -> 1390: after_corrupt_files(Config, "DECISION.LOG", head, repair). 1391: 1392: after_corrupt_files_decision_log_tail(suite) -> []; 1393: after_corrupt_files_decision_log_tail(Config) when is_list(Config) -> 1394: after_corrupt_files(Config, "DECISION.LOG", tail, repair). 1395: 1396: after_corrupt_files_latest_log_head(suite) -> []; 1397: after_corrupt_files_latest_log_head(Config) when is_list(Config) -> 1398: after_corrupt_files(Config, "LATEST.LOG", head, repair). 1399: 1400: after_corrupt_files_latest_log_tail(suite) -> []; 1401: after_corrupt_files_latest_log_tail(Config) when is_list(Config) -> 1402: after_corrupt_files(Config, "LATEST.LOG", tail, repair). 1403: 1404: after_corrupt_files_table_dat_head(suite) -> []; 1405: after_corrupt_files_table_dat_head(Config) when is_list(Config) -> 1406: after_corrupt_files(Config, "rec_files.DAT", head, crash). 1407: 1408: after_corrupt_files_table_dat_tail(suite) -> []; 1409: after_corrupt_files_table_dat_tail(Config) when is_list(Config) -> 1410: after_corrupt_files(Config, "rec_files.DAT", tail, repair). 1411: 1412: after_corrupt_files_schema_dat_head(suite) -> []; 1413: after_corrupt_files_schema_dat_head(Config) when is_list(Config) -> 1414: after_corrupt_files(Config, "schema.DAT", head, crash). 1415: 1416: after_corrupt_files_schema_dat_tail(suite) -> []; 1417: after_corrupt_files_schema_dat_tail(Config) when is_list(Config) -> 1418: after_corrupt_files(Config, "schema.DAT", tail, crash). 1419: 1420: 1421: 1422: %%% BUGBUG: We should also write testcase's for autorepair=false i.e. 1423: %%% not the standard case! 1424: after_corrupt_files(Config, File, Where, Behaviour) -> 1425: [Node] = ?acquire_nodes(1, Config ++ [{tc_timeout, timer:minutes(2)}]), 1426: Tab = rec_files, 1427: Def = [{disc_only_copies, [Node]}], 1428: ?match({atomic, ok}, mnesia:create_table(Tab, Def)), 1429: insert_data(Tab, 100), 1430: Dir = mnesia:system_info(directory), 1431: mnesia_test_lib:kill_mnesia([Node]), 1432: timer:sleep(timer:seconds(10)), % Let dets finish whatever it does 1433: 1434: DirFile = Dir ++ "/" ++ File, 1435: 1436: {ok, Fd} = file:open(DirFile, read_write), 1437: {ok, FileInfo} = file:read_file_info(DirFile), 1438: case Where of 1439: head -> 1440: ?match({ok, _NewP}, file:position(Fd, {bof, 1})), 1441: ?match(ok, file:write(Fd, [255, 255, 255, 255, 255, 255, 255, 255, 254])), 1442: ok; 1443: tail -> 1444: Size = FileInfo#file_info.size, 1445: Half = Size div 2, 1446: 1447: ?dl(" Size = ~p Half = ~p ", [Size, Half]), 1448: ?match({ok, _NewP}, file:position(Fd, {bof, Half})), 1449: ?match(ok, file:truncate(Fd)), 1450: ok 1451: end, 1452: ?match(ok, file:close(Fd)), 1453: 1454: ?warning("++++++SOME OF THE after_corrupt* TEST CASES WILL INTENTIONALLY CRASH MNESIA+++++++~n", []), 1455: Pid = spawn_link(?MODULE, mymnesia_start, [self()]), 1456: receive 1457: {Pid, ok} -> 1458: ?match(ok, mnesia:wait_for_tables([schema, Tab], 10000)), 1459: ?match(ok, verify_data(Tab, 100)), 1460: case mnesia_monitor:get_env(auto_repair) of 1461: false -> 1462: ?error("Mnesia should have crashed in ~p ~p ~n", 1463: [File, Where]); 1464: true -> 1465: ok 1466: end, 1467: ?verify_mnesia([Node], []); 1468: {Pid, {error, ED}} -> 1469: case {mnesia_monitor:get_env(auto_repair), Behaviour} of 1470: {true, repair} -> 1471: ?error("Mnesia crashed with ~p: in ~p ~p ~n", 1472: [ED, File, Where]); 1473: _ -> %% Every other can crash! 1474: ok 1475: end, 1476: ?verify_mnesia([], [Node]); 1477: Msg -> 1478: ?error("~p ~p: Got ~p during start of Mnesia~n", 1479: [File, Where, Msg]) 1480: end. 1481: 1482: mymnesia_start(Tester) -> 1483: Res = mnesia:start(), 1484: unlink(Tester), 1485: Tester ! {self(), Res}. 1486: 1487: verify_data(_, 0) -> ok; 1488: verify_data(Tab, N) -> 1489: Actual = mnesia:dirty_read({Tab, N}), 1490: Expected = [{Tab, N, N}], 1491: if 1492: Expected == Actual -> 1493: verify_data(Tab, N - 1); 1494: true -> 1495: mnesia:schema(Tab), 1496: {not_equal, node(), Expected, Actual} 1497: end. 1498: 1499: insert_data(_Tab, 0) -> ok; 1500: insert_data(Tab, N) -> 1501: ok = mnesia:sync_dirty(fun() -> mnesia:write({Tab, N, N}) end), 1502: insert_data(Tab, N-1). 1503: 1504: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1505: 1506: disc_less(doc) -> 1507: ["Here is a simple test case of a simple recovery of a disc less node. " 1508: "However a lot more test cases involving disc less nodes should " 1509: "be written"]; 1510: disc_less(suite) -> []; 1511: disc_less(Config) when is_list(Config) -> 1512: [Node1, Node2, Node3] = Nodes = ?acquire_nodes(3, Config), 1513: case mnesia_test_lib:diskless(Config) of 1514: true -> skip; 1515: false -> 1516: ?match({atomic, ok}, mnesia:change_table_copy_type(schema, Node3, ram_copies)) 1517: end, 1518: Tab1 = disc_less1, 1519: Tab2 = disc_less2, 1520: Tab3 = disc_less3, 1521: Def1 = [{ram_copies, [Node3]}, {disc_copies, [Node1, Node2]}], 1522: Def2 = [{ram_copies, [Node3]}, {disc_copies, [Node1]}], 1523: Def3 = [{ram_copies, [Node3]}, {disc_copies, [Node2]}], 1524: ?match({atomic, ok}, mnesia:create_table(Tab1, Def1)), 1525: ?match({atomic, ok}, mnesia:create_table(Tab2, Def2)), 1526: ?match({atomic, ok}, mnesia:create_table(Tab3, Def3)), 1527: insert_data(Tab1, 100), 1528: insert_data(Tab2, 100), 1529: insert_data(Tab3, 100), 1530: 1531: mnesia_test_lib:kill_mnesia([Node1, Node2]), 1532: timer:sleep(500), 1533: mnesia_test_lib:kill_mnesia([Node3]), 1534: ?match(ok, rpc:call(Node1, mnesia, start, [])), 1535: ?match(ok, rpc:call(Node2, mnesia, start, [])), 1536: 1537: timer:sleep(500), 1538: ?match(ok, rpc:call(Node3, mnesia, start, [[{extra_db_nodes, [Node1, Node2]}]])), 1539: ?match(ok, rpc:call(Node3, mnesia, wait_for_tables, [[Tab1, Tab2, Tab3], 20000])), 1540: 1541: ?match(ok, rpc:call(Node3, ?MODULE, verify_data, [Tab1, 100])), 1542: ?match(ok, rpc:call(Node3, ?MODULE, verify_data, [Tab2, 100])), 1543: ?match(ok, rpc:call(Node3, ?MODULE, verify_data, [Tab3, 100])), 1544: 1545: 1546: ?match(ok, rpc:call(Node2, ?MODULE, verify_data, [Tab1, 100])), 1547: ?match(ok, rpc:call(Node2, ?MODULE, verify_data, [Tab2, 100])), 1548: ?match(ok, rpc:call(Node2, ?MODULE, verify_data, [Tab3, 100])), 1549: 1550: ?match(ok, rpc:call(Node1, ?MODULE, verify_data, [Tab1, 100])), 1551: ?match(ok, rpc:call(Node1, ?MODULE, verify_data, [Tab2, 100])), 1552: ?match(ok, rpc:call(Node1, ?MODULE, verify_data, [Tab3, 100])), 1553: ?verify_mnesia(Nodes, []). 1554: 1555: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1556: 1557: system_upgrade(doc) -> 1558: ["Test on-line and off-line upgrade of the Mnesia application"]. 1559: 1560: garb_decision(doc) -> 1561: ["Test that decisions are garbed correctly."]; 1562: garb_decision(suite) -> []; 1563: garb_decision(Config) when is_list(Config) -> 1564: [Node1, Node2, Node3] = Nodes = ?acquire_nodes(3, Config), 1565: check_garb(Nodes), 1566: ?match({atomic, ok},mnesia:create_table(a, [{disc_copies, Nodes}])), 1567: check_garb(Nodes), 1568: ?match({atomic, ok},mnesia:create_table(b, [{ram_copies, Nodes}])), 1569: check_garb(Nodes), 1570: ?match({atomic, ok},mnesia:create_table(c, [{ram_copies, [Node1, Node3]}, 1571: {disc_copies, [Node2]}])), 1572: check_garb(Nodes), 1573: ?match({atomic, ok},mnesia:create_table(d, [{disc_copies, [Node1, Node3]}, 1574: {ram_copies, [Node2]}])), 1575: check_garb(Nodes), 1576: 1577: W = fun(Tab) -> mnesia:write({Tab,1,1}) end, 1578: A = fun(Tab) -> mnesia:write({Tab,1,1}), exit(1) end, 1579: 1580: ?match({atomic, ok}, mnesia:transaction(W,[a])), 1581: check_garb(Nodes), 1582: ?match({atomic, ok}, mnesia:transaction(W,[b])), 1583: check_garb(Nodes), 1584: ?match({atomic, ok}, mnesia:transaction(W,[c])), 1585: check_garb(Nodes), 1586: ?match({atomic, ok}, mnesia:transaction(W,[d])), 1587: check_garb(Nodes), 1588: ?match({aborted,1}, mnesia:transaction(A,[a])), 1589: check_garb(Nodes), 1590: ?match({aborted,1}, mnesia:transaction(A,[b])), 1591: check_garb(Nodes), 1592: ?match({aborted,1}, mnesia:transaction(A,[c])), 1593: check_garb(Nodes), 1594: ?match({aborted,1}, mnesia:transaction(A,[d])), 1595: check_garb(Nodes), 1596: 1597: rpc:call(Node2, mnesia, lkill, []), 1598: ?match({atomic, ok}, mnesia:transaction(W,[a])), 1599: ?match({atomic, ok}, mnesia:transaction(W,[b])), 1600: ?match({atomic, ok}, mnesia:transaction(W,[c])), 1601: ?match({atomic, ok}, mnesia:transaction(W,[d])), 1602: check_garb(Nodes), 1603: ?match([], mnesia_test_lib:start_mnesia([Node2])), 1604: check_garb(Nodes), 1605: timer:sleep(2000), 1606: check_garb(Nodes), 1607: %%%%%% Check transient_decision logs %%%%% 1608: 1609: ?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync 1610: [{atomic, ok} = mnesia:transaction(W,[a]) || _ <- lists:seq(1,30)], 1611: ?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync 1612: TD0 = mnesia_lib:val(latest_transient_decision), 1613: ?match(0, ets:info(TD0, size)), 1614: {atomic, ok} = mnesia:transaction(W,[a]), 1615: ?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync 1616: ?match(TD0, mnesia_lib:val(latest_transient_decision)), 1617: [{atomic, ok} = mnesia:transaction(W,[a]) || _ <- lists:seq(1,30)], 1618: ?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync 1619: ?match(false, TD0 =:= mnesia_lib:val(latest_transient_decision)), 1620: ?match(true, lists:member(TD0, mnesia_lib:val(previous_transient_decisions))), 1621: ?verify_mnesia(Nodes, []). 1622: 1623: check_garb(Nodes) -> 1624: rpc:multicall(Nodes, sys, get_status, [mnesia_recover]), 1625: ?match({_, []},rpc:multicall(Nodes, erlang, apply, [fun check_garb/0, []])). 1626: 1627: check_garb() -> 1628: try 1629: Ds = ets:tab2list(mnesia_decision), 1630: Check = fun({trans_tid,serial, _}) -> false; 1631: ({mnesia_down,_,_,_}) -> false; 1632: (_Else) -> true 1633: end, 1634: Node = node(), 1635: ?match({Node, []}, {node(), lists:filter(Check, Ds)}) 1636: catch _:_ -> ok 1637: end, 1638: ok.