1: %% 2: %% %CopyrightBegin% 3: %% 4: %% Copyright Ericsson AB 1997-2013. All Rights Reserved. 5: %% 6: %% The contents of this file are subject to the Erlang Public License, 7: %% Version 1.1, (the "License"); you may not use this file except in 8: %% compliance with the License. You should have received a copy of the 9: %% Erlang Public License along with this software. If not, it can be 10: %% retrieved online at http://www.erlang.org/. 11: %% 12: %% Software distributed under the License is distributed on an "AS IS" 13: %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 14: %% the License for the specific language governing rights and limitations 15: %% under the License. 16: %% 17: %% %CopyrightEnd% 18: %% 19: 20: %% 21: -module(mnesia_consistency_test). 22: -author('hakan@erix.ericsson.se'). 23: -compile([export_all]). 24: 25: -include("mnesia_test_lib.hrl"). 26: 27: init_per_testcase(Func, Conf) -> 28: mnesia_test_lib:init_per_testcase(Func, Conf). 29: 30: end_per_testcase(Func, Conf) -> 31: mnesia_test_lib:end_per_testcase(Func, Conf). 32: 33: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 34: all() -> 35: [{group, consistency_after_restart}, 36: {group, consistency_after_dump_tables}, 37: {group, consistency_after_add_replica}, 38: {group, consistency_after_del_replica}, 39: {group, consistency_after_move_replica}, 40: {group, consistency_after_transform_table}, 41: consistency_after_change_table_copy_type, 42: {group, consistency_after_restore}, 43: consistency_after_rename_of_node, 44: {group, checkpoint_retainer_consistency}, 45: {group, backup_consistency}]. 46: 47: groups() -> 48: [{consistency_after_restart, [], 49: [consistency_after_restart_1_ram, 50: consistency_after_restart_1_disc, 51: consistency_after_restart_1_disc_only, 52: consistency_after_restart_2_ram, 53: consistency_after_restart_2_disc, 54: consistency_after_restart_2_disc_only]}, 55: {consistency_after_dump_tables, [], 56: [consistency_after_dump_tables_1_ram, 57: consistency_after_dump_tables_2_ram]}, 58: {consistency_after_add_replica, [], 59: [consistency_after_add_replica_2_ram, 60: consistency_after_add_replica_2_disc, 61: consistency_after_add_replica_2_disc_only, 62: consistency_after_add_replica_3_ram, 63: consistency_after_add_replica_3_disc, 64: consistency_after_add_replica_3_disc_only]}, 65: {consistency_after_del_replica, [], 66: [consistency_after_del_replica_2_ram, 67: consistency_after_del_replica_2_disc, 68: consistency_after_del_replica_2_disc_only, 69: consistency_after_del_replica_3_ram, 70: consistency_after_del_replica_3_disc, 71: consistency_after_del_replica_3_disc_only]}, 72: {consistency_after_move_replica, [], 73: [consistency_after_move_replica_2_ram, 74: consistency_after_move_replica_2_disc, 75: consistency_after_move_replica_2_disc_only, 76: consistency_after_move_replica_3_ram, 77: consistency_after_move_replica_3_disc, 78: consistency_after_move_replica_3_disc_only]}, 79: {consistency_after_transform_table, [], 80: [consistency_after_transform_table_ram, 81: consistency_after_transform_table_disc, 82: consistency_after_transform_table_disc_only]}, 83: {consistency_after_fallback, [], 84: [consistency_after_fallback_2_ram, 85: consistency_after_fallback_2_disc, 86: consistency_after_fallback_2_disc_only, 87: consistency_after_fallback_3_ram, 88: consistency_after_fallback_3_disc, 89: consistency_after_fallback_3_disc_only]}, 90: {consistency_after_restore, [], 91: [consistency_after_restore_clear_ram, 92: consistency_after_restore_clear_disc, 93: consistency_after_restore_clear_disc_only, 94: consistency_after_restore_recreate_ram, 95: consistency_after_restore_recreate_disc, 96: consistency_after_restore_recreate_disc_only]}, 97: {checkpoint_retainer_consistency, [], 98: [{group, updates_during_checkpoint_activation}, 99: {group, updates_during_checkpoint_iteration}, 100: {group, load_table_with_activated_checkpoint}, 101: {group, add_table_copy_to_table_checkpoint}, 102: {group, consistency_after_fallback} 103: ]}, 104: {updates_during_checkpoint_activation, [], 105: [updates_during_checkpoint_activation_1_ram, 106: updates_during_checkpoint_activation_1_disc, 107: updates_during_checkpoint_activation_1_disc_only, 108: updates_during_checkpoint_activation_2_ram, 109: updates_during_checkpoint_activation_2_disc, 110: updates_during_checkpoint_activation_2_disc_only, 111: updates_during_checkpoint_activation_3_ram, 112: updates_during_checkpoint_activation_3_disc, 113: updates_during_checkpoint_activation_3_disc_only]}, 114: {updates_during_checkpoint_iteration, [], 115: [updates_during_checkpoint_iteration_2_ram, 116: updates_during_checkpoint_iteration_2_disc, 117: updates_during_checkpoint_iteration_2_disc_only]}, 118: {load_table_with_activated_checkpoint, [], 119: [load_table_with_activated_checkpoint_ram, 120: load_table_with_activated_checkpoint_disc, 121: load_table_with_activated_checkpoint_disc_only]}, 122: {add_table_copy_to_table_checkpoint, [], 123: [add_table_copy_to_table_checkpoint_ram, 124: add_table_copy_to_table_checkpoint_disc, 125: add_table_copy_to_table_checkpoint_disc_only]}, 126: {backup_consistency, [], 127: [{group, interupted_install_fallback}, 128: {group, interupted_uninstall_fallback}, 129: {group, mnesia_down_during_backup_causes_switch}, 130: {group, mnesia_down_during_backup_causes_abort}, 131: {group, schema_transactions_during_backup}]}, 132: {interupted_install_fallback, [], 133: [inst_fallback_process_dies, fatal_when_inconsistency]}, 134: {interupted_uninstall_fallback, [], [after_delete]}, 135: {mnesia_down_during_backup_causes_switch, [], 136: [cause_switch_before, cause_switch_after]}, 137: {mnesia_down_during_backup_causes_abort, [], 138: [cause_abort_before, cause_abort_after]}, 139: {schema_transactions_during_backup, [], 140: [change_schema_before, change_schema_after]}]. 141: 142: init_per_group(_GroupName, Config) -> 143: Config. 144: 145: end_per_group(_GroupName, Config) -> 146: Config. 147: 148: 149: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 150: % 151: % stolen from mnesia_tpcb.erl: 152: 153: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 154: %%% Account record, total size must be at least 100 bytes 155: 156: -define(ACCOUNT_FILLER, 157: {123456789012345678901234567890123456789012345678901234567890, 158: 123456789012345678901234567890123456789012345678901234567890, 159: 123456789012345678901234567890123456789012345678901234}). 160: 161: -record(account, 162: { 163: id = 0, %% Unique account id 164: branch_id = 0, %% Branch where the account is held 165: balance = 0, %% Account balance 166: filler = ?ACCOUNT_FILLER %% Gap filler to ensure size >= 100 bytes 167: }). 168: 169: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 170: %%% Branch record, total size must be at least 100 bytes 171: 172: -define(BRANCH_FILLER, 173: {123456789012345678901234567890123456789012345678901234567890, 174: 123456789012345678901234567890123456789012345678901234567890, 175: 123456789012345678901234567890123456789012345678901234567890}). 176: 177: 178: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 179: %%% Teller record, total size must be at least 100 bytes 180: 181: -define(TELLER_FILLER, 182: {123456789012345678901234567890123456789012345678901234567890, 183: 123456789012345678901234567890123456789012345678901234567890, 184: 1234567890123456789012345678901234567890123456789012345678}). 185: 186: 187: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 188: %%% History record, total size must be at least 50 bytes 189: 190: -define(HISTORY_FILLER, 1234567890). 191: 192: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 193: -record(tab_config, 194: { 195: db_nodes = [node()], 196: replica_nodes = [node()], 197: replica_type = ram_copies, 198: use_running_mnesia = false, 199: n_branches = 1, 200: n_tellers_per_branch = 10, %% Must be 10 201: n_accounts_per_branch = 100000, %% Must be 100000 202: branch_filler = ?BRANCH_FILLER, 203: account_filler = ?ACCOUNT_FILLER, 204: teller_filler = ?TELLER_FILLER 205: }). 206: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 207: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 208: % 209: % stolen from mnesia_tpcb.erl: 210: 211: list2rec(List, Fields, DefaultTuple) -> 212: [Name|Defaults] = tuple_to_list(DefaultTuple), 213: List2 = list2rec(List, Fields, Defaults, []), 214: list_to_tuple([Name] ++ List2). 215: 216: list2rec(_List, [], [], Acc) -> 217: Acc; 218: list2rec(List, [F|Fields], [D|Defaults], Acc) -> 219: {Val, List2} = 220: case lists:keysearch(F, 1, List) of 221: false -> 222: {D, List}; 223: {value, {F, NewVal}} -> 224: {NewVal, lists:keydelete(F, 1, List)} 225: end, 226: list2rec(List2, Fields, Defaults, Acc ++ [Val]). 227: 228: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 229: 230: tpcb_config(ReplicaType, _NodeConfig, Nodes, NoDriverNodes) -> 231: [{n_branches, 10}, 232: {n_drivers_per_node, 10}, 233: {replica_nodes, Nodes}, 234: {driver_nodes, Nodes -- NoDriverNodes}, 235: {use_running_mnesia, true}, 236: {report_interval, infinity}, 237: {n_accounts_per_branch, 100}, 238: {replica_type, ReplicaType}, 239: {reuse_history_id, true}]. 240: 241: %% Stolen from mnesia_tpcb:dist 242: tpcb_config_dist(ReplicaType, _NodeConfig, Nodes, _Config) -> 243: [{db_nodes, Nodes}, 244: {driver_nodes, Nodes}, 245: {replica_nodes, Nodes}, 246: {n_drivers_per_node, 10}, 247: {n_branches, 1}, 248: {use_running_mnesia, true}, 249: {n_accounts_per_branch, 10}, 250: {replica_type, ReplicaType}, 251: {stop_after, timer:minutes(15)}, 252: {report_interval, timer:seconds(10)}, 253: {reuse_history_id, true}]. 254: 255: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 256: % 257: % stolen from mnesia_recovery_test.erl: 258: 259: receive_messages([]) -> []; 260: receive_messages(ListOfMsgs) -> 261: receive 262: {Pid, Msg} -> 263: case lists:member(Msg, ListOfMsgs) of 264: false -> 265: ?warning("I (~p) have received unexpected msg~n ~p ~n", 266: [self(),{Pid, Msg}]), 267: receive_messages(ListOfMsgs); 268: true -> 269: ?verbose("I (~p) got msg ~p from ~p ~n", [self(),Msg, Pid]), 270: [{Pid, Msg} | receive_messages(ListOfMsgs -- [Msg])] 271: end; 272: Else -> ?warning("Recevied unexpected Msg~n ~p ~n", [Else]) 273: after timer:minutes(3) -> 274: ?error("Timeout in receive msgs while waiting for ~p~n", 275: [ListOfMsgs]) 276: end. 277: 278: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 279: 280: consistency_after_restart_1_ram(suite) -> []; 281: consistency_after_restart_1_ram(Config) when is_list(Config) -> 282: consistency_after_restart(ram_copies, 2, Config). 283: 284: consistency_after_restart_1_disc(suite) -> []; 285: consistency_after_restart_1_disc(Config) when is_list(Config) -> 286: consistency_after_restart(disc_copies, 2, Config). 287: 288: consistency_after_restart_1_disc_only(suite) -> []; 289: consistency_after_restart_1_disc_only(Config) when is_list(Config) -> 290: consistency_after_restart(disc_only_copies, 2, Config). 291: 292: consistency_after_restart_2_ram(suite) -> []; 293: consistency_after_restart_2_ram(Config) when is_list(Config) -> 294: consistency_after_restart(ram_copies, 3, Config). 295: 296: consistency_after_restart_2_disc(suite) -> []; 297: consistency_after_restart_2_disc(Config) when is_list(Config) -> 298: consistency_after_restart(disc_copies, 3, Config). 299: 300: consistency_after_restart_2_disc_only(suite) -> []; 301: consistency_after_restart_2_disc_only(Config) when is_list(Config) -> 302: consistency_after_restart(disc_only_copies, 3, Config). 303: 304: consistency_after_restart(ReplicaType, NodeConfig, Config) -> 305: [Node1 | _] = Nodes = ?acquire_nodes(NodeConfig, Config), 306: {success, [A]} = ?start_activities([Node1]), 307: ?log("consistency_after_restart with ~p on ~p~n", 308: [ReplicaType, Nodes]), 309: TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, [Node1]), 310: mnesia_tpcb:init(TpcbConfig), 311: A ! fun () -> mnesia_tpcb:run(TpcbConfig) end, 312: timer:sleep(timer:seconds(10)), 313: mnesia_test_lib:kill_mnesia([Node1]), 314: %% Start and wait for tables to be loaded on all nodes 315: timer:sleep(timer:seconds(3)), 316: ?match([], mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller, history])), 317: mnesia_tpcb:stop(), 318: ?match(ok, mnesia_tpcb:verify_tabs()), 319: ?verify_mnesia(Nodes, []). 320: 321: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 322: 323: consistency_after_dump_tables_1_ram(suite) -> []; 324: consistency_after_dump_tables_1_ram(Config) when is_list(Config) -> 325: consistency_after_dump_tables(ram_copies, 1, Config). 326: 327: consistency_after_dump_tables_2_ram(suite) -> []; 328: consistency_after_dump_tables_2_ram(Config) when is_list(Config) -> 329: consistency_after_dump_tables(ram_copies, 2, Config). 330: 331: consistency_after_dump_tables(ReplicaType, NodeConfig, Config) -> 332: [Node1 | _] = Nodes = ?acquire_nodes(NodeConfig, Config), 333: {success, [A]} = ?start_activities([Node1]), 334: ?log("consistency_after_dump_tables with ~p on ~p~n", 335: [ReplicaType, Nodes]), 336: TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []), 337: mnesia_tpcb:init(TpcbConfig), 338: A ! fun() -> mnesia_tpcb:run(TpcbConfig) end, 339: timer:sleep(timer:seconds(10)), 340: ?match({atomic, ok}, rpc:call(Node1, mnesia, dump_tables, 341: [[branch, teller, account, history]])), 342: mnesia_tpcb:stop(), 343: ?match(ok, mnesia_tpcb:verify_tabs()), 344: 345: mnesia_test_lib:kill_mnesia(Nodes), 346: timer:sleep(timer:seconds(1)), 347: ?match([], mnesia_test_lib:start_mnesia(Nodes,[account, branch, 348: teller, history])), 349: mnesia_tpcb:stop(), 350: ?match(ok, mnesia_tpcb:verify_tabs()), 351: ?verify_mnesia(Nodes, []). 352: 353: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 354: 355: consistency_after_add_replica_2_ram(suite) -> []; 356: consistency_after_add_replica_2_ram(Config) when is_list(Config) -> 357: consistency_after_add_replica(ram_copies, 2, Config). 358: 359: consistency_after_add_replica_2_disc(suite) -> []; 360: consistency_after_add_replica_2_disc(Config) when is_list(Config) -> 361: consistency_after_add_replica(disc_copies, 2, Config). 362: 363: consistency_after_add_replica_2_disc_only(suite) -> []; 364: consistency_after_add_replica_2_disc_only(Config) when is_list(Config) -> 365: consistency_after_add_replica(disc_only_copies, 2, Config). 366: 367: consistency_after_add_replica_3_ram(suite) -> []; 368: consistency_after_add_replica_3_ram(Config) when is_list(Config) -> 369: consistency_after_add_replica(ram_copies, 3, Config). 370: 371: consistency_after_add_replica_3_disc(suite) -> []; 372: consistency_after_add_replica_3_disc(Config) when is_list(Config) -> 373: consistency_after_add_replica(disc_copies, 3, Config). 374: 375: consistency_after_add_replica_3_disc_only(suite) -> []; 376: consistency_after_add_replica_3_disc_only(Config) when is_list(Config) -> 377: consistency_after_add_replica(disc_only_copies, 3, Config). 378: 379: consistency_after_add_replica(ReplicaType, NodeConfig, Config) -> 380: Nodes0 = ?acquire_nodes(NodeConfig, Config), 381: AddNode = lists:last(Nodes0), 382: Nodes = Nodes0 -- [AddNode], 383: Node1 = hd(Nodes), 384: {success, [A]} = ?start_activities([Node1]), 385: ?log("consistency_after_add_replica with ~p on ~p~n", 386: [ReplicaType, Nodes0]), 387: TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []), 388: mnesia_tpcb:init(TpcbConfig), 389: A ! fun () -> mnesia_tpcb:run(TpcbConfig) end, 390: timer:sleep(timer:seconds(10)), 391: ?match({atomic, ok}, mnesia:add_table_copy(account, AddNode, ReplicaType)), 392: mnesia_tpcb:stop(), 393: ?match(ok, mnesia_tpcb:verify_tabs()), 394: ?verify_mnesia(Nodes0, []). 395: 396: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 397: 398: consistency_after_del_replica_2_ram(suite) -> []; 399: consistency_after_del_replica_2_ram(Config) when is_list(Config) -> 400: consistency_after_del_replica(ram_copies, 2, Config). 401: 402: consistency_after_del_replica_2_disc(suite) -> []; 403: consistency_after_del_replica_2_disc(Config) when is_list(Config) -> 404: consistency_after_del_replica(disc_copies, 2, Config). 405: 406: consistency_after_del_replica_2_disc_only(suite) -> []; 407: consistency_after_del_replica_2_disc_only(Config) when is_list(Config) -> 408: consistency_after_del_replica(disc_only_copies, 2, Config). 409: 410: consistency_after_del_replica_3_ram(suite) -> []; 411: consistency_after_del_replica_3_ram(Config) when is_list(Config) -> 412: consistency_after_del_replica(ram_copies, 3, Config). 413: 414: consistency_after_del_replica_3_disc(suite) -> []; 415: consistency_after_del_replica_3_disc(Config) when is_list(Config) -> 416: consistency_after_del_replica(disc_copies, 3, Config). 417: 418: consistency_after_del_replica_3_disc_only(suite) -> []; 419: consistency_after_del_replica_3_disc_only(Config) when is_list(Config) -> 420: consistency_after_del_replica(disc_only_copies, 3, Config). 421: 422: consistency_after_del_replica(ReplicaType, NodeConfig, Config) -> 423: Nodes = ?acquire_nodes(NodeConfig, Config), 424: Node1 = hd(Nodes), 425: Node2 = lists:last(Nodes), 426: {success, [A]} = ?start_activities([Node1]), 427: ?log("consistency_after_del_replica with ~p on ~p~n", 428: [ReplicaType, Nodes]), 429: TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []), 430: mnesia_tpcb:init(TpcbConfig), 431: A ! fun () -> mnesia_tpcb:run(TpcbConfig) end, 432: timer:sleep(timer:seconds(10)), 433: ?match({atomic, ok}, mnesia:del_table_copy(account, Node2)), 434: mnesia_tpcb:stop(), 435: ?match(ok, mnesia_tpcb:verify_tabs()), 436: ?verify_mnesia(Nodes, []). 437: 438: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 439: 440: consistency_after_move_replica_2_ram(suite) -> []; 441: consistency_after_move_replica_2_ram(Config) when is_list(Config) -> 442: consistency_after_move_replica(ram_copies, 2, Config). 443: 444: consistency_after_move_replica_2_disc(suite) -> []; 445: consistency_after_move_replica_2_disc(Config) when is_list(Config) -> 446: consistency_after_move_replica(disc_copies, 2, Config). 447: 448: consistency_after_move_replica_2_disc_only(suite) -> []; 449: consistency_after_move_replica_2_disc_only(Config) when is_list(Config) -> 450: consistency_after_move_replica(disc_only_copies, 2, Config). 451: 452: consistency_after_move_replica_3_ram(suite) -> []; 453: consistency_after_move_replica_3_ram(Config) when is_list(Config) -> 454: consistency_after_move_replica(ram_copies, 3, Config). 455: 456: consistency_after_move_replica_3_disc(suite) -> []; 457: consistency_after_move_replica_3_disc(Config) when is_list(Config) -> 458: consistency_after_move_replica(disc_copies, 3, Config). 459: 460: consistency_after_move_replica_3_disc_only(suite) -> []; 461: consistency_after_move_replica_3_disc_only(Config) when is_list(Config) -> 462: consistency_after_move_replica(disc_only_copies, 3, Config). 463: 464: consistency_after_move_replica(ReplicaType, NodeConfig, Config) -> 465: Nodes = ?acquire_nodes(NodeConfig, Config ++ [{tc_timeout, timer:minutes(10)}]), 466: Node1 = hd(Nodes), 467: Node2 = lists:last(Nodes), 468: {success, [A]} = ?start_activities([Node1]), 469: ?log("consistency_after_move_replica with ~p on ~p~n", 470: [ReplicaType, Nodes]), 471: TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes -- [Node2], []), 472: mnesia_tpcb:init(TpcbConfig), 473: A ! fun () -> mnesia_tpcb:run(TpcbConfig) end, 474: timer:sleep(timer:seconds(10)), 475: ?match({atomic, ok}, mnesia:move_table_copy(account, Node1, Node2)), 476: ?log("First move completed from node ~p to ~p ~n", [Node1, Node2]), 477: ?match({atomic, ok}, mnesia:move_table_copy(account, Node2, Node1)), 478: mnesia_tpcb:stop(), 479: ?match(ok, mnesia_tpcb:verify_tabs()), 480: ?verify_mnesia(Nodes, []). 481: 482: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 483: 484: 485: consistency_after_transform_table_ram(suite) -> []; 486: consistency_after_transform_table_ram(Config) when is_list(Config) -> 487: consistency_after_transform_table(ram_copies, Config). 488: 489: consistency_after_transform_table_disc(suite) -> []; 490: consistency_after_transform_table_disc(Config) when is_list(Config) -> 491: consistency_after_transform_table(disc_copies, Config). 492: 493: consistency_after_transform_table_disc_only(suite) -> []; 494: consistency_after_transform_table_disc_only(Config) when is_list(Config) -> 495: consistency_after_transform_table(disc_only_copies, Config). 496: 497: consistency_after_transform_table(Type, Config) -> 498: Nodes = [N1, N2,_N3] = ?acquire_nodes(3, Config), 499: 500: ?match({atomic, ok}, mnesia:create_table(tab1, [{index, [3]}, {Type, [N1]}])), 501: ?match({atomic, ok}, mnesia:create_table(tab2, [{index, [3]}, {Type, [N1,N2]}])), 502: ?match({atomic, ok}, mnesia:create_table(tab3, [{index, [3]}, {Type, Nodes}])), 503: ?match({atomic, ok}, mnesia:create_table(empty, [{index, [3]},{Type, Nodes}])), 504: 505: Tabs = lists:sort([tab1, tab2, tab3, empty]), 506: 507: [[mnesia:dirty_write({Tab, N, N}) || N <- lists:seq(1,10)] || 508: Tab <- Tabs -- [empty, tab4]], 509: mnesia:dump_log(), 510: 511: Ok = lists:duplicate(4, {atomic, ok}), 512: ?match(Ok, [mnesia:transform_table(Tab, fun({T, N, N}) -> {T, N, N, ok} end, 513: [k,a,n]) || Tab <- Tabs]), 514: [?match([k,a,n], mnesia:table_info(Tab, attributes)) || Tab <- Tabs], 515: 516: Filter = fun(Tab) -> mnesia:foldl(fun(A, Acc) when size(A) == 3 -> [A|Acc]; 517: (A, Acc) when size(A) == 4 -> Acc 518: end, [], Tab) 519: end, 520: 521: ?match([[],[],[],[]], [element(2,mnesia:transaction(Filter, [Tab])) || Tab <- Tabs]), 522: 523: mnesia_test_lib:kill_mnesia(Nodes), 524: mnesia_test_lib:start_mnesia(Nodes, Tabs), 525: 526: ?match([Tabs, Tabs, Tabs], 527: [lists:sort(rpc:call(Node, mnesia,system_info, [tables]) -- [schema]) || Node <- Nodes]), 528: 529: ?match([[],[],[],[]], [element(2,mnesia:transaction(Filter, [Tab])) || Tab <- Tabs]), 530: [?match([k,a,n], mnesia:table_info(Tab, attributes)) || Tab <- Tabs], 531: 532: ?verify_mnesia(Nodes, []). 533: 534: 535: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 536: consistency_after_change_table_copy_type(doc) -> 537: ["Check that the database is consistent after change of copy type.", 538: " While applications are updating the involved tables. "]. 539: 540: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 541: 542: consistency_after_fallback_2_ram(suite) -> []; 543: consistency_after_fallback_2_ram(Config) when is_list(Config) -> 544: consistency_after_fallback(ram_copies, 2, Config). 545: 546: consistency_after_fallback_2_disc(suite) -> []; 547: consistency_after_fallback_2_disc(Config) when is_list(Config) -> 548: consistency_after_fallback(disc_copies, 2, Config). 549: 550: consistency_after_fallback_2_disc_only(suite) -> []; 551: consistency_after_fallback_2_disc_only(Config) when is_list(Config) -> 552: consistency_after_fallback(disc_only_copies, 2, Config). 553: 554: consistency_after_fallback_3_ram(suite) -> []; 555: consistency_after_fallback_3_ram(Config) when is_list(Config) -> 556: consistency_after_fallback(ram_copies, 3, Config). 557: 558: consistency_after_fallback_3_disc(suite) -> []; 559: consistency_after_fallback_3_disc(Config) when is_list(Config) -> 560: consistency_after_fallback(disc_copies, 3, Config). 561: 562: consistency_after_fallback_3_disc_only(suite) -> []; 563: consistency_after_fallback_3_disc_only(Config) when is_list(Config) -> 564: consistency_after_fallback(disc_only_copies, 3, Config). 565: 566: consistency_after_fallback(ReplicaType, NodeConfig, Config) -> 567: %%?verbose("Starting consistency_after_fallback2 at ~p~n", [self()]), 568: Delay = 5, 569: Nodes = ?acquire_nodes(NodeConfig, [{tc_timeout, timer:minutes(10)} | Config]), 570: Node1 = hd(Nodes), 571: %%?verbose("Mnesia info: ~p~n", [mnesia:info()]), 572: 573: {success, [A]} = ?start_activities([Node1]), 574: ?log("consistency_after_fallback with ~p on ~p~n", 575: [ReplicaType, Nodes]), 576: TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []), 577: mnesia_tpcb:init(TpcbConfig), 578: A ! fun () -> mnesia_tpcb:run(TpcbConfig) end, 579: timer:sleep(timer:seconds(Delay)), 580: 581: %% Make a backup 582: ?verbose("Doing backup~n", []), 583: ?match(ok, mnesia:backup(consistency_after_fallback2)), 584: 585: %% Install the backup as a fallback 586: ?verbose("Doing fallback~n", []), 587: ?match(ok, mnesia:install_fallback(consistency_after_fallback2)), 588: timer:sleep(timer:seconds(Delay)), 589: 590: %% Stop tpcb 591: ?verbose("Stopping TPC-B~n", []), 592: mnesia_tpcb:stop(), 593: ?match(ok, mnesia_tpcb:verify_tabs()), 594: 595: %% Stop and then start mnesia and check table consistency 596: %%?verbose("Restarting Mnesia~n", []), 597: mnesia_test_lib:kill_mnesia(Nodes), 598: mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller,history]), 599: 600: ?match(ok, mnesia_tpcb:verify_tabs()), 601: if 602: ReplicaType == ram_copies -> 603: %% Test that change_table_copy work i.e. no account.dcd file exists. 604: ?match({atomic, ok}, mnesia:change_table_copy_type(account, node(), disc_copies)); 605: true -> 606: ignore 607: end, 608: file:delete(consistency_after_fallback2), 609: ?verify_mnesia(Nodes, []). 610: 611: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 612: 613: consistency_after_restore_clear_ram(suite) -> []; 614: consistency_after_restore_clear_ram(Config) when is_list(Config) -> 615: consistency_after_restore(ram_copies, clear_tables, Config). 616: 617: consistency_after_restore_clear_disc(suite) -> []; 618: consistency_after_restore_clear_disc(Config) when is_list(Config) -> 619: consistency_after_restore(disc_copies, clear_tables, Config). 620: 621: consistency_after_restore_clear_disc_only(suite) -> []; 622: consistency_after_restore_clear_disc_only(Config) when is_list(Config) -> 623: consistency_after_restore(disc_only_copies, clear_tables, Config). 624: 625: consistency_after_restore_recreate_ram(suite) -> []; 626: consistency_after_restore_recreate_ram(Config) when is_list(Config) -> 627: consistency_after_restore(ram_copies, recreate_tables, Config). 628: 629: consistency_after_restore_recreate_disc(suite) -> []; 630: consistency_after_restore_recreate_disc(Config) when is_list(Config) -> 631: consistency_after_restore(disc_copies, recreate_tables, Config). 632: 633: consistency_after_restore_recreate_disc_only(suite) -> []; 634: consistency_after_restore_recreate_disc_only(Config) when is_list(Config) -> 635: consistency_after_restore(disc_only_copies, recreate_tables, Config). 636: 637: consistency_after_restore(ReplicaType, Op, Config) -> 638: Delay = 1, 639: Nodes = ?acquire_nodes(3, [{tc_timeout, timer:minutes(10)} | Config]), 640: [Node1, Node2, _Node3] = Nodes, 641: File = "cons_backup_restore", 642: 643: ?log("consistency_after_restore with ~p on ~p~n", 644: [ReplicaType, Nodes]), 645: Tabs = [carA, carB, carC, carD], 646: 647: ?match({atomic, ok}, mnesia:create_table(carA, [{ReplicaType, Nodes}])), 648: ?match({atomic, ok}, mnesia:create_table(carB, [{ReplicaType, Nodes -- [Node1]}])), 649: ?match({atomic, ok}, mnesia:create_table(carC, [{ReplicaType, Nodes -- [Node2]}])), 650: ?match({atomic, ok}, mnesia:create_table(carD, [{ReplicaType, [Node2]}])), 651: 652: NList = lists:seq(0, 20), 653: [lists:foreach(fun(E) -> ok = mnesia:dirty_write({Tab, E, 1}) end, NList) || 654: Tab <- Tabs], 655: 656: {ok, Name, _} = mnesia:activate_checkpoint([{max, [schema | Tabs]}, 657: {ram_overrides_dump, true}]), 658: ?verbose("Doing backup~n", []), 659: ?match(ok, mnesia:backup_checkpoint(Name, File)), 660: ?match(ok, mnesia:deactivate_checkpoint(Name)), 661: 662: [lists:foreach(fun(E) -> ok = mnesia:dirty_write({Tab, E, 2}) end, NList) || 663: Tab <- Tabs], 664: 665: Pids1 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carA, Op]), ok} || _ <- lists:seq(1, 5)], 666: Pids2 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carB, Op]), ok} || _ <- lists:seq(1, 5)], 667: Pids3 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carC, Op]), ok} || _ <- lists:seq(1, 5)], 668: Pids4 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carD, Op]), ok} || _ <- lists:seq(1, 5)], 669: 670: AllPids = Pids1 ++ Pids2 ++ Pids3 ++ Pids4, 671: 672: Restore = fun(F, Args) -> 673: case mnesia:restore(F, Args) of 674: {atomic, List} -> lists:sort(List); 675: Else -> Else 676: end 677: end, 678: 679: timer:sleep(timer:seconds(Delay)), %% Let changers grab locks 680: ?verbose("Doing restore~n", []), 681: ?match(Tabs, Restore(File, [{default_op, Op}])), 682: 683: timer:sleep(timer:seconds(Delay)), %% Let em die 684: 685: ?match_multi_receive(AllPids), 686: 687: case ?match(ok, restore_verify_tabs(Tabs)) of 688: {success, ok} -> 689: file:delete(File); 690: _ -> 691: {T, M, S} = time(), 692: File2 = ?flat_format("consistency_error~w~w~w.BUP", [T, M, S]), 693: file:rename(File, File2) 694: end, 695: ?verify_mnesia(Nodes, []). 696: 697: change_tab(Father, Tab, Test) -> 698: Key = random:uniform(20), 699: Update = fun() -> 700: case mnesia:read({Tab, Key}) of 701: [{Tab, Key, 1}] -> 702: quit; 703: [{Tab, Key, _N}] -> 704: mnesia:write({Tab, Key, 3}) 705: end 706: end, 707: case mnesia:transaction(Update) of 708: {atomic, quit} -> 709: exit(ok); 710: {aborted, {no_exists, Tab}} when Test == recreate_tables ->%% I'll allow this 711: change_tab(Father, Tab, Test); 712: {atomic, ok} -> 713: change_tab(Father, Tab, Test) 714: end. 715: 716: restore_verify_tabs([Tab | R]) -> 717: ?match({atomic, ok}, 718: mnesia:transaction(fun() -> mnesia:foldl(fun({_, _, 1}, ok) -> 719: ok; 720: (Else, Acc) -> 721: [Else|Acc] 722: end, ok, Tab) 723: end)), 724: restore_verify_tabs(R); 725: restore_verify_tabs([]) -> 726: ok. 727: 728: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 729: consistency_after_rename_of_node(doc) -> 730: ["Skipped because it is an unimportant case."]. 731: 732: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 733: 734: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 735: 736: updates_during_checkpoint_activation_1_ram(suite) -> []; 737: updates_during_checkpoint_activation_1_ram(Config) when is_list(Config) -> 738: updates_during_checkpoint_activation(ram_copies, 1, Config). 739: 740: updates_during_checkpoint_activation_1_disc(suite) -> []; 741: updates_during_checkpoint_activation_1_disc(Config) when is_list(Config) -> 742: updates_during_checkpoint_activation(disc_copies, 1, Config). 743: 744: updates_during_checkpoint_activation_1_disc_only(suite) -> []; 745: updates_during_checkpoint_activation_1_disc_only(Config) when is_list(Config) -> 746: updates_during_checkpoint_activation(disc_only_copies, 1, Config). 747: 748: updates_during_checkpoint_activation_2_ram(suite) -> []; 749: updates_during_checkpoint_activation_2_ram(Config) when is_list(Config) -> 750: updates_during_checkpoint_activation(ram_copies, 2, Config). 751: 752: updates_during_checkpoint_activation_2_disc(suite) -> []; 753: updates_during_checkpoint_activation_2_disc(Config) when is_list(Config) -> 754: updates_during_checkpoint_activation(disc_copies, 2, Config). 755: 756: updates_during_checkpoint_activation_2_disc_only(suite) -> []; 757: updates_during_checkpoint_activation_2_disc_only(Config) when is_list(Config) -> 758: updates_during_checkpoint_activation(disc_only_copies, 2, Config). 759: 760: updates_during_checkpoint_activation_3_ram(suite) -> []; 761: updates_during_checkpoint_activation_3_ram(Config) when is_list(Config) -> 762: updates_during_checkpoint_activation(ram_copies, 3, Config). 763: 764: updates_during_checkpoint_activation_3_disc(suite) -> []; 765: updates_during_checkpoint_activation_3_disc(Config) when is_list(Config) -> 766: updates_during_checkpoint_activation(disc_copies, 3, Config). 767: 768: updates_during_checkpoint_activation_3_disc_only(suite) -> []; 769: updates_during_checkpoint_activation_3_disc_only(Config) when is_list(Config) -> 770: updates_during_checkpoint_activation(disc_only_copies, 3, Config). 771: 772: updates_during_checkpoint_activation(ReplicaType,NodeConfig,Config) -> 773: %%?verbose("updates_during_checkpoint_activation2 at ~p~n", [self()]), 774: Delay = 5, 775: Nodes = ?acquire_nodes(NodeConfig, Config), 776: Node1 = hd(Nodes), 777: %%?verbose("Mnesia info: ~p~n", [mnesia:info()]), 778: 779: {success, [A]} = ?start_activities([Node1]), 780: ?log("consistency_after_fallback with ~p on ~p~n", 781: [ReplicaType, Nodes]), 782: TpcbConfig = tpcb_config_dist(ReplicaType, NodeConfig, Nodes, Config), 783: %%TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes), 784: mnesia_tpcb:init(TpcbConfig), 785: A ! fun () -> mnesia_tpcb:run(TpcbConfig) end, 786: timer:sleep(timer:seconds(Delay)), 787: 788: {ok, CPName, _NodeList} = 789: mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, 790: {ram_overrides_dump, true}]), 791: timer:sleep(timer:seconds(Delay)), 792: 793: %% Stop tpcb 794: ?verbose("Stopping TPC-B~n", []), 795: mnesia_tpcb:stop(), 796: ?match(ok, mnesia_tpcb:verify_tabs()), 797: 798: ?match(ok, mnesia:backup_checkpoint(CPName, 799: updates_during_checkpoint_activation2)), 800: timer:sleep(timer:seconds(Delay)), 801: 802: ?match(ok, mnesia:install_fallback(updates_during_checkpoint_activation2)), 803: 804: %% Stop and then start mnesia and check table consistency 805: %%?verbose("Restarting Mnesia~n", []), 806: mnesia_test_lib:kill_mnesia(Nodes), 807: file:delete(updates_during_checkpoint_activation2), 808: mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller, history]), 809: 810: ?match(ok, mnesia_tpcb:verify_tabs()), 811: ?verify_mnesia(Nodes, []). 812: 813: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 814: 815: updates_during_checkpoint_iteration_2_ram(suite) -> []; 816: updates_during_checkpoint_iteration_2_ram(Config) when is_list(Config) -> 817: updates_during_checkpoint_iteration(ram_copies, 2, Config). 818: 819: updates_during_checkpoint_iteration_2_disc(suite) -> []; 820: updates_during_checkpoint_iteration_2_disc(Config) when is_list(Config) -> 821: updates_during_checkpoint_iteration(disc_copies, 2, Config). 822: 823: updates_during_checkpoint_iteration_2_disc_only(suite) -> []; 824: updates_during_checkpoint_iteration_2_disc_only(Config) when is_list(Config) -> 825: updates_during_checkpoint_iteration(disc_only_copies, 2, Config). 826: 827: updates_during_checkpoint_iteration(ReplicaType,NodeConfig,Config) -> 828: %?verbose("updates_during_checkpoint_iteration2 at ~p~n", [self()]), 829: Delay = 5, 830: Nodes = ?acquire_nodes(NodeConfig, Config), 831: Node1 = hd(Nodes), 832: %?verbose("Mnesia info: ~p~n", [mnesia:info()]), 833: File = updates_during_checkpoint_iteration2, 834: {success, [A]} = ?start_activities([Node1]), 835: ?log("updates_during_checkpoint_iteration with ~p on ~p~n", 836: [ReplicaType, Nodes]), 837: TpcbConfig = tpcb_config_dist(ReplicaType, NodeConfig, Nodes, Config), 838: %%TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes), 839: TpcbConfigRec = list2rec(TpcbConfig, 840: record_info(fields,tab_config), 841: #tab_config{}), 842: mnesia_tpcb:init(TpcbConfig), 843: ?match(ok, mnesia_tpcb:verify_tabs()), 844: 845: {ok, CPName, _NodeList} = 846: mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, 847: {ram_overrides_dump,true}]), 848: A ! fun () -> mnesia:backup_checkpoint(CPName, File) end, 849: 850: do_changes_during_backup(TpcbConfigRec), 851: 852: ?match_receive({A,ok}), 853: 854: timer:sleep(timer:seconds(Delay)), 855: ?match(ok, mnesia:install_fallback(File)), 856: timer:sleep(timer:seconds(Delay)), 857: 858: ?match({error,{"Bad balance",_,_}}, mnesia_tpcb:verify_tabs()), 859: 860: mnesia_test_lib:kill_mnesia(Nodes), 861: mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller, history]), 862: 863: ?match(ok, mnesia_tpcb:verify_tabs()), 864: 865: ?match(ok, file:delete(File)), 866: ?verify_mnesia(Nodes, []). 867: 868: do_changes_during_backup(TpcbConfig) -> 869: loop_branches(TpcbConfig#tab_config.n_branches, 870: TpcbConfig#tab_config.n_accounts_per_branch). 871: 872: loop_branches(N_br,N_acc) when N_br >= 1 -> 873: loop_accounts(N_br,N_acc), 874: loop_branches(N_br-1,N_acc); 875: loop_branches(_,_) -> done. 876: 877: loop_accounts(N_br, N_acc) when N_acc >= 1 -> 878: A = #account{id=N_acc, branch_id=N_br, balance = 4711}, 879: ok = mnesia:dirty_write(A), 880: loop_accounts(N_br, N_acc-1); 881: 882: loop_accounts(_,_) -> done. 883: 884: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 885: 886: load_table_with_activated_checkpoint_ram(suite) -> []; 887: load_table_with_activated_checkpoint_ram(Config) when is_list(Config) -> 888: load_table_with_activated_checkpoint(ram_copies, Config). 889: 890: load_table_with_activated_checkpoint_disc(suite) -> []; 891: load_table_with_activated_checkpoint_disc(Config) when is_list(Config) -> 892: load_table_with_activated_checkpoint(disc_copies, Config). 893: 894: load_table_with_activated_checkpoint_disc_only(suite) -> []; 895: load_table_with_activated_checkpoint_disc_only(Config) when is_list(Config) -> 896: load_table_with_activated_checkpoint(disc_only_copies, Config). 897: 898: load_table_with_activated_checkpoint(Type, Config) -> 899: Nodes = ?acquire_nodes(2, Config), 900: Node1 = hd(Nodes), 901: Tab = load_test, 902: Def = [{attributes, [key, value]}, 903: {Type, Nodes}], %% ??? important that RAM ??? 904: 905: ?match({atomic, ok}, mnesia:create_table(Tab, Def)), 906: ?match(ok, mnesia:dirty_write({Tab, 1, 4711})), 907: ?match(ok, mnesia:dirty_write({Tab, 2, 42})), 908: ?match(ok, mnesia:dirty_write({Tab, 3, 256})), 909: 910: timer:sleep(timer:seconds(1)), 911: 912: {ok, CPName, _NodeList} = 913: mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, 914: {ram_overrides_dump,true}]), 915: 916: mnesia_test_lib:stop_mnesia([Node1]), 917: mnesia_test_lib:start_mnesia([Node1],[Tab]), 918: %%--- check, whether the checkpiont is attached to both replicas 919: {success, [A,B]} = ?start_activities(Nodes), 920: 921: A ! fun () -> 922: mnesia:table_info(Tab,checkpoints) 923: end, 924: ?match_receive({A,[CPName]}), 925: 926: B ! fun () -> 927: mnesia:table_info(Tab,checkpoints) 928: end, 929: ?match_receive({B,[CPName]}), 930: 931: %%--- check, whether both retainers are consistent 932: ?match(ok, mnesia:dirty_write({Tab, 1, 815})), 933: A ! fun () -> 934: mnesia:backup_checkpoint(CPName, load_table_a) 935: end, 936: ?match_receive({A,ok}), 937: B ! fun () -> 938: mnesia:backup_checkpoint(CPName, load_table_b) 939: end, 940: ?match_receive({B,ok}), 941: 942: Mod = mnesia_backup, %% Assume local files 943: List_a = view(load_table_a, Mod), 944: List_b = view(load_table_b, Mod), 945: 946: ?match(List_a, List_b), 947: 948: ?match(ok,file:delete(load_table_a)), 949: ?match(ok,file:delete(load_table_b)), 950: ?verify_mnesia(Nodes, []). 951: 952: view(Source, Mod) -> 953: View = fun(Item, Acc) -> 954: ?verbose("tab - item : ~p ~n",[Item]), 955: case Item of 956: {schema, Tab, Cs} -> %% Remove cookie information 957: NewCs = lists:keyreplace(cookie, 1, Cs, 958: {cookie, skip_cookie}), 959: Item2 = {schema, Tab, NewCs}, 960: {[Item], [Item2|Acc]}; 961: _ -> 962: {[Item], [Item|Acc]} 963: end 964: end, 965: {ok,TabList} = 966: mnesia:traverse_backup(Source, Mod, dummy, read_only, View, []), 967: lists:sort(TabList). 968: 969: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 970: 971: add_table_copy_to_table_checkpoint_ram(suite) -> []; 972: add_table_copy_to_table_checkpoint_ram(Config) when is_list(Config) -> 973: add_table_copy_to_table_with_activated_checkpoint(ram_copies, Config). 974: 975: add_table_copy_to_table_checkpoint_disc(suite) -> []; 976: add_table_copy_to_table_checkpoint_disc(Config) when is_list(Config) -> 977: add_table_copy_to_table_with_activated_checkpoint(disc_copies, Config). 978: 979: add_table_copy_to_table_checkpoint_disc_only(suite) -> []; 980: add_table_copy_to_table_checkpoint_disc_only(Config) when is_list(Config) -> 981: add_table_copy_to_table_with_activated_checkpoint(disc_only_copies, Config). 982: 983: add_table_copy_to_table_with_activated_checkpoint(Type,Config) -> 984: Nodes = ?acquire_nodes(2, Config), 985: %?verbose("NODES = ~p ~n",[Nodes]), 986: [Node1,Node2] = Nodes, 987: 988: Tab = add_test, 989: Def = [{attributes, [key, value]}, 990: {Type, [Node1]}], %% ??? important that RAM ??? 991: 992: ?match({atomic, ok}, mnesia:create_table(Tab, Def)), 993: ?match(ok, mnesia:dirty_write({Tab, 1, 4711})), 994: ?match(ok, mnesia:dirty_write({Tab, 2, 42})), 995: ?match(ok, mnesia:dirty_write({Tab, 3, 256})), 996: 997: {ok, CPName, _NodeList} = 998: mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, 999: {ram_overrides_dump,true}]), 1000: 1001: ?match({atomic,ok},mnesia:add_table_copy(Tab,Node2,ram_copies)), 1002: 1003: %%--- check, whether the checkpiont is attached to both replicas 1004: {success, [A,B]} = ?start_activities(Nodes), 1005: 1006: A ! fun () -> 1007: mnesia:table_info(Tab,checkpoints) 1008: end, 1009: ?match_receive({A,[CPName]}), 1010: 1011: B ! fun () -> 1012: mnesia:table_info(Tab,checkpoints) 1013: end, 1014: ?match_receive({B,[CPName]}), 1015: 1016: %%--- check, whether both retainers are consistent 1017: 1018: ?match(ok, mnesia:dirty_write({Tab, 1, 815})), 1019: ?match(ok, mnesia:dirty_write({Tab, 2, 815})), 1020: 1021: A ! fun () -> 1022: mnesia:backup_checkpoint(CPName, add_table_a) 1023: end, 1024: ?match_receive({A,ok}), 1025: B ! fun () -> 1026: mnesia:backup_checkpoint(CPName, add_table_b) 1027: end, 1028: ?match_receive({B,ok}), 1029: 1030: Mod = mnesia_backup, %% Assume local files 1031: 1032: List_a = view(add_table_a, Mod), 1033: List_b = view(add_table_b, Mod), 1034: 1035: ?match(List_a, List_b), 1036: 1037: ?match(ok,file:delete(add_table_a)), 1038: ?match(ok, file:delete(add_table_b)), 1039: ?verify_mnesia(Nodes, []). 1040: 1041: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1042: 1043: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1044: 1045: inst_fallback_process_dies(suite) -> 1046: []; 1047: inst_fallback_process_dies(Config) when is_list(Config) -> 1048: ?is_debug_compiled, 1049: 1050: Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1051: {success, [A,_B,_C]} = ?start_activities(Nodes), 1052: 1053: TestPid = self(), 1054: DebugId = {mnesia_bup, fallback_receiver_loop, pre_swap}, 1055: DebugFun = 1056: fun(PrevContext, _EvalContext) -> 1057: ?verbose("fallback_receiver_loop - pre_swap pid ~p #~p~n", 1058: [self(),PrevContext]), 1059: TestPid ! {self(),fallback_preswap}, 1060: case receive_messages([fallback_continue]) of 1061: [{TestPid,fallback_continue}] -> 1062: ?deactivate_debug_fun(DebugId), 1063: PrevContext+1 1064: end 1065: end, 1066: ?activate_debug_fun(DebugId, DebugFun, 1), 1067: 1068: Tab = install_table, 1069: Def = [{attributes, [key, value]}, {disc_copies, Nodes}], 1070: 1071: ?match({atomic, ok}, mnesia:create_table(Tab, Def)), 1072: 1073: ?match(ok, mnesia:dirty_write({Tab, 1, 4711})), 1074: ?match(ok, mnesia:dirty_write({Tab, 2, 42})), 1075: ?match(ok, mnesia:dirty_write({Tab, 3, 256})), 1076: 1077: {ok, CPName, _NodeList} = 1078: mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, 1079: {ram_overrides_dump,true}]), 1080: 1081: ?match(ok, mnesia:backup_checkpoint(CPName, install_backup)), 1082: 1083: A ! fun() -> mnesia:install_fallback(install_backup) end, 1084: [{AnsPid,fallback_preswap}] = receive_messages([fallback_preswap]), 1085: exit(A, kill), 1086: AnsPid ! {self(), fallback_continue}, 1087: ?match_receive({'EXIT', A, killed}), 1088: timer:sleep(2000), %% Wait till fallback is installed everywhere 1089: 1090: mnesia_test_lib:kill_mnesia(Nodes), 1091: ?verbose("~n---->Mnesia is stopped everywhere<-----~n", []), 1092: ?match([], mnesia_test_lib:start_mnesia(Nodes,[Tab])), 1093: 1094: check_data(Nodes, Tab), 1095: ?match(ok, file:delete(install_backup)), 1096: ?verify_mnesia(Nodes, []). 1097: 1098: check_data([N1 | R], Tab) -> 1099: ?match([{Tab, 1, 4711}], rpc:call(N1, mnesia, dirty_read, [{Tab, 1}])), 1100: ?match([{Tab, 2, 42}], rpc:call(N1, mnesia, dirty_read, [{Tab, 2}])), 1101: ?match([{Tab, 3, 256}], rpc:call(N1, mnesia, dirty_read, [{Tab, 3}])), 1102: check_data(R, Tab); 1103: check_data([], _Tab) -> 1104: ok. 1105: 1106: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1107: 1108: fatal_when_inconsistency(suite) -> 1109: []; 1110: fatal_when_inconsistency(Config) when is_list(Config) -> 1111: ?is_debug_compiled, 1112: 1113: [Node1, Node2, Node3] = Nodes = 1114: ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), 1115: {success, [A,_B,_C]} = ?start_activities(Nodes), 1116: 1117: TestPid = self(), 1118: DebugId = {mnesia_bup, fallback_receiver_loop, pre_swap}, 1119: DebugFun = 1120: fun(PrevContext, _EvalContext) -> 1121: ?verbose("fallback_receiver_loop - pre_swap pid ~p #~p~n", 1122: [self(),PrevContext]), 1123: TestPid ! {self(),fallback_preswap}, 1124: case receive_messages([fallback_continue]) of 1125: [{TestPid,fallback_continue}] -> 1126: ?deactivate_debug_fun(DebugId), 1127: PrevContext+1 1128: end 1129: end, 1130: ?activate_debug_fun(DebugId, DebugFun, 1), 1131: 1132: Tab = install_table, 1133: Def = [{attributes, [key, value]}, {disc_copies, Nodes}], 1134: 1135: ?match({atomic, ok}, mnesia:create_table(Tab, Def)), 1136: 1137: ?match(ok, mnesia:dirty_write({Tab, 1, 4711})), 1138: ?match(ok, mnesia:dirty_write({Tab, 2, 42})), 1139: ?match(ok, mnesia:dirty_write({Tab, 3, 256})), 1140: 1141: {ok, CPName, _NodeList} = 1142: mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, 1143: {ram_overrides_dump,true}]), 1144: 1145: ?match(ok, mnesia:backup_checkpoint(CPName, install_backup)), 1146: ?match(ok, mnesia:dirty_write({Tab, 2, 42424242})), 1147: 1148: A ! fun() -> 1149: mnesia:install_fallback(install_backup) 1150: end, 1151: 1152: [{AnsPid,fallback_preswap}] = receive_messages([fallback_preswap]), 1153: exit(AnsPid, kill), %% Kill install-fallback on local node will 1154: AnsPid ! {self(), fallback_continue}, 1155: ?deactivate_debug_fun(DebugId), 1156: 1157: ?match_receive({A,{error,{"Cannot install fallback", 1158: {'EXIT',AnsPid,killed}}}}), 1159: mnesia_test_lib:kill_mnesia(Nodes), 1160: ?verbose("EXPECTING FATAL from 2 nodes WITH CORE DUMP~n", []), 1161: 1162: ?match([], mnesia_test_lib:start_mnesia([Node1],[])), 1163: is_running(Node1, yes), 1164: ?match([{Node2, mnesia, _}], mnesia_test_lib:start_mnesia([Node2],[])), 1165: is_running(Node2, no), 1166: ?match([{Node3, mnesia, _}], mnesia_test_lib:start_mnesia([Node3],[])), 1167: is_running(Node3, no), 1168: mnesia_test_lib:kill_mnesia(Nodes), 1169: 1170: ?match(ok, mnesia:install_fallback(install_backup)), 1171: mnesia_test_lib:start_mnesia(Nodes,[Tab]), 1172: 1173: check_data(Nodes, Tab), 1174: 1175: ?match(ok,file:delete(install_backup)), 1176: ?verify_mnesia(Nodes, []). 1177: 1178: is_running(Node, Shouldbe) -> 1179: timer:sleep(1000), 1180: Running = rpc:call(Node, mnesia, system_info, [is_running]), 1181: case Running of 1182: Shouldbe -> ok; 1183: _ -> is_running(Node, Shouldbe) 1184: end. 1185: 1186: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1187: 1188: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1189: 1190: after_delete(doc) -> 1191: ["interrupt the uninstall after deletion of ", 1192: "fallback files - there shall be no fallback"]; 1193: after_delete(suite) -> []; 1194: after_delete(Config) when is_list(Config) -> 1195: do_uninstall(Config, post_delete). 1196: 1197: %%%%%%%%%%%%%%%%%%%%%%%%% 1198: 1199: do_uninstall(Config,DebugPoint) -> 1200: ?is_debug_compiled, 1201: 1202: Nodes = ?acquire_nodes(3, Config), 1203: %%?verbose("NODES = ~p ~n",[Nodes]), 1204: 1205: {success, [P1,P2,P3]} = ?start_activities(Nodes), 1206: 1207: NP1 = node(P1), 1208: NP2 = node(P2), 1209: 1210: {A,B,C} = case node() of 1211: NP1 -> 1212: %%?verbose("first case ~n"), 1213: {P3,P2,P1}; 1214: NP2 -> 1215: %%?verbose("second case ~n"), 1216: {P3, P1, P2}; 1217: _ -> 1218: { P1, P2, P3} 1219: end, 1220: 1221: Node1 = node(A), 1222: Node2 = node(B), 1223: Node3 = node(C), 1224: 1225: ?verbose(" A pid:~p node:~p ~n",[A,Node1]), 1226: ?verbose(" B pid:~p node:~p ~n",[B,Node2]), 1227: ?verbose(" C pid:~p node:~p ~n",[C,Node3]), 1228: 1229: 1230: TestPid = self(), 1231: %%?verbose("TestPid : ~p~n",[TestPid]), 1232: DebugId = {mnesia_bup, uninstall_fallback2, DebugPoint}, 1233: DebugFun = fun(PrevContext, _EvalContext) -> 1234: ?verbose("uninstall_fallback pid ~p #~p~n" 1235: ,[self(),PrevContext]), 1236: TestPid ! {self(),uninstall_predelete}, 1237: case receive_messages([uninstall_continue]) of 1238: [{TestPid,uninstall_continue}] -> 1239: ?deactivate_debug_fun(DebugId), 1240: %%?verbose("uninstall_fallback continues~n"), 1241: PrevContext+1 1242: end 1243: end, 1244: ?remote_activate_debug_fun(Node1,DebugId, DebugFun, 1), 1245: 1246: Tab = install_table, 1247: Def = [{attributes, [key, value]}, 1248: {ram_copies, Nodes}], %% necessary to test different types ??? 1249: 1250: ?match({atomic, ok}, mnesia:create_table(Tab, Def)), 1251: 1252: ?match(ok, mnesia:dirty_write({Tab, 1, 4711})), 1253: ?match(ok, mnesia:dirty_write({Tab, 2, 42})), 1254: ?match(ok, mnesia:dirty_write({Tab, 3, 256})), 1255: 1256: {ok, CPName, _NodeList} = 1257: mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, 1258: {ram_overrides_dump,true}]), 1259: 1260: ?match(ok, mnesia:backup_checkpoint(CPName,install_backup)), 1261: timer:sleep(timer:seconds(1)), 1262: 1263: A ! fun () -> 1264: mnesia:install_fallback(install_backup) 1265: end, 1266: ?match_receive({A,ok}), 1267: 1268: A ! fun () -> 1269: mnesia:uninstall_fallback() 1270: end, 1271: %% 1272: %% catch the debug entry in mnesia and kill one Mnesia node 1273: %% 1274: 1275: 1276: [{AnsPid,uninstall_predelete}] = receive_messages([uninstall_predelete]), 1277: 1278: ?verbose("AnsPid : ~p~n",[AnsPid]), 1279: 1280: mnesia_test_lib:kill_mnesia([Node2]), 1281: timer:sleep(timer:seconds(1)), 1282: 1283: AnsPid ! {self(),uninstall_continue}, 1284: 1285: ?match_receive({A,ok}), 1286: 1287: mnesia_test_lib:kill_mnesia(Nodes) , 1288: mnesia_test_lib:start_mnesia(Nodes,[Tab]), 1289: 1290: A ! fun () -> 1291: R1 = mnesia:dirty_read({Tab,1}), 1292: R2 = mnesia:dirty_read({Tab,2}), 1293: R3 = mnesia:dirty_read({Tab,3}), 1294: {R1,R2,R3} 1295: end, 1296: ?match_receive({ A, {[],[],[]} }), 1297: 1298: B ! fun () -> 1299: R1 = mnesia:dirty_read({Tab,1}), 1300: R2 = mnesia:dirty_read({Tab,2}), 1301: R3 = mnesia:dirty_read({Tab,3}), 1302: {R1,R2,R3} 1303: end, 1304: ?match_receive({ B, {[],[],[]} }), 1305: 1306: C ! fun () -> 1307: R1 = mnesia:dirty_read({Tab,1}), 1308: R2 = mnesia:dirty_read({Tab,2}), 1309: R3 = mnesia:dirty_read({Tab,3}), 1310: {R1,R2,R3} 1311: end, 1312: ?match_receive({ C, {[],[],[]} }), 1313: 1314: ?match(ok,file:delete(install_backup)), 1315: ?verify_mnesia(Nodes, []). 1316: 1317: 1318: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1319: 1320: %%%%%%%%%%%%%%% 1321: 1322: cause_switch_before(doc) -> 1323: ["interrupt the backup before iterating the retainer"]; 1324: cause_switch_before(suite) -> []; 1325: cause_switch_before(Config) when is_list(Config) -> 1326: do_something_during_backup(cause_switch,pre,Config). 1327: 1328: %%%%%%%%%%%%%%% 1329: 1330: cause_switch_after(doc) -> 1331: ["interrupt the backup after iterating the retainer"]; 1332: cause_switch_after(suite) -> []; 1333: cause_switch_after(Config) when is_list(Config) -> 1334: do_something_during_backup(cause_switch,post,Config). 1335: 1336: 1337: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1338: 1339: %%%%%%%%%%%%%%%%%% 1340: 1341: cause_abort_before(doc) -> 1342: ["interrupt the backup before iterating the retainer"]; 1343: 1344: cause_abort_before(suite) -> []; 1345: cause_abort_before(Config) when is_list(Config) -> 1346: do_something_during_backup(cause_abort,pre,Config). 1347: 1348: %%%%%%%%%%%%%%%%%% 1349: 1350: cause_abort_after(doc) -> 1351: ["interrupt the backup after iterating the retainer"]; 1352: 1353: cause_abort_after(suite) -> []; 1354: cause_abort_after(Config) when is_list(Config) -> 1355: do_something_during_backup(cause_abort,post,Config). 1356: 1357: 1358: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1359: 1360: %%%%%%%%%%%%% 1361: 1362: change_schema_before(doc) -> 1363: ["interrupt the backup before iterating the retainer"]; 1364: change_schema_before(suite) -> []; 1365: change_schema_before(Config) when is_list(Config) -> 1366: do_something_during_backup(change_schema,pre,Config). 1367: 1368: %%%%%%%%%%%%%%%% 1369: 1370: change_schema_after(doc) -> 1371: ["interrupt the backup after iterating the retainer"]; 1372: change_schema_after(suite) -> []; 1373: change_schema_after(Config) when is_list(Config) -> 1374: do_something_during_backup(change_schema,post,Config). 1375: 1376: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 1377: 1378: do_something_during_backup(Action,DebugPoint,Config) -> 1379: ?is_debug_compiled, 1380: 1381: Nodes = ?acquire_nodes(3, Config), 1382: 1383: {success, [A,B,C]} = ?start_activities(Nodes), 1384: 1385: Node1 = node(A), 1386: Node2 = node(B), 1387: Node3 = node(C), 1388: 1389: TestPid = self(), 1390: %%?verbose("TestPid : ~p~n",[TestPid]), 1391: 1392: Tab = interrupt_table, 1393: Bak = interrupt_backup, 1394: Def = [{attributes, [key, value]}, 1395: {ram_copies, [Node2,Node3]}], 1396: %% necessary to test different types ??? 1397: 1398: ?match({atomic, ok}, mnesia:create_table(Tab, Def)), 1399: 1400: 1401: 1402: DebugId = {mnesia_log, tab_copier, DebugPoint}, 1403: DebugFun = fun(PrevContext, EvalContext) -> 1404: ?verbose("interrupt backup pid ~p #~p ~n context ~p ~n" 1405: ,[self(),PrevContext,EvalContext]), 1406: TestPid ! {self(),interrupt_backup_pre}, 1407: global:set_lock({{lock_for_backup, Tab}, self()}, 1408: Nodes, 1409: infinity), 1410: 1411: %%?verbose("interrupt backup - continues ~n"), 1412: ?deactivate_debug_fun(DebugId), 1413: PrevContext+1 1414: end, 1415: ?remote_activate_debug_fun(Node1,DebugId, DebugFun, 1), 1416: 1417: ?match(ok, mnesia:dirty_write({Tab, 1, 4711})), 1418: ?match(ok, mnesia:dirty_write({Tab, 2, 42})), 1419: ?match(ok, mnesia:dirty_write({Tab, 3, 256})), 1420: 1421: {ok, CPName, _NodeList} = 1422: mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, 1423: {ram_overrides_dump,true}]), 1424: 1425: A ! fun () -> 1426: %%?verbose("node: ~p pid: ~p ~n",[node(),self()]), 1427: mnesia:table_info(Tab,where_to_read) 1428: end, 1429: 1430: ReadNode_a = receive { A, ReadNode_a_tmp } -> ReadNode_a_tmp end, 1431: ?verbose("ReadNode ~p ~n",[ReadNode_a]), 1432: 1433: global:set_lock({{lock_for_backup, Tab}, self()}, Nodes, infinity), 1434: 1435: A ! fun () -> %% A shall perform the backup, so the test proc is 1436: %% able to do further actions in between 1437: mnesia:backup_checkpoint(CPName, Bak) 1438: end, 1439: 1440: %% catch the debug function of mnesia, stop the backup process 1441: %% kill the node ReadNode_a and continue the backup process 1442: %% As there is a second replica of the table, the backup shall continue 1443: 1444: case receive_messages([interrupt_backup_pre]) of 1445: [{_AnsPid,interrupt_backup_pre}] -> ok 1446: end, 1447: 1448: case Action of 1449: cause_switch -> 1450: mnesia_test_lib:kill_mnesia([ReadNode_a]), 1451: timer:sleep(timer:seconds(1)); 1452: cause_abort -> 1453: mnesia_test_lib:kill_mnesia([Node2,Node3]), 1454: timer:sleep(timer:seconds(1)); 1455: change_schema -> 1456: Tab2 = second_interrupt_table, 1457: Def2 = [{attributes, [key, value]}, 1458: {ram_copies, Nodes}], 1459: 1460: ?match({atomic, ok}, mnesia:create_table(Tab2, Def2)) 1461: end, 1462: 1463: %% AnsPid ! {self(),interrupt_backup_continue}, 1464: global:del_lock({{lock_for_backup, Tab}, self()}, Nodes), 1465: 1466: case Action of 1467: cause_abort -> 1468: 1469: %% answer of A when finishing the backup 1470: ?match_receive({A,{error, _}}), 1471: 1472: ?match({error,{"Cannot install fallback",_}}, 1473: mnesia:install_fallback(Bak)); 1474: _ -> %% cause_switch, change_schema 1475: 1476: ?match_receive({A,ok}), %% answer of A when finishing the backup 1477: 1478: %% send a fun to that node where mnesia is still running 1479: WritePid = case ReadNode_a of 1480: Node2 -> C; %% node(C) == Node3 1481: Node3 -> B 1482: end, 1483: WritePid ! fun () -> 1484: ?match(ok, mnesia:dirty_write({Tab, 1, 815})), 1485: ?match(ok, mnesia:dirty_write({Tab, 2, 816})), 1486: ok 1487: end, 1488: ?match_receive({ WritePid, ok }), 1489: ?match(ok, mnesia:install_fallback(Bak)) 1490: end, 1491: 1492: %% Stop and then start mnesia and check table consistency 1493: %%?verbose("Restarting Mnesia~n", []), 1494: mnesia_test_lib:kill_mnesia(Nodes), 1495: mnesia_test_lib:start_mnesia(Nodes,[Tab]), 1496: 1497: case Action of 1498: cause_switch -> 1499: %% the backup should exist 1500: cross_check_tables([A,B,C],Tab,{[{Tab,1,4711}], 1501: [{Tab,2,42}], 1502: [{Tab,3,256}] }), 1503: ?match(ok,file:delete(Bak)); 1504: cause_abort -> 1505: %% the backup should NOT exist 1506: cross_check_tables([A,B,C],Tab,{[],[],[]}), 1507: %% file does not exist 1508: ?match({error, _},file:delete(Bak)); 1509: change_schema -> 1510: %% the backup should exist 1511: cross_check_tables([A,B,C],Tab,{[{Tab,1,4711}], 1512: [{Tab,2,42}], 1513: [{Tab,3,256}] }), 1514: ?match(ok,file:delete(Bak)) 1515: end, 1516: ?verify_mnesia(Nodes, []). 1517: 1518: %% check the contents of the table 1519: cross_check_tables([],_tab,_elements) -> ok; 1520: cross_check_tables([Pid|Rest],Tab,{Val1,Val2,Val3}) -> 1521: Pid ! fun () -> 1522: R1 = mnesia:dirty_read({Tab,1}), 1523: R2 = mnesia:dirty_read({Tab,2}), 1524: R3 = mnesia:dirty_read({Tab,3}), 1525: {R1,R2,R3} 1526: end, 1527: ?match_receive({ Pid, {Val1, Val2, Val3 } }), 1528: cross_check_tables(Rest,Tab,{Val1,Val2,Val3} ).