1: %%
    2: %% %CopyrightBegin%
    3: %%
    4: %% Copyright Ericsson AB 1997-2013. All Rights Reserved.
    5: %%
    6: %% The contents of this file are subject to the Erlang Public License,
    7: %% Version 1.1, (the "License"); you may not use this file except in
    8: %% compliance with the License. You should have received a copy of the
    9: %% Erlang Public License along with this software. If not, it can be
   10: %% retrieved online at http://www.erlang.org/.
   11: %%
   12: %% Software distributed under the License is distributed on an "AS IS"
   13: %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
   14: %% the License for the specific language governing rights and limitations
   15: %% under the License.
   16: %%
   17: %% %CopyrightEnd%
   18: %%
   19: 
   20: %%
   21: -module(mnesia_consistency_test).
   22: -author('hakan@erix.ericsson.se').
   23: -compile([export_all]).
   24: 
   25: -include("mnesia_test_lib.hrl").
   26: 
   27: init_per_testcase(Func, Conf) ->
   28:     mnesia_test_lib:init_per_testcase(Func, Conf).
   29: 
   30: end_per_testcase(Func, Conf) ->
   31:     mnesia_test_lib:end_per_testcase(Func, Conf).
   32: 
   33: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
   34: all() -> 
   35:     [{group, consistency_after_restart},
   36:      {group, consistency_after_dump_tables},
   37:      {group, consistency_after_add_replica},
   38:      {group, consistency_after_del_replica},
   39:      {group, consistency_after_move_replica},
   40:      {group, consistency_after_transform_table},
   41:      consistency_after_change_table_copy_type,
   42:      {group, consistency_after_restore},
   43:      consistency_after_rename_of_node,
   44:      {group, checkpoint_retainer_consistency},
   45:      {group, backup_consistency}].
   46: 
   47: groups() -> 
   48:     [{consistency_after_restart, [],
   49:       [consistency_after_restart_1_ram,
   50:        consistency_after_restart_1_disc,
   51:        consistency_after_restart_1_disc_only,
   52:        consistency_after_restart_2_ram,
   53:        consistency_after_restart_2_disc,
   54:        consistency_after_restart_2_disc_only]},
   55:      {consistency_after_dump_tables, [],
   56:       [consistency_after_dump_tables_1_ram,
   57:        consistency_after_dump_tables_2_ram]},
   58:      {consistency_after_add_replica, [],
   59:       [consistency_after_add_replica_2_ram,
   60:        consistency_after_add_replica_2_disc,
   61:        consistency_after_add_replica_2_disc_only,
   62:        consistency_after_add_replica_3_ram,
   63:        consistency_after_add_replica_3_disc,
   64:        consistency_after_add_replica_3_disc_only]},
   65:      {consistency_after_del_replica, [],
   66:       [consistency_after_del_replica_2_ram,
   67:        consistency_after_del_replica_2_disc,
   68:        consistency_after_del_replica_2_disc_only,
   69:        consistency_after_del_replica_3_ram,
   70:        consistency_after_del_replica_3_disc,
   71:        consistency_after_del_replica_3_disc_only]},
   72:      {consistency_after_move_replica, [],
   73:       [consistency_after_move_replica_2_ram,
   74:        consistency_after_move_replica_2_disc,
   75:        consistency_after_move_replica_2_disc_only,
   76:        consistency_after_move_replica_3_ram,
   77:        consistency_after_move_replica_3_disc,
   78:        consistency_after_move_replica_3_disc_only]},
   79:      {consistency_after_transform_table, [],
   80:       [consistency_after_transform_table_ram,
   81:        consistency_after_transform_table_disc,
   82:        consistency_after_transform_table_disc_only]},
   83:      {consistency_after_fallback, [],
   84:       [consistency_after_fallback_2_ram,
   85:        consistency_after_fallback_2_disc,
   86:        consistency_after_fallback_2_disc_only,
   87:        consistency_after_fallback_3_ram,
   88:        consistency_after_fallback_3_disc,
   89:        consistency_after_fallback_3_disc_only]},
   90:      {consistency_after_restore, [],
   91:       [consistency_after_restore_clear_ram,
   92:        consistency_after_restore_clear_disc,
   93:        consistency_after_restore_clear_disc_only,
   94:        consistency_after_restore_recreate_ram,
   95:        consistency_after_restore_recreate_disc,
   96:        consistency_after_restore_recreate_disc_only]},
   97:      {checkpoint_retainer_consistency, [],
   98:       [{group, updates_during_checkpoint_activation},
   99:        {group, updates_during_checkpoint_iteration},
  100:        {group, load_table_with_activated_checkpoint},
  101:        {group, add_table_copy_to_table_checkpoint},
  102:        {group, consistency_after_fallback}
  103:       ]},
  104:      {updates_during_checkpoint_activation, [],
  105:       [updates_during_checkpoint_activation_1_ram,
  106:        updates_during_checkpoint_activation_1_disc,
  107:        updates_during_checkpoint_activation_1_disc_only,
  108:        updates_during_checkpoint_activation_2_ram,
  109:        updates_during_checkpoint_activation_2_disc,
  110:        updates_during_checkpoint_activation_2_disc_only,
  111:        updates_during_checkpoint_activation_3_ram,
  112:        updates_during_checkpoint_activation_3_disc,
  113:        updates_during_checkpoint_activation_3_disc_only]},
  114:      {updates_during_checkpoint_iteration, [],
  115:       [updates_during_checkpoint_iteration_2_ram,
  116:        updates_during_checkpoint_iteration_2_disc,
  117:        updates_during_checkpoint_iteration_2_disc_only]},
  118:      {load_table_with_activated_checkpoint, [],
  119:       [load_table_with_activated_checkpoint_ram,
  120:        load_table_with_activated_checkpoint_disc,
  121:        load_table_with_activated_checkpoint_disc_only]},
  122:      {add_table_copy_to_table_checkpoint, [],
  123:       [add_table_copy_to_table_checkpoint_ram,
  124:        add_table_copy_to_table_checkpoint_disc,
  125:        add_table_copy_to_table_checkpoint_disc_only]},
  126:      {backup_consistency, [],
  127:       [{group, interupted_install_fallback},
  128:        {group, interupted_uninstall_fallback},
  129:        {group, mnesia_down_during_backup_causes_switch},
  130:        {group, mnesia_down_during_backup_causes_abort},
  131:        {group, schema_transactions_during_backup}]},
  132:      {interupted_install_fallback, [],
  133:       [inst_fallback_process_dies, fatal_when_inconsistency]},
  134:      {interupted_uninstall_fallback, [], [after_delete]},
  135:      {mnesia_down_during_backup_causes_switch, [],
  136:       [cause_switch_before, cause_switch_after]},
  137:      {mnesia_down_during_backup_causes_abort, [],
  138:       [cause_abort_before, cause_abort_after]},
  139:      {schema_transactions_during_backup, [],
  140:       [change_schema_before, change_schema_after]}].
  141: 
  142: init_per_group(_GroupName, Config) ->
  143:     Config.
  144: 
  145: end_per_group(_GroupName, Config) ->
  146:     Config.
  147: 
  148: 
  149: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  150: %
  151: %  stolen from mnesia_tpcb.erl:
  152: 
  153: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  154: %%% Account record, total size must be at least 100 bytes
  155: 
  156: -define(ACCOUNT_FILLER,
  157:         {123456789012345678901234567890123456789012345678901234567890,
  158:          123456789012345678901234567890123456789012345678901234567890,
  159:          123456789012345678901234567890123456789012345678901234}).
  160: 
  161: -record(account,
  162:        {
  163:         id           = 0, %% Unique account id
  164:         branch_id    = 0, %% Branch where the account is held
  165:         balance      = 0, %% Account balance
  166:         filler       = ?ACCOUNT_FILLER  %% Gap filler to ensure size >= 100 bytes
  167:        }).
  168: 
  169: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  170: %%% Branch record, total size must be at least 100 bytes
  171: 
  172: -define(BRANCH_FILLER,
  173:         {123456789012345678901234567890123456789012345678901234567890,
  174:          123456789012345678901234567890123456789012345678901234567890,
  175:          123456789012345678901234567890123456789012345678901234567890}).
  176: 
  177: 
  178: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  179: %%% Teller record, total size must be at least 100 bytes
  180: 
  181: -define(TELLER_FILLER,
  182:         {123456789012345678901234567890123456789012345678901234567890,
  183:          123456789012345678901234567890123456789012345678901234567890,
  184:          1234567890123456789012345678901234567890123456789012345678}).
  185: 
  186: 
  187: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  188: %%% History record, total size must be at least 50 bytes
  189: 
  190: -define(HISTORY_FILLER, 1234567890).
  191: 
  192: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  193: -record(tab_config,
  194:         {
  195:          db_nodes = [node()],
  196:          replica_nodes = [node()],
  197:          replica_type = ram_copies,
  198:          use_running_mnesia = false,
  199:          n_branches = 1,
  200:          n_tellers_per_branch = 10, %% Must be 10
  201:          n_accounts_per_branch = 100000, %% Must be 100000
  202:          branch_filler = ?BRANCH_FILLER,
  203:          account_filler = ?ACCOUNT_FILLER,
  204:          teller_filler = ?TELLER_FILLER
  205:         }).
  206: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  207: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  208: %
  209: %  stolen from mnesia_tpcb.erl:
  210: 
  211: list2rec(List, Fields, DefaultTuple) ->
  212:     [Name|Defaults] = tuple_to_list(DefaultTuple),
  213:     List2 = list2rec(List, Fields, Defaults, []),
  214:     list_to_tuple([Name] ++ List2).
  215: 
  216: list2rec(_List, [], [], Acc) ->
  217:     Acc;
  218: list2rec(List, [F|Fields], [D|Defaults], Acc) ->
  219:     {Val, List2} =
  220:         case lists:keysearch(F, 1, List) of
  221:             false ->
  222:                 {D, List};
  223:             {value, {F, NewVal}} ->
  224:                 {NewVal, lists:keydelete(F, 1, List)}
  225:         end,
  226:     list2rec(List2, Fields, Defaults, Acc ++ [Val]).
  227: 
  228: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  229: 
  230: tpcb_config(ReplicaType, _NodeConfig, Nodes, NoDriverNodes) ->
  231:     [{n_branches, 10},
  232:      {n_drivers_per_node, 10},
  233:      {replica_nodes, Nodes},
  234:      {driver_nodes, Nodes -- NoDriverNodes},
  235:      {use_running_mnesia, true},
  236:      {report_interval, infinity},
  237:      {n_accounts_per_branch, 100},
  238:      {replica_type, ReplicaType},
  239:      {reuse_history_id, true}].
  240: 
  241: %% Stolen from mnesia_tpcb:dist
  242: tpcb_config_dist(ReplicaType, _NodeConfig, Nodes, _Config) ->
  243:     [{db_nodes, Nodes},
  244:      {driver_nodes, Nodes},
  245:      {replica_nodes, Nodes},
  246:      {n_drivers_per_node, 10},
  247:      {n_branches, 1},
  248:      {use_running_mnesia, true},
  249:      {n_accounts_per_branch, 10},
  250:      {replica_type, ReplicaType},
  251:      {stop_after, timer:minutes(15)},
  252:      {report_interval, timer:seconds(10)},
  253:      {reuse_history_id, true}].
  254: 
  255: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  256: %
  257: %  stolen from mnesia_recovery_test.erl:
  258: 
  259: receive_messages([]) -> [];
  260: receive_messages(ListOfMsgs) ->
  261:     receive 
  262:         {Pid, Msg} ->     
  263:             case lists:member(Msg, ListOfMsgs) of
  264:                 false -> 
  265:                     ?warning("I (~p) have received unexpected msg~n ~p ~n",
  266:                         [self(),{Pid, Msg}]),
  267:                     receive_messages(ListOfMsgs);
  268:                 true -> 
  269:                     ?verbose("I (~p) got msg ~p from ~p ~n", [self(),Msg, Pid]),
  270:                     [{Pid, Msg} | receive_messages(ListOfMsgs -- [Msg])]
  271:             end;
  272:         Else -> ?warning("Recevied unexpected Msg~n ~p ~n", [Else])
  273:     after timer:minutes(3) -> 
  274:             ?error("Timeout in receive msgs while waiting for ~p~n", 
  275:                    [ListOfMsgs])
  276:     end.  
  277: 
  278: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  279: 
  280: consistency_after_restart_1_ram(suite) -> [];
  281: consistency_after_restart_1_ram(Config) when is_list(Config) ->
  282:     consistency_after_restart(ram_copies, 2, Config).
  283: 
  284: consistency_after_restart_1_disc(suite) -> [];
  285: consistency_after_restart_1_disc(Config) when is_list(Config) ->
  286:     consistency_after_restart(disc_copies, 2, Config).
  287: 
  288: consistency_after_restart_1_disc_only(suite) -> [];
  289: consistency_after_restart_1_disc_only(Config) when is_list(Config) ->
  290:     consistency_after_restart(disc_only_copies, 2, Config).
  291: 
  292: consistency_after_restart_2_ram(suite) -> [];
  293: consistency_after_restart_2_ram(Config) when is_list(Config) ->
  294:     consistency_after_restart(ram_copies, 3, Config).
  295: 
  296: consistency_after_restart_2_disc(suite) -> [];
  297: consistency_after_restart_2_disc(Config) when is_list(Config) ->
  298:     consistency_after_restart(disc_copies, 3, Config).
  299: 
  300: consistency_after_restart_2_disc_only(suite) -> [];
  301: consistency_after_restart_2_disc_only(Config) when is_list(Config) ->
  302:     consistency_after_restart(disc_only_copies, 3, Config).
  303: 
  304: consistency_after_restart(ReplicaType, NodeConfig, Config) ->
  305:     [Node1 | _] = Nodes = ?acquire_nodes(NodeConfig, Config),
  306:     {success, [A]} = ?start_activities([Node1]),
  307:     ?log("consistency_after_restart with ~p on ~p~n",
  308:          [ReplicaType, Nodes]),
  309:     TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, [Node1]),
  310:     mnesia_tpcb:init(TpcbConfig),
  311:     A ! fun () -> mnesia_tpcb:run(TpcbConfig) end,
  312:     timer:sleep(timer:seconds(10)),
  313:     mnesia_test_lib:kill_mnesia([Node1]),
  314:     %% Start and wait for tables to be loaded on all nodes
  315:     timer:sleep(timer:seconds(3)),
  316:     ?match([], mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller, history])), 
  317:     mnesia_tpcb:stop(),
  318:     ?match(ok, mnesia_tpcb:verify_tabs()),
  319:     ?verify_mnesia(Nodes, []).
  320: 
  321: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  322: 
  323: consistency_after_dump_tables_1_ram(suite) -> [];
  324: consistency_after_dump_tables_1_ram(Config) when is_list(Config) ->
  325:      consistency_after_dump_tables(ram_copies, 1, Config).
  326: 
  327: consistency_after_dump_tables_2_ram(suite) -> [];
  328: consistency_after_dump_tables_2_ram(Config) when is_list(Config) ->
  329:     consistency_after_dump_tables(ram_copies, 2, Config).
  330: 
  331: consistency_after_dump_tables(ReplicaType, NodeConfig, Config) ->
  332:     [Node1 | _] = Nodes = ?acquire_nodes(NodeConfig, Config),
  333:     {success, [A]} = ?start_activities([Node1]),
  334:     ?log("consistency_after_dump_tables with ~p on ~p~n",
  335:          [ReplicaType, Nodes]),
  336:     TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []),
  337:     mnesia_tpcb:init(TpcbConfig),
  338:     A ! fun() -> mnesia_tpcb:run(TpcbConfig) end,
  339:     timer:sleep(timer:seconds(10)),
  340:     ?match({atomic, ok}, rpc:call(Node1, mnesia, dump_tables,
  341:                         [[branch, teller, account, history]])),
  342:     mnesia_tpcb:stop(),
  343:     ?match(ok, mnesia_tpcb:verify_tabs()),
  344: 
  345:     mnesia_test_lib:kill_mnesia(Nodes),
  346:     timer:sleep(timer:seconds(1)),
  347:     ?match([], mnesia_test_lib:start_mnesia(Nodes,[account, branch,
  348: 						   teller, history])),
  349:     mnesia_tpcb:stop(),
  350:     ?match(ok, mnesia_tpcb:verify_tabs()),
  351:     ?verify_mnesia(Nodes, []).
  352: 
  353: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  354: 
  355: consistency_after_add_replica_2_ram(suite) -> [];
  356: consistency_after_add_replica_2_ram(Config) when is_list(Config) ->
  357:     consistency_after_add_replica(ram_copies, 2, Config).
  358: 
  359: consistency_after_add_replica_2_disc(suite) -> [];
  360: consistency_after_add_replica_2_disc(Config) when is_list(Config) ->
  361:     consistency_after_add_replica(disc_copies, 2, Config).
  362: 
  363: consistency_after_add_replica_2_disc_only(suite) -> [];
  364: consistency_after_add_replica_2_disc_only(Config) when is_list(Config) ->
  365:     consistency_after_add_replica(disc_only_copies, 2, Config).
  366: 
  367: consistency_after_add_replica_3_ram(suite) -> [];
  368: consistency_after_add_replica_3_ram(Config) when is_list(Config) ->
  369:     consistency_after_add_replica(ram_copies, 3, Config).
  370: 
  371: consistency_after_add_replica_3_disc(suite) -> [];
  372: consistency_after_add_replica_3_disc(Config) when is_list(Config) ->
  373:     consistency_after_add_replica(disc_copies, 3, Config).
  374: 
  375: consistency_after_add_replica_3_disc_only(suite) -> [];
  376: consistency_after_add_replica_3_disc_only(Config) when is_list(Config) ->
  377:     consistency_after_add_replica(disc_only_copies, 3, Config).
  378: 
  379: consistency_after_add_replica(ReplicaType, NodeConfig, Config) ->
  380:     Nodes0 = ?acquire_nodes(NodeConfig, Config),
  381:     AddNode = lists:last(Nodes0),
  382:     Nodes = Nodes0 -- [AddNode],
  383:     Node1 = hd(Nodes),
  384:     {success, [A]} = ?start_activities([Node1]),
  385:     ?log("consistency_after_add_replica with ~p on ~p~n",
  386:          [ReplicaType, Nodes0]),
  387:     TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []),
  388:     mnesia_tpcb:init(TpcbConfig),
  389:     A ! fun () -> mnesia_tpcb:run(TpcbConfig) end,
  390:     timer:sleep(timer:seconds(10)),
  391:     ?match({atomic, ok}, mnesia:add_table_copy(account, AddNode, ReplicaType)),
  392:     mnesia_tpcb:stop(),
  393:     ?match(ok, mnesia_tpcb:verify_tabs()),
  394:     ?verify_mnesia(Nodes0, []).
  395: 
  396: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  397: 
  398: consistency_after_del_replica_2_ram(suite) -> [];
  399: consistency_after_del_replica_2_ram(Config) when is_list(Config) ->
  400:     consistency_after_del_replica(ram_copies, 2, Config).
  401: 
  402: consistency_after_del_replica_2_disc(suite) -> [];
  403: consistency_after_del_replica_2_disc(Config) when is_list(Config) ->
  404:     consistency_after_del_replica(disc_copies, 2, Config).
  405: 
  406: consistency_after_del_replica_2_disc_only(suite) -> [];
  407: consistency_after_del_replica_2_disc_only(Config) when is_list(Config) ->
  408:     consistency_after_del_replica(disc_only_copies, 2, Config).
  409: 
  410: consistency_after_del_replica_3_ram(suite) -> [];
  411: consistency_after_del_replica_3_ram(Config) when is_list(Config) ->
  412:     consistency_after_del_replica(ram_copies, 3, Config).
  413: 
  414: consistency_after_del_replica_3_disc(suite) -> [];
  415: consistency_after_del_replica_3_disc(Config) when is_list(Config) ->
  416:     consistency_after_del_replica(disc_copies, 3, Config).
  417: 
  418: consistency_after_del_replica_3_disc_only(suite) -> [];
  419: consistency_after_del_replica_3_disc_only(Config) when is_list(Config) ->
  420:     consistency_after_del_replica(disc_only_copies, 3, Config).
  421: 
  422: consistency_after_del_replica(ReplicaType, NodeConfig, Config) ->
  423:     Nodes = ?acquire_nodes(NodeConfig, Config),
  424:     Node1 = hd(Nodes),
  425:     Node2 = lists:last(Nodes),
  426:     {success, [A]} = ?start_activities([Node1]),
  427:     ?log("consistency_after_del_replica with ~p on ~p~n",
  428:          [ReplicaType, Nodes]),
  429:     TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []),
  430:     mnesia_tpcb:init(TpcbConfig),
  431:     A ! fun () -> mnesia_tpcb:run(TpcbConfig) end,
  432:     timer:sleep(timer:seconds(10)),
  433:     ?match({atomic, ok}, mnesia:del_table_copy(account, Node2)),
  434:     mnesia_tpcb:stop(),
  435:     ?match(ok, mnesia_tpcb:verify_tabs()),
  436:     ?verify_mnesia(Nodes, []).
  437: 
  438: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  439: 
  440: consistency_after_move_replica_2_ram(suite) -> [];
  441: consistency_after_move_replica_2_ram(Config) when is_list(Config) ->
  442:     consistency_after_move_replica(ram_copies, 2, Config).
  443: 
  444: consistency_after_move_replica_2_disc(suite) -> [];
  445: consistency_after_move_replica_2_disc(Config) when is_list(Config) ->
  446:     consistency_after_move_replica(disc_copies, 2, Config).
  447: 
  448: consistency_after_move_replica_2_disc_only(suite) -> [];
  449: consistency_after_move_replica_2_disc_only(Config) when is_list(Config) ->
  450:     consistency_after_move_replica(disc_only_copies, 2, Config).
  451: 
  452: consistency_after_move_replica_3_ram(suite) -> [];
  453: consistency_after_move_replica_3_ram(Config) when is_list(Config) ->
  454:     consistency_after_move_replica(ram_copies, 3, Config).
  455: 
  456: consistency_after_move_replica_3_disc(suite) -> [];
  457: consistency_after_move_replica_3_disc(Config) when is_list(Config) ->
  458:     consistency_after_move_replica(disc_copies, 3, Config).
  459: 
  460: consistency_after_move_replica_3_disc_only(suite) -> [];
  461: consistency_after_move_replica_3_disc_only(Config) when is_list(Config) ->
  462:     consistency_after_move_replica(disc_only_copies, 3, Config).
  463: 
  464: consistency_after_move_replica(ReplicaType, NodeConfig, Config) ->
  465:     Nodes = ?acquire_nodes(NodeConfig, Config ++ [{tc_timeout, timer:minutes(10)}]),
  466:     Node1 = hd(Nodes),
  467:     Node2 = lists:last(Nodes),
  468:     {success, [A]} = ?start_activities([Node1]),
  469:     ?log("consistency_after_move_replica with ~p on ~p~n",
  470:          [ReplicaType, Nodes]),
  471:     TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes -- [Node2], []),
  472:     mnesia_tpcb:init(TpcbConfig),
  473:     A ! fun () -> mnesia_tpcb:run(TpcbConfig) end,
  474:     timer:sleep(timer:seconds(10)),
  475:     ?match({atomic, ok}, mnesia:move_table_copy(account, Node1, Node2)),    
  476:     ?log("First move completed from node ~p to ~p ~n", [Node1, Node2]),
  477:     ?match({atomic, ok}, mnesia:move_table_copy(account, Node2, Node1)),
  478:     mnesia_tpcb:stop(),
  479:     ?match(ok, mnesia_tpcb:verify_tabs()),
  480:     ?verify_mnesia(Nodes, []).
  481: 
  482: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  483: 
  484: 
  485: consistency_after_transform_table_ram(suite) -> [];
  486: consistency_after_transform_table_ram(Config) when is_list(Config) ->
  487:     consistency_after_transform_table(ram_copies, Config).
  488: 
  489: consistency_after_transform_table_disc(suite) -> [];
  490: consistency_after_transform_table_disc(Config) when is_list(Config) ->
  491:     consistency_after_transform_table(disc_copies, Config).
  492: 
  493: consistency_after_transform_table_disc_only(suite) -> [];
  494: consistency_after_transform_table_disc_only(Config) when is_list(Config) ->
  495:     consistency_after_transform_table(disc_only_copies, Config).
  496: 
  497: consistency_after_transform_table(Type, Config) ->
  498:     Nodes = [N1, N2,_N3] = ?acquire_nodes(3, Config),
  499: 
  500:     ?match({atomic, ok}, mnesia:create_table(tab1, [{index, [3]}, {Type, [N1]}])),
  501:     ?match({atomic, ok}, mnesia:create_table(tab2, [{index, [3]}, {Type, [N1,N2]}])),
  502:     ?match({atomic, ok}, mnesia:create_table(tab3, [{index, [3]}, {Type, Nodes}])),
  503:     ?match({atomic, ok}, mnesia:create_table(empty, [{index, [3]},{Type, Nodes}])),
  504:     
  505:     Tabs = lists:sort([tab1, tab2, tab3, empty]),
  506:     
  507:     [[mnesia:dirty_write({Tab, N, N}) || N <- lists:seq(1,10)] || 
  508: 	Tab <- Tabs -- [empty, tab4]],
  509:     mnesia:dump_log(),
  510:     
  511:     Ok = lists:duplicate(4, {atomic, ok}),
  512:     ?match(Ok, [mnesia:transform_table(Tab, fun({T, N, N}) ->  {T, N, N, ok} end,
  513: 	[k,a,n]) || Tab <- Tabs]),
  514:     [?match([k,a,n], mnesia:table_info(Tab, attributes)) || Tab <- Tabs],
  515: 
  516:     Filter = fun(Tab) -> mnesia:foldl(fun(A, Acc) when size(A) == 3 -> [A|Acc];
  517: 					 (A, Acc) when size(A) == 4 -> Acc
  518: 				      end, [], Tab)
  519: 	     end,    
  520: 	
  521:     ?match([[],[],[],[]], [element(2,mnesia:transaction(Filter, [Tab])) || Tab <- Tabs]),
  522:     
  523:     mnesia_test_lib:kill_mnesia(Nodes),
  524:     mnesia_test_lib:start_mnesia(Nodes, Tabs),
  525:     
  526:     ?match([Tabs, Tabs, Tabs], 
  527: 	[lists:sort(rpc:call(Node, mnesia,system_info, [tables]) -- [schema]) || Node <- Nodes]),
  528:     
  529:     ?match([[],[],[],[]], [element(2,mnesia:transaction(Filter, [Tab])) || Tab <- Tabs]),
  530:     [?match([k,a,n], mnesia:table_info(Tab, attributes)) || Tab <- Tabs],
  531: 
  532:     ?verify_mnesia(Nodes, []).
  533: 
  534: 
  535: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  536: consistency_after_change_table_copy_type(doc) ->
  537:     ["Check that the database is consistent after change of copy type.",
  538:      " While applications are updating the involved tables. "].
  539: 
  540: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  541: 
  542: consistency_after_fallback_2_ram(suite) -> [];
  543: consistency_after_fallback_2_ram(Config) when is_list(Config) ->
  544:     consistency_after_fallback(ram_copies, 2, Config).
  545: 
  546: consistency_after_fallback_2_disc(suite) -> [];
  547: consistency_after_fallback_2_disc(Config) when is_list(Config) ->
  548:     consistency_after_fallback(disc_copies, 2, Config).
  549: 
  550: consistency_after_fallback_2_disc_only(suite) -> [];
  551: consistency_after_fallback_2_disc_only(Config) when is_list(Config) ->
  552:     consistency_after_fallback(disc_only_copies, 2, Config).
  553: 
  554: consistency_after_fallback_3_ram(suite) -> [];
  555: consistency_after_fallback_3_ram(Config) when is_list(Config) ->
  556:     consistency_after_fallback(ram_copies, 3, Config).
  557: 
  558: consistency_after_fallback_3_disc(suite) -> [];
  559: consistency_after_fallback_3_disc(Config) when is_list(Config) ->
  560:     consistency_after_fallback(disc_copies, 3, Config).
  561: 
  562: consistency_after_fallback_3_disc_only(suite) -> [];
  563: consistency_after_fallback_3_disc_only(Config) when is_list(Config) ->
  564:     consistency_after_fallback(disc_only_copies, 3, Config).
  565: 
  566: consistency_after_fallback(ReplicaType, NodeConfig, Config) ->
  567:     %%?verbose("Starting consistency_after_fallback2 at ~p~n", [self()]),
  568:     Delay = 5,
  569:     Nodes = ?acquire_nodes(NodeConfig, [{tc_timeout, timer:minutes(10)} | Config]),
  570:     Node1 = hd(Nodes),
  571:     %%?verbose("Mnesia info: ~p~n", [mnesia:info()]),
  572: 
  573:     {success, [A]} = ?start_activities([Node1]),
  574:     ?log("consistency_after_fallback with ~p on ~p~n",
  575:          [ReplicaType, Nodes]),
  576:     TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []),
  577:     mnesia_tpcb:init(TpcbConfig),
  578:     A ! fun () -> mnesia_tpcb:run(TpcbConfig) end,
  579:     timer:sleep(timer:seconds(Delay)),
  580:     
  581:     %% Make a backup
  582:     ?verbose("Doing backup~n", []),
  583:     ?match(ok, mnesia:backup(consistency_after_fallback2)),
  584:     
  585:     %% Install the backup as a fallback
  586:     ?verbose("Doing fallback~n", []),
  587:     ?match(ok, mnesia:install_fallback(consistency_after_fallback2)),
  588:     timer:sleep(timer:seconds(Delay)),
  589: 
  590:     %% Stop tpcb
  591:     ?verbose("Stopping TPC-B~n", []),
  592:     mnesia_tpcb:stop(),    
  593:     ?match(ok, mnesia_tpcb:verify_tabs()),
  594:     
  595:     %% Stop and then start mnesia and check table consistency
  596:     %%?verbose("Restarting Mnesia~n", []),
  597:     mnesia_test_lib:kill_mnesia(Nodes),
  598:     mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller,history]),
  599:     
  600:     ?match(ok, mnesia_tpcb:verify_tabs()),
  601:     if 
  602: 	ReplicaType == ram_copies ->
  603: 	    %% Test that change_table_copy work i.e. no account.dcd file exists.
  604: 	    ?match({atomic, ok}, mnesia:change_table_copy_type(account, node(), disc_copies));
  605: 	true -> 
  606: 	    ignore
  607:     end,
  608:     file:delete(consistency_after_fallback2),
  609:     ?verify_mnesia(Nodes, []).
  610: 
  611: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  612: 
  613: consistency_after_restore_clear_ram(suite) -> [];
  614: consistency_after_restore_clear_ram(Config) when is_list(Config) ->
  615:     consistency_after_restore(ram_copies, clear_tables, Config).
  616: 
  617: consistency_after_restore_clear_disc(suite) -> [];
  618: consistency_after_restore_clear_disc(Config) when is_list(Config) ->
  619:     consistency_after_restore(disc_copies, clear_tables, Config).
  620: 
  621: consistency_after_restore_clear_disc_only(suite) -> [];
  622: consistency_after_restore_clear_disc_only(Config) when is_list(Config) ->
  623:     consistency_after_restore(disc_only_copies, clear_tables, Config).
  624: 
  625: consistency_after_restore_recreate_ram(suite) -> [];
  626: consistency_after_restore_recreate_ram(Config) when is_list(Config) ->
  627:     consistency_after_restore(ram_copies, recreate_tables, Config).
  628: 
  629: consistency_after_restore_recreate_disc(suite) -> [];
  630: consistency_after_restore_recreate_disc(Config) when is_list(Config) ->
  631:     consistency_after_restore(disc_copies, recreate_tables, Config).
  632: 
  633: consistency_after_restore_recreate_disc_only(suite) -> [];
  634: consistency_after_restore_recreate_disc_only(Config) when is_list(Config) ->
  635:     consistency_after_restore(disc_only_copies, recreate_tables, Config).
  636: 
  637: consistency_after_restore(ReplicaType, Op, Config) ->
  638:     Delay = 1,
  639:     Nodes = ?acquire_nodes(3, [{tc_timeout, timer:minutes(10)} | Config]),
  640:     [Node1, Node2, _Node3] = Nodes,
  641:     File = "cons_backup_restore",
  642:     
  643:     ?log("consistency_after_restore with ~p on ~p~n",
  644:          [ReplicaType, Nodes]),
  645:     Tabs = [carA, carB, carC, carD],
  646:     
  647:     ?match({atomic, ok}, mnesia:create_table(carA, [{ReplicaType, Nodes}])),
  648:     ?match({atomic, ok}, mnesia:create_table(carB, [{ReplicaType, Nodes -- [Node1]}])),
  649:     ?match({atomic, ok}, mnesia:create_table(carC, [{ReplicaType, Nodes -- [Node2]}])),
  650:     ?match({atomic, ok}, mnesia:create_table(carD, [{ReplicaType, [Node2]}])),
  651: 
  652:     NList = lists:seq(0, 20),
  653:     [lists:foreach(fun(E) -> ok = mnesia:dirty_write({Tab, E, 1}) end, NList) ||
  654: 	Tab <- Tabs],
  655:     
  656:     {ok, Name, _} = mnesia:activate_checkpoint([{max, [schema | Tabs]}, 
  657: 						{ram_overrides_dump, true}]),
  658:     ?verbose("Doing backup~n", []),
  659:     ?match(ok, mnesia:backup_checkpoint(Name, File)),
  660:     ?match(ok, mnesia:deactivate_checkpoint(Name)),
  661:     
  662:     [lists:foreach(fun(E) -> ok = mnesia:dirty_write({Tab, E, 2}) end, NList) ||
  663: 	Tab <- Tabs],
  664:     
  665:     Pids1 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carA, Op]), ok} || _ <- lists:seq(1, 5)],
  666:     Pids2 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carB, Op]), ok} || _ <- lists:seq(1, 5)],
  667:     Pids3 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carC, Op]), ok} || _ <- lists:seq(1, 5)],
  668:     Pids4 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carD, Op]), ok} || _ <- lists:seq(1, 5)],
  669:     
  670:     AllPids = Pids1 ++ Pids2 ++ Pids3 ++ Pids4,
  671:     
  672:     Restore = fun(F, Args) ->
  673: 		      case mnesia:restore(F, Args) of
  674: 			  {atomic, List} -> lists:sort(List);
  675: 			  Else -> Else
  676: 		      end
  677: 	      end,
  678:     
  679:     timer:sleep(timer:seconds(Delay)),  %% Let changers grab locks
  680:     ?verbose("Doing restore~n", []),
  681:     ?match(Tabs, Restore(File, [{default_op, Op}])),
  682: 
  683:     timer:sleep(timer:seconds(Delay)),  %% Let em die
  684: 
  685:     ?match_multi_receive(AllPids),
  686: 
  687:     case ?match(ok, restore_verify_tabs(Tabs)) of 
  688: 	{success, ok} -> 
  689: 	    file:delete(File);
  690: 	_ -> 
  691: 	    {T, M, S} = time(),
  692: 	    File2 = ?flat_format("consistency_error~w~w~w.BUP", [T, M, S]),
  693: 	    file:rename(File, File2)
  694:     end,
  695:     ?verify_mnesia(Nodes, []).
  696: 
  697: change_tab(Father, Tab, Test) ->
  698:     Key = random:uniform(20),
  699:     Update = fun() ->
  700: 		     case mnesia:read({Tab, Key}) of
  701: 			 [{Tab, Key, 1}] -> 
  702: 			     quit;
  703: 			 [{Tab, Key, _N}] ->				       
  704: 			     mnesia:write({Tab, Key, 3})
  705: 		     end
  706: 	     end,
  707:     case mnesia:transaction(Update) of
  708: 	{atomic, quit} ->
  709: 	    exit(ok);
  710: 	{aborted, {no_exists, Tab}} when Test == recreate_tables ->%% I'll allow this 
  711: 	    change_tab(Father, Tab, Test);
  712: 	{atomic, ok} ->
  713: 	    change_tab(Father, Tab, Test)
  714:     end.
  715: 
  716: restore_verify_tabs([Tab | R]) ->
  717:     ?match({atomic, ok}, 
  718: 	   mnesia:transaction(fun() -> mnesia:foldl(fun({_, _, 1}, ok) -> 
  719: 							    ok;
  720: 						       (Else, Acc) ->
  721: 							    [Else|Acc]
  722: 						    end, ok, Tab) 
  723: 			      end)),
  724:     restore_verify_tabs(R);
  725: restore_verify_tabs([]) ->
  726:     ok.
  727: 
  728: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  729: consistency_after_rename_of_node(doc) ->
  730:     ["Skipped because it is an unimportant case."].
  731: 
  732: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  733: 
  734: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  735: 
  736: updates_during_checkpoint_activation_1_ram(suite) -> [];
  737: updates_during_checkpoint_activation_1_ram(Config) when is_list(Config) ->
  738:     updates_during_checkpoint_activation(ram_copies, 1, Config).
  739: 
  740: updates_during_checkpoint_activation_1_disc(suite) -> [];
  741: updates_during_checkpoint_activation_1_disc(Config) when is_list(Config) ->
  742:     updates_during_checkpoint_activation(disc_copies, 1, Config).
  743: 
  744: updates_during_checkpoint_activation_1_disc_only(suite) -> [];
  745: updates_during_checkpoint_activation_1_disc_only(Config) when is_list(Config) ->
  746:     updates_during_checkpoint_activation(disc_only_copies, 1, Config).
  747: 
  748: updates_during_checkpoint_activation_2_ram(suite) -> [];
  749: updates_during_checkpoint_activation_2_ram(Config) when is_list(Config) ->
  750:     updates_during_checkpoint_activation(ram_copies, 2, Config).
  751: 
  752: updates_during_checkpoint_activation_2_disc(suite) -> [];
  753: updates_during_checkpoint_activation_2_disc(Config) when is_list(Config) ->
  754:     updates_during_checkpoint_activation(disc_copies, 2, Config).
  755: 
  756: updates_during_checkpoint_activation_2_disc_only(suite) -> [];
  757: updates_during_checkpoint_activation_2_disc_only(Config) when is_list(Config) ->
  758:     updates_during_checkpoint_activation(disc_only_copies, 2, Config).
  759: 
  760: updates_during_checkpoint_activation_3_ram(suite) -> [];
  761: updates_during_checkpoint_activation_3_ram(Config) when is_list(Config) ->
  762:     updates_during_checkpoint_activation(ram_copies, 3, Config).
  763: 
  764: updates_during_checkpoint_activation_3_disc(suite) -> [];
  765: updates_during_checkpoint_activation_3_disc(Config) when is_list(Config) ->
  766:     updates_during_checkpoint_activation(disc_copies, 3, Config).
  767: 
  768: updates_during_checkpoint_activation_3_disc_only(suite) -> [];
  769: updates_during_checkpoint_activation_3_disc_only(Config) when is_list(Config) ->
  770:     updates_during_checkpoint_activation(disc_only_copies, 3, Config).
  771: 
  772: updates_during_checkpoint_activation(ReplicaType,NodeConfig,Config) ->
  773:     %%?verbose("updates_during_checkpoint_activation2 at ~p~n", [self()]),
  774:     Delay = 5,
  775:     Nodes = ?acquire_nodes(NodeConfig, Config),
  776:     Node1 = hd(Nodes),
  777:     %%?verbose("Mnesia info: ~p~n", [mnesia:info()]),
  778: 
  779:     {success, [A]} = ?start_activities([Node1]),
  780:     ?log("consistency_after_fallback with ~p on ~p~n",
  781:          [ReplicaType, Nodes]),
  782:     TpcbConfig = tpcb_config_dist(ReplicaType, NodeConfig, Nodes, Config),
  783:     %%TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes),
  784:     mnesia_tpcb:init(TpcbConfig),
  785:     A ! fun () -> mnesia_tpcb:run(TpcbConfig) end,
  786:     timer:sleep(timer:seconds(Delay)),
  787: 
  788:     {ok, CPName, _NodeList} =
  789:         mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
  790: 				    {ram_overrides_dump, true}]),
  791:     timer:sleep(timer:seconds(Delay)),
  792: 
  793:     %% Stop tpcb
  794:     ?verbose("Stopping TPC-B~n", []),
  795:     mnesia_tpcb:stop(),
  796:     ?match(ok, mnesia_tpcb:verify_tabs()),
  797:     
  798:     ?match(ok, mnesia:backup_checkpoint(CPName, 
  799: 					updates_during_checkpoint_activation2)),
  800:     timer:sleep(timer:seconds(Delay)),
  801: 
  802:     ?match(ok, mnesia:install_fallback(updates_during_checkpoint_activation2)),
  803: 
  804:     %% Stop and then start mnesia and check table consistency
  805:     %%?verbose("Restarting Mnesia~n", []),
  806:     mnesia_test_lib:kill_mnesia(Nodes),
  807:     file:delete(updates_during_checkpoint_activation2),
  808:     mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller, history]),
  809: 
  810:     ?match(ok, mnesia_tpcb:verify_tabs()),
  811:     ?verify_mnesia(Nodes, []).
  812: 
  813: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  814: 
  815: updates_during_checkpoint_iteration_2_ram(suite) -> [];
  816: updates_during_checkpoint_iteration_2_ram(Config) when is_list(Config) ->
  817:     updates_during_checkpoint_iteration(ram_copies, 2, Config).
  818: 
  819: updates_during_checkpoint_iteration_2_disc(suite) -> [];
  820: updates_during_checkpoint_iteration_2_disc(Config) when is_list(Config) ->
  821:     updates_during_checkpoint_iteration(disc_copies, 2, Config).
  822: 
  823: updates_during_checkpoint_iteration_2_disc_only(suite) -> [];
  824: updates_during_checkpoint_iteration_2_disc_only(Config) when is_list(Config) ->
  825:     updates_during_checkpoint_iteration(disc_only_copies, 2, Config).
  826: 
  827: updates_during_checkpoint_iteration(ReplicaType,NodeConfig,Config) ->
  828:    %?verbose("updates_during_checkpoint_iteration2 at ~p~n", [self()]),
  829:     Delay = 5,
  830:     Nodes = ?acquire_nodes(NodeConfig, Config),
  831:     Node1 = hd(Nodes),
  832:    %?verbose("Mnesia info: ~p~n", [mnesia:info()]),
  833:     File = updates_during_checkpoint_iteration2,
  834:     {success, [A]} = ?start_activities([Node1]),
  835:     ?log("updates_during_checkpoint_iteration with ~p on ~p~n",
  836:          [ReplicaType, Nodes]),
  837:     TpcbConfig = tpcb_config_dist(ReplicaType, NodeConfig, Nodes, Config),
  838:     %%TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes),
  839:     TpcbConfigRec = list2rec(TpcbConfig, 
  840:                                          record_info(fields,tab_config),
  841:                                          #tab_config{}),
  842:     mnesia_tpcb:init(TpcbConfig),
  843:     ?match(ok, mnesia_tpcb:verify_tabs()),
  844: 
  845:     {ok, CPName, _NodeList} =
  846:         mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
  847:                                     {ram_overrides_dump,true}]),  
  848:     A ! fun () -> mnesia:backup_checkpoint(CPName, File) end,
  849: 
  850:     do_changes_during_backup(TpcbConfigRec),
  851: 
  852:     ?match_receive({A,ok}),
  853: 
  854:     timer:sleep(timer:seconds(Delay)), 
  855:     ?match(ok, mnesia:install_fallback(File)),
  856:     timer:sleep(timer:seconds(Delay)),
  857: 
  858:     ?match({error,{"Bad balance",_,_}}, mnesia_tpcb:verify_tabs()),
  859:  
  860:     mnesia_test_lib:kill_mnesia(Nodes),
  861:     mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller, history]),
  862: 
  863:     ?match(ok, mnesia_tpcb:verify_tabs()),
  864: 
  865:     ?match(ok, file:delete(File)),
  866:     ?verify_mnesia(Nodes, []).
  867: 
  868: do_changes_during_backup(TpcbConfig) ->
  869:     loop_branches(TpcbConfig#tab_config.n_branches,
  870:                   TpcbConfig#tab_config.n_accounts_per_branch).
  871: 
  872: loop_branches(N_br,N_acc) when N_br >= 1 ->
  873:    loop_accounts(N_br,N_acc),
  874:    loop_branches(N_br-1,N_acc);
  875: loop_branches(_,_) -> done.
  876: 
  877: loop_accounts(N_br, N_acc) when N_acc >= 1 ->
  878:     A = #account{id=N_acc, branch_id=N_br, balance = 4711},
  879:     ok = mnesia:dirty_write(A),
  880:     loop_accounts(N_br, N_acc-1);
  881: 
  882: loop_accounts(_,_) -> done.
  883: 
  884: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  885: 
  886: load_table_with_activated_checkpoint_ram(suite) -> [];
  887: load_table_with_activated_checkpoint_ram(Config) when is_list(Config) ->
  888:     load_table_with_activated_checkpoint(ram_copies, Config).
  889: 
  890: load_table_with_activated_checkpoint_disc(suite) -> [];
  891: load_table_with_activated_checkpoint_disc(Config) when is_list(Config) ->
  892:     load_table_with_activated_checkpoint(disc_copies, Config).
  893: 
  894: load_table_with_activated_checkpoint_disc_only(suite) -> [];
  895: load_table_with_activated_checkpoint_disc_only(Config) when is_list(Config) ->
  896:     load_table_with_activated_checkpoint(disc_only_copies, Config).
  897: 
  898: load_table_with_activated_checkpoint(Type, Config) ->
  899:     Nodes = ?acquire_nodes(2, Config),
  900:     Node1 = hd(Nodes),
  901:     Tab = load_test,
  902:     Def = [{attributes, [key, value]},
  903:            {Type, Nodes}],            %%  ??? important that RAM  ???
  904:     
  905:     ?match({atomic, ok}, mnesia:create_table(Tab, Def)),
  906:     ?match(ok, mnesia:dirty_write({Tab, 1, 4711})),
  907:     ?match(ok, mnesia:dirty_write({Tab, 2, 42})),
  908:     ?match(ok, mnesia:dirty_write({Tab, 3, 256})),
  909:     
  910:     timer:sleep(timer:seconds(1)), 
  911:     
  912:     {ok, CPName, _NodeList} =
  913:         mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
  914:                                     {ram_overrides_dump,true}]),
  915:     
  916:     mnesia_test_lib:stop_mnesia([Node1]),    
  917:     mnesia_test_lib:start_mnesia([Node1],[Tab]),      
  918:     %%--- check, whether the checkpiont is attached to both replicas
  919:     {success, [A,B]} = ?start_activities(Nodes),
  920:     
  921:     A ! fun () ->
  922: 		mnesia:table_info(Tab,checkpoints)
  923: 	end,
  924:     ?match_receive({A,[CPName]}),
  925:     
  926:     B ! fun () ->
  927: 		mnesia:table_info(Tab,checkpoints)
  928: 	end,
  929:     ?match_receive({B,[CPName]}),    
  930:     
  931:     %%--- check, whether both retainers are consistent    
  932:     ?match(ok, mnesia:dirty_write({Tab, 1, 815})),       
  933:     A ! fun () ->
  934: 		mnesia:backup_checkpoint(CPName, load_table_a)
  935: 	end,
  936:     ?match_receive({A,ok}),
  937:     B ! fun () ->
  938: 		mnesia:backup_checkpoint(CPName, load_table_b)
  939: 	end,
  940:     ?match_receive({B,ok}),
  941:     
  942:     Mod = mnesia_backup, %% Assume local files
  943:     List_a =  view(load_table_a, Mod),
  944:     List_b =  view(load_table_b, Mod),
  945:     
  946:     ?match(List_a, List_b),
  947:     
  948:     ?match(ok,file:delete(load_table_a)),
  949:     ?match(ok,file:delete(load_table_b)),
  950:     ?verify_mnesia(Nodes, []).
  951: 
  952: view(Source, Mod) ->
  953:     View = fun(Item, Acc) ->
  954:                    ?verbose("tab - item : ~p ~n",[Item]),
  955: 		   case Item of
  956: 		       {schema, Tab, Cs} ->  %% Remove cookie information
  957: 			   NewCs = lists:keyreplace(cookie, 1, Cs,
  958: 						    {cookie, skip_cookie}),
  959: 			   Item2 = {schema, Tab, NewCs},
  960: 			   {[Item], [Item2|Acc]};
  961: 		       _ ->   
  962: 			   {[Item], [Item|Acc]}
  963: 		   end
  964:            end,
  965:     {ok,TabList} =  
  966: 	mnesia:traverse_backup(Source, Mod, dummy, read_only, View, []),
  967:     lists:sort(TabList).
  968: 
  969: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  970: 
  971: add_table_copy_to_table_checkpoint_ram(suite) -> [];
  972: add_table_copy_to_table_checkpoint_ram(Config) when is_list(Config) ->
  973:     add_table_copy_to_table_with_activated_checkpoint(ram_copies, Config).
  974: 
  975: add_table_copy_to_table_checkpoint_disc(suite) -> [];
  976: add_table_copy_to_table_checkpoint_disc(Config) when is_list(Config) ->
  977:     add_table_copy_to_table_with_activated_checkpoint(disc_copies, Config).
  978: 
  979: add_table_copy_to_table_checkpoint_disc_only(suite) -> [];
  980: add_table_copy_to_table_checkpoint_disc_only(Config) when is_list(Config) ->
  981:     add_table_copy_to_table_with_activated_checkpoint(disc_only_copies, Config).
  982: 
  983: add_table_copy_to_table_with_activated_checkpoint(Type,Config) -> 
  984:     Nodes = ?acquire_nodes(2, Config),
  985: 						%?verbose("NODES = ~p ~n",[Nodes]),
  986:     [Node1,Node2] = Nodes,
  987: 
  988:     Tab = add_test,
  989:     Def = [{attributes, [key, value]},
  990:            {Type, [Node1]}],            %%  ??? important that RAM  ???
  991: 
  992:     ?match({atomic, ok}, mnesia:create_table(Tab, Def)),
  993:     ?match(ok, mnesia:dirty_write({Tab, 1, 4711})),
  994:     ?match(ok, mnesia:dirty_write({Tab, 2, 42})),
  995:     ?match(ok, mnesia:dirty_write({Tab, 3, 256})),
  996: 
  997:     {ok, CPName, _NodeList} =
  998:         mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
  999:                                     {ram_overrides_dump,true}]),
 1000: 
 1001:     ?match({atomic,ok},mnesia:add_table_copy(Tab,Node2,ram_copies)),
 1002: 
 1003:     %%--- check, whether the checkpiont is attached to both replicas
 1004:     {success, [A,B]} = ?start_activities(Nodes),
 1005: 
 1006:     A ! fun () ->
 1007: 		mnesia:table_info(Tab,checkpoints)
 1008: 	end,
 1009:     ?match_receive({A,[CPName]}),
 1010: 
 1011:     B ! fun () ->
 1012: 		mnesia:table_info(Tab,checkpoints)
 1013: 	end,
 1014:     ?match_receive({B,[CPName]}),
 1015: 
 1016:     %%--- check, whether both retainers are consistent
 1017: 
 1018:     ?match(ok, mnesia:dirty_write({Tab, 1, 815})),
 1019:     ?match(ok, mnesia:dirty_write({Tab, 2, 815})),
 1020: 
 1021:     A ! fun () ->
 1022: 		mnesia:backup_checkpoint(CPName, add_table_a)
 1023: 	end,
 1024:     ?match_receive({A,ok}),
 1025:     B ! fun () ->
 1026: 		mnesia:backup_checkpoint(CPName, add_table_b)
 1027: 	end,
 1028:     ?match_receive({B,ok}),
 1029: 
 1030:     Mod = mnesia_backup, %% Assume local files
 1031: 
 1032:     List_a = view(add_table_a, Mod),
 1033:     List_b = view(add_table_b, Mod),
 1034: 
 1035:     ?match(List_a, List_b),
 1036: 
 1037:     ?match(ok,file:delete(add_table_a)),
 1038:     ?match(ok, file:delete(add_table_b)),
 1039:     ?verify_mnesia(Nodes, []).
 1040: 
 1041: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1042: 
 1043: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1044: 
 1045: inst_fallback_process_dies(suite) -> 
 1046:     [];
 1047: inst_fallback_process_dies(Config) when is_list(Config) ->
 1048:     ?is_debug_compiled,
 1049:     
 1050:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1051:     {success, [A,_B,_C]} = ?start_activities(Nodes),
 1052:     
 1053:     TestPid = self(),
 1054:     DebugId = {mnesia_bup, fallback_receiver_loop, pre_swap},
 1055:     DebugFun =
 1056:         fun(PrevContext, _EvalContext) ->
 1057:                 ?verbose("fallback_receiver_loop -  pre_swap pid ~p  #~p~n",
 1058:                      [self(),PrevContext]),
 1059:                 TestPid ! {self(),fallback_preswap},
 1060: 		case receive_messages([fallback_continue]) of
 1061: 		    [{TestPid,fallback_continue}] ->
 1062: 			?deactivate_debug_fun(DebugId),
 1063: 			PrevContext+1
 1064: 		end
 1065:         end,
 1066:     ?activate_debug_fun(DebugId, DebugFun, 1),
 1067:         
 1068:     Tab = install_table,
 1069:     Def = [{attributes, [key, value]}, {disc_copies, Nodes}], 
 1070:     
 1071:     ?match({atomic, ok}, mnesia:create_table(Tab, Def)),
 1072:     
 1073:     ?match(ok, mnesia:dirty_write({Tab, 1, 4711})),
 1074:     ?match(ok, mnesia:dirty_write({Tab, 2, 42})),
 1075:     ?match(ok, mnesia:dirty_write({Tab, 3, 256})),
 1076:     
 1077:     {ok, CPName, _NodeList} =
 1078:         mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
 1079:                                     {ram_overrides_dump,true}]),
 1080:     
 1081:     ?match(ok, mnesia:backup_checkpoint(CPName, install_backup)),
 1082:     
 1083:     A ! fun() -> mnesia:install_fallback(install_backup) end,    
 1084:     [{AnsPid,fallback_preswap}] = receive_messages([fallback_preswap]),
 1085:     exit(A, kill),
 1086:     AnsPid ! {self(), fallback_continue},
 1087:     ?match_receive({'EXIT', A, killed}), 
 1088:     timer:sleep(2000),  %% Wait till fallback is installed everywhere
 1089: 
 1090:     mnesia_test_lib:kill_mnesia(Nodes),    
 1091:     ?verbose("~n---->Mnesia is stopped everywhere<-----~n", []),
 1092:     ?match([], mnesia_test_lib:start_mnesia(Nodes,[Tab])),
 1093:     
 1094:     check_data(Nodes, Tab),
 1095:     ?match(ok, file:delete(install_backup)),
 1096:     ?verify_mnesia(Nodes, []).
 1097: 
 1098: check_data([N1 | R], Tab) ->
 1099:     ?match([{Tab, 1, 4711}], rpc:call(N1, mnesia, dirty_read, [{Tab, 1}])),
 1100:     ?match([{Tab, 2, 42}],   rpc:call(N1, mnesia, dirty_read, [{Tab, 2}])),
 1101:     ?match([{Tab, 3, 256}],  rpc:call(N1, mnesia, dirty_read, [{Tab, 3}])),
 1102:     check_data(R, Tab);
 1103: check_data([], _Tab) -> 
 1104:     ok.
 1105: 
 1106: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1107: 
 1108: fatal_when_inconsistency(suite) -> 
 1109:     [];
 1110: fatal_when_inconsistency(Config) when is_list(Config) ->
 1111:     ?is_debug_compiled,
 1112:     
 1113:     [Node1, Node2, Node3] = Nodes = 
 1114: 	?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
 1115:     {success, [A,_B,_C]} = ?start_activities(Nodes),
 1116:     
 1117:     TestPid = self(),
 1118:     DebugId = {mnesia_bup, fallback_receiver_loop, pre_swap},
 1119:     DebugFun =
 1120:         fun(PrevContext, _EvalContext) ->
 1121:                 ?verbose("fallback_receiver_loop -  pre_swap pid ~p  #~p~n",
 1122:                      [self(),PrevContext]),
 1123:                 TestPid ! {self(),fallback_preswap},
 1124: 		case receive_messages([fallback_continue])  of
 1125: 		    [{TestPid,fallback_continue}] ->
 1126: 			?deactivate_debug_fun(DebugId),
 1127: 			PrevContext+1
 1128: 		end
 1129:         end,
 1130:     ?activate_debug_fun(DebugId, DebugFun, 1),
 1131:         
 1132:     Tab = install_table,
 1133:     Def = [{attributes, [key, value]}, {disc_copies, Nodes}], 
 1134:     
 1135:     ?match({atomic, ok}, mnesia:create_table(Tab, Def)),
 1136:     
 1137:     ?match(ok, mnesia:dirty_write({Tab, 1, 4711})),
 1138:     ?match(ok, mnesia:dirty_write({Tab, 2, 42})),
 1139:     ?match(ok, mnesia:dirty_write({Tab, 3, 256})),
 1140:     
 1141:     {ok, CPName, _NodeList} =
 1142:         mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
 1143:                                     {ram_overrides_dump,true}]),
 1144:     
 1145:     ?match(ok, mnesia:backup_checkpoint(CPName, install_backup)),
 1146:     ?match(ok, mnesia:dirty_write({Tab, 2, 42424242})),
 1147:     
 1148:     A ! fun() ->
 1149:                 mnesia:install_fallback(install_backup)
 1150:         end,    
 1151: 
 1152:     [{AnsPid,fallback_preswap}] = receive_messages([fallback_preswap]),
 1153:     exit(AnsPid, kill),  %% Kill install-fallback on local node will 
 1154:     AnsPid ! {self(), fallback_continue},
 1155:     ?deactivate_debug_fun(DebugId),
 1156:         
 1157:     ?match_receive({A,{error,{"Cannot install fallback",
 1158: 			      {'EXIT',AnsPid,killed}}}}),
 1159:     mnesia_test_lib:kill_mnesia(Nodes),
 1160:     ?verbose("EXPECTING FATAL from 2 nodes WITH CORE DUMP~n", []),
 1161:     
 1162:     ?match([], mnesia_test_lib:start_mnesia([Node1],[])),
 1163:     is_running(Node1, yes),
 1164:     ?match([{Node2, mnesia, _}], mnesia_test_lib:start_mnesia([Node2],[])),
 1165:     is_running(Node2, no),
 1166:     ?match([{Node3, mnesia, _}], mnesia_test_lib:start_mnesia([Node3],[])),
 1167:     is_running(Node3, no),
 1168:     mnesia_test_lib:kill_mnesia(Nodes),
 1169:     
 1170:     ?match(ok, mnesia:install_fallback(install_backup)),    
 1171:     mnesia_test_lib:start_mnesia(Nodes,[Tab]),
 1172:     
 1173:     check_data(Nodes, Tab),
 1174:     
 1175:     ?match(ok,file:delete(install_backup)),
 1176:     ?verify_mnesia(Nodes, []).
 1177: 
 1178: is_running(Node, Shouldbe) ->
 1179:     timer:sleep(1000),
 1180:     Running = rpc:call(Node, mnesia, system_info, [is_running]),
 1181:     case Running of
 1182: 	Shouldbe -> ok;
 1183: 	_  -> is_running(Node, Shouldbe)
 1184:     end.
 1185: 
 1186: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1187: 
 1188: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1189: 
 1190: after_delete(doc) ->
 1191:     ["interrupt the uninstall after deletion of ",
 1192:      "fallback files - there shall be no fallback"];
 1193: after_delete(suite) -> [];
 1194: after_delete(Config) when is_list(Config) ->
 1195:         do_uninstall(Config, post_delete).
 1196: 
 1197: %%%%%%%%%%%%%%%%%%%%%%%%%
 1198: 
 1199: do_uninstall(Config,DebugPoint) ->
 1200:     ?is_debug_compiled,
 1201:     
 1202:     Nodes = ?acquire_nodes(3, Config),
 1203:     %%?verbose("NODES = ~p ~n",[Nodes]),
 1204:     
 1205:     {success, [P1,P2,P3]} = ?start_activities(Nodes),
 1206:     
 1207:     NP1 = node(P1),
 1208:     NP2 = node(P2),
 1209:     
 1210:     {A,B,C} = case node() of
 1211: 		  NP1 ->
 1212: 		      %%?verbose("first case ~n"),
 1213: 		      {P3,P2,P1};
 1214: 		  NP2 ->
 1215: 		      %%?verbose("second case ~n"),
 1216: 		      {P3, P1, P2};
 1217: 		  _  ->
 1218: 		      { P1, P2, P3}
 1219: 	      end,
 1220:     
 1221:     Node1 = node(A),
 1222:     Node2 = node(B),
 1223:     Node3 = node(C),
 1224:     
 1225:     ?verbose(" A   pid:~p  node:~p ~n",[A,Node1]),
 1226:     ?verbose(" B   pid:~p  node:~p ~n",[B,Node2]),
 1227:     ?verbose(" C   pid:~p  node:~p ~n",[C,Node3]),
 1228:     
 1229:     
 1230:     TestPid = self(),
 1231:     %%?verbose("TestPid : ~p~n",[TestPid]),
 1232:     DebugId = {mnesia_bup, uninstall_fallback2, DebugPoint},
 1233:     DebugFun = fun(PrevContext, _EvalContext) ->
 1234: 		       ?verbose("uninstall_fallback pid ~p  #~p~n"
 1235: 				,[self(),PrevContext]),
 1236: 		       TestPid ! {self(),uninstall_predelete},
 1237: 		       case receive_messages([uninstall_continue]) of
 1238: 			   [{TestPid,uninstall_continue}] ->
 1239: 			       ?deactivate_debug_fun(DebugId),
 1240: 			       %%?verbose("uninstall_fallback continues~n"),
 1241: 			       PrevContext+1
 1242: 		       end
 1243: 	       end,
 1244:     ?remote_activate_debug_fun(Node1,DebugId, DebugFun, 1),
 1245:     
 1246:     Tab = install_table,
 1247:     Def = [{attributes, [key, value]},
 1248:            {ram_copies, Nodes}],     %% necessary to test different types ???  
 1249:     
 1250:     ?match({atomic, ok}, mnesia:create_table(Tab, Def)),
 1251:     
 1252:     ?match(ok, mnesia:dirty_write({Tab, 1, 4711})),
 1253:     ?match(ok, mnesia:dirty_write({Tab, 2, 42})),
 1254:     ?match(ok, mnesia:dirty_write({Tab, 3, 256})),
 1255:     
 1256:     {ok, CPName, _NodeList} =
 1257:         mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
 1258:                                     {ram_overrides_dump,true}]),
 1259:     
 1260:     ?match(ok, mnesia:backup_checkpoint(CPName,install_backup)),
 1261:     timer:sleep(timer:seconds(1)), 
 1262:     
 1263:     A ! fun () ->
 1264: 		mnesia:install_fallback(install_backup)
 1265: 	end,
 1266:     ?match_receive({A,ok}),
 1267:     
 1268:     A ! fun () ->
 1269: 		mnesia:uninstall_fallback()
 1270: 	end,
 1271:     %%
 1272:     %%  catch the debug entry in mnesia and kill one Mnesia node
 1273:     %%
 1274:     
 1275:     
 1276:     [{AnsPid,uninstall_predelete}] = receive_messages([uninstall_predelete]),
 1277:     
 1278:     ?verbose("AnsPid : ~p~n",[AnsPid]),
 1279:     
 1280:     mnesia_test_lib:kill_mnesia([Node2]),
 1281:     timer:sleep(timer:seconds(1)), 
 1282:     
 1283:     AnsPid ! {self(),uninstall_continue},
 1284:     
 1285:     ?match_receive({A,ok}),
 1286:     
 1287:     mnesia_test_lib:kill_mnesia(Nodes) ,
 1288:     mnesia_test_lib:start_mnesia(Nodes,[Tab]),
 1289:     
 1290:     A ! fun () ->
 1291: 		R1 = mnesia:dirty_read({Tab,1}),
 1292: 		R2 = mnesia:dirty_read({Tab,2}),
 1293: 		R3 = mnesia:dirty_read({Tab,3}),
 1294: 		{R1,R2,R3}
 1295: 	end,    
 1296:     ?match_receive({ A, {[],[],[]} }),
 1297:     
 1298:     B ! fun () ->
 1299: 		R1 = mnesia:dirty_read({Tab,1}),
 1300: 		R2 = mnesia:dirty_read({Tab,2}),
 1301: 		R3 = mnesia:dirty_read({Tab,3}),
 1302: 		{R1,R2,R3}
 1303: 	end,
 1304:     ?match_receive({ B, {[],[],[]} }),
 1305:     
 1306:     C ! fun () ->
 1307: 		R1 = mnesia:dirty_read({Tab,1}),
 1308: 		R2 = mnesia:dirty_read({Tab,2}),
 1309: 		R3 = mnesia:dirty_read({Tab,3}),
 1310: 		{R1,R2,R3}
 1311: 	end,
 1312:     ?match_receive({ C, {[],[],[]} }),
 1313:     
 1314:     ?match(ok,file:delete(install_backup)),
 1315:     ?verify_mnesia(Nodes, []).
 1316: 
 1317: 
 1318: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1319: 
 1320: %%%%%%%%%%%%%%%
 1321: 
 1322: cause_switch_before(doc)  ->
 1323:       ["interrupt the backup before iterating the retainer"];
 1324: cause_switch_before(suite)  ->  [];
 1325: cause_switch_before(Config) when is_list(Config) ->
 1326:    do_something_during_backup(cause_switch,pre,Config).
 1327: 
 1328: %%%%%%%%%%%%%%%
 1329: 
 1330: cause_switch_after(doc)  ->
 1331:       ["interrupt the backup after iterating the retainer"];
 1332: cause_switch_after(suite)  ->  [];
 1333: cause_switch_after(Config) when is_list(Config) ->
 1334:    do_something_during_backup(cause_switch,post,Config).
 1335: 
 1336: 
 1337: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1338: 
 1339: %%%%%%%%%%%%%%%%%%
 1340: 
 1341: cause_abort_before(doc) ->
 1342:       ["interrupt the backup before iterating the retainer"];
 1343: 
 1344: cause_abort_before(suite) ->  [];
 1345: cause_abort_before(Config) when is_list(Config) ->
 1346:    do_something_during_backup(cause_abort,pre,Config).
 1347: 
 1348: %%%%%%%%%%%%%%%%%%
 1349: 
 1350: cause_abort_after(doc) ->
 1351:       ["interrupt the backup after iterating the retainer"];
 1352: 
 1353: cause_abort_after(suite) ->  [];
 1354: cause_abort_after(Config) when is_list(Config) ->
 1355:    do_something_during_backup(cause_abort,post,Config).
 1356: 
 1357: 
 1358: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1359: 
 1360: %%%%%%%%%%%%%
 1361: 
 1362: change_schema_before(doc) ->
 1363:       ["interrupt the backup before iterating the retainer"];
 1364: change_schema_before(suite) -> [];
 1365: change_schema_before(Config) when is_list(Config) ->
 1366:    do_something_during_backup(change_schema,pre,Config).
 1367: 
 1368: %%%%%%%%%%%%%%%%
 1369: 
 1370: change_schema_after(doc) ->
 1371:       ["interrupt the backup after iterating the retainer"];
 1372: change_schema_after(suite) -> [];
 1373: change_schema_after(Config) when is_list(Config) ->
 1374:    do_something_during_backup(change_schema,post,Config).
 1375: 
 1376: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1377: 
 1378: do_something_during_backup(Action,DebugPoint,Config) ->
 1379:     ?is_debug_compiled,
 1380:     
 1381:     Nodes = ?acquire_nodes(3, Config),
 1382:     
 1383:     {success, [A,B,C]} = ?start_activities(Nodes),
 1384:     
 1385:     Node1 = node(A),
 1386:     Node2 = node(B),
 1387:     Node3 = node(C),
 1388:     
 1389:     TestPid = self(),
 1390:     %%?verbose("TestPid : ~p~n",[TestPid]),
 1391:     
 1392:     Tab = interrupt_table,
 1393:     Bak = interrupt_backup,
 1394:     Def = [{attributes, [key, value]},
 1395:            {ram_copies, [Node2,Node3]}],  
 1396:     %% necessary to test different types ???  
 1397:     
 1398:     ?match({atomic, ok}, mnesia:create_table(Tab, Def)),
 1399:     
 1400:     
 1401:     
 1402:     DebugId = {mnesia_log, tab_copier, DebugPoint},
 1403:     DebugFun = fun(PrevContext, EvalContext) ->
 1404: 		       ?verbose("interrupt backup  pid ~p  #~p ~n context ~p ~n"
 1405: 			    ,[self(),PrevContext,EvalContext]),
 1406: 		       TestPid ! {self(),interrupt_backup_pre},
 1407: 		       global:set_lock({{lock_for_backup, Tab}, self()},
 1408: 				       Nodes,
 1409: 				       infinity),
 1410: 		       		       
 1411: 		       %%?verbose("interrupt backup - continues ~n"),
 1412: 		       ?deactivate_debug_fun(DebugId),
 1413: 		       PrevContext+1
 1414: 	       end,
 1415:     ?remote_activate_debug_fun(Node1,DebugId, DebugFun, 1),
 1416:     
 1417:     ?match(ok, mnesia:dirty_write({Tab, 1, 4711})),
 1418:     ?match(ok, mnesia:dirty_write({Tab, 2, 42})),
 1419:     ?match(ok, mnesia:dirty_write({Tab, 3, 256})),
 1420:     
 1421:     {ok, CPName, _NodeList} =
 1422:         mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
 1423:                                     {ram_overrides_dump,true}]),
 1424:     
 1425:     A ! fun () ->
 1426: 		%%?verbose("node: ~p pid: ~p ~n",[node(),self()]),
 1427: 		mnesia:table_info(Tab,where_to_read)
 1428: 	end,
 1429: 
 1430:     ReadNode_a = receive { A, ReadNode_a_tmp } -> ReadNode_a_tmp end,
 1431:     ?verbose("ReadNode ~p ~n",[ReadNode_a]),
 1432:     
 1433:     global:set_lock({{lock_for_backup, Tab}, self()}, Nodes, infinity),
 1434:     
 1435:     A ! fun () ->    %% A shall perform the backup, so the test proc is
 1436: 		%% able to do further actions in between
 1437: 		mnesia:backup_checkpoint(CPName, Bak)
 1438: 	end,
 1439:     
 1440:     %% catch the debug function of mnesia, stop the backup process 
 1441:     %% kill the node ReadNode_a and continue the backup process
 1442:     %% As there is a second replica of the table, the backup shall continue
 1443: 
 1444:     case receive_messages([interrupt_backup_pre]) of
 1445: 	[{_AnsPid,interrupt_backup_pre}] -> ok
 1446:     end,
 1447:     
 1448:     case Action of
 1449:         cause_switch ->
 1450: 	    mnesia_test_lib:kill_mnesia([ReadNode_a]),
 1451: 	    timer:sleep(timer:seconds(1));
 1452:         cause_abort ->
 1453: 	    mnesia_test_lib:kill_mnesia([Node2,Node3]),
 1454: 	    timer:sleep(timer:seconds(1));
 1455:         change_schema ->
 1456: 	    Tab2 = second_interrupt_table,
 1457: 	    Def2 = [{attributes, [key, value]},
 1458: 		    {ram_copies, Nodes}],  
 1459: 	    
 1460: 	    ?match({atomic, ok}, mnesia:create_table(Tab2, Def2))
 1461:     end,
 1462:     
 1463:     %%    AnsPid ! {self(),interrupt_backup_continue},
 1464:     global:del_lock({{lock_for_backup, Tab}, self()}, Nodes),
 1465:     
 1466:     case Action of
 1467:         cause_abort -> 
 1468: 	    
 1469:             %% answer of A when finishing the backup
 1470:             ?match_receive({A,{error, _}}), 
 1471: 	    
 1472:             ?match({error,{"Cannot install fallback",_}},
 1473:                    mnesia:install_fallback(Bak)); 
 1474:         _ ->      %% cause_switch, change_schema
 1475: 	    
 1476:             ?match_receive({A,ok}), %% answer of A when finishing the backup
 1477: 	    
 1478:             %% send a fun to that node where mnesia is still running
 1479:             WritePid = case ReadNode_a of
 1480: 			   Node2 -> C; %%   node(C) == Node3
 1481: 			   Node3 -> B
 1482:                        end,
 1483:             WritePid ! fun () ->
 1484: 			       ?match(ok, mnesia:dirty_write({Tab, 1, 815})),
 1485: 			       ?match(ok, mnesia:dirty_write({Tab, 2, 816})),
 1486: 			       ok
 1487:                        end,
 1488:             ?match_receive({ WritePid, ok }),  
 1489:             ?match(ok, mnesia:install_fallback(Bak))
 1490:     end,        
 1491:     
 1492:     %% Stop and then start mnesia and check table consistency
 1493:     %%?verbose("Restarting Mnesia~n", []),
 1494:     mnesia_test_lib:kill_mnesia(Nodes),
 1495:     mnesia_test_lib:start_mnesia(Nodes,[Tab]),
 1496:     
 1497:     case Action of
 1498:         cause_switch ->
 1499: 	    %% the backup should exist
 1500: 	    cross_check_tables([A,B,C],Tab,{[{Tab,1,4711}],
 1501: 					    [{Tab,2,42}],
 1502: 					    [{Tab,3,256}] }),
 1503: 	    ?match(ok,file:delete(Bak));
 1504:         cause_abort ->
 1505: 	    %% the backup should  NOT exist
 1506: 	    cross_check_tables([A,B,C],Tab,{[],[],[]}),
 1507: 	    %% file does not exist 
 1508: 	    ?match({error, _},file:delete(Bak)); 
 1509:         change_schema ->
 1510: 	    %% the backup should exist
 1511: 	    cross_check_tables([A,B,C],Tab,{[{Tab,1,4711}],
 1512: 					    [{Tab,2,42}],
 1513: 					    [{Tab,3,256}] }),
 1514: 	    ?match(ok,file:delete(Bak))
 1515:     end,
 1516:     ?verify_mnesia(Nodes, []).
 1517: 
 1518: %% check the contents of the table
 1519: cross_check_tables([],_tab,_elements) -> ok;
 1520: cross_check_tables([Pid|Rest],Tab,{Val1,Val2,Val3}) ->
 1521:     Pid ! fun () ->
 1522:               R1 = mnesia:dirty_read({Tab,1}),
 1523:               R2 = mnesia:dirty_read({Tab,2}),
 1524:               R3 = mnesia:dirty_read({Tab,3}),
 1525:               {R1,R2,R3}
 1526:             end,
 1527:     ?match_receive({ Pid, {Val1, Val2, Val3 } }),
 1528:     cross_check_tables(Rest,Tab,{Val1,Val2,Val3} ).