1: %%
    2: %% %CopyrightBegin%
    3: %%
    4: %% Copyright Ericsson AB 1997-2013. All Rights Reserved.
    5: %%
    6: %% The contents of this file are subject to the Erlang Public License,
    7: %% Version 1.1, (the "License"); you may not use this file except in
    8: %% compliance with the License. You should have received a copy of the
    9: %% Erlang Public License along with this software. If not, it can be
   10: %% retrieved online at http://www.erlang.org/.
   11: %%
   12: %% Software distributed under the License is distributed on an "AS IS"
   13: %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
   14: %% the License for the specific language governing rights and limitations
   15: %% under the License.
   16: %%
   17: %% %CopyrightEnd%
   18: %%
   19: 
   20: %%
   21: -module(mnesia_durability_test).
   22: -author('hakan@erix.ericsson.se').
   23: -compile([export_all]).
   24: -include("mnesia_test_lib.hrl").
   25: 
   26: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
   27: 
   28: init_per_testcase(Func, Conf) ->
   29:     mnesia_test_lib:init_per_testcase(Func, Conf).
   30: 
   31: end_per_testcase(Func, Conf) ->
   32:     mnesia_test_lib:end_per_testcase(Func, Conf).
   33: 
   34: -record(test_rec,{key,val}).
   35: 
   36: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
   37: 
   38: all() -> 
   39:     [{group, load_tables},
   40:      {group, durability_of_dump_tables},
   41:      durability_of_disc_copies,
   42:      durability_of_disc_only_copies].
   43: 
   44: groups() -> 
   45:     [{load_tables, [],
   46:       [load_latest_data, load_local_contents_directly,
   47:        load_directly_when_all_are_ram_copiesA,
   48:        load_directly_when_all_are_ram_copiesB,
   49:        {group, late_load_when_all_are_ram_copies_on_ram_nodes},
   50:        load_when_last_replica_becomes_available,
   51:        load_when_we_have_down_from_all_other_replica_nodes,
   52:        late_load_transforms_into_disc_load,
   53:        late_load_leads_to_hanging,
   54:        force_load_when_nobody_intents_to_load,
   55:        force_load_when_someone_has_decided_to_load,
   56:        force_load_when_someone_else_already_has_loaded,
   57:        force_load_when_we_has_loaded,
   58:        force_load_on_a_non_local_table,
   59:        force_load_when_the_table_does_not_exist,
   60:        {group, load_tables_with_master_tables}]},
   61:      {late_load_when_all_are_ram_copies_on_ram_nodes, [],
   62:       [late_load_when_all_are_ram_copies_on_ram_nodes1,
   63:        late_load_when_all_are_ram_copies_on_ram_nodes2]},
   64:      {load_tables_with_master_tables, [],
   65:       [master_nodes, starting_master_nodes,
   66:        master_on_non_local_tables,
   67:        remote_force_load_with_local_master_node]},
   68:      {durability_of_dump_tables, [],
   69:       [dump_ram_copies, dump_disc_copies, dump_disc_only]}].
   70: 
   71: init_per_group(_GroupName, Config) ->
   72:     Config.
   73: 
   74: end_per_group(_GroupName, Config) ->
   75:     Config.
   76: 
   77: 
   78: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
   79: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
   80: 
   81: load_latest_data(doc) ->
   82:     ["Base functionality, verify that the latest data is loaded"];
   83: load_latest_data(suite) -> [];
   84: load_latest_data(Config) when is_list(Config) ->
   85:     [N1,N2,N3] = Nodes = ?acquire_nodes(3, Config),
   86:     %%Create a replicated local table
   87:     ?match({atomic,ok}, mnesia:create_table(t0, [{disc_copies,[N1,N2]}])),
   88:     ?match({atomic,ok}, mnesia:create_table(t1, [{disc_copies,[N1,N2]}])),
   89:     ?match({atomic,ok}, mnesia:create_table(t2, [{disc_copies,[N1,N2]}])),
   90:     ?match({atomic,ok}, mnesia:create_table(t3, [{disc_copies,[N1,N2]}])),
   91:     ?match({atomic,ok}, mnesia:create_table(t4, [{disc_copies,[N1,N2]}])),
   92:     ?match({atomic,ok}, mnesia:create_table(t5, [{disc_copies,[N1,N2]}])),
   93:     Rec1 = {t1, test, ok},
   94:     Rec2 = {t1, test, 2},
   95: 
   96:     ?match([], mnesia_test_lib:kill_mnesia([N1])),
   97:     ?match(ok, rpc:call(N2, mnesia, dirty_write, [Rec2])),
   98:     ?match([], mnesia_test_lib:kill_mnesia([N2])),
   99:     ?match([], mnesia_test_lib:kill_mnesia([N3])),
  100:    
  101:     ?match([], mnesia_test_lib:start_mnesia([N1], [])),   
  102:     %% Should wait for N2
  103:     ?match({timeout, [t1]}, rpc:call(N1, mnesia, wait_for_tables, [[t1], 1000])),
  104:     ?match([], mnesia_test_lib:start_mnesia([N3], [])),   
  105:     ?match({timeout, [t1]}, rpc:call(N1, mnesia, wait_for_tables, [[t1], 1000])),
  106: 
  107: 
  108:     ?match([], mnesia_test_lib:start_mnesia([N2], [])),   
  109:     ?match(ok, rpc:call(N2, mnesia, wait_for_tables, [[t1], 10000])),
  110:     ?match(ok, rpc:call(N1, mnesia, wait_for_tables, [[t1], 10000])),
  111:     %% We should find the record
  112:     ?match([Rec2], rpc:call(N1, mnesia, dirty_read, [t1, test])),
  113:     ?match([Rec2], rpc:call(N2, mnesia, dirty_read, [t1, test])),
  114:     
  115:     %% ok, lets switch order    
  116:     ?match(ok, mnesia:dirty_delete_object(Rec1)),
  117:     ?match(ok, mnesia:dirty_delete_object(Rec2)),
  118:     %% redo
  119: 
  120:     ?match([], mnesia_test_lib:kill_mnesia([N2])),
  121:     ?match(ok, mnesia:dirty_write(Rec1)),
  122:     ?match([], mnesia_test_lib:kill_mnesia([N3])),
  123:     ?match([], mnesia_test_lib:kill_mnesia([N1])),
  124: 
  125:     ?match([], mnesia_test_lib:start_mnesia([N2], [])),   
  126:     %% Should wait for N1
  127:     ?match({timeout, [t1]}, rpc:call(N2, mnesia, wait_for_tables, [[t1], 1000])),
  128:     ?match([], mnesia_test_lib:start_mnesia([N3], [])),   
  129:     ?match({timeout, [t1]}, rpc:call(N2, mnesia, wait_for_tables, [[t1], 1000])),
  130:     ?match([], mnesia_test_lib:start_mnesia([N1], [])),
  131:     ?match(ok, rpc:call(N2, mnesia, wait_for_tables, [[t1], 10000])),
  132:     ?match(ok, rpc:call(N1, mnesia, wait_for_tables, [[t1], 10000])),
  133:     %% We should find the record
  134:     ?match([Rec1], rpc:call(N1, mnesia, dirty_read, [t1, test])),
  135:     ?match([Rec1], rpc:call(N2, mnesia, dirty_read, [t1, test])),
  136:     
  137:     ?verify_mnesia(Nodes, []).
  138: 
  139: 
  140: load_local_contents_directly(doc) ->
  141:     ["Local contents shall always be loaded. Check this by having a local ",
  142:      "table on two nodes N1, N2, stopping N1 before N2, an then verifying ",
  143:      "that N1 can start without N2 being started."];
  144: load_local_contents_directly(suite) -> [];
  145: load_local_contents_directly(Config) when is_list(Config) ->
  146:     [N1, N2] = Nodes = ?acquire_nodes(2, Config),
  147:     %%Create a replicated local table
  148:     ?match({atomic,ok},
  149:            mnesia:create_table(test_rec,
  150:                                [{local_content,true},
  151:                                 {disc_copies,Nodes},
  152:                                 {attributes,record_info(fields,test_rec)}]
  153:                               ) ),
  154:     %%Verify that it has local contents.
  155:     ?match( true, mnesia:table_info(test_rec,local_content) ),
  156:     %%Helper Funs
  157:     Write_one = fun(Value) -> mnesia:write(#test_rec{key=1,val=Value}) end,
  158:     Read_one  = fun(Key)   -> mnesia:read( {test_rec, Key}) end,
  159:     %%Write a value one N1 that we may test against later
  160:     ?match({atomic,ok},
  161:            rpc:call( N1, mnesia, transaction, [Write_one,[11]] ) ),
  162:     %%Stop Mnesia on N1
  163:     %?match([], mnesia_test_lib:stop_mnesia([N1])),
  164:     ?match([], mnesia_test_lib:kill_mnesia([N1])),
  165: 
  166:     %%Write a value on N2, same key but a different value
  167:     ?match({atomic,ok},
  168:            rpc:call( N2, mnesia, transaction, [Write_one,[22]] ) ),
  169:     %%Stop Mnesia on N2
  170:     %?match([], mnesia_test_lib:stop_mnesia([N2])),
  171:     ?match([], mnesia_test_lib:kill_mnesia([N2])),
  172: 
  173:     %%Restart Mnesia on N1 verify that we can read from it without
  174:     %%starting Mnesia on N2.
  175:     ?match(ok, rpc:call(N1, mnesia, start, [])),
  176:     ?match(ok, rpc:call(N1, mnesia, wait_for_tables, [[test_rec], 30000])),
  177:     %%Read back the value
  178:     ?match( {atomic,[#test_rec{key=1,val=11}]},
  179:            rpc:call(N1, mnesia, transaction, [Read_one,[1]] ) ),
  180:     %%Restart Mnesia on N2 and verify the contents there.
  181:     ?match(ok, rpc:call(N2, mnesia, start, [])),
  182:     ?match(ok, rpc:call(N2, mnesia, wait_for_tables, [[test_rec], 30000])),
  183:     ?match( {atomic,[#test_rec{key=1,val=22}]},
  184:            rpc:call(N2, mnesia, transaction, [Read_one,[1]] ) ),
  185:     %%Check that the start of Mnesai on N2 did not affect the contents on N1
  186:     ?match( {atomic,[#test_rec{key=1,val=11}]},
  187:            rpc:call(N1, mnesia, transaction, [Read_one,[1]] ) ),
  188:     ?verify_mnesia(Nodes, []).
  189: 
  190: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  191: 
  192: load_directly_when_all_are_ram_copiesA(doc) ->
  193:     ["Tables that are RAM copies only shall also be loaded directly. ",
  194:      "1. N1 and N2 has RAM copies of a table, stop N1 before N2. ",
  195:      "2. When N1 starts he shall have access to the table ",
  196:      "   without having to start N2" ];
  197: load_directly_when_all_are_ram_copiesA(suite) -> [];
  198: load_directly_when_all_are_ram_copiesA(Config) when is_list(Config) ->
  199:     [N1, N2] = Nodes = ?acquire_nodes(2, Config),
  200:     
  201:     ?match({atomic,ok},
  202:            mnesia:create_table(test_rec,
  203:                                [{ram_copies,Nodes},
  204:                                 {attributes,record_info(fields,test_rec)}]
  205:                               ) ),
  206:     ?match( Nodes, mnesia:table_info(test_rec,ram_copies) ),
  207:     ?match( [], mnesia:table_info(test_rec,disc_copies) ),
  208:     ?match( [], mnesia:table_info(test_rec,disc_only_copies) ),
  209:     Write_one = fun(Value) -> mnesia:write(#test_rec{key=2,val=Value}) end,
  210:     Read_one  = fun()      -> mnesia:read({test_rec,2}) end,
  211:     %%Write a value one N1 that we may test against later
  212:     ?match({atomic,ok},
  213:            rpc:call( N1, mnesia, transaction, [Write_one,[11]] ) ),
  214:     %%Stop Mnesia on N1
  215:     ?match([], mnesia_test_lib:kill_mnesia([N1])),
  216:     %%Write a value and check result (on N2; not possible on N1
  217:     %%since Mnesia is stopped there).
  218:     ?match({atomic,ok}, rpc:call(N2,mnesia,transaction,[Write_one,[22]]) ),
  219:     ?match({atomic,[#test_rec{key=2,val=22}]},
  220:            rpc:call(N2,mnesia,transaction,[Read_one]) ),
  221:     %%Stop Mnesia on N2
  222:     ?match([], mnesia_test_lib:kill_mnesia([N2])),
  223:     %%Restart Mnesia on N1 verify that we can access test_rec from
  224:     %%N1 without starting Mnesia on N2.
  225:     ?match(ok, rpc:call(N1, mnesia, start, [])),
  226:     ?match(ok, rpc:call(N1, mnesia, wait_for_tables, [[test_rec], 30000])),
  227:     ?match({atomic,[]}, rpc:call(N1,mnesia,transaction,[Read_one])),
  228:     ?match({atomic,ok}, rpc:call(N1,mnesia,transaction,[Write_one,[33]])),
  229:     ?match({atomic,[#test_rec{key=2,val=33}]},
  230:            rpc:call(N1,mnesia,transaction,[Read_one])),
  231:     %%Restart Mnesia on N2 and verify the contents there.
  232:     ?match([], mnesia_test_lib:start_mnesia([N2], [test_rec])),
  233:     ?match( {atomic,[#test_rec{key=2,val=33}]},
  234:            rpc:call(N2, mnesia, transaction, [Read_one] ) ),
  235:     %%Check that the start of Mnesai on N2 did not affect the contents on N1
  236:     ?match( {atomic,[#test_rec{key=2,val=33}]},
  237:            rpc:call(N1, mnesia, transaction, [Read_one] ) ),
  238:     ?verify_mnesia(Nodes, []).
  239: 
  240: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  241: 
  242: load_directly_when_all_are_ram_copiesB(doc) ->
  243:     ["Tables that are RAM copies only shall be loaded from a replicat ",
  244:      "when possible. ",
  245:      "1. N1 and N2 has RAM copies of a table, stop N1 before N2.",
  246:      "2. Now start N2 first and then N1, N1 shall then load the table ",
  247:      "   from N2."];
  248: load_directly_when_all_are_ram_copiesB(suite) -> [];
  249: load_directly_when_all_are_ram_copiesB(Config) when is_list(Config) ->
  250:     [N1, N2] = Nodes = ?acquire_nodes(2, Config),
  251:     ?match({atomic,ok},
  252:            mnesia:create_table(test_rec,
  253:                                [{ram_copies,Nodes},
  254:                                 {attributes,record_info(fields,test_rec)}]
  255:                               ) ),
  256:     ?match( Nodes, mnesia:table_info(test_rec,ram_copies) ),
  257:     ?match( [], mnesia:table_info(test_rec,disc_copies) ),
  258:     ?match( [], mnesia:table_info(test_rec,disc_only_copies) ),
  259:     Write_one = fun(Value) -> mnesia:write(#test_rec{key=3,val=Value}) end,
  260:     Read_one  = fun()      -> mnesia:read( {test_rec, 3}) end,
  261:     %%Write a value one N1 that we may test against later
  262:     ?match({atomic,ok},
  263:            rpc:call( N1, mnesia, transaction, [Write_one,[11]] ) ),
  264:     ?match({atomic,[#test_rec{key=3,val=11}]},
  265:            rpc:call(N2,mnesia,transaction,[Read_one]) ),
  266:     %%Stop Mnesia on N1
  267:     ?match([], mnesia_test_lib:kill_mnesia([N1])),
  268:     %%Write a value and check result (on N2; not possible on N1
  269:     %%since Mnesia is stopped there).
  270:     ?match({atomic,ok}, rpc:call(N2,mnesia,transaction,[Write_one,[22]]) ),
  271:     ?match({atomic,[#test_rec{key=3,val=22}]},
  272:            rpc:call(N2,mnesia,transaction,[Read_one]) ),
  273:     %%Stop Mnesia on N2
  274:     ?match([], mnesia_test_lib:kill_mnesia([N2])),
  275:     %%Restart Mnesia on N2 verify that we can access test_rec from
  276:     %%N2 without starting Mnesia on N1.
  277:     ?match(ok, rpc:call(N2, mnesia, start, [])),
  278:     ?match(ok, rpc:call(N2, mnesia, wait_for_tables, [[test_rec], 30000])),
  279:     ?match({atomic,[]}, rpc:call(N2,mnesia,transaction,[Read_one])),
  280:     ?match({atomic,ok}, rpc:call(N2,mnesia,transaction,[Write_one,[33]])),
  281:     ?match({atomic,[#test_rec{key=3,val=33}]},
  282:            rpc:call(N2,mnesia,transaction,[Read_one])),
  283:     %%Restart Mnesia on N1 and verify the contents there.
  284:     ?match([], mnesia_test_lib:start_mnesia([N1], [test_rec])),
  285:     ?match( {atomic,[#test_rec{key=3,val=33}]},
  286:            rpc:call(N1,mnesia,transaction,[Read_one])),
  287:     %%Check that the start of Mnesai on N1 did not affect the contents on N2
  288:     ?match( {atomic,[#test_rec{key=3,val=33}]},
  289:            rpc:call(N2,mnesia,transaction,[Read_one])),
  290:     ?verify_mnesia(Nodes, []).
  291: 
  292: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  293: 
  294: 
  295: late_load_when_all_are_ram_copies_on_ram_nodes1(suite) -> [];
  296: late_load_when_all_are_ram_copies_on_ram_nodes1(Config) when is_list(Config) ->
  297:     [N1, N2] = mnesia_test_lib:prepare_test_case([{init_test_case, [mnesia]},
  298: 						  delete_schema,
  299: 						  {reload_appls, [mnesia]}],
  300: 						 2, Config, ?FILE, ?LINE),
  301:     Res = late_load_when_all_are_ram_copies_on_ram_nodes(N1, [N2], Config),
  302:     mnesia_test_lib:prepare_test_case([{reload_appls, [mnesia]}],
  303: 				      2, Config, ?FILE, ?LINE),
  304:     Res.
  305: 
  306: late_load_when_all_are_ram_copies_on_ram_nodes2(suite) -> [];
  307: late_load_when_all_are_ram_copies_on_ram_nodes2(Config) when is_list(Config) ->
  308:     [N1, N2, N3] = mnesia_test_lib:prepare_test_case([{init_test_case, [mnesia]},
  309: 						      delete_schema,
  310: 						      {reload_appls, [mnesia]}],
  311: 						     3, Config, ?FILE, ?LINE),
  312:     Res = late_load_when_all_are_ram_copies_on_ram_nodes(N1, [N2, N3], Config),
  313:     mnesia_test_lib:prepare_test_case([{reload_appls, [mnesia]}],
  314: 				      3, Config, ?FILE, ?LINE),
  315:     Res.
  316: 
  317: late_load_when_all_are_ram_copies_on_ram_nodes(DiscNode, RamNs, _Config)
  318:         when DiscNode == node() ->
  319:     ?match(ok, mnesia:create_schema([DiscNode])),
  320:     ?match(ok, mnesia:start()),
  321:     Nodes = [DiscNode | RamNs],
  322:     Extra = [{extra_db_nodes, Nodes}],
  323:     Ok = [ok || _ <- RamNs],
  324:     ?match({Ok, []}, rpc:multicall(RamNs, mnesia, start, [Extra])),
  325:     ?match([], wait_until_running(Nodes)),
  326: 
  327:     LastRam = lists:last(RamNs),
  328:     %% ?match({atomic, ok},
  329:     %%	   mnesia:add_table_copy(schema, LastRam, ram_copies)),
  330:     Def = [{ram_copies, RamNs}, {attributes, record_info(fields, test_rec)}],
  331:     ?match({atomic,ok}, mnesia:create_table(test_rec, Def)),
  332:     ?verify_mnesia(Nodes, []),
  333:     ?match([], mnesia_test_lib:stop_mnesia(RamNs)),
  334:     ?match(stopped, mnesia:stop()),
  335:     ?match(ok, mnesia:start()),
  336: 
  337:     Rec1 = #test_rec{key=3, val=33},
  338:     Rec2 = #test_rec{key=4, val=44},
  339:     
  340:     FirstRam = hd(RamNs),
  341:     ?match(ok, rpc:call(FirstRam, mnesia, start, [Extra])),
  342:     ?match(ok, rpc:call(FirstRam, mnesia, wait_for_tables,
  343: 			[[test_rec], 30000])),
  344:     ?match(ok, rpc:call(FirstRam, mnesia, dirty_write,[Rec1])),
  345:     ?match(ok, mnesia:wait_for_tables([test_rec], 30000)),
  346:     mnesia:dirty_write(Rec2),
  347: 
  348:     if
  349: 	FirstRam /= LastRam ->
  350: 	    ?match(ok, rpc:call(LastRam, mnesia, start, [Extra])),
  351: 	    ?match(ok, rpc:call(LastRam, mnesia, wait_for_tables,
  352: 				[[test_rec], 30000]));
  353: 	true ->
  354: 	    ignore
  355:     end,
  356:     ?match([Rec1], rpc:call(LastRam, mnesia, dirty_read, [{test_rec, 3}])),
  357:     ?match([Rec2], rpc:call(LastRam, mnesia, dirty_read, [{test_rec, 4}])),
  358:     ?verify_mnesia(Nodes, []).
  359: 
  360: wait_until_running(Nodes) ->
  361:     wait_until_running(Nodes, 30).
  362: 
  363: wait_until_running(Nodes, Times) when Times > 0->
  364:     Alive = mnesia:system_info(running_db_nodes),
  365:     case Nodes -- Alive of
  366: 	[] ->
  367: 	    [];
  368: 	Remaining ->
  369: 	    timer:sleep(timer:seconds(1)),
  370: 	    wait_until_running(Remaining, Times - 1)
  371:     end;
  372: wait_until_running(Nodes, _) ->
  373:     Nodes.
  374: 
  375: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  376: load_when_last_replica_becomes_available(doc) ->
  377:     ["Check that when all Mnesia nodes die at the same instant, then the ",
  378:      "replicated table shall be accessible when the last node is started ",
  379:      "again.",
  380:      "Checked by cheating. Start Mnesia on N1, N2, N3. Have a table ",
  381:      "replicated on disc on all three nodes, fill in some transactions, ",
  382:      "install a fallback. Restart mnesia on all nodes"
  383:      "This is the cheat and it simulates that all nodes died at the same ",
  384:      "time. Check that the table is only accessible after the last node ",
  385:      "has come up."];
  386: load_when_last_replica_becomes_available(suite) -> [];
  387: load_when_last_replica_becomes_available(Config) when is_list(Config) ->
  388:     [N1, N2, N3] = Nodes = ?acquire_nodes(3, Config),
  389:     ?match({atomic,ok},
  390:            mnesia:create_table(test_rec,
  391:                                [{disc_copies,Nodes},
  392:                                 {attributes,record_info(fields,test_rec)}]
  393:                               ) ),
  394:     ?match( [], mnesia:table_info(test_rec,ram_copies) ),
  395:     ?match( Nodes, mnesia:table_info(test_rec,disc_copies) ),
  396:     ?match( [], mnesia:table_info(test_rec,disc_only_copies) ),
  397:     Write_one = fun(Key,Val)->mnesia:write(#test_rec{key=Key,val=Val}) end,
  398:     Read_one  = fun(Key)    ->mnesia:read( {test_rec, Key}) end,
  399:     %%Write one value from each node.
  400:     ?match({atomic,ok},rpc:call(N1,mnesia,transaction,[Write_one,[1,11]])),
  401:     ?match({atomic,ok},rpc:call(N2,mnesia,transaction,[Write_one,[2,22]])),
  402:     ?match({atomic,ok},rpc:call(N3,mnesia,transaction,[Write_one,[3,33]])),
  403:     %%Check the values
  404:     ?match({atomic,[#test_rec{key=1,val=11}]},
  405:            rpc:call(N2,mnesia,transaction,[Read_one,[1]]) ),
  406:     ?match({atomic,[#test_rec{key=2,val=22}]},
  407:            rpc:call(N3,mnesia,transaction,[Read_one,[2]]) ),
  408:     ?match({atomic,[#test_rec{key=3,val=33}]},
  409:            rpc:call(N1,mnesia,transaction,[Read_one,[3]]) ),
  410:     
  411:     ?match(ok, mnesia:backup("test_last_replica")),
  412:     ?match(ok, mnesia:install_fallback("test_last_replica")),
  413:     file:delete("test_last_replica"),
  414:     %%Stop Mnesia on all three nodes   
  415:     ?match([], mnesia_test_lib:kill_mnesia(Nodes)),
  416: 
  417:     %%Start Mnesia on one node, make sure that test_rec is not available
  418:     ?match(ok, rpc:call(N2, mnesia, start, [])),
  419:     ?match({timeout,[test_rec]},
  420:            rpc:call(N2, mnesia, wait_for_tables, [[test_rec], 10000])),
  421:     ?match(ok, rpc:call(N1, mnesia, start, [])),
  422:     ?match({timeout,[test_rec]},
  423:            rpc:call(N1, mnesia, wait_for_tables, [[test_rec], 10000])),
  424:     %%Start the third node
  425:     ?match(ok, rpc:call(N3, mnesia, start, [])),
  426:     %%Make sure that the table is loaded everywhere
  427:     ?match(ok, rpc:call(N3, mnesia, wait_for_tables, [[test_rec], 30000])),
  428:     ?match(ok, rpc:call(N2, mnesia, wait_for_tables, [[test_rec], 30000])),
  429:     ?match(ok, rpc:call(N1, mnesia, wait_for_tables, [[test_rec], 30000])),
  430: 
  431:     %%Check the values
  432:     ?match({atomic,[#test_rec{key=1,val=11}]},
  433:            rpc:call(N2,mnesia,transaction,[Read_one,[1]]) ),
  434:     ?match({atomic,[#test_rec{key=2,val=22}]},
  435:            rpc:call(N3,mnesia,transaction,[Read_one,[2]]) ),
  436:     ?match({atomic,[#test_rec{key=3,val=33}]},
  437:            rpc:call(N1,mnesia,transaction,[Read_one,[3]]) ),
  438:     ?verify_mnesia(Nodes, []).
  439: 
  440: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  441: 
  442: load_when_we_have_down_from_all_other_replica_nodes(doc) ->
  443:     ["The table can be loaded if this node was the last one surviving. ",
  444:      "Check this by having N1, N2, N3 and a table replicated on all those ",
  445:      "nodes. Then kill them in the N1, N2, N3 order. Then start N3 and ",
  446:      "verify that the table is available with correct contents."];
  447: load_when_we_have_down_from_all_other_replica_nodes(suite) -> [];
  448: load_when_we_have_down_from_all_other_replica_nodes(Config) when is_list(Config) ->
  449:     [N1, N2, N3] = Nodes = ?acquire_nodes(3, Config),
  450:     ?match({atomic,ok},
  451:            mnesia:create_table(test_rec,
  452:                                [{disc_copies,Nodes},
  453:                                 {attributes,record_info(fields,test_rec)}]
  454:                               ) ),
  455:     ?match( [], mnesia:table_info(test_rec,ram_copies) ),
  456:     ?match( Nodes, mnesia:table_info(test_rec,disc_copies) ),
  457:     ?match( [], mnesia:table_info(test_rec,disc_only_copies) ),
  458:     Write_one = fun(Key,Val)->mnesia:write(#test_rec{key=Key,val=Val}) end,
  459:     Read_one  = fun(Key)    ->mnesia:read( {test_rec, Key}) end,
  460:     %%Write one value from each node.
  461:     ?match({atomic,ok},rpc:call(N1,mnesia,transaction,[Write_one,[1,111]])),
  462:     ?match({atomic,ok},rpc:call(N2,mnesia,transaction,[Write_one,[2,222]])),
  463:     ?match({atomic,ok},rpc:call(N3,mnesia,transaction,[Write_one,[3,333]])),
  464:     %%Check the values
  465:     ?match({atomic,[#test_rec{key=1,val=111}]},
  466:            rpc:call(N2,mnesia,transaction,[Read_one,[1]]) ),
  467:     ?match({atomic,[#test_rec{key=2,val=222}]},
  468:            rpc:call(N3,mnesia,transaction,[Read_one,[2]]) ),
  469:     ?match({atomic,[#test_rec{key=3,val=333}]},
  470:            rpc:call(N1,mnesia,transaction,[Read_one,[3]]) ),
  471:     %%Stop Mnesia on all three nodes
  472:     ?match([], mnesia_test_lib:kill_mnesia([N1])),
  473:     ?match({atomic,ok},rpc:call(N2,mnesia,transaction,[Write_one,[22,22]])),
  474:     ?match([], mnesia_test_lib:kill_mnesia([N2])),
  475:     ?match({atomic,ok},rpc:call(N3,mnesia,transaction,[Write_one,[33,33]])),
  476:     ?match([], mnesia_test_lib:kill_mnesia([N3])),
  477:     ?verbose("Mnesia stoped on all three nodes.~n",[]),
  478: 
  479:     %%Start Mnesia on N3; wait for 'test_rec' table to load
  480:     ?match(ok, rpc:call(N3, mnesia, start, [])),
  481:     ?match(ok, rpc:call(N3, mnesia, wait_for_tables, [[test_rec], 30000])),
  482: 
  483:     %%Check the values
  484:     ?match({atomic,[#test_rec{key=1,val=111}]},
  485:            rpc:call(N3,mnesia,transaction,[Read_one,[1]]) ),
  486:     ?match({atomic,[#test_rec{key=2,val=222}]},
  487:            rpc:call(N3,mnesia,transaction,[Read_one,[2]]) ),
  488:     ?match({atomic,[#test_rec{key=3,val=333}]},
  489:            rpc:call(N3,mnesia,transaction,[Read_one,[3]]) ),
  490:     ?match({atomic,[#test_rec{key=22,val=22}]},
  491:            rpc:call(N3,mnesia,transaction,[Read_one,[22]]) ),
  492:     ?match({atomic,[#test_rec{key=33,val=33}]},
  493:            rpc:call(N3,mnesia,transaction,[Read_one,[33]]) ),
  494:     ?verify_mnesia([N3], [N1, N2]).
  495: 
  496: 
  497: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  498: 
  499: late_load_transforms_into_disc_load(doc) ->
  500:     ["Difficult case that needs instrumentation of Mnesia.",
  501:      "A table is force loaded, and Mnesia decides to load it from another ",
  502:      "Mnesia node because it is avaliable there. The other Mnesia node then ",
  503:      "dies in mid copy which shall make the first Mnesia node to really ",
  504:      "force load from disc.",
  505:      "Check this by starting N1 and N2 and replicating a table between ",
  506:      "them. Then kill N1 before N2. The idea is to start N2 first, then ",
  507:      "N1 and then do a force load on N1. This force load will load from ",
  508:      "N2 BUT N2 must be killed after the decision to load from it has ",
  509:      "been made. tricky."];
  510: 
  511: late_load_transforms_into_disc_load(suite) ->  [];
  512: late_load_transforms_into_disc_load(Config) when is_list(Config) ->
  513:     ?is_debug_compiled,
  514:     
  515:     [Node1, Node2] = Nodes = ?acquire_nodes(2, Config),
  516:     
  517:     {success, [A, B]} = ?start_activities(Nodes),
  518:     
  519:     ?match(Node1, node(A)),
  520:     ?match(Node2, node(B)),
  521: 
  522:     Tab = late_load_table,
  523:     Def = [{attributes, [key, value]},
  524:            {disc_copies, Nodes}],  
  525:     ?match({atomic, ok}, mnesia:create_table(Tab, Def)),
  526:     ?match(ok, mnesia:dirty_write({Tab, 111, 4711})),
  527:     ?match(ok, mnesia:dirty_write({Tab, 222, 42})),
  528: 
  529:     TestPid = self(),
  530:     DebugId = {mnesia_loader, do_get_network_copy},
  531:     DebugFun = fun(PrevContext, EvalContext) ->
  532: 		       ?verbose("interrupt late load,  pid ~p  #~p ~n context ~p ~n",
  533: 			    [self(),PrevContext,EvalContext]),
  534: 		       
  535: 		       mnesia_test_lib:kill_mnesia([Node2]),
  536: 		       TestPid ! {self(),debug_fun_was_called},
  537: 
  538: 		       ?verbose("interrupt late_log - continues ~n",[]),
  539: 		       ?deactivate_debug_fun(DebugId),
  540: 		       PrevContext+1
  541: 	       end,
  542:     ?remote_activate_debug_fun(Node1,DebugId, DebugFun, 1),
  543:     
  544:     %% kill mnesia on node1
  545:     mnesia_test_lib:kill_mnesia([Node1]),    
  546:     %% wait a while, so  that mnesia is really down 
  547:     timer:sleep(timer:seconds(1)), 
  548: 
  549:     ?match(ok, rpc:call(Node2, mnesia, dirty_write, [{Tab, 222, 815}])),
  550: 
  551:     %% start Mnesia on node1
  552:     ?match(ok,mnesia:start()),
  553:     ?match(yes, mnesia:force_load_table(Tab)),
  554:     ?match(ok, mnesia:wait_for_tables([Tab],timer:seconds(30))),
  555:     
  556:     receive_messages([debug_fun_was_called]),
  557:     
  558:     check_tables([A],[{Tab,111},{Tab,222}],[[{Tab,111,4711}],[{Tab,222,42}]]),
  559:     ?verify_mnesia([Node1], [Node2]).
  560: 
  561: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  562: 
  563: late_load_leads_to_hanging(doc) ->
  564:     ["Difficult case that needs instrumentation of Mnesia.",
  565:      "A table is loaded, and Mnesia decides to load it from another ",
  566:      "Mnesia node because it has the latest copy there. ",
  567:      "The other Mnesia node then ",
  568:      "dies in mid copy which shall make the first Mnesia node not to ",
  569:      "force load from disc but to wait for the other node to come up again",
  570:      "Check this by starting N1 and N2 and replicating a table between ",
  571:      "them. Then kill N1 before N2. The idea is to start N2 first, then ",
  572:      "N1. This load will load from ",
  573:      "N2 BUT N2 must be killed after the decision to load from it has ",
  574:      "been made. tricky."];
  575: 
  576: late_load_leads_to_hanging(suite) ->  [];
  577: late_load_leads_to_hanging(Config) when is_list(Config) ->
  578:     ?is_debug_compiled,
  579: 
  580:     [Node1, Node2] = Nodes = ?acquire_nodes(2, Config),
  581:     
  582:     Tab = late_load_table,
  583:     Def = [{attributes, [key, value]},
  584:            {disc_copies, Nodes}],  
  585: 
  586:     ?match({atomic, ok}, mnesia:create_table(Tab, Def)),
  587:     ?match(ok, mnesia:dirty_write({Tab, 111, 4711})),
  588:     ?match(ok, mnesia:dirty_write({Tab, 222, 42})),
  589: 
  590:     DebugId = {mnesia_loader, do_get_network_copy},
  591:     DebugFun = fun(PrevContext, EvalContext) ->
  592: 		       ?verbose("interrupt late load,  pid ~p  #~p ~n context ~p ~n",
  593: 			    [self(), PrevContext, EvalContext]),
  594: 		       mnesia_test_lib:kill_mnesia([Node2]),
  595: 		       ?verbose("interrupt late load - continues ~n",[]),
  596: 		       ?deactivate_debug_fun(DebugId),
  597: 		       PrevContext+1
  598: 	       end,
  599: 
  600:     ?remote_activate_debug_fun(Node1,DebugId, DebugFun, 1),
  601:     mnesia_test_lib:kill_mnesia([Node1]),
  602:     %% wait a while, so  that mnesia is really down 
  603:     timer:sleep(timer:seconds(1)), 
  604:     
  605:     ?match(ok, rpc:call(Node2, mnesia, dirty_write, [{Tab, 333, 666}])),
  606:     
  607:     %% start Mnesia on node1
  608:     ?match(ok, mnesia:start()),
  609: 
  610:     ?match({timeout, [Tab]}, mnesia:wait_for_tables([Tab], timer:seconds(2))),
  611: 
  612:     ?match({'EXIT', {aborted, _}}, mnesia:dirty_read({Tab, 222})),    
  613:     %% mnesia on node1 is waiting for node2 coming up 
  614:     
  615:     ?match(ok, rpc:call(Node2, mnesia, start, [])),
  616:     ?match(ok, mnesia:wait_for_tables([Tab], timer:seconds(30))),
  617:     ?match([{Tab, 333, 666}], mnesia:dirty_read({Tab, 333})),   
  618:     ?verify_mnesia([Node2, Node1], []).
  619: 
  620: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  621: 
  622: force_load_when_nobody_intents_to_load(doc) ->
  623:     ["Normal force load. Start N1 N2, kill in N1, N2 order. Start N1 do ",
  624:      "force load. Did it work?"];
  625: force_load_when_nobody_intents_to_load(suite) -> [];
  626: force_load_when_nobody_intents_to_load(Config) when is_list(Config) ->
  627:     [N1, N2] = Nodes = ?acquire_nodes(2, Config),
  628:     Table = test_rec,
  629:     Trec1a = #test_rec{key=1,val=111},
  630:     Trec1b = #test_rec{key=1,val=333},
  631:     Trec2a = #test_rec{key=2,val=222},
  632:     Trec3a = #test_rec{key=3,val=333},
  633:     Trec3b = #test_rec{key=3,val=666},
  634: 
  635:     ?match({atomic,ok}, rpc:call(N1, mnesia,create_table,
  636:                                  [Table,
  637:                                   [{disc_copies,Nodes},
  638:                                    {attributes,record_info(fields,test_rec)}
  639:                                   ] ] ) ),
  640:     ?match( [], mnesia:table_info(Table,ram_copies) ),
  641:     ?match( Nodes, mnesia:table_info(Table,disc_copies) ),
  642:     ?match( [], mnesia:table_info(Table,disc_only_copies) ),
  643:     Write_one = fun(Rec) -> mnesia:write(Rec) end,
  644:     Read_one  = fun(Key) -> mnesia:read({Table, Key}) end,
  645:     %%Write one value
  646:     ?match({atomic,ok},rpc:call(N1,mnesia,transaction,[Write_one,[Trec1a]])),
  647:     %%Check it
  648:     ?match({atomic,[Trec1a]},rpc:call(N2,mnesia,transaction,[Read_one,[1]]) ),
  649:     %%Shut down mnesia on N1
  650:     ?match([], mnesia_test_lib:stop_mnesia([N1])),
  651:     %%Write and check value while N1 is down
  652:     ?match({atomic,ok},rpc:call(N2,mnesia,transaction,[Write_one,[Trec1b]])),
  653:     ?match({atomic,ok},rpc:call(N2,mnesia,transaction,[Write_one,[Trec2a]])),
  654:     ?match({atomic,ok},rpc:call(N2,mnesia,transaction,[Write_one,[Trec3a]])),
  655:     ?match({aborted,{node_not_running,N1}},
  656:            rpc:call(N1,mnesia,transaction,[Read_one,[2]]) ),
  657:     ?match({atomic,[Trec1b]},rpc:call(N2,mnesia,transaction,[Read_one,[1]]) ),
  658:     ?match({atomic,[Trec2a]},rpc:call(N2,mnesia,transaction,[Read_one,[2]]) ),
  659:     ?match({atomic,[Trec3a]},rpc:call(N2,mnesia,transaction,[Read_one,[3]]) ),
  660:     %%Shut down Mnesia on N2
  661:     ?match([], mnesia_test_lib:stop_mnesia([N2])),
  662: 
  663:     %%Restart Mnesia on N1
  664:     ?match(ok, rpc:call(N1, mnesia, start, [])),
  665:     %%Check that table is not available (waiting for N2)
  666:     ?match({timeout,[Table]},
  667:            rpc:call(N1, mnesia, wait_for_tables, [[Table], 3000])),
  668: 
  669:     %%Force load on N1
  670:     ?match(yes,rpc:call(N1,mnesia,force_load_table,[Table])),
  671:     %%Check values
  672:     ?match({atomic,[Trec1a]},rpc:call(N1,mnesia,transaction,[Read_one,[1]]) ),
  673:     ?match({atomic,[]},     rpc:call(N1,mnesia,transaction,[Read_one,[2]]) ),
  674:     ?match({atomic,[]},     rpc:call(N1,mnesia,transaction,[Read_one,[3]]) ),
  675:     %%Write a value for key=3
  676:     ?match({atomic,ok},rpc:call(N1,mnesia,transaction,[Write_one,[Trec3b]])),
  677: 
  678:     %%Restart N2 and check values
  679:     ?match(ok, rpc:call(N2, mnesia, start, [])),
  680:     ?match(ok, rpc:call(N2, mnesia, wait_for_tables, [[Table], 30000])),
  681: 
  682:     ?match({atomic,[Trec1a]},rpc:call(N1,mnesia,transaction,[Read_one,[1]]) ),
  683:     ?match({atomic,[Trec1a]},rpc:call(N2,mnesia,transaction,[Read_one,[1]]) ),
  684: 
  685:     ?match({atomic,[]},rpc:call(N1,mnesia,transaction,[Read_one,[2]]) ),
  686:     ?match({atomic,[]},rpc:call(N2,mnesia,transaction,[Read_one,[2]]) ),
  687: 
  688:     ?match({atomic,[Trec3b]},rpc:call(N1,mnesia,transaction,[Read_one,[3]]) ),
  689:     ?match({atomic,[Trec3b]},rpc:call(N2,mnesia,transaction,[Read_one,[3]]) ),
  690:     ?verify_mnesia(Nodes, []).
  691: 
  692: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  693: 
  694: force_load_when_someone_has_decided_to_load(doc) ->
  695:     ["Difficult case that needs instrumentation of Mnesia.",
  696:      "Start N1 and N2, replicate table, kill in N1, N2 order. Start N2 ",
  697:      "and start N1 before N2 has really loaded the table but after N2 has ",
  698:      "decided to load it."];
  699: 
  700: force_load_when_someone_has_decided_to_load(suite) -> [];
  701: force_load_when_someone_has_decided_to_load(Config) when is_list(Config) ->
  702:     ?is_debug_compiled,
  703:     
  704:     [Node1, Node2] = Nodes = ?acquire_nodes(2, Config),   
  705:     {success, [A, B]} = ?start_activities(Nodes),        
  706:     ?match(Node1, node(A)), %% Just to check :)
  707:     ?match(Node2, node(B)),
  708: 
  709:     Tab = late_load_table,
  710:     Def = [{attributes, [key, value]}, {disc_copies, Nodes}],  
  711: 
  712:     ?match({atomic, ok}, mnesia:create_table(Tab, Def)),
  713:     ?match(ok, mnesia:dirty_write({Tab, 111, 4711})),
  714:     ?match(ok, mnesia:dirty_write({Tab, 222, 42})),
  715: 
  716:     Self = self(),
  717:     DebugId = {mnesia_controller, late_disc_load},
  718:     DebugFun = fun(PrevContext, EvalContext) ->
  719: 		       ?verbose("interrupt late disc load,
  720:                              pid ~p  #~p ~n context ~p ~n",
  721: 			    [self(),PrevContext,EvalContext]),
  722: 		       Self ! {self(), fun_in_postion},
  723: 		       wait_for_signal(),
  724: 		       ?verbose("interrupt late disc load - continues ~n",[]),
  725: 		       ?deactivate_debug_fun(DebugId),
  726: 		       PrevContext+1
  727: 	       end,
  728:  
  729:     %% kill mnesia on node1
  730:     mnesia_test_lib:kill_mnesia([Node1]),
  731:     %% wait a while, so  that mnesia is really down 
  732:     timer:sleep(timer:seconds(1)), 
  733: 
  734:     ?match(ok, rpc:call(Node2, mnesia, dirty_write, [{Tab, 222, 815}])),
  735:     %% kill mnesia on node2
  736:     mnesia_test_lib:kill_mnesia([Node2]),
  737:     %% wait a while, so that mnesia is really down 
  738:     timer:sleep(timer:seconds(1)), 
  739: 
  740:     ?remote_activate_debug_fun(Node2,DebugId, DebugFun, 1),
  741: 
  742:     B ! fun() -> mnesia:start() end,    
  743:     [{Mnesia_Pid, fun_in_postion}] = receive_messages([fun_in_postion]),    
  744:     
  745:     %% start Mnesia on node1
  746:     A ! fun() -> mnesia:start() end, 
  747:     ?match_receive(timeout),
  748: % Got some problem with this testcase when we modified mnesia init 
  749: % These test cases are very implementation dependent!
  750: %    A ! fun() -> mnesia:wait_for_tables([Tab], 3000) end,
  751: %    ?match_receive({A, {timeout, [Tab]}}),
  752:     A ! fun() -> mnesia:force_load_table(Tab) end,
  753:     ?match_receive(timeout),
  754: 
  755:     Mnesia_Pid ! continue,
  756:     ?match_receive({B, ok}),
  757:     ?match_receive({A, ok}),
  758:     ?match_receive({A, yes}),
  759: 
  760:     B ! fun() -> mnesia:wait_for_tables([Tab], 10000) end,
  761:     ?match_receive({B, ok}),
  762:     ?match(ok, mnesia:wait_for_tables([Tab], timer:seconds(30))),
  763:     ?match([{Tab, 222, 815}], mnesia:dirty_read({Tab, 222})),
  764:     ?verify_mnesia(Nodes, []).
  765: 
  766: wait_for_signal() ->
  767:     receive 
  768: 	continue -> ok
  769:     %% Don't eat any other mnesia internal msg's
  770:     after 
  771: 	timer:minutes(2) -> ?error("Timedout in wait_for_signal~n", [])
  772:     end.
  773: 	
  774: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  775: 
  776: force_load_when_someone_else_already_has_loaded(doc) ->
  777:     ["Normal case. Do a force load when somebody else has loaded the table. ",
  778:      "Start N1, N2, kill in N1, N2 order. Start N2 load the table, start N1 ",
  779:      "force load. Did it work? (i.e: did N1 load the table from N2 as that",
  780:      "one is the latest version and it is available on N2)"];
  781: 
  782: force_load_when_someone_else_already_has_loaded(suite) -> [];
  783: force_load_when_someone_else_already_has_loaded(Config) when is_list(Config) ->
  784:     [N1, N2] = Nodes = ?acquire_nodes(2, Config),
  785:     Table = test_rec,
  786:     Trec1 = #test_rec{key=1,val=111},
  787:     Trec2 = #test_rec{key=1,val=222},
  788: 
  789:     ?match({atomic,ok}, rpc:call(N1, mnesia,create_table,
  790:                                  [Table,
  791:                                   [{disc_copies,Nodes},
  792:                                    {attributes,record_info(fields,test_rec)}
  793:                                   ] ] ) ),
  794:     ?match( [], mnesia:table_info(Table,ram_copies) ),
  795:     ?match( Nodes, mnesia:table_info(Table,disc_copies) ),
  796:     ?match( [], mnesia:table_info(Table,disc_only_copies) ),
  797:     Write_one = fun(Rec) -> mnesia:write(Rec) end,
  798:     Read_one  = fun(Key) -> mnesia:read({Table, Key}) end,
  799:     %%Write one value
  800:     ?match({atomic,ok},rpc:call(N1,mnesia,transaction,[Write_one,[Trec1]])),
  801:     %%Check it
  802:     ?match({atomic,[Trec1]},rpc:call(N2,mnesia,transaction,[Read_one,[1]]) ),
  803:     %%Shut down mnesia
  804:     ?match([], mnesia_test_lib:stop_mnesia([N1])),
  805:     timer:sleep(500),
  806:     ?match([], mnesia_test_lib:stop_mnesia([N2])),
  807:     %%Restart Mnesia on N2;wait for tables to load
  808:     ?match(ok, rpc:call(N2, mnesia, start, [])),
  809:     ?match(ok, rpc:call(N2, mnesia, wait_for_tables, [[test_rec], 30000])),
  810:     %%Write one value
  811:     ?match({atomic,ok},rpc:call(N2,mnesia,transaction,[Write_one,[Trec2]])),
  812:     %%Start on N1; force load
  813:     ?match(ok, rpc:call(N1, mnesia, start, [])),
  814:     %%Force load from file
  815:     ?match(yes, rpc:call(N1,mnesia,force_load_table,[Table])),
  816:     %%Check the value
  817:     ?match({atomic,[Trec2]},rpc:call(N1,mnesia,transaction,[Read_one,[1]]) ),
  818:        %%               === there must be a Trec2 here !!!!
  819:     ?verify_mnesia(Nodes, []).
  820: 
  821: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  822: 
  823: force_load_when_we_has_loaded(doc) ->
  824:     ["Force load a table we already have loaded"];
  825: force_load_when_we_has_loaded(suite) -> [];
  826: force_load_when_we_has_loaded(Config) when is_list(Config) ->
  827:     [N1] = Nodes = ?acquire_nodes(1, Config),
  828:     Table = test_rec,
  829:     Trec1 = #test_rec{key=1,val=111},
  830:     Trec2 = #test_rec{key=1,val=222},
  831: 
  832:     ?match({atomic,ok}, rpc:call(N1, mnesia,create_table,
  833:                                  [Table,
  834:                                   [{disc_copies,Nodes},
  835:                                    {attributes,record_info(fields,test_rec)}
  836:                                   ] ] ) ),
  837:     ?match( [], mnesia:table_info(Table,ram_copies) ),
  838:     ?match( Nodes, mnesia:table_info(Table,disc_copies) ),
  839:     ?match( [], mnesia:table_info(Table,disc_only_copies) ),
  840:     Write_one = fun(Rec) -> mnesia:write(Rec) end,
  841:     Read_one  = fun(Key) -> mnesia:read({Table, Key}) end,
  842:     %%Write one value
  843:     ?match({atomic,ok},rpc:call(N1,mnesia,transaction,[Write_one,[Trec1]])),
  844:     %%Check it
  845:     ?match({atomic,[Trec1]},rpc:call(N1,mnesia,transaction,[Read_one,[1]]) ),
  846:     %%Shut down mnesia
  847:     ?match([], mnesia_test_lib:stop_mnesia(Nodes)),
  848:     %%Restart Mnesia;wait for tables to load
  849:     ?match([], mnesia_test_lib:start_mnesia(Nodes, [Table])),
  850:     %%Write one value
  851:     ?match({atomic,ok},rpc:call(N1,mnesia,transaction,[Write_one,[Trec2]])),
  852:     %%Force load from file
  853:     ?match(yes, rpc:call(N1,mnesia,force_load_table,[Table])),
  854:     %%Check the value
  855:     ?match({atomic,[Trec2]},rpc:call(N1,mnesia,transaction,[Read_one,[1]]) ),
  856:     ?verify_mnesia(Nodes, []).
  857: 
  858: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  859: 
  860: force_load_on_a_non_local_table(doc) ->
  861:     ["This is NOT allowed, the test case is a negative test",
  862:      "Force load on a table that isn't replicated on this node."];
  863: force_load_on_a_non_local_table(suite) -> [];
  864: force_load_on_a_non_local_table(Config) when is_list(Config) ->
  865:     [N1, N2, N3] = Nodes = ?acquire_nodes( 3, Config),
  866:     TableNodes = lists:sublist(Nodes,2),
  867:     Table = test_rec,
  868:     Trec1 = #test_rec{key=1,val=11},
  869: 
  870:     ?match({atomic,ok}, rpc:call(N1, mnesia,create_table,
  871:                                  [Table,
  872:                                   [{disc_copies,TableNodes},
  873:                                    {attributes,record_info(fields,test_rec)}
  874:                                   ] ] ) ),
  875:     ?match( [], mnesia:table_info(Table,ram_copies) ),
  876:     ?match( TableNodes, mnesia:table_info(Table,disc_copies) ),
  877:     ?match( [], mnesia:table_info(Table,disc_only_copies) ),
  878:     Write_one = fun(Rec) -> mnesia:write(Rec) end,
  879:     Read_one  = fun(Key) -> mnesia:read({Table, Key}) end,
  880:     %%Write one value
  881:     ?match({atomic,ok},rpc:call(N1,mnesia,transaction,[Write_one,[Trec1]])),
  882:     %%Check it from the other nodes
  883:     ?match({atomic,[Trec1]},rpc:call(N2,mnesia,transaction,[Read_one,[1]]) ),
  884:     ?match({atomic,[Trec1]},rpc:call(N3,mnesia,transaction,[Read_one,[1]]) ),
  885: 
  886:     %%Make sure that Table is non-local
  887:     ?match_inverse(N3, rpc:call(N3,mnesia,table_info,[Table,where_to_read])),
  888:     %%Try to force load it
  889:     ?match(yes, rpc:call(N3,mnesia,force_load_table,[Table])),
  890:     ?verify_mnesia(Nodes, []).
  891: 
  892: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  893: 
  894: force_load_when_the_table_does_not_exist(doc) ->
  895:     ["This is NOT allowed, the test case is a negative test",
  896:      "Force load on a table that doesn't exist."];
  897: force_load_when_the_table_does_not_exist(suite) -> [];
  898: force_load_when_the_table_does_not_exist(Config) when is_list(Config) ->
  899:     Nodes = ?acquire_nodes( 2, Config),
  900:     
  901:     %%Dummy table
  902:     ?match({atomic,ok},
  903:            mnesia:create_table(test_rec,
  904:                                [{disc_copies,Nodes},
  905:                                 {attributes,record_info(fields,test_rec)}]
  906:                               ) ),
  907:     ?match( [], mnesia:table_info(test_rec,ram_copies) ),
  908:     ?match( Nodes, mnesia:table_info(test_rec,disc_copies) ),
  909:     ?match( [], mnesia:table_info(test_rec,disc_only_copies) ),
  910:     Tab = dummy,
  911:     %%Make sure that Tab is an unknown table
  912:     ?match( false, lists:member(Tab,mnesia:system_info(tables)) ),
  913:     ?match( {error, {no_exists, Tab}}, mnesia:force_load_table(Tab) ),
  914:     ?verify_mnesia(Nodes, []).
  915: 
  916: 
  917: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  918: 
  919: 
  920: 
  921: -define(SDwrite(Tup), fun() -> mnesia:write(Tup) end).			            
  922: 
  923: master_nodes(suite) -> [];
  924: master_nodes(Config) when is_list(Config) ->
  925:     [A, B, C] = Nodes = ?acquire_nodes(3, Config),
  926:     Tab = test_table_master_nodes,    
  927:     ?match({atomic,ok}, mnesia:create_table(Tab, [{disc_copies, Nodes}])),
  928: 
  929:     %% Test one: Master A and the table should be loaded from A
  930: 
  931:     ?match(ok, rpc:call(A, mnesia, set_master_nodes, [Tab, [A]])),
  932:     ?match({atomic, ok}, mnesia:sync_transaction(?SDwrite({Tab, 1, init}))),
  933:     
  934:     mnesia_test_lib:stop_mnesia([A]),
  935:     ?match({atomic, ok}, rpc:call(B, mnesia, sync_transaction, [?SDwrite({Tab, 1, updated})])),
  936:     ?match(ok, rpc:call(A, mnesia, start, [])),
  937:     ?match(ok, rpc:call(A, mnesia, wait_for_tables, [[Tab], 3000])),
  938:     
  939:     ?match([{Tab, 1, init}], rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
  940:     ?match([{Tab, 1, updated}], rpc:call(B, mnesia, dirty_read, [{Tab, 1}])),
  941:     ?match([{Tab, 1, updated}], rpc:call(C, mnesia, dirty_read, [{Tab, 1}])),
  942:     
  943:     %% Test 2: Master [A,B] and B is Up the table should be loaded from B
  944: 
  945:     ?match(ok, rpc:call(A, mnesia, set_master_nodes, [Tab, [A, B]])),
  946:     ?match({atomic, ok}, mnesia:sync_transaction(?SDwrite({Tab, 1, init}))),
  947:     
  948:     mnesia_test_lib:stop_mnesia([A]),
  949:     ?match({atomic, ok}, rpc:call(B, mnesia, sync_transaction, [?SDwrite({Tab, 1, updated})])),
  950:     ?match(ok, rpc:call(A, mnesia, start, [])),
  951:     ?match(ok, rpc:call(A, mnesia, wait_for_tables, [[Tab], 3000])),
  952:     
  953:     ?match([{Tab, 1, updated}], rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
  954:     ?match([{Tab, 1, updated}], rpc:call(B, mnesia, dirty_read, [{Tab, 1}])),
  955:     ?match([{Tab, 1, updated}], rpc:call(C, mnesia, dirty_read, [{Tab, 1}])),
  956: 
  957:     %% Test 3: Master [A,B] and B is down the table should be loaded from A
  958: 
  959:     ?match(ok, rpc:call(A, mnesia, set_master_nodes, [Tab, [A, B]])),
  960:     ?match({atomic, ok}, mnesia:sync_transaction(?SDwrite({Tab, 1, init}))),
  961:     
  962:     mnesia_test_lib:stop_mnesia([A]),
  963:     ?match({atomic, ok}, rpc:call(B, mnesia, sync_transaction, [?SDwrite({Tab, 1, updated})])),
  964:     mnesia_test_lib:stop_mnesia([B]),
  965:     ?match(ok, rpc:call(A, mnesia, start, [])),
  966:     ?match(ok, rpc:call(A, mnesia, wait_for_tables, [[Tab], 3000])),
  967:     
  968:     ?match(ok, rpc:call(B, mnesia, start, [])),
  969:     ?match(ok, rpc:call(B, mnesia, wait_for_tables, [[Tab], 3000])),
  970:     
  971:     ?match([{Tab, 1, init}], rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
  972:     ?match([{Tab, 1, _Unknown}], rpc:call(B, mnesia, dirty_read, [{Tab, 1}])),
  973:     ?match([{Tab, 1, updated}], rpc:call(C, mnesia, dirty_read, [{Tab, 1}])),
  974:     
  975:     %% Test 4: Master [B] and B is Up the table should be loaded from B
  976: 
  977:     ?match(ok, rpc:call(A, mnesia, set_master_nodes, [Tab, [B]])),
  978:     ?match({atomic, ok}, mnesia:sync_transaction(?SDwrite({Tab, 1, init}))),
  979:     
  980:     mnesia_test_lib:stop_mnesia([A]),
  981:     ?match({atomic, ok}, rpc:call(B, mnesia, sync_transaction, [?SDwrite({Tab, 1, updated})])),
  982:     ?match(ok, rpc:call(A, mnesia, start, [])),
  983:     ?match(ok, rpc:call(A, mnesia, wait_for_tables, [[Tab], 3000])),
  984:     
  985:     ?match([{Tab, 1, updated}], rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
  986:     ?match([{Tab, 1, updated}], rpc:call(B, mnesia, dirty_read, [{Tab, 1}])),
  987:     ?match([{Tab, 1, updated}], rpc:call(C, mnesia, dirty_read, [{Tab, 1}])),
  988:     
  989:     %% Test 5: Master [B] and B is down the table should not be loaded
  990: 
  991:     ?match(ok, rpc:call(A, mnesia, set_master_nodes, [Tab, [B]])),
  992:     ?match({atomic, ok}, mnesia:sync_transaction(?SDwrite({Tab, 1, init}))),
  993:     
  994:     mnesia_test_lib:stop_mnesia([A]),
  995:     ?match({atomic, ok}, rpc:call(B, mnesia, sync_transaction, [?SDwrite({Tab, 1, updated})])),
  996:     mnesia_test_lib:stop_mnesia([B]),
  997:     ?match({atomic, ok}, rpc:call(C, mnesia, sync_transaction, [?SDwrite({Tab, 1, update_2})])),
  998:     ?match(ok, rpc:call(A, mnesia, start, [])),
  999:     ?match({timeout, [Tab]}, rpc:call(A, mnesia, wait_for_tables, [[Tab], 2000])),
 1000: 
 1001:     %% Test 6: Force load on table that couldn't be loaded due to master 
 1002:     %%         table setttings, loads other active replicas i.e. from C
 1003: 
 1004:     ?match(yes, rpc:call(A, mnesia, force_load_table, [Tab])),
 1005:     ?match(ok, rpc:call(A, mnesia, wait_for_tables, [[Tab], 3000])),
 1006:     
 1007:     ?match(ok, rpc:call(B, mnesia, start, [])),
 1008:     ?match(ok, rpc:call(B, mnesia, wait_for_tables, [[Tab], 3000])),
 1009:     
 1010:     ?match([{Tab, 1, update_2}], rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
 1011:     ?match([{Tab, 1, update_2}], rpc:call(B, mnesia, dirty_read, [{Tab, 1}])),
 1012:     ?match([{Tab, 1, update_2}], rpc:call(C, mnesia, dirty_read, [{Tab, 1}])),
 1013: 
 1014:     %% Test 7: Master [B] and B is down the table should not be loaded, 
 1015:     %%         force_load when there are no active replicas availible 
 1016:     %%         should generate a load of a local table
 1017: 
 1018:     ?match(ok, rpc:call(A, mnesia, set_master_nodes, [Tab, [B]])),
 1019:     ?match({atomic, ok}, mnesia:sync_transaction(?SDwrite({Tab, 1, init}))),
 1020:     
 1021:     mnesia_test_lib:stop_mnesia([A]),
 1022:     ?match({atomic, ok}, rpc:call(B, mnesia, sync_transaction, [?SDwrite({Tab, 1, updated})])),
 1023:     mnesia_test_lib:stop_mnesia([B, C]),
 1024:     ?match(ok, rpc:call(A, mnesia, start, [])),
 1025:     ?match({timeout, [Tab]}, rpc:call(A, mnesia, wait_for_tables, [[Tab], 2000])),
 1026: 
 1027:     ?match(yes, rpc:call(A, mnesia, force_load_table, [Tab])),
 1028:     ?match([{Tab, 1, init}], rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
 1029: 
 1030:     ?verify_mnesia([A], [B,C]).
 1031: 
 1032: starting_master_nodes(suite) -> [];
 1033: starting_master_nodes(doc) -> 
 1034:     ["Complementory to TEST 5 and 6 above, if the master node (B) starts"
 1035:      " and loads the table it should be loaded on the waiting node (A) "];
 1036: starting_master_nodes(Config) when is_list(Config) ->
 1037:     [A, B, C] = Nodes = ?acquire_nodes(3, Config),
 1038:     Tab = starting_master_nodes,    
 1039:     ?match({atomic,ok}, mnesia:create_table(Tab, [{disc_copies, Nodes}])),
 1040:     %% Start by checking TEST 5 above.
 1041:     
 1042:     ?match(ok, rpc:call(A, mnesia, set_master_nodes, [Tab, [B]])),
 1043:     ?match({atomic, ok}, mnesia:sync_transaction(?SDwrite({Tab, 1, init}))),
 1044:     mnesia_test_lib:stop_mnesia([A]),
 1045:     ?match({atomic, ok}, rpc:call(B, mnesia, sync_transaction, [?SDwrite({Tab, 1, updated})])),
 1046:     mnesia_test_lib:stop_mnesia([B]),
 1047:     ?match({atomic, ok}, rpc:call(C, mnesia, sync_transaction, [?SDwrite({Tab, 1, update_2})])),
 1048:     
 1049:     ?match(ok, rpc:call(A, mnesia, start, [])),
 1050:     ?match({timeout, [Tab]}, rpc:call(A, mnesia, wait_for_tables, [[Tab], 2000])),
 1051:     %% Start the B node and the table should be loaded on A!
 1052:     ?match(ok, rpc:call(B, mnesia, start, [])),
 1053:     ?match(ok, rpc:call(B, mnesia, wait_for_tables, [[Tab], 3000])),
 1054:     ?match(ok, rpc:call(A, mnesia, wait_for_tables, [[Tab], 3000])),
 1055:     
 1056:     ?verify_mnesia([A,B,C], []).
 1057: 
 1058: 
 1059: master_on_non_local_tables(suite) -> [];
 1060: master_on_non_local_tables(Config) when is_list(Config) ->
 1061:     [A, B, C] = Nodes = ?acquire_nodes(3, Config),
 1062:     Tab = test_table_non_local,    
 1063:     ?match({atomic,ok}, mnesia:create_table(Tab, [{disc_copies, [B, C]}])),
 1064: 
 1065:     ?match(ok, rpc:call(A, mnesia, set_master_nodes, [Tab, [B]])),
 1066:     ?match({atomic, ok}, mnesia:sync_transaction(?SDwrite({Tab, 1, init}))),
 1067: 
 1068:     %% Test 1: Test that table info are updated when master node comes up
 1069:     
 1070:     mnesia_test_lib:stop_mnesia([A, B]),
 1071:     ?match({atomic, ok}, rpc:call(C, mnesia, sync_transaction, [?SDwrite({Tab, 1, updated})])),   
 1072:     ?match(ok, rpc:call(A, mnesia, start, [])),
 1073:     
 1074:     ?match({timeout, [Tab]}, rpc:call(A, mnesia, wait_for_tables, [[Tab], 2000])),
 1075:     ErrorRead = {badrpc,{'EXIT', {aborted,{no_exists,[test_table_non_local,1]}}}},
 1076:     ErrorWrite = {badrpc,{'EXIT', {aborted,{no_exists,test_table_non_local}}}},
 1077:     ?match(ErrorRead, rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
 1078:     ?match(ErrorWrite, rpc:call(A, mnesia, dirty_write, [{Tab, 1, updated_twice}])),
 1079:     
 1080:     ?match(ok, rpc:call(B, mnesia, start, [])),
 1081:     ?match(ok, rpc:call(A, mnesia, wait_for_tables, [[Tab], 2000])),
 1082:     
 1083:     ?match([{Tab, 1, updated}], rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
 1084:     ?match(B, rpc:call(A, mnesia, table_info, [Tab, where_to_read])),
 1085:     ?match({atomic, ok}, rpc:call(A, mnesia, sync_transaction, [?SDwrite({Tab, 1, init})])),
 1086: 
 1087:     %% Test 2: Test that table info are updated after force_load
 1088: 
 1089:     mnesia_test_lib:stop_mnesia([A, B]),
 1090:     ?match({atomic, ok}, rpc:call(C, mnesia, sync_transaction, [?SDwrite({Tab, 1, updated})])),   
 1091:     ?match(ok, rpc:call(A, mnesia, start, [])),
 1092:     
 1093:     ?match({timeout, [Tab]}, rpc:call(A, mnesia, wait_for_tables, [[Tab], 2000])),
 1094:     ?match(yes, rpc:call(A, mnesia, force_load_table, [Tab])),
 1095:     ?match(C, rpc:call(A, mnesia, table_info, [Tab, where_to_read])),
 1096: 
 1097:     ?match([{Tab, 1, updated}], rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
 1098:     ?match({atomic, ok}, rpc:call(A, mnesia, sync_transaction, [?SDwrite({Tab, 1, updated_twice})])),
 1099:     
 1100:     ?match(ok, rpc:call(B, mnesia, start, [])),
 1101:     ?match(ok, rpc:call(B, mnesia, wait_for_tables, [[Tab], 10000])),
 1102: 
 1103:     ?match([{Tab, 1, updated_twice}], rpc:call(A, mnesia, dirty_read, [{Tab, 1}])),
 1104:     ?match([{Tab, 1, updated_twice}], rpc:call(B, mnesia, dirty_read, [{Tab, 1}])),
 1105:     ?match([{Tab, 1, updated_twice}], rpc:call(C, mnesia, dirty_read, [{Tab, 1}])),
 1106:     
 1107:     ?verify_mnesia(Nodes, []).
 1108: 
 1109: remote_force_load_with_local_master_node(doc) ->
 1110:     ["Force load a table on a remote node while the ",
 1111:      "local node is down. Start the local node and ",
 1112:      "verfify that the tables is loaded from disc locally "
 1113:      "if the local node has itself as master node and ",
 1114:      "the remote node has both the local and remote node ",
 1115:      "as master nodes"];
 1116: remote_force_load_with_local_master_node(suite) -> [];
 1117: remote_force_load_with_local_master_node(Config) when is_list(Config) ->
 1118:     [A, B] = Nodes = ?acquire_nodes(2, Config),
 1119: 
 1120:     Tab = remote_force_load_with_local_master_node,
 1121:     ?match({atomic,ok}, mnesia:create_table(Tab, [{disc_copies, Nodes}])),
 1122:     ?match(ok, rpc:call(A, mnesia, set_master_nodes, [Tab, [A, B]])),
 1123:     ?match(ok, rpc:call(B, mnesia, set_master_nodes, [Tab, [B]])),
 1124:     
 1125:     W = fun(Who) -> mnesia:write({Tab, who, Who}) end,
 1126:     ?match({atomic, ok}, rpc:call(A,mnesia, sync_transaction, [W, [a]])),
 1127:     ?match(stopped, rpc:call(A, mnesia, stop, [])),
 1128:     ?match({atomic, ok}, rpc:call(B, mnesia, sync_transaction, [W, [b]])),
 1129:     ?match(stopped, rpc:call(B, mnesia, stop, [])),
 1130: 
 1131:     ?match(ok, rpc:call(A, mnesia, start, [])),
 1132:     ?match(ok, rpc:call(A, mnesia, wait_for_tables, [[Tab], 3000])),
 1133:     ?match([{Tab, who, a}], rpc:call(A, mnesia, dirty_read, [{Tab, who}])),
 1134:     
 1135:     ?match(ok, rpc:call(B, mnesia, start, [])),
 1136:     ?match(ok, rpc:call(B, mnesia, wait_for_tables, [[Tab], 3000])),
 1137:     ?match([{Tab, who, b}], rpc:call(B, mnesia, dirty_read, [{Tab, who}])),
 1138: 
 1139:     ?verify_mnesia(Nodes, []).
 1140: 
 1141: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1142: 
 1143: 
 1144: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1145: 
 1146: dump_ram_copies(doc) -> 
 1147:     ["Check that ram_copies tables are loaded with the"
 1148:      "contents that had been dumped before Mnesia",
 1149:      "was restarted. " ];
 1150: dump_ram_copies(suite)  -> [];
 1151: dump_ram_copies(Config) when is_list(Config)  ->
 1152:     Nodes = ?acquire_nodes(3, Config),
 1153:     {success, [P1,P2,P3]} = ?start_activities(Nodes),
 1154:     
 1155:     NP1 = node(P1),
 1156:     NP2 = node(P2),
 1157:     
 1158:     {A,B,C} = case node() of
 1159: 		  NP1 ->
 1160: 						%?verbose("first case ~n"),
 1161: 		      {P3,P2,P1};
 1162: 		  NP2 ->
 1163: 						%?verbose("second case ~n"),
 1164: 		      {P3,P1,P2};
 1165: 		  _  ->
 1166: 		      {P1,P2,P3}
 1167: 	      end,
 1168:     
 1169:     Node1 = node(A),
 1170:     Node2 = node(B),
 1171:     Node3 = node(C),
 1172:     
 1173:     ?verbose(" A   pid:~p  node:~p ~n",[A,Node1]),
 1174:     ?verbose(" B   pid:~p  node:~p ~n",[B,Node2]),
 1175:     ?verbose(" C   pid:~p  node:~p ~n",[C,Node3]),
 1176:     
 1177:     
 1178:     %%  ram copies table on 2 nodes
 1179:     
 1180:     Tab = dump_table,
 1181:     Def = [{attributes, [key, value]},
 1182:            {ram_copies, [Node1,Node2]}],      
 1183:     
 1184:     ?match({atomic, ok}, mnesia:create_table(Tab, Def)),
 1185:     
 1186:     ?match(ok, mnesia:dirty_write({Tab, 1, 4711})),
 1187:     ?match(ok, mnesia:dirty_write({Tab, 2, 42})),
 1188:     ?match(ok, mnesia:dirty_write({Tab, 3, 256})),
 1189:     
 1190:     %%  dump the table
 1191:     
 1192:     ?match( {atomic,ok}, mnesia:dump_tables([Tab])),
 1193:     
 1194:     %% perform updates (they shall be lost after kill Mnesia )
 1195:     
 1196:     ?match(ok, mnesia:dirty_write({Tab, 1, 815})),
 1197:     ?match(ok, mnesia:dirty_write({Tab, 2, 915})),
 1198:     
 1199:     %% add another replica on node3
 1200:     mnesia:add_table_copy(Tab,Node3,ram_copies),
 1201:     
 1202:     %% all 3 replicas shall have the new contents
 1203:     cross_check_tables([A,B,C],Tab,
 1204:                        {[{Tab,1,815}],[{Tab,2,915}],[{Tab,3,256}]}),
 1205:     
 1206:     %% kill mnesia on node 3
 1207:     mnesia_test_lib:kill_mnesia([Node3]),
 1208:     
 1209:     %% wait a while, so  that mnesia is really down 
 1210:     timer:sleep(timer:seconds(2)), 
 1211:     
 1212:     mnesia_test_lib:kill_mnesia([Node1,Node2]),  %% kill them as well
 1213:     timer:sleep(timer:seconds(2)), 
 1214:     
 1215:     %% start Mnesia only on node 3
 1216:     ?verbose("starting mnesia on Node3~n",[]),
 1217:     
 1218:     %% test_lib:mnesia_start doesnt work, because it waits
 1219:     %% for the schema on all nodes ... ???
 1220:     ?match(ok,rpc:call(Node3,mnesia,start,[]) ),
 1221:     ?match(ok,rpc:call(Node3,mnesia,wait_for_tables,
 1222: 		       [[Tab],timer:seconds(30)]   ) ),
 1223:     
 1224:     %% node3  shall have the conents of the dump
 1225:     cross_check_tables([C],Tab,{[{Tab,1,4711}],[{Tab,2,42}],[{Tab,3,256}]}),
 1226:     
 1227:     %% start Mnesia on the other 2 nodes, too
 1228:     mnesia_test_lib:start_mnesia([Node1,Node2],[Tab]),
 1229:     
 1230:     cross_check_tables([A,B,C],Tab,
 1231: 		       {[{Tab,1,4711}],[{Tab,2,42}],[{Tab,3,256}]}),
 1232:     ?verify_mnesia(Nodes, []).
 1233: 
 1234: %% check the contents of the table
 1235: 
 1236: cross_check_tables([],_tab,_elements) -> ok;
 1237: cross_check_tables([Pid|Rest],Tab,{Val1,Val2,Val3}) ->
 1238:     Pid ! fun () ->
 1239:               R1 = mnesia:dirty_read({Tab,1}),
 1240:               R2 = mnesia:dirty_read({Tab,2}),
 1241:               R3 = mnesia:dirty_read({Tab,3}),
 1242:               {R1,R2,R3}
 1243:             end,
 1244:     ?match_receive({ Pid, {Val1, Val2, Val3 } }),
 1245:     cross_check_tables(Rest,Tab,{Val1,Val2,Val3} ).   
 1246: 
 1247: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1248: 
 1249: %% Should be in evil test suite !!!
 1250: 
 1251: dump_disc_copies(doc) -> 
 1252:       ["Check that it is not possible to dump disc_copies tables"];
 1253: dump_disc_copies(suite)  -> [];
 1254: dump_disc_copies(Config) when is_list(Config)  ->
 1255:     do_dump_copies(Config, disc_copies).
 1256: 
 1257: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1258: 
 1259: %% Should be in evil test suite !!!
 1260: dump_disc_only(doc) -> 
 1261:        ["Check that it is not possible to dump disc_only_copies tables"];
 1262: dump_disc_only(suite)  -> [];
 1263: dump_disc_only(Config) when is_list(Config)  ->
 1264:     do_dump_copies(Config,disc_only_copies).
 1265: 
 1266: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1267: do_dump_copies(Config,Copies) ->
 1268:     [Node1] = Nodes = ?acquire_nodes(1, Config),
 1269: 
 1270:     Tab = dump_copies,
 1271:     Def = [{attributes, [key, value]},
 1272:            {Copies, [Node1]}],      
 1273:     
 1274:     ?match({atomic, ok}, mnesia:create_table(Tab, Def)),
 1275: 
 1276:     ?match(ok, mnesia:dirty_write({Tab, 1, 4711})),
 1277:     ?match(ok, mnesia:dirty_write({Tab, 2, 42})),
 1278:     ?match(ok, mnesia:dirty_write({Tab, 3, 256})),
 1279: 
 1280:     %%  dump the table
 1281:     ?match( {aborted, {"Only allowed on ram_copies",Tab,[Node1]}},
 1282:               mnesia:dump_tables([Tab])),
 1283: 
 1284:     ?match(ok, mnesia:dirty_write({Tab, 1, 815})),
 1285:     ?match(ok, mnesia:dirty_write({Tab, 2, 915})),
 1286: 
 1287:     %% kill mnesia on node1
 1288:     mnesia_test_lib:kill_mnesia([Node1]),
 1289:     
 1290:     %% wait a while, so  that mnesia is really down 
 1291:     timer:sleep(timer:seconds(1)), 
 1292: 
 1293:     mnesia_test_lib:start_mnesia([Node1],[Tab]),
 1294:     
 1295:     ?match([{Tab, 1, 815}], mnesia:dirty_read({Tab,1}) ),
 1296:     ?match([{Tab, 2, 915}], mnesia:dirty_read({Tab,2}) ),
 1297:     ?match([{Tab, 3, 256}], mnesia:dirty_read({Tab,3}) ),
 1298:     ?verify_mnesia(Nodes, []).
 1299: 
 1300: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1301: durability_of_disc_copies(doc) -> 
 1302:        ["Perform all possible kinds of updates on tables and check"
 1303:         "whether no data is lost after a restart of Mnesia.",
 1304:         "This test is done for disc_copies"];
 1305: 
 1306: durability_of_disc_copies(suite) -> [];
 1307: durability_of_disc_copies(Config) when is_list(Config) -> 
 1308:        do_disc_durability(Config,disc_copies).
 1309: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1310: 
 1311: durability_of_disc_only_copies(doc) -> 
 1312:        ["Perform all possible kinds of updates on tables and check"
 1313:         "whether no data is lost after a restart of Mnesia.",
 1314:         "This test is done for disc_only_copies"];
 1315: durability_of_disc_only_copies(suite) -> [];
 1316: durability_of_disc_only_copies(Config) when is_list(Config) -> 
 1317:        do_disc_durability(Config,disc_only_copies).
 1318: 
 1319: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1320: 
 1321: do_disc_durability(Config,CopyType) ->
 1322:     Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(1)}]),
 1323:     {success, [A,B,C]} = ?start_activities(Nodes),
 1324: 
 1325:     Tab_set = disc_durability_set,
 1326:     Def_set = [{attributes, [key, value]},
 1327: 	       {CopyType, Nodes}],  
 1328: 
 1329:     Tab_bag = disc_durability_bag,
 1330:     Def_bag = [{attributes, [key, value]},
 1331:                {type, bag},
 1332:                {CopyType, Nodes}],  
 1333: 
 1334:     ?match({atomic, ok}, mnesia:create_table(Tab_set, Def_set)),
 1335:     ?match({atomic, ok}, mnesia:create_table(Tab_bag, Def_bag)),
 1336: 
 1337:     %% do updates     
 1338:     ?match({atomic, ok},
 1339: 	   mnesia:transaction(fun()->
 1340: 				      mnesia:write({Tab_set, 11, 1111}),
 1341: 				      mnesia:write({Tab_set, 22, 2222}),
 1342: 				      mnesia:write({Tab_set, 33, 3333}),
 1343: 				      mnesia:write({Tab_set, 55, 5555})
 1344: 			      end)),
 1345:     mnesia:dirty_write({Tab_set, 44, 4444}),
 1346: 
 1347:     ?match({atomic, ok},
 1348: 	   mnesia:transaction(fun()->
 1349: 				      mnesia:write({Tab_bag, 11, a_1111}),
 1350: 				      mnesia:write({Tab_bag, 11, b_1111}),
 1351: 				      mnesia:write({Tab_bag, 22, a_2222}),
 1352: 				      mnesia:write({Tab_bag, 22, b_2222}),
 1353: 				      mnesia:write({Tab_bag, 33, a_3333}),
 1354: 				      mnesia:write({Tab_bag, 33, b_3333})
 1355: 			      end)),  
 1356:     ?match({atomic, ok},
 1357: 	   mnesia:transaction(fun()-> mnesia:delete({Tab_set, 22}) end)),  
 1358:     ?match(ok, mnesia:dirty_delete({Tab_set, 33})),
 1359:     ?match(5558, mnesia:dirty_update_counter({Tab_set, 55}, 3)),
 1360:     ?match({atomic, ok}, 
 1361: 	   mnesia:transaction(fun()->
 1362: 				      mnesia:delete_object({Tab_bag, 22, b_2222})
 1363: 			      end)),  
 1364:     ?match(ok, mnesia:dirty_delete_object({Tab_bag, 33, b_3333})),
 1365:     ?match(10, mnesia:dirty_update_counter({Tab_set, counter}, 10)),
 1366:     ?match({atomic, ok},  % Also syncs update_counter
 1367: 	   mnesia:sync_transaction(fun() -> mnesia:write({Tab_set,66,6666}) end)),  
 1368:     
 1369:     Updated = {[[{Tab_set,counter,10}],
 1370: 		[{Tab_set,counter,10}],
 1371: 		[{Tab_set,counter,10}]],[]},
 1372:     ?match(Updated, rpc:multicall(Nodes, mnesia, dirty_read, [Tab_set,counter])),
 1373:     
 1374:     %%  kill mnesia on all nodes, start it again and check the data
 1375:     mnesia_test_lib:kill_mnesia(Nodes),
 1376:     mnesia_test_lib:start_mnesia(Nodes,[Tab_set,Tab_bag]),
 1377:     
 1378:     ?log("Flushed ~p ~n", [mnesia_test_lib:flush()]), %% Debugging strange msgs..
 1379:     ?log("Processes ~p ~p ~p~n", [A,B,C]),
 1380:     check_tables([A,B,C], 
 1381: 		 [{Tab_set,11}, {Tab_set,22},{Tab_set,33},
 1382: 		  {Tab_set,44},{Tab_set,55}, {Tab_set,66},
 1383: 		  {Tab_bag,11}, {Tab_bag,22},{Tab_bag,33},
 1384: 		  {Tab_set, counter}],
 1385: 		 [[{Tab_set, 11, 1111}], [], [], [{Tab_set, 44, 4444}],
 1386: 		  [{Tab_set, 55, 5558}], [{Tab_set, 66, 6666}],
 1387: 		  lists:sort([{Tab_bag, 11, a_1111},{Tab_bag, 11, b_1111}]),
 1388: 		  [{Tab_bag, 22, a_2222}], [{Tab_bag, 33, a_3333}],
 1389: 		  [{Tab_set, counter, 10}]]),
 1390: 
 1391:     timer:sleep(1000), %% Debugging strange msgs..
 1392:     ?log("Flushed ~p ~n", [mnesia_test_lib:flush()]),
 1393:     ?verify_mnesia(Nodes, []).
 1394: 
 1395: %% check the contents of the table
 1396: %%
 1397: %%  all the processes in the PidList shall find all 
 1398: %%  table entries in ValList
 1399: 
 1400: check_tables([],_vallist,_resultList) -> ok;
 1401: check_tables([Pid|Rest],ValList,ResultList) ->
 1402:     Pid ! fun () ->
 1403:               check_values(ValList)
 1404:             end,
 1405:     ?match_receive({ Pid, ResultList }),
 1406:     check_tables(Rest,ValList,ResultList).
 1407: 
 1408: check_values([]) -> [];
 1409: check_values([{Tab,Key}|Rest]) ->
 1410:         Ret = lists:sort(mnesia:dirty_read({Tab,Key})),
 1411:         [Ret|check_values(Rest)].
 1412: 
 1413: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 1414: %
 1415: %  stolen from mnesia_recovery_test.erl:
 1416: 
 1417: receive_messages([]) -> [];
 1418: receive_messages(ListOfMsgs) ->
 1419:     receive 
 1420:         timeout ->     
 1421:             case lists:member(timeout, ListOfMsgs) of
 1422:                 false -> 
 1423:                     ?warning("I (~p) have received unexpected msg~n ~p ~n",
 1424:                         [self(),timeout]),
 1425:                     receive_messages(ListOfMsgs);
 1426:                 true -> 
 1427:                     ?verbose("I (~p) got msg ~p  ~n", [self(),timeout]),
 1428:                     [ timeout | receive_messages(ListOfMsgs -- [timeout])]
 1429:             end;
 1430: 
 1431:         {Pid, Msg} ->     
 1432:             case lists:member(Msg, ListOfMsgs) of
 1433:                 false -> 
 1434:                     ?warning("I (~p) have received unexpected msg~n ~p ~n",
 1435:                         [self(),{Pid, Msg}]),
 1436:                     receive_messages(ListOfMsgs);
 1437:                 true -> 
 1438:                     ?verbose("I (~p) got msg ~p from ~p ~n", [self(),Msg, Pid]),
 1439:                     [{Pid, Msg} | receive_messages(ListOfMsgs -- [Msg])]
 1440:             end;
 1441: 
 1442:         Else -> ?warning("Recevied unexpected Msg~n ~p ~n", [Else])
 1443:     after timer:seconds(40) -> 
 1444:             ?error("Timeout in receive msgs while waiting for ~p~n", 
 1445:                    [ListOfMsgs])
 1446:     end.  
 1447: