1: %%
    2: %% %CopyrightBegin%
    3: %%
    4: %% Copyright Ericsson AB 1997-2011. All Rights Reserved.
    5: %%
    6: %% The contents of this file are subject to the Erlang Public License,
    7: %% Version 1.1, (the "License"); you may not use this file except in
    8: %% compliance with the License. You should have received a copy of the
    9: %% Erlang Public License along with this software. If not, it can be
   10: %% retrieved online at http://www.erlang.org/.
   11: %%
   12: %% Software distributed under the License is distributed on an "AS IS"
   13: %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
   14: %% the License for the specific language governing rights and limitations
   15: %% under the License.
   16: %%
   17: %% %CopyrightEnd%
   18: %%
   19: 
   20: %%
   21: -module(mnesia_atomicity_test).
   22: -author('hakan@erix.ericsson.se').
   23: -author('rossi@erix.ericsson.se').
   24: -compile([export_all]).
   25: -include("mnesia_test_lib.hrl").
   26: 
   27: init_per_testcase(Func, Conf) ->
   28:     mnesia_test_lib:init_per_testcase(Func, Conf).
   29: 
   30: end_per_testcase(Func, Conf) ->
   31:     mnesia_test_lib:end_per_testcase(Func, Conf).
   32: 
   33: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
   34: all() ->
   35:     [explicit_abort_in_middle_of_trans,
   36:      runtime_error_in_middle_of_trans,
   37:      kill_self_in_middle_of_trans, throw_in_middle_of_trans,
   38:      {group, mnesia_down_in_middle_of_trans}].
   39: 
   40: groups() ->
   41:     [{mnesia_down_in_middle_of_trans, [],
   42:       [mnesia_down_during_infinite_trans,
   43:        {group, lock_waiter}, {group, restart_check}]},
   44:      {lock_waiter, [],
   45:       [lock_waiter_sw_r, lock_waiter_sw_rt, lock_waiter_sw_wt,
   46:        lock_waiter_wr_r, lock_waiter_srw_r, lock_waiter_sw_sw,
   47:        lock_waiter_sw_w, lock_waiter_sw_wr, lock_waiter_sw_srw,
   48:        lock_waiter_wr_wt, lock_waiter_srw_wt,
   49:        lock_waiter_wr_sw, lock_waiter_srw_sw, lock_waiter_wr_w,
   50:        lock_waiter_srw_w, lock_waiter_r_sw, lock_waiter_r_w,
   51:        lock_waiter_r_wt, lock_waiter_rt_sw, lock_waiter_rt_w,
   52:        lock_waiter_rt_wt, lock_waiter_wr_wr,
   53:        lock_waiter_srw_srw, lock_waiter_wt_r, lock_waiter_wt_w,
   54:        lock_waiter_wt_rt, lock_waiter_wt_wt, lock_waiter_wt_wr,
   55:        lock_waiter_wt_srw, lock_waiter_wt_sw, lock_waiter_w_wr,
   56:        lock_waiter_w_srw, lock_waiter_w_sw, lock_waiter_w_r,
   57:        lock_waiter_w_w, lock_waiter_w_rt, lock_waiter_w_wt]},
   58:      {restart_check, [],
   59:       [restart_r_one, restart_w_one, restart_rt_one,
   60:        restart_wt_one, restart_wr_one, restart_sw_one,
   61:        restart_r_two, restart_w_two, restart_rt_two,
   62:        restart_wt_two, restart_wr_two, restart_sw_two]}].
   63: 
   64: init_per_group(_GroupName, Config) ->
   65:     Config.
   66: 
   67: end_per_group(_GroupName, Config) ->
   68:     Config.
   69: 
   70: 
   71: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
   72: explicit_abort_in_middle_of_trans(suite) -> [];
   73: explicit_abort_in_middle_of_trans(Config) when is_list(Config) ->
   74:     [Node1] = Nodes = ?acquire_nodes(1, Config),
   75:     Tab = explicit_abort_in_middle_of_trans,
   76: 
   77:     Rec1A = {Tab, 1, a},
   78:     Rec1B = {Tab, 1, b},
   79: 
   80:     ?match({atomic, ok}, mnesia:create_table([{name, Tab},
   81: 					     {ram_copies, [Node1]}])),
   82:     %% Start a transaction on one node
   83:     {success, [A]} = ?start_activities([Node1]),
   84: 
   85:     %% store an object in the Tab - first tranaction
   86:     ?start_transactions([A]),
   87:     A ! fun() ->
   88: 		mnesia:write(Rec1A)	% returns ok when successful
   89: 	 end,
   90:     ?match_receive({A, ok}),
   91:     A ! end_trans,
   92:     ?match_receive({A, {atomic, end_trans}}),
   93: 
   94:     %% second transaction: store some new objects and abort before the
   95:     %% transaction is finished -> the new changes should be invisable
   96:     ?start_transactions([A]),
   97:     A ! fun() ->
   98: 		mnesia:write(Rec1B),
   99: 		exit(abort_by_purpose) %does that stop the process A ???
  100: 	 end,
  101:     ?match_receive({A, {aborted, abort_by_purpose}}),
  102: 
  103: 
  104:     %?match_receive({A, {'EXIT', Pid, normal}}), % A died and sends EXIT
  105: 
  106: 
  107:     %% Start a second  transactionprocess, after the first failed
  108:     {success, [B]} = ?start_activities([Node1]),
  109: 
  110:     %% check, whether the interupted transaction had no influence on the db
  111:     ?start_transactions([B]),
  112:     B ! fun() ->
  113: 		?match([Rec1A], mnesia:read({Tab, 1})),
  114: 		ok
  115: 	 end,
  116:     ?match_receive({B, ok}),
  117:     B ! end_trans,
  118:     ?match_receive({B, {atomic, end_trans}}),
  119: 
  120:     ?verify_mnesia(Nodes, []).
  121: 
  122: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  123: runtime_error_in_middle_of_trans(suite) -> [];
  124: runtime_error_in_middle_of_trans(Config) when is_list(Config) ->
  125:     [Node1] = Nodes = ?acquire_nodes(1, Config),
  126:     Tab = runtime_error_in_middle_of_trans,
  127: 
  128:     Rec1A = {Tab, 1, a},
  129:     Rec1B = {Tab, 1, b},
  130:     Rec1C = {Tab, 1, c},
  131: 
  132:     ?match({atomic, ok}, mnesia:create_table([{name, Tab},
  133: 					     {ram_copies, [Node1]}])),
  134:     %% Start a transaction on one node
  135:     {success, [A]} = ?start_activities([Node1]),
  136: 
  137:     %% store an object in the Tab - first tranaction
  138:     ?start_transactions([A]),
  139:     A ! fun() ->
  140: 		mnesia:write(Rec1A)	% returns ok when successful
  141: 	 end,
  142:     ?match_receive({A, ok}),
  143:     A ! end_trans,
  144:     ?match_receive({A, {atomic, end_trans}}),
  145: 
  146:     %% second transaction: store some new objects and abort before the
  147:     %% transaction is finished -> the new changes should be invisable
  148:     ?start_transactions([A]),
  149:     A ! fun() ->
  150: 		mnesia:write(Rec1B),
  151:                 erlang:error(foo), % that should provoke a runtime error
  152: 		mnesia:write(Rec1C)
  153: 	 end,
  154:     ?match_receive({A, {aborted, _Reason}}),
  155: 
  156:     %?match_receive({A, {'EXIT', Msg1}), % A died and sends EXIT
  157: 
  158: 
  159:     %% Start a second  transactionprocess, after the first failed
  160:     {success, [B]} = ?start_activities([Node1]),
  161: 
  162:     %% check, whether the interupted transaction had no influence on the db
  163:     ?start_transactions([B]),
  164:     B ! fun() ->
  165: 		?match([Rec1A], mnesia:read({Tab, 1})),
  166: 		ok
  167: 	 end,
  168:     ?match_receive({B, ok}),
  169:     B ! end_trans,
  170:     ?match_receive({B, {atomic, end_trans}}),
  171: 
  172:     ?verify_mnesia(Nodes, []).
  173: 
  174: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  175: kill_self_in_middle_of_trans(suite) -> [];
  176: kill_self_in_middle_of_trans(Config) when is_list(Config) ->
  177:     [Node1] = Nodes = ?acquire_nodes(1, Config),
  178:     Tab = kill_self_in_middle_of_trans,
  179: 
  180:     Rec1A = {Tab, 1, a},
  181:     Rec1B = {Tab, 1, b},
  182:     Rec1C = {Tab, 1, c},
  183: 
  184:     ?match({atomic, ok}, mnesia:create_table([{name, Tab},
  185: 					     {ram_copies, [Node1]}])),
  186:     %% Start a transaction on one node
  187:     {success, [A]} = ?start_activities([Node1]),
  188: 
  189:     %% store an object in the Tab - first tranaction
  190:     ?start_transactions([A]),
  191:     A ! fun() ->
  192: 		mnesia:write(Rec1A)	% returns ok when successful
  193: 	 end,
  194:     ?match_receive({A, ok}),
  195:     A ! end_trans,
  196:     ?match_receive({A, {atomic, end_trans}}),
  197: 
  198:     %% second transaction: store some new objects and abort before the
  199:     %% transaction is finished -> the new changes should be invisable
  200:     ?start_transactions([A]),
  201:     A ! fun() ->
  202: 		mnesia:write(Rec1B),
  203:                 exit(self(), kill), % that should kill the process himself
  204: 		                        %   - poor guy !
  205: 		mnesia:write(Rec1C)
  206: 	 end,
  207:     %%
  208:     %% exit(.., kill) : the transaction can't trap this error - thus no
  209:     %%                 proper result can be send by the test server
  210: 
  211:     % ?match_receive({A, {aborted, Reason}}),
  212: 
  213:     ?match_receive({'EXIT', _Pid, killed}), % A is killed and sends EXIT
  214: 
  215:     %% Start a second  transactionprocess, after the first failed
  216:     {success, [B]} = ?start_activities([Node1]),
  217: 
  218:     %% check, whether the interupted transaction had no influence on the db
  219:     ?start_transactions([B]),
  220:     B ! fun() ->
  221: 		?match([Rec1A], mnesia:read({Tab, 1})),
  222: 		ok
  223: 	 end,
  224:     ?match_receive({B, ok}),
  225:     B ! end_trans,
  226:     ?match_receive({B, {atomic, end_trans}}),
  227: 
  228:     ?verify_mnesia(Nodes, []).
  229: 
  230: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  231: throw_in_middle_of_trans(suite) -> [];
  232: throw_in_middle_of_trans(Config) when is_list(Config) ->
  233:     [Node1] = Nodes = ?acquire_nodes(1, Config),
  234:     Tab = throw_in_middle_of_trans,
  235: 
  236:     Rec1A = {Tab, 1, a},
  237:     Rec1B = {Tab, 1, b},
  238:     Rec1C = {Tab, 1, c},
  239: 
  240:     ?match({atomic, ok}, mnesia:create_table([{name, Tab},
  241: 					     {ram_copies, [Node1]}])),
  242:     %% Start a transaction on one node
  243:     {success, [A]} = ?start_activities([Node1]),
  244: 
  245:     %% store an object in the Tab - first tranaction
  246:     ?start_transactions([A]),
  247:     A ! fun() ->
  248: 		mnesia:write(Rec1A)	% returns ok when successful
  249: 	 end,
  250:     ?match_receive({A, ok}),
  251:     A ! end_trans,
  252:     ?match_receive({A, {atomic, end_trans}}),
  253: 
  254:     %% second transaction: store some new objects and abort before the
  255:     %% transaction is finished -> the new changes should be invisable
  256:     ?start_transactions([A]),
  257:     A ! fun() ->
  258: 		mnesia:write(Rec1B),
  259:                 throw(exit_transactian_by_a_throw),
  260: 		mnesia:write(Rec1C)
  261: 	 end,
  262:     ?match_receive({A, {aborted, {throw, exit_transactian_by_a_throw}}}),
  263:     % A ! end_trans, % is A still alive ?
  264:     % ?match_receive({A, {atomic, end_trans}}), % {'EXIT', Pid, normal}
  265: 
  266:     %?match_receive({A, {'EXIT', Pid, normal}}), % A died and sends EXIT
  267: 
  268:     %% Start a second  transactionprocess, after the first failed
  269:     {success, [B]} = ?start_activities([Node1]),
  270: 
  271:     %% check, whether the interupted transaction had no influence on the db
  272:     ?start_transactions([B]),
  273:     B ! fun() ->
  274: 		?match([Rec1A], mnesia:read({Tab, 1})),
  275: 		ok
  276: 	 end,
  277:     ?match_receive({B, ok}),
  278:     B ! end_trans,
  279:     ?match_receive({B, {atomic, end_trans}}),
  280: 
  281:     ?verify_mnesia(Nodes, []).
  282: 
  283: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  284: 
  285: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  286: mnesia_down_during_infinite_trans(suite) -> [];
  287: mnesia_down_during_infinite_trans(Config) when is_list(Config) ->
  288:     [Node1, Node2]  = ?acquire_nodes(2, Config),
  289:     Tab = mnesia_down_during_infinite_trans,
  290: 
  291:     ?match({atomic, ok},
  292: 	   mnesia:create_table([{name, Tab}, {ram_copies, [Node1, Node2]}])),
  293:     %% Start a transaction on one node
  294:     {success, [A2, A1]} = ?start_activities([Node2, Node1]),
  295:     %% Start order of the transactions are important
  296:     %% We also needs to sync the tid counter
  297:     ?match({atomic, ok},
  298: 	   mnesia:transaction(fun() -> mnesia:write({Tab, 1, test_ok}) end)),
  299:     mnesia_test_lib:start_sync_transactions([A2, A1]),
  300: 
  301:     %% Obtain a write lock and wait forever
  302:     RecA = {Tab, 1, test_not_ok},
  303:     A1 ! fun() -> mnesia:write(RecA) end,
  304:     ?match_receive({A1, ok}),
  305: 
  306:     A1 ! fun() -> process_flag(trap_exit, true), timer:sleep(infinity) end,
  307:     ?match_receive(timeout),
  308: 
  309:     %% Try to get read lock, but gets queued
  310:     A2 ! fun() -> mnesia:read({Tab, 1}) end,
  311:     ?match_receive(timeout),
  312: 
  313:     %% Kill Mnesia on other node
  314:     mnesia_test_lib:kill_mnesia([Node1]),
  315: 
  316:     %% Second transaction gets the read lock
  317:     ?match_receive({A2, [{Tab, 1, test_ok}]}),
  318:     exit(A1, kill), % Needed since we trap exit
  319: 
  320:     ?verify_mnesia([Node2], [Node1]).
  321: 
  322: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  323: 
  324: lock_waiter_sw_r(suite) -> [];
  325: lock_waiter_sw_r(Config) when is_list(Config) ->
  326:     start_lock_waiter(sw, r, Config).
  327: 
  328: lock_waiter_sw_rt(suite) -> [];
  329: lock_waiter_sw_rt(Config) when is_list(Config) ->
  330:     start_lock_waiter(sw, rt, Config).
  331: 
  332: lock_waiter_sw_wt(suite) -> [];
  333: lock_waiter_sw_wt(Config) when is_list(Config) ->
  334:     start_lock_waiter(sw, wt,Config).
  335: 
  336: lock_waiter_wr_r(suite) -> [];
  337: lock_waiter_wr_r(Config) when is_list(Config) ->
  338:     start_lock_waiter(wr, r, Config).
  339: 
  340: lock_waiter_srw_r(suite) -> [];
  341: lock_waiter_srw_r(Config) when is_list(Config) ->
  342:     start_lock_waiter(srw, r, Config).
  343: 
  344: lock_waiter_sw_sw(suite) -> [];
  345: lock_waiter_sw_sw(Config) when is_list(Config) ->
  346:     start_lock_waiter(sw, sw,Config).
  347: 
  348: lock_waiter_srw_srw(suite) -> [];
  349: lock_waiter_srw_srw(Config) when is_list(Config) ->
  350:     start_lock_waiter(srw, srw,Config).
  351: 
  352: lock_waiter_wr_wr(suite) -> [];
  353: lock_waiter_wr_wr(Config) when is_list(Config) ->
  354:     start_lock_waiter(wr, wr,Config).
  355: 
  356: lock_waiter_sw_w(suite) -> [];
  357: lock_waiter_sw_w(Config) when is_list(Config) ->
  358:     start_lock_waiter(sw, w,Config).
  359: 
  360: lock_waiter_sw_wr(suite) -> [];
  361: lock_waiter_sw_wr(Config) when is_list(Config) ->
  362:     start_lock_waiter(sw, wr,Config).
  363: 
  364: lock_waiter_sw_srw(suite) -> [];
  365: lock_waiter_sw_srw(Config) when is_list(Config) ->
  366:     start_lock_waiter(sw, srw,Config).
  367: 
  368: lock_waiter_wr_wt(suite) -> [];
  369: lock_waiter_wr_wt(Config) when is_list(Config) ->
  370:     start_lock_waiter(wr, wt,Config).
  371: 
  372: lock_waiter_srw_wt(suite) -> [];
  373: lock_waiter_srw_wt(Config) when is_list(Config) ->
  374:     start_lock_waiter(srw, wt,Config).
  375: 
  376: lock_waiter_wr_sw(suite) -> [];
  377: lock_waiter_wr_sw(Config) when is_list(Config) ->
  378:     start_lock_waiter(wr, sw,Config).
  379: 
  380: lock_waiter_srw_sw(suite) -> [];
  381: lock_waiter_srw_sw(Config) when is_list(Config) ->
  382:     start_lock_waiter(srw, sw,Config).
  383: 
  384: lock_waiter_wr_w(suite) -> [];
  385: lock_waiter_wr_w(Config) when is_list(Config) ->
  386:     start_lock_waiter(wr, w,Config).
  387: 
  388: lock_waiter_srw_w(suite) -> [];
  389: lock_waiter_srw_w(Config) when is_list(Config) ->
  390:     start_lock_waiter(srw, w,Config).
  391: 
  392: lock_waiter_r_sw(suite) -> [];
  393: lock_waiter_r_sw(Config) when is_list(Config) ->
  394:     start_lock_waiter(r, sw,Config).
  395: 
  396: lock_waiter_r_w(suite) -> [];
  397: lock_waiter_r_w(Config) when is_list(Config) ->
  398:     start_lock_waiter(r, w,Config).
  399: 
  400: lock_waiter_r_wt(suite) -> [];
  401: lock_waiter_r_wt(Config) when is_list(Config) ->
  402:     start_lock_waiter(r, wt,Config).
  403: 
  404: lock_waiter_rt_sw(suite) -> [];
  405: lock_waiter_rt_sw(Config) when is_list(Config) ->
  406:     start_lock_waiter(rt, sw,Config).
  407: 
  408: lock_waiter_rt_w(suite) -> [];
  409: lock_waiter_rt_w(Config) when is_list(Config) ->
  410:      start_lock_waiter(rt, w,Config).
  411: 
  412: lock_waiter_rt_wt(suite) -> [];
  413: lock_waiter_rt_wt(Config) when is_list(Config) ->
  414:     start_lock_waiter(rt, wt,Config).
  415: 
  416: lock_waiter_wt_r(suite) -> [];
  417: lock_waiter_wt_r(Config) when is_list(Config) ->
  418:     start_lock_waiter(wt, r,Config).
  419: 
  420: lock_waiter_wt_w(suite) -> [];
  421: lock_waiter_wt_w(Config) when is_list(Config) ->
  422:     start_lock_waiter(wt, w,Config).
  423: 
  424: lock_waiter_wt_rt(suite) -> [];
  425: lock_waiter_wt_rt(Config) when is_list(Config) ->
  426:     start_lock_waiter(wt, rt,Config).
  427: 
  428: lock_waiter_wt_wt(suite) -> [];
  429: lock_waiter_wt_wt(Config) when is_list(Config) ->
  430:     start_lock_waiter(wt, wt,Config).
  431: 
  432: lock_waiter_wt_wr(suite) -> [];
  433: lock_waiter_wt_wr(Config) when is_list(Config) ->
  434:     start_lock_waiter(wt, wr,Config).
  435: 
  436: lock_waiter_wt_srw(suite) -> [];
  437: lock_waiter_wt_srw(Config) when is_list(Config) ->
  438:     start_lock_waiter(wt, srw,Config).
  439: 
  440: lock_waiter_wt_sw(suite) -> [];
  441: lock_waiter_wt_sw(Config) when is_list(Config) ->
  442:     start_lock_waiter(wt, sw,Config).
  443: 
  444: lock_waiter_w_wr(suite) -> [];
  445: lock_waiter_w_wr(Config) when is_list(Config) ->
  446:     start_lock_waiter(w, wr, Config).
  447: 
  448: lock_waiter_w_srw(suite) -> [];
  449: lock_waiter_w_srw(Config) when is_list(Config) ->
  450:     start_lock_waiter(w, srw, Config).
  451: 
  452: lock_waiter_w_sw(suite) -> [];
  453: lock_waiter_w_sw(Config) when is_list(Config) ->
  454:     start_lock_waiter(w, sw, Config).
  455: 
  456: lock_waiter_w_r(suite) -> [];
  457: lock_waiter_w_r(Config) when is_list(Config) ->
  458:     start_lock_waiter(w, r, Config).
  459: 
  460: lock_waiter_w_w(suite) -> [];
  461: lock_waiter_w_w(Config) when is_list(Config) ->
  462:     start_lock_waiter(w, w, Config).
  463: 
  464: lock_waiter_w_rt(suite) -> [];
  465: lock_waiter_w_rt(Config) when is_list(Config) ->
  466:     start_lock_waiter(w, rt, Config).
  467: 
  468: lock_waiter_w_wt(suite) -> [];
  469: lock_waiter_w_wt(Config) when is_list(Config) ->
  470:     start_lock_waiter(w, wt, Config).
  471: 
  472: start_lock_waiter(BlockOpA, BlockOpB, Config) ->
  473:     [N1, N2] = Nodes = ?acquire_nodes(2, Config),
  474: 
  475:     TabName = mk_tab_name(lock_waiter_),
  476:     ?match({atomic, ok}, mnesia:create_table(TabName,
  477: 					     [{ram_copies, [N1, N2]}])),
  478: 
  479:     %% initialize the table with object {1, c} - when there
  480:     %% is a read transaction, the read will find that  value
  481:     ?match({atomic, ok}, mnesia:sync_transaction(fun() -> mnesia:write({TabName, 1, c}) end)),
  482:     rpc:call(N2, ?MODULE, sync_tid_release, []),
  483: 
  484:     Tester = self(),
  485:     Fun_A =fun() ->
  486: 		   NewCounter = incr_restart_counter(),
  487: 		   if
  488: 		       NewCounter == 1 ->
  489: 			   Tester ! go_ahead_test,
  490: 			   receive go_ahead -> ok end;
  491: 		      true -> ok
  492: 		   end,
  493: 		   lock_waiter_fun(BlockOpA, TabName, a),
  494: 		   NewCounter
  495: 	   end,
  496: 
  497:     %% it's not possible to just spawn the transaction, because
  498:     %% the result shall be evaluated
  499:     A = spawn_link(N1, ?MODULE, perform_restarted_transaction, [Fun_A]),
  500: 
  501:     ?match(ok, receive go_ahead_test -> ok after 10000 -> timeout end),
  502: 
  503:     mnesia_test_lib:sync_trans_tid_serial([N1, N2]),
  504: 
  505:     Fun_B = fun() ->
  506: 		    lock_waiter_fun(BlockOpB, TabName, b),
  507: 		    A ! go_ahead,
  508: 		    wait(infinity)
  509: 	    end,
  510: 
  511:     B = spawn_link(N2, mnesia, transaction, [Fun_B, 100]),
  512: 
  513:     io:format("waiting for A (~p on ~p) to be in the queue ~n", [A, [N1, N2]]),
  514:     wait_for_a(A, [N1, N2]),
  515: 
  516:     io:format("Queus ~p~n",
  517: 	      [[{N,rpc:call(N, mnesia, system_info, [lock_queue])} || N <- Nodes]]),
  518: 
  519:     KillNode = node(B),
  520:     io:format("A was in the queue, time to kill Mnesia on B's node (~p on ~p)~n",
  521: 	      [B, KillNode]),
  522: 
  523:     mnesia_test_lib:kill_mnesia([KillNode]), % kill mnesia of fun B
  524: 
  525:     %% Read Ops does not need to be restarted
  526:     ExpectedCounter =
  527: 	if
  528: 	    BlockOpA == sw, BlockOpB == w -> 1;
  529: 	    BlockOpA == sw, BlockOpB == wt -> 1;
  530: 	    BlockOpA == sw, BlockOpB == wr -> 1;
  531: 	    BlockOpA == srw, BlockOpB == w -> 1;
  532: 	    BlockOpA == srw, BlockOpB == wt -> 1;
  533: 	    BlockOpA == srw, BlockOpB == wr -> 1;
  534: 	    BlockOpA == r,  BlockOpB /= sw -> 1;
  535: 	    BlockOpA == rt, BlockOpB /= sw -> 1;
  536: 	    true -> 2
  537: 	end,
  538:     receive {'EXIT', B, _} -> ok
  539:     after 3000 -> ?error("Timeout~n", []) end,
  540:     receive {'EXIT', A, Exp1} -> ?match({atomic, ExpectedCounter}, Exp1)
  541:     after 3000 -> ?error("Timeout~n", []) end,
  542: 
  543:     %% the expected result depends on the transaction of
  544:     %% fun A - when that doesn't change the object in the
  545:     %% table (e.g. it is a read) then the predefined
  546:     %% value {Tabname, 1, c} is expected to be the result here
  547:     ExpectedResult =
  548: 	case BlockOpA of
  549: 	    w -> {TabName, 1, a};
  550: 	    sw ->{TabName, 1, a};
  551: 	    _all_other -> {TabName, 1, c}
  552: 	end,
  553: 
  554:     ?match({atomic, [ExpectedResult]},
  555: 	   mnesia:transaction(fun() -> mnesia:read({TabName, 1}) end, 100)),
  556:     ?verify_mnesia([N1], [N2]).
  557: 
  558: mk_tab_name(Prefix) ->
  559:     {Mega, Sec, Micro} = erlang:now(),
  560:     list_to_atom(lists:concat([Prefix , Mega, '_', Sec, '_', Micro])).
  561: 
  562: lock_waiter_fun(Op, TabName, Val) ->
  563:     case Op of
  564: 	rt ->  mnesia:read_lock_table(TabName);
  565: 	wt ->  mnesia:write_lock_table(TabName);
  566: 	r  ->  mnesia:read({TabName, 1});
  567: 	w ->   mnesia:write({TabName, 1, Val});
  568: 	wr ->  mnesia:wread({TabName, 1});
  569: 	srw -> mnesia:read(TabName, 1, sticky_write);
  570: 	sw ->  mnesia:s_write({TabName, 1, Val})
  571:     end.
  572: 
  573: wait_for_a(Pid, Nodes) ->
  574:     wait_for_a(Pid, Nodes, 5).
  575: 
  576: wait_for_a(_P, _N, 0) ->
  577:     ?error("Timeout while waiting for lock on a~n", []);
  578: 
  579: wait_for_a(Pid, Nodes, Count) ->
  580:     %%    io:format("WAIT_FOR_A ~p ON ~w ~n", [Pid, Nodes]),
  581:     List = [rpc:call(N, mnesia, system_info, [lock_queue]) || N <- Nodes],
  582:     Q = lists:append(List),
  583:     check_q(Pid, Q, Nodes, Count).
  584: 
  585: check_q(Pid, [{{_Oid,_Tid}, _Op, Pid, _WFT} | _Tail], _N, _Count) ->
  586:     ok;
  587: check_q(Pid, [{_Oid, _Op, Pid, _Tid, _WFT} | _Tail], _N, _Count) ->
  588:     ok;
  589: check_q(Pid, [_ | Tail], N, Count) ->
  590:     check_q(Pid, Tail, N, Count);
  591: check_q(Pid, [], N, Count) ->
  592:     timer:sleep(500),
  593:     wait_for_a(Pid, N, Count - 1).
  594: 
  595: perform_restarted_transaction (Fun_Trans) ->
  596:     %% the result of the transaction shall be:
  597:     %%  - undefined (if the transaction was never executed)
  598:     %%  - Times ( number of times that the transaction has been executed)
  599: 
  600:     Result = mnesia:transaction(Fun_Trans, 100),
  601:     exit(Result).
  602: 
  603: %% Returns new val
  604: incr_restart_counter() ->
  605:     NewCount =
  606: 	case get(count_restart_of_transaction) of
  607: 	    undefined -> 1;
  608: 	    OldCount -> OldCount + 1
  609: 	end,
  610:     put(count_restart_of_transaction, NewCount),
  611:     NewCount.
  612: 
  613: wait(Mseconds) ->
  614:     receive
  615:     after Mseconds -> ok
  616:     end.
  617: 
  618: 
  619: %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  620: 
  621: restart_r_one(suite) -> [];
  622: restart_r_one(Config) when is_list(Config) ->
  623:     start_restart_check(r, one, Config).
  624: 
  625: restart_w_one(suite) -> [];
  626: restart_w_one(Config) when is_list(Config) ->
  627:     start_restart_check(w, one, Config).
  628: 
  629: restart_rt_one(suite) -> [];
  630: restart_rt_one(Config) when is_list(Config) ->
  631:     start_restart_check(rt, one, Config).
  632: 
  633: restart_wt_one(suite) -> [];
  634: restart_wt_one(Config) when is_list(Config) ->
  635:     start_restart_check(wt, one, Config).
  636: 
  637: restart_wr_one(suite) -> [];
  638: restart_wr_one(Config) when is_list(Config) ->
  639:     start_restart_check(wr, one, Config).
  640: 
  641: restart_sw_one(suite) -> [];
  642: restart_sw_one(Config) when is_list(Config) ->
  643:     start_restart_check(sw, one, Config).
  644: 
  645: restart_r_two(suite) -> [];
  646: restart_r_two(Config) when is_list(Config) ->
  647:     start_restart_check(r, two, Config).
  648: 
  649: restart_w_two(suite) -> [];
  650: restart_w_two(Config) when is_list(Config) ->
  651:     start_restart_check(w, two, Config).
  652: 
  653: restart_rt_two(suite) -> [];
  654: restart_rt_two(Config) when is_list(Config) ->
  655:     start_restart_check(rt, two, Config).
  656: 
  657: restart_wt_two(suite) -> [];
  658: restart_wt_two(Config) when is_list(Config) ->
  659:     start_restart_check(wt, two, Config).
  660: 
  661: restart_wr_two(suite) -> [];
  662: restart_wr_two(Config) when is_list(Config) ->
  663:     start_restart_check(wr, two, Config).
  664: 
  665: restart_sw_two(suite) -> [];
  666: restart_sw_two(Config) when is_list(Config) ->
  667:     start_restart_check(sw, two, Config).
  668: 
  669: start_restart_check(RestartOp, ReplicaNeed, Config) ->
  670:     [N1, N2, N3] = Nodes = ?acquire_nodes(3, Config),
  671: 
  672:     {TabName, _TabNodes} = create_restart_table(ReplicaNeed, Nodes),
  673: 
  674:     %% initialize the table with object {1, c} - when there
  675:     %% is a read transaction, the read will find that  value
  676:     ?match({atomic, ok}, mnesia:sync_transaction(fun() -> mnesia:write({TabName, 1, c}) end)),
  677: 
  678:     %% Really sync tid_release
  679:     rpc:multicall([N2,N3], ?MODULE, sync_tid_release, []),
  680:     Coord = self(),
  681: 
  682:     Fun_A = fun() ->
  683: 		    NewCounter = incr_restart_counter(),
  684: 		    case NewCounter of
  685: 			1 ->
  686: 			    mnesia:write({TabName, 1, d}),
  687: 			    %% send a message to the test proc
  688: 			    Coord ! {self(),fun_a_is_blocked},
  689: 			    receive go_ahead -> ok end;
  690: 			_ ->
  691: 			    %% the fun will NOT be blocked here
  692: 			    restart_fun_A(RestartOp, TabName)
  693: 		    end,
  694: 		    NewCounter
  695: 	    end,
  696: 
  697:     A = spawn_link(N1, ?MODULE, perform_restarted_transaction, [Fun_A]),
  698:     ?match_receive({A,fun_a_is_blocked}),
  699: 
  700:     %% mnesia shall be killed at that node, where A is reading
  701:     %% the information from
  702:     kill_where_to_read(TabName, N1, [N2, N3]),
  703: 
  704:     %% wait some time to let mnesia go down and spread those news around
  705:     %% fun A shall be able to finish its job before being restarted
  706:     wait(500),
  707:     A ! go_ahead,
  708: 
  709:     %% the sticky write doesnt work on remote nodes !!!
  710:     ExpectedMsg =
  711: 	case RestartOp of
  712: 	    sw when ReplicaNeed == two ->
  713: 		{'EXIT',A,{aborted,  {not_local, TabName}}};
  714: 	    _all_other ->
  715: 		case ReplicaNeed of
  716: 		    one ->
  717: 			{'EXIT',A,{aborted,  {no_exists, TabName}}};
  718: 		    two ->
  719: 			{'EXIT',A,{atomic, 2}}
  720: 		end
  721: 	end,
  722: 
  723:     ?match_receive(ExpectedMsg),
  724: 
  725:     %% now mnesia has to be started again on the node KillNode
  726:     %% because the next test suite will need it
  727:     ?match([], mnesia_test_lib:start_mnesia(Nodes, [TabName])),
  728: 
  729: 
  730:     %%  the expected result depends on the transaction of
  731:     %%  fun A - when that doesnt change the object in the
  732:     %%  table (e.g. it is a read) then the predefined
  733:     %%  value {Tabname, 1, c} is expected to be the result here
  734: 
  735:     ExpectedResult =
  736: 	case ReplicaNeed of
  737: 	    one ->
  738: 		[];
  739: 	    two ->
  740: 		case RestartOp of
  741: 		    w -> [{TabName, 1, a}];
  742: 		    _ ->[ {TabName, 1, c}]
  743: 		end
  744: 	end,
  745: 
  746:     ?match({atomic, ExpectedResult},
  747: 	   mnesia:transaction(fun() -> mnesia:read({TabName, 1}) end,100)),
  748:     ?verify_mnesia(Nodes, []).
  749: 
  750: create_restart_table(ReplicaNeed, [_N1, N2, N3]) ->
  751:     TabNodes =
  752: 	case ReplicaNeed of
  753: 	    one ->  [N2];
  754: 	    two ->  [N2, N3]
  755: 	end,
  756:     TabName = mk_tab_name(restart_check_),
  757:     ?match({atomic, ok}, mnesia:create_table(TabName, [{ram_copies, TabNodes}])),
  758:     {TabName, TabNodes}.
  759: 
  760: restart_fun_A(Op, TabName) ->
  761:     case Op of
  762: 	rt -> mnesia:read_lock_table(TabName);
  763: 	wt -> mnesia:write_lock_table(TabName);
  764: 	r ->  mnesia:read( {TabName, 1});
  765: 	w ->  mnesia:write({TabName, 1, a});
  766: 	wr -> mnesia:wread({TabName, 1});
  767: 	sw -> mnesia:s_write({TabName, 1, a})
  768:     end.
  769: 
  770: kill_where_to_read(TabName, N1, Nodes) ->
  771:     Read = rpc:call(N1,mnesia,table_info, [TabName, where_to_read]),
  772:     case lists:member(Read, Nodes) of
  773: 	true ->
  774: 	    mnesia_test_lib:kill_mnesia([Read]);
  775: 	false ->
  776: 	    ?error("Fault while killing Mnesia: ~p~n", [Read]),
  777: 	    mnesia_test_lib:kill_mnesia(Nodes)
  778:     end.
  779: 
  780: sync_tid_release() ->
  781:     sys:get_status(whereis(mnesia_tm)),
  782:     sys:get_status(whereis(mnesia_locker)),
  783:     ok.
  784: