diff options
Diffstat (limited to 'lib/mnesia/test/mnesia_consistency_test.erl')
| -rw-r--r-- | lib/mnesia/test/mnesia_consistency_test.erl | 1612 |
1 files changed, 1612 insertions, 0 deletions
diff --git a/lib/mnesia/test/mnesia_consistency_test.erl b/lib/mnesia/test/mnesia_consistency_test.erl new file mode 100644 index 0000000000..ffe8ab7ac3 --- /dev/null +++ b/lib/mnesia/test/mnesia_consistency_test.erl @@ -0,0 +1,1612 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%% +-module(mnesia_consistency_test). +-author('[email protected]'). +-compile([export_all]). + +-include("mnesia_test_lib.hrl"). + +init_per_testcase(Func, Conf) -> + mnesia_test_lib:init_per_testcase(Func, Conf). + +fin_per_testcase(Func, Conf) -> + mnesia_test_lib:fin_per_testcase(Func, Conf). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +all(doc) -> + ["Verify transaction consistency", + "Consistency is the property of the application that requires any", + "execution of the transaction to take the database from one", + "consistent state to another. Verify that the database is", + "consistent at any point in time.", + "Verify for various configurations.", + " Verify for both set and bag"]; +all(suite) -> + [ + consistency_after_restart, + consistency_after_dump_tables, + consistency_after_add_replica, + consistency_after_del_replica, + consistency_after_move_replica, + consistency_after_transform_table, + consistency_after_change_table_copy_type, + consistency_after_fallback, + consistency_after_restore, + consistency_after_rename_of_node, + checkpoint_retainer_consistency, + backup_consistency + ]. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% +% stolen from mnesia_tpcb.erl: + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Account record, total size must be at least 100 bytes + +-define(ACCOUNT_FILLER, + {123456789012345678901234567890123456789012345678901234567890, + 123456789012345678901234567890123456789012345678901234567890, + 123456789012345678901234567890123456789012345678901234}). + +-record(account, + { + id = 0, %% Unique account id + branch_id = 0, %% Branch where the account is held + balance = 0, %% Account balance + filler = ?ACCOUNT_FILLER %% Gap filler to ensure size >= 100 bytes + }). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Branch record, total size must be at least 100 bytes + +-define(BRANCH_FILLER, + {123456789012345678901234567890123456789012345678901234567890, + 123456789012345678901234567890123456789012345678901234567890, + 123456789012345678901234567890123456789012345678901234567890}). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Teller record, total size must be at least 100 bytes + +-define(TELLER_FILLER, + {123456789012345678901234567890123456789012345678901234567890, + 123456789012345678901234567890123456789012345678901234567890, + 1234567890123456789012345678901234567890123456789012345678}). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% History record, total size must be at least 50 bytes + +-define(HISTORY_FILLER, 1234567890). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +-record(tab_config, + { + db_nodes = [node()], + replica_nodes = [node()], + replica_type = ram_copies, + use_running_mnesia = false, + n_branches = 1, + n_tellers_per_branch = 10, %% Must be 10 + n_accounts_per_branch = 100000, %% Must be 100000 + branch_filler = ?BRANCH_FILLER, + account_filler = ?ACCOUNT_FILLER, + teller_filler = ?TELLER_FILLER + }). +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% +% stolen from mnesia_tpcb.erl: + +list2rec(List, Fields, DefaultTuple) -> + [Name|Defaults] = tuple_to_list(DefaultTuple), + List2 = list2rec(List, Fields, Defaults, []), + list_to_tuple([Name] ++ List2). + +list2rec(_List, [], [], Acc) -> + Acc; +list2rec(List, [F|Fields], [D|Defaults], Acc) -> + {Val, List2} = + case lists:keysearch(F, 1, List) of + false -> + {D, List}; + {value, {F, NewVal}} -> + {NewVal, lists:keydelete(F, 1, List)} + end, + list2rec(List2, Fields, Defaults, Acc ++ [Val]). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +tpcb_config(ReplicaType, _NodeConfig, Nodes, NoDriverNodes) -> + [{n_branches, 10}, + {n_drivers_per_node, 10}, + {replica_nodes, Nodes}, + {driver_nodes, Nodes -- NoDriverNodes}, + {use_running_mnesia, true}, + {report_interval, infinity}, + {n_accounts_per_branch, 100}, + {replica_type, ReplicaType}, + {reuse_history_id, true}]. + +%% Stolen from mnesia_tpcb:dist +tpcb_config_dist(ReplicaType, _NodeConfig, Nodes, _Config) -> + [{db_nodes, Nodes}, + {driver_nodes, Nodes}, + {replica_nodes, Nodes}, + {n_drivers_per_node, 10}, + {n_branches, 1}, + {use_running_mnesia, true}, + {n_accounts_per_branch, 10}, + {replica_type, ReplicaType}, + {stop_after, timer:minutes(15)}, + {report_interval, timer:seconds(10)}, + {reuse_history_id, true}]. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% +% stolen from mnesia_recovery_test.erl: + +receive_messages([]) -> []; +receive_messages(ListOfMsgs) -> + receive + {Pid, Msg} -> + case lists:member(Msg, ListOfMsgs) of + false -> + ?warning("I (~p) have received unexpected msg~n ~p ~n", + [self(),{Pid, Msg}]), + receive_messages(ListOfMsgs); + true -> + ?verbose("I (~p) got msg ~p from ~p ~n", [self(),Msg, Pid]), + [{Pid, Msg} | receive_messages(ListOfMsgs -- [Msg])] + end; + Else -> ?warning("Recevied unexpected Msg~n ~p ~n", [Else]) + after timer:minutes(3) -> + ?error("Timeout in receive msgs while waiting for ~p~n", + [ListOfMsgs]) + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +consistency_after_restart(suite) -> + [ + consistency_after_restart_1_ram, + consistency_after_restart_1_disc, + consistency_after_restart_1_disc_only, + consistency_after_restart_2_ram, + consistency_after_restart_2_disc, + consistency_after_restart_2_disc_only + ]. + +consistency_after_restart_1_ram(suite) -> []; +consistency_after_restart_1_ram(Config) when is_list(Config) -> + consistency_after_restart(ram_copies, 2, Config). + +consistency_after_restart_1_disc(suite) -> []; +consistency_after_restart_1_disc(Config) when is_list(Config) -> + consistency_after_restart(disc_copies, 2, Config). + +consistency_after_restart_1_disc_only(suite) -> []; +consistency_after_restart_1_disc_only(Config) when is_list(Config) -> + consistency_after_restart(disc_only_copies, 2, Config). + +consistency_after_restart_2_ram(suite) -> []; +consistency_after_restart_2_ram(Config) when is_list(Config) -> + consistency_after_restart(ram_copies, 3, Config). + +consistency_after_restart_2_disc(suite) -> []; +consistency_after_restart_2_disc(Config) when is_list(Config) -> + consistency_after_restart(disc_copies, 3, Config). + +consistency_after_restart_2_disc_only(suite) -> []; +consistency_after_restart_2_disc_only(Config) when is_list(Config) -> + consistency_after_restart(disc_only_copies, 3, Config). + +consistency_after_restart(ReplicaType, NodeConfig, Config) -> + [Node1 | _] = Nodes = ?acquire_nodes(NodeConfig, Config), + {success, [A]} = ?start_activities([Node1]), + ?log("consistency_after_restart with ~p on ~p~n", + [ReplicaType, Nodes]), + TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, [Node1]), + mnesia_tpcb:init(TpcbConfig), + A ! fun () -> mnesia_tpcb:run(TpcbConfig) end, + timer:sleep(timer:seconds(10)), + mnesia_test_lib:kill_mnesia([Node1]), + %% Start and wait for tables to be loaded on all nodes + timer:sleep(timer:seconds(3)), + ?match([], mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller, history])), + mnesia_tpcb:stop(), + ?match(ok, mnesia_tpcb:verify_tabs()), + ?verify_mnesia(Nodes, []). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +consistency_after_dump_tables(suite) -> + [ + consistency_after_dump_tables_1_ram, + consistency_after_dump_tables_2_ram + ]. + +consistency_after_dump_tables_1_ram(suite) -> []; +consistency_after_dump_tables_1_ram(Config) when is_list(Config) -> + consistency_after_dump_tables(ram_copies, 1, Config). + +consistency_after_dump_tables_2_ram(suite) -> []; +consistency_after_dump_tables_2_ram(Config) when is_list(Config) -> + consistency_after_dump_tables(ram_copies, 2, Config). + +consistency_after_dump_tables(ReplicaType, NodeConfig, Config) -> + [Node1 | _] = Nodes = ?acquire_nodes(NodeConfig, Config), + {success, [A]} = ?start_activities([Node1]), + ?log("consistency_after_dump_tables with ~p on ~p~n", + [ReplicaType, Nodes]), + TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []), + mnesia_tpcb:init(TpcbConfig), + A ! fun() -> mnesia_tpcb:run(TpcbConfig) end, + timer:sleep(timer:seconds(10)), + ?match({atomic, ok}, rpc:call(Node1, mnesia, dump_tables, + [[branch, teller, account, history]])), + mnesia_tpcb:stop(), + ?match(ok, mnesia_tpcb:verify_tabs()), + + mnesia_test_lib:kill_mnesia(Nodes), + timer:sleep(timer:seconds(1)), + ?match([], mnesia_test_lib:start_mnesia(Nodes,[account, branch, + teller, history])), + mnesia_tpcb:stop(), + ?match(ok, mnesia_tpcb:verify_tabs()), + ?verify_mnesia(Nodes, []). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +consistency_after_add_replica(suite) -> + [ + consistency_after_add_replica_2_ram, + consistency_after_add_replica_2_disc, + consistency_after_add_replica_2_disc_only, + consistency_after_add_replica_3_ram, + consistency_after_add_replica_3_disc, + consistency_after_add_replica_3_disc_only + ]. + +consistency_after_add_replica_2_ram(suite) -> []; +consistency_after_add_replica_2_ram(Config) when is_list(Config) -> + consistency_after_add_replica(ram_copies, 2, Config). + +consistency_after_add_replica_2_disc(suite) -> []; +consistency_after_add_replica_2_disc(Config) when is_list(Config) -> + consistency_after_add_replica(disc_copies, 2, Config). + +consistency_after_add_replica_2_disc_only(suite) -> []; +consistency_after_add_replica_2_disc_only(Config) when is_list(Config) -> + consistency_after_add_replica(disc_only_copies, 2, Config). + +consistency_after_add_replica_3_ram(suite) -> []; +consistency_after_add_replica_3_ram(Config) when is_list(Config) -> + consistency_after_add_replica(ram_copies, 3, Config). + +consistency_after_add_replica_3_disc(suite) -> []; +consistency_after_add_replica_3_disc(Config) when is_list(Config) -> + consistency_after_add_replica(disc_copies, 3, Config). + +consistency_after_add_replica_3_disc_only(suite) -> []; +consistency_after_add_replica_3_disc_only(Config) when is_list(Config) -> + consistency_after_add_replica(disc_only_copies, 3, Config). + +consistency_after_add_replica(ReplicaType, NodeConfig, Config) -> + Nodes0 = ?acquire_nodes(NodeConfig, Config), + AddNode = lists:last(Nodes0), + Nodes = Nodes0 -- [AddNode], + Node1 = hd(Nodes), + {success, [A]} = ?start_activities([Node1]), + ?log("consistency_after_add_replica with ~p on ~p~n", + [ReplicaType, Nodes0]), + TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []), + mnesia_tpcb:init(TpcbConfig), + A ! fun () -> mnesia_tpcb:run(TpcbConfig) end, + timer:sleep(timer:seconds(10)), + ?match({atomic, ok}, mnesia:add_table_copy(account, AddNode, ReplicaType)), + mnesia_tpcb:stop(), + ?match(ok, mnesia_tpcb:verify_tabs()), + ?verify_mnesia(Nodes0, []). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +consistency_after_del_replica(suite) -> + [ + consistency_after_del_replica_2_ram, + consistency_after_del_replica_2_disc, + consistency_after_del_replica_2_disc_only, + consistency_after_del_replica_3_ram, + consistency_after_del_replica_3_disc, + consistency_after_del_replica_3_disc_only + ]. + +consistency_after_del_replica_2_ram(suite) -> []; +consistency_after_del_replica_2_ram(Config) when is_list(Config) -> + consistency_after_del_replica(ram_copies, 2, Config). + +consistency_after_del_replica_2_disc(suite) -> []; +consistency_after_del_replica_2_disc(Config) when is_list(Config) -> + consistency_after_del_replica(disc_copies, 2, Config). + +consistency_after_del_replica_2_disc_only(suite) -> []; +consistency_after_del_replica_2_disc_only(Config) when is_list(Config) -> + consistency_after_del_replica(disc_only_copies, 2, Config). + +consistency_after_del_replica_3_ram(suite) -> []; +consistency_after_del_replica_3_ram(Config) when is_list(Config) -> + consistency_after_del_replica(ram_copies, 3, Config). + +consistency_after_del_replica_3_disc(suite) -> []; +consistency_after_del_replica_3_disc(Config) when is_list(Config) -> + consistency_after_del_replica(disc_copies, 3, Config). + +consistency_after_del_replica_3_disc_only(suite) -> []; +consistency_after_del_replica_3_disc_only(Config) when is_list(Config) -> + consistency_after_del_replica(disc_only_copies, 3, Config). + +consistency_after_del_replica(ReplicaType, NodeConfig, Config) -> + Nodes = ?acquire_nodes(NodeConfig, Config), + Node1 = hd(Nodes), + Node2 = lists:last(Nodes), + {success, [A]} = ?start_activities([Node1]), + ?log("consistency_after_del_replica with ~p on ~p~n", + [ReplicaType, Nodes]), + TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []), + mnesia_tpcb:init(TpcbConfig), + A ! fun () -> mnesia_tpcb:run(TpcbConfig) end, + timer:sleep(timer:seconds(10)), + ?match({atomic, ok}, mnesia:del_table_copy(account, Node2)), + mnesia_tpcb:stop(), + ?match(ok, mnesia_tpcb:verify_tabs()), + ?verify_mnesia(Nodes, []). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +consistency_after_move_replica(suite) -> + [ + consistency_after_move_replica_2_ram, + consistency_after_move_replica_2_disc, + consistency_after_move_replica_2_disc_only, + consistency_after_move_replica_3_ram, + consistency_after_move_replica_3_disc, + consistency_after_move_replica_3_disc_only + ]. + +consistency_after_move_replica_2_ram(suite) -> []; +consistency_after_move_replica_2_ram(Config) when is_list(Config) -> + consistency_after_move_replica(ram_copies, 2, Config). + +consistency_after_move_replica_2_disc(suite) -> []; +consistency_after_move_replica_2_disc(Config) when is_list(Config) -> + consistency_after_move_replica(disc_copies, 2, Config). + +consistency_after_move_replica_2_disc_only(suite) -> []; +consistency_after_move_replica_2_disc_only(Config) when is_list(Config) -> + consistency_after_move_replica(disc_only_copies, 2, Config). + +consistency_after_move_replica_3_ram(suite) -> []; +consistency_after_move_replica_3_ram(Config) when is_list(Config) -> + consistency_after_move_replica(ram_copies, 3, Config). + +consistency_after_move_replica_3_disc(suite) -> []; +consistency_after_move_replica_3_disc(Config) when is_list(Config) -> + consistency_after_move_replica(disc_copies, 3, Config). + +consistency_after_move_replica_3_disc_only(suite) -> []; +consistency_after_move_replica_3_disc_only(Config) when is_list(Config) -> + consistency_after_move_replica(disc_only_copies, 3, Config). + +consistency_after_move_replica(ReplicaType, NodeConfig, Config) -> + Nodes = ?acquire_nodes(NodeConfig, Config ++ [{tc_timeout, timer:minutes(10)}]), + Node1 = hd(Nodes), + Node2 = lists:last(Nodes), + {success, [A]} = ?start_activities([Node1]), + ?log("consistency_after_move_replica with ~p on ~p~n", + [ReplicaType, Nodes]), + TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes -- [Node2], []), + mnesia_tpcb:init(TpcbConfig), + A ! fun () -> mnesia_tpcb:run(TpcbConfig) end, + timer:sleep(timer:seconds(10)), + ?match({atomic, ok}, mnesia:move_table_copy(account, Node1, Node2)), + ?log("First move completed from node ~p to ~p ~n", [Node1, Node2]), + ?match({atomic, ok}, mnesia:move_table_copy(account, Node2, Node1)), + mnesia_tpcb:stop(), + ?match(ok, mnesia_tpcb:verify_tabs()), + ?verify_mnesia(Nodes, []). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +consistency_after_transform_table(doc) -> + ["Check that the database is consistent after transform_table.", + " While applications are updating the involved tables. "]; + +consistency_after_transform_table(suite) -> + [ + consistency_after_transform_table_ram, + consistency_after_transform_table_disc, + consistency_after_transform_table_disc_only + ]. + + +consistency_after_transform_table_ram(suite) -> []; +consistency_after_transform_table_ram(Config) when is_list(Config) -> + consistency_after_transform_table(ram_copies, Config). + +consistency_after_transform_table_disc(suite) -> []; +consistency_after_transform_table_disc(Config) when is_list(Config) -> + consistency_after_transform_table(disc_copies, Config). + +consistency_after_transform_table_disc_only(suite) -> []; +consistency_after_transform_table_disc_only(Config) when is_list(Config) -> + consistency_after_transform_table(disc_only_copies, Config). + +consistency_after_transform_table(Type, Config) -> + Nodes = [N1, N2,_N3] = ?acquire_nodes(3, Config), + + ?match({atomic, ok}, mnesia:create_table(tab1, [{index, [3]}, {Type, [N1]}])), + ?match({atomic, ok}, mnesia:create_table(tab2, [{index, [3]}, {Type, [N1,N2]}])), + ?match({atomic, ok}, mnesia:create_table(tab3, [{index, [3]}, {Type, Nodes}])), + ?match({atomic, ok}, mnesia:create_table(empty, [{index, [3]},{Type, Nodes}])), + + Tabs = lists:sort([tab1, tab2, tab3, empty]), + + [[mnesia:dirty_write({Tab, N, N}) || N <- lists:seq(1,10)] || + Tab <- Tabs -- [empty, tab4]], + mnesia:dump_log(), + + Ok = lists:duplicate(4, {atomic, ok}), + ?match(Ok, [mnesia:transform_table(Tab, fun({T, N, N}) -> {T, N, N, ok} end, + [k,a,n]) || Tab <- Tabs]), + [?match([k,a,n], mnesia:table_info(Tab, attributes)) || Tab <- Tabs], + + Filter = fun(Tab) -> mnesia:foldl(fun(A, Acc) when size(A) == 3 -> [A|Acc]; + (A, Acc) when size(A) == 4 -> Acc + end, [], Tab) + end, + + ?match([[],[],[],[]], [element(2,mnesia:transaction(Filter, [Tab])) || Tab <- Tabs]), + + mnesia_test_lib:kill_mnesia(Nodes), + mnesia_test_lib:start_mnesia(Nodes, Tabs), + + ?match([Tabs, Tabs, Tabs], + [lists:sort(rpc:call(Node, mnesia,system_info, [tables]) -- [schema]) || Node <- Nodes]), + + ?match([[],[],[],[]], [element(2,mnesia:transaction(Filter, [Tab])) || Tab <- Tabs]), + [?match([k,a,n], mnesia:table_info(Tab, attributes)) || Tab <- Tabs], + + ?verify_mnesia(Nodes, []). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +consistency_after_change_table_copy_type(doc) -> + ["Check that the database is consistent after change of copy type.", + " While applications are updating the involved tables. "]. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +consistency_after_fallback(doc) -> + ["Check that installed fallbacks are consistent. Check this by starting ", + "some nodes, run tpcb on them, take a backup at any time, install it ", + "as a fallback, kill all nodes, start mnesia again and check for ", + "any inconsistencies"]; +consistency_after_fallback(suite) -> + [ + consistency_after_fallback_2_ram, + consistency_after_fallback_2_disc, + consistency_after_fallback_2_disc_only, + consistency_after_fallback_3_ram, + consistency_after_fallback_3_disc + , consistency_after_fallback_3_disc_only + ]. + +consistency_after_fallback_2_ram(suite) -> []; +consistency_after_fallback_2_ram(Config) when is_list(Config) -> + consistency_after_fallback(ram_copies, 2, Config). + +consistency_after_fallback_2_disc(suite) -> []; +consistency_after_fallback_2_disc(Config) when is_list(Config) -> + consistency_after_fallback(disc_copies, 2, Config). + +consistency_after_fallback_2_disc_only(suite) -> []; +consistency_after_fallback_2_disc_only(Config) when is_list(Config) -> + consistency_after_fallback(disc_only_copies, 2, Config). + +consistency_after_fallback_3_ram(suite) -> []; +consistency_after_fallback_3_ram(Config) when is_list(Config) -> + consistency_after_fallback(ram_copies, 3, Config). + +consistency_after_fallback_3_disc(suite) -> []; +consistency_after_fallback_3_disc(Config) when is_list(Config) -> + consistency_after_fallback(disc_copies, 3, Config). + +consistency_after_fallback_3_disc_only(suite) -> []; +consistency_after_fallback_3_disc_only(Config) when is_list(Config) -> + consistency_after_fallback(disc_only_copies, 3, Config). + +consistency_after_fallback(ReplicaType, NodeConfig, Config) -> + %%?verbose("Starting consistency_after_fallback2 at ~p~n", [self()]), + Delay = 5, + Nodes = ?acquire_nodes(NodeConfig, [{tc_timeout, timer:minutes(10)} | Config]), + Node1 = hd(Nodes), + %%?verbose("Mnesia info: ~p~n", [mnesia:info()]), + + {success, [A]} = ?start_activities([Node1]), + ?log("consistency_after_fallback with ~p on ~p~n", + [ReplicaType, Nodes]), + TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []), + mnesia_tpcb:init(TpcbConfig), + A ! fun () -> mnesia_tpcb:run(TpcbConfig) end, + timer:sleep(timer:seconds(Delay)), + + %% Make a backup + ?verbose("Doing backup~n", []), + ?match(ok, mnesia:backup(consistency_after_fallback2)), + + %% Install the backup as a fallback + ?verbose("Doing fallback~n", []), + ?match(ok, mnesia:install_fallback(consistency_after_fallback2)), + timer:sleep(timer:seconds(Delay)), + + %% Stop tpcb + ?verbose("Stopping TPC-B~n", []), + mnesia_tpcb:stop(), + ?match(ok, mnesia_tpcb:verify_tabs()), + + %% Stop and then start mnesia and check table consistency + %%?verbose("Restarting Mnesia~n", []), + mnesia_test_lib:kill_mnesia(Nodes), + mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller,history]), + + ?match(ok, mnesia_tpcb:verify_tabs()), + if + ReplicaType == ram_copies -> + %% Test that change_table_copy work i.e. no account.dcd file exists. + ?match({atomic, ok}, mnesia:change_table_copy_type(account, node(), disc_copies)); + true -> + ignore + end, + file:delete(consistency_after_fallback2), + ?verify_mnesia(Nodes, []). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +consistency_after_restore(doc) -> + ["Verify consistency after restore operations."]; + +consistency_after_restore(suite) -> + [ + consistency_after_restore_clear_ram, + consistency_after_restore_clear_disc, + consistency_after_restore_clear_disc_only, + consistency_after_restore_recreate_ram, + consistency_after_restore_recreate_disc, + consistency_after_restore_recreate_disc_only + ]. + +consistency_after_restore_clear_ram(suite) -> []; +consistency_after_restore_clear_ram(Config) when is_list(Config) -> + consistency_after_restore(ram_copies, clear_tables, Config). + +consistency_after_restore_clear_disc(suite) -> []; +consistency_after_restore_clear_disc(Config) when is_list(Config) -> + consistency_after_restore(disc_copies, clear_tables, Config). + +consistency_after_restore_clear_disc_only(suite) -> []; +consistency_after_restore_clear_disc_only(Config) when is_list(Config) -> + consistency_after_restore(disc_only_copies, clear_tables, Config). + +consistency_after_restore_recreate_ram(suite) -> []; +consistency_after_restore_recreate_ram(Config) when is_list(Config) -> + consistency_after_restore(ram_copies, recreate_tables, Config). + +consistency_after_restore_recreate_disc(suite) -> []; +consistency_after_restore_recreate_disc(Config) when is_list(Config) -> + consistency_after_restore(disc_copies, recreate_tables, Config). + +consistency_after_restore_recreate_disc_only(suite) -> []; +consistency_after_restore_recreate_disc_only(Config) when is_list(Config) -> + consistency_after_restore(disc_only_copies, recreate_tables, Config). + +consistency_after_restore(ReplicaType, Op, Config) -> + Delay = 1, + Nodes = ?acquire_nodes(3, [{tc_timeout, timer:minutes(10)} | Config]), + [Node1, Node2, _Node3] = Nodes, + File = "cons_backup_restore", + + ?log("consistency_after_restore with ~p on ~p~n", + [ReplicaType, Nodes]), + Tabs = [carA, carB, carC, carD], + + ?match({atomic, ok}, mnesia:create_table(carA, [{ReplicaType, Nodes}])), + ?match({atomic, ok}, mnesia:create_table(carB, [{ReplicaType, Nodes -- [Node1]}])), + ?match({atomic, ok}, mnesia:create_table(carC, [{ReplicaType, Nodes -- [Node2]}])), + ?match({atomic, ok}, mnesia:create_table(carD, [{ReplicaType, [Node2]}])), + + NList = lists:seq(0, 20), + [lists:foreach(fun(E) -> ok = mnesia:dirty_write({Tab, E, 1}) end, NList) || + Tab <- Tabs], + + {ok, Name, _} = mnesia:activate_checkpoint([{max, [schema | Tabs]}, + {ram_overrides_dump, true}]), + ?verbose("Doing backup~n", []), + ?match(ok, mnesia:backup_checkpoint(Name, File)), + ?match(ok, mnesia:deactivate_checkpoint(Name)), + + [lists:foreach(fun(E) -> ok = mnesia:dirty_write({Tab, E, 2}) end, NList) || + Tab <- Tabs], + + Pids1 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carA, Op]), ok} || _ <- lists:seq(1, 5)], + Pids2 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carB, Op]), ok} || _ <- lists:seq(1, 5)], + Pids3 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carC, Op]), ok} || _ <- lists:seq(1, 5)], + Pids4 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carD, Op]), ok} || _ <- lists:seq(1, 5)], + + AllPids = Pids1 ++ Pids2 ++ Pids3 ++ Pids4, + + Restore = fun(F, Args) -> + case mnesia:restore(F, Args) of + {atomic, List} -> lists:sort(List); + Else -> Else + end + end, + + timer:sleep(timer:seconds(Delay)), %% Let changers grab locks + ?verbose("Doing restore~n", []), + ?match(Tabs, Restore(File, [{default_op, Op}])), + + timer:sleep(timer:seconds(Delay)), %% Let em die + + ?match_multi_receive(AllPids), + + case ?match(ok, restore_verify_tabs(Tabs)) of + {success, ok} -> + file:delete(File); + _ -> + {T, M, S} = time(), + File2 = ?flat_format("consistency_error~w~w~w.BUP", [T, M, S]), + file:rename(File, File2) + end, + ?verify_mnesia(Nodes, []). + +change_tab(Father, Tab, Test) -> + Key = random:uniform(20), + Update = fun() -> + case mnesia:read({Tab, Key}) of + [{Tab, Key, 1}] -> + quit; + [{Tab, Key, _N}] -> + mnesia:write({Tab, Key, 3}) + end + end, + case mnesia:transaction(Update) of + {atomic, quit} -> + exit(ok); + {aborted, {no_exists, Tab}} when Test == recreate_tables ->%% I'll allow this + change_tab(Father, Tab, Test); + {atomic, ok} -> + change_tab(Father, Tab, Test) + end. + +restore_verify_tabs([Tab | R]) -> + ?match({atomic, ok}, + mnesia:transaction(fun() -> mnesia:foldl(fun({_, _, 1}, ok) -> + ok; + (Else, Acc) -> + [Else|Acc] + end, ok, Tab) + end)), + restore_verify_tabs(R); +restore_verify_tabs([]) -> + ok. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +consistency_after_rename_of_node(doc) -> + ["Skipped because it is an unimportant case."]. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +checkpoint_retainer_consistency(doc) -> + ["Verify that the contents of a checkpoint retainer has the expected", + "contents in various situations."]; +checkpoint_retainer_consistency(suite) -> + [ + updates_during_checkpoint_activation, + updates_during_checkpoint_iteration, + load_table_with_activated_checkpoint, + add_table_copy_to_table_with_activated_checkpoint + ]. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +updates_during_checkpoint_activation(doc) -> + ["Perform updates while the checkpoint getting activated", + "and verify that all checkpoint retainers associated with", + "different replicas of the same table really has the same", + "contents."]; +updates_during_checkpoint_activation(suite) -> + [ + updates_during_checkpoint_activation_2_ram, + updates_during_checkpoint_activation_2_disc, + updates_during_checkpoint_activation_2_disc_only, + updates_during_checkpoint_activation_3_ram, + updates_during_checkpoint_activation_3_disc + , updates_during_checkpoint_activation_3_disc_only + ]. + +updates_during_checkpoint_activation_2_ram(suite) -> []; +updates_during_checkpoint_activation_2_ram(Config) when is_list(Config) -> + updates_during_checkpoint_activation(ram_copies, 2, Config). + +updates_during_checkpoint_activation_2_disc(suite) -> []; +updates_during_checkpoint_activation_2_disc(Config) when is_list(Config) -> + updates_during_checkpoint_activation(disc_copies, 2, Config). + +updates_during_checkpoint_activation_2_disc_only(suite) -> []; +updates_during_checkpoint_activation_2_disc_only(Config) when is_list(Config) -> + updates_during_checkpoint_activation(disc_only_copies, 2, Config). + +updates_during_checkpoint_activation_3_ram(suite) -> []; +updates_during_checkpoint_activation_3_ram(Config) when is_list(Config) -> + updates_during_checkpoint_activation(ram_copies, 3, Config). + +updates_during_checkpoint_activation_3_disc(suite) -> []; +updates_during_checkpoint_activation_3_disc(Config) when is_list(Config) -> + updates_during_checkpoint_activation(disc_copies, 3, Config). + +updates_during_checkpoint_activation_3_disc_only(suite) -> []; +updates_during_checkpoint_activation_3_disc_only(Config) when is_list(Config) -> + updates_during_checkpoint_activation(disc_only_copies, 3, Config). + +updates_during_checkpoint_activation(ReplicaType,NodeConfig,Config) -> + %%?verbose("updates_during_checkpoint_activation2 at ~p~n", [self()]), + Delay = 5, + Nodes = ?acquire_nodes(NodeConfig, Config), + Node1 = hd(Nodes), + %%?verbose("Mnesia info: ~p~n", [mnesia:info()]), + + {success, [A]} = ?start_activities([Node1]), + ?log("consistency_after_fallback with ~p on ~p~n", + [ReplicaType, Nodes]), + TpcbConfig = tpcb_config_dist(ReplicaType, NodeConfig, Nodes, Config), + %%TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes), + mnesia_tpcb:init(TpcbConfig), + A ! fun () -> mnesia_tpcb:run(TpcbConfig) end, + timer:sleep(timer:seconds(Delay)), + + {ok, CPName, _NodeList} = + mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}]), + timer:sleep(timer:seconds(Delay)), + + %% Stop tpcb + ?verbose("Stopping TPC-B~n", []), + mnesia_tpcb:stop(), + ?match(ok, mnesia_tpcb:verify_tabs()), + + ?match(ok, mnesia:backup_checkpoint(CPName, + updates_during_checkpoint_activation2)), + timer:sleep(timer:seconds(Delay)), + + ?match(ok, mnesia:install_fallback(updates_during_checkpoint_activation2)), + + %% Stop and then start mnesia and check table consistency + %%?verbose("Restarting Mnesia~n", []), + mnesia_test_lib:kill_mnesia(Nodes), + file:delete(updates_during_checkpoint_activation2), + mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller, history]), + + ?match(ok, mnesia_tpcb:verify_tabs()), + ?verify_mnesia(Nodes, []). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +updates_during_checkpoint_iteration(doc) -> + ["Perform updates while someone is iterating over a checkpoint", + "and verify that the iterator really finds the expected data", + "regardless of ongoing upates."]; + +updates_during_checkpoint_iteration(suite) -> + [ + updates_during_checkpoint_iteration_2_ram, + updates_during_checkpoint_iteration_2_disc + , updates_during_checkpoint_iteration_2_disc_only + ]. + +updates_during_checkpoint_iteration_2_ram(suite) -> []; +updates_during_checkpoint_iteration_2_ram(Config) when is_list(Config) -> + updates_during_checkpoint_iteration(ram_copies, 2, Config). + +updates_during_checkpoint_iteration_2_disc(suite) -> []; +updates_during_checkpoint_iteration_2_disc(Config) when is_list(Config) -> + updates_during_checkpoint_iteration(disc_copies, 2, Config). + +updates_during_checkpoint_iteration_2_disc_only(suite) -> []; +updates_during_checkpoint_iteration_2_disc_only(Config) when is_list(Config) -> + updates_during_checkpoint_iteration(disc_only_copies, 2, Config). + +updates_during_checkpoint_iteration(ReplicaType,NodeConfig,Config) -> + %?verbose("updates_during_checkpoint_iteration2 at ~p~n", [self()]), + Delay = 5, + Nodes = ?acquire_nodes(NodeConfig, Config), + Node1 = hd(Nodes), + %?verbose("Mnesia info: ~p~n", [mnesia:info()]), + File = updates_during_checkpoint_iteration2, + {success, [A]} = ?start_activities([Node1]), + ?log("updates_during_checkpoint_iteration with ~p on ~p~n", + [ReplicaType, Nodes]), + TpcbConfig = tpcb_config_dist(ReplicaType, NodeConfig, Nodes, Config), + %%TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes), + TpcbConfigRec = list2rec(TpcbConfig, + record_info(fields,tab_config), + #tab_config{}), + mnesia_tpcb:init(TpcbConfig), + ?match(ok, mnesia_tpcb:verify_tabs()), + + {ok, CPName, _NodeList} = + mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, + {ram_overrides_dump,true}]), + A ! fun () -> mnesia:backup_checkpoint(CPName, File) end, + + do_changes_during_backup(TpcbConfigRec), + + ?match_receive({A,ok}), + + timer:sleep(timer:seconds(Delay)), + ?match(ok, mnesia:install_fallback(File)), + timer:sleep(timer:seconds(Delay)), + + ?match({error,{"Bad balance",_,_}}, mnesia_tpcb:verify_tabs()), + + mnesia_test_lib:kill_mnesia(Nodes), + mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller, history]), + + ?match(ok, mnesia_tpcb:verify_tabs()), + + ?match(ok, file:delete(File)), + ?verify_mnesia(Nodes, []). + +do_changes_during_backup(TpcbConfig) -> + loop_branches(TpcbConfig#tab_config.n_branches, + TpcbConfig#tab_config.n_accounts_per_branch). + +loop_branches(N_br,N_acc) when N_br >= 1 -> + loop_accounts(N_br,N_acc), + loop_branches(N_br-1,N_acc); +loop_branches(_,_) -> done. + +loop_accounts(N_br, N_acc) when N_acc >= 1 -> + A = #account{id=N_acc, branch_id=N_br, balance = 4711}, + ok = mnesia:dirty_write(A), + loop_accounts(N_br, N_acc-1); + +loop_accounts(_,_) -> done. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +load_table_with_activated_checkpoint(doc) -> + ["Load a table with a checkpoint attached to it and verify that the", + "newly loaded replica also gets a checkpoint retainer attached to it", + "and that it is consistent with the original retainer."]; + +load_table_with_activated_checkpoint(suite) -> + [ + load_table_with_activated_checkpoint_ram, + load_table_with_activated_checkpoint_disc, + load_table_with_activated_checkpoint_disc_only + ]. + +load_table_with_activated_checkpoint_ram(suite) -> []; +load_table_with_activated_checkpoint_ram(Config) when is_list(Config) -> + load_table_with_activated_checkpoint(ram_copies, Config). + +load_table_with_activated_checkpoint_disc(suite) -> []; +load_table_with_activated_checkpoint_disc(Config) when is_list(Config) -> + load_table_with_activated_checkpoint(disc_copies, Config). + +load_table_with_activated_checkpoint_disc_only(suite) -> []; +load_table_with_activated_checkpoint_disc_only(Config) when is_list(Config) -> + load_table_with_activated_checkpoint(disc_only_copies, Config). + +load_table_with_activated_checkpoint(Type, Config) -> + Nodes = ?acquire_nodes(2, Config), + Node1 = hd(Nodes), + Tab = load_test, + Def = [{attributes, [key, value]}, + {Type, Nodes}], %% ??? important that RAM ??? + + ?match({atomic, ok}, mnesia:create_table(Tab, Def)), + ?match(ok, mnesia:dirty_write({Tab, 1, 4711})), + ?match(ok, mnesia:dirty_write({Tab, 2, 42})), + ?match(ok, mnesia:dirty_write({Tab, 3, 256})), + + timer:sleep(timer:seconds(1)), + + {ok, CPName, _NodeList} = + mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, + {ram_overrides_dump,true}]), + + mnesia_test_lib:stop_mnesia([Node1]), + mnesia_test_lib:start_mnesia([Node1],[Tab]), + %%--- check, whether the checkpiont is attached to both replicas + {success, [A,B]} = ?start_activities(Nodes), + + A ! fun () -> + mnesia:table_info(Tab,checkpoints) + end, + ?match_receive({A,[CPName]}), + + B ! fun () -> + mnesia:table_info(Tab,checkpoints) + end, + ?match_receive({B,[CPName]}), + + %%--- check, whether both retainers are consistent + ?match(ok, mnesia:dirty_write({Tab, 1, 815})), + A ! fun () -> + mnesia:backup_checkpoint(CPName, load_table_a) + end, + ?match_receive({A,ok}), + B ! fun () -> + mnesia:backup_checkpoint(CPName, load_table_b) + end, + ?match_receive({B,ok}), + + Mod = mnesia_backup, %% Assume local files + List_a = view(load_table_a, Mod), + List_b = view(load_table_b, Mod), + + ?match(List_a, List_b), + + ?match(ok,file:delete(load_table_a)), + ?match(ok,file:delete(load_table_b)), + ?verify_mnesia(Nodes, []). + +view(Source, Mod) -> + View = fun(Item, Acc) -> + ?verbose("tab - item : ~p ~n",[Item]), + case Item of + {schema, Tab, Cs} -> %% Remove cookie information + NewCs = lists:keyreplace(cookie, 1, Cs, + {cookie, skip_cookie}), + Item2 = {schema, Tab, NewCs}, + {[Item], [Item2|Acc]}; + _ -> + {[Item], [Item|Acc]} + end + end, + {ok,TabList} = + mnesia:traverse_backup(Source, Mod, dummy, read_only, View, []), + lists:sort(TabList). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +add_table_copy_to_table_with_activated_checkpoint(doc) -> + ["Add a replica to a table with a checkpoint attached to it", + "and verify that the new replica also gets a checkpoint", + "retainer attached to it and that it is consistent with the", + "original retainer."]; + +add_table_copy_to_table_with_activated_checkpoint(suite) -> + [ + add_table_copy_to_table_with_activated_checkpoint_ram, + add_table_copy_to_table_with_activated_checkpoint_disc, + add_table_copy_to_table_with_activated_checkpoint_disc_only + ]. + +add_table_copy_to_table_with_activated_checkpoint_ram(suite) -> []; +add_table_copy_to_table_with_activated_checkpoint_ram(Config) when is_list(Config) -> + add_table_copy_to_table_with_activated_checkpoint(ram_copies, Config). + +add_table_copy_to_table_with_activated_checkpoint_disc(suite) -> []; +add_table_copy_to_table_with_activated_checkpoint_disc(Config) when is_list(Config) -> + add_table_copy_to_table_with_activated_checkpoint(disc_copies, Config). + +add_table_copy_to_table_with_activated_checkpoint_disc_only(suite) -> []; +add_table_copy_to_table_with_activated_checkpoint_disc_only(Config) when is_list(Config) -> + add_table_copy_to_table_with_activated_checkpoint(disc_only_copies, Config). + +add_table_copy_to_table_with_activated_checkpoint(Type,Config) -> + Nodes = ?acquire_nodes(2, Config), + %?verbose("NODES = ~p ~n",[Nodes]), + [Node1,Node2] = Nodes, + + Tab = add_test, + Def = [{attributes, [key, value]}, + {Type, [Node1]}], %% ??? important that RAM ??? + + ?match({atomic, ok}, mnesia:create_table(Tab, Def)), + ?match(ok, mnesia:dirty_write({Tab, 1, 4711})), + ?match(ok, mnesia:dirty_write({Tab, 2, 42})), + ?match(ok, mnesia:dirty_write({Tab, 3, 256})), + + {ok, CPName, _NodeList} = + mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, + {ram_overrides_dump,true}]), + + ?match({atomic,ok},mnesia:add_table_copy(Tab,Node2,ram_copies)), + + %%--- check, whether the checkpiont is attached to both replicas + {success, [A,B]} = ?start_activities(Nodes), + + A ! fun () -> + mnesia:table_info(Tab,checkpoints) + end, + ?match_receive({A,[CPName]}), + + B ! fun () -> + mnesia:table_info(Tab,checkpoints) + end, + ?match_receive({B,[CPName]}), + + %%--- check, whether both retainers are consistent + + ?match(ok, mnesia:dirty_write({Tab, 1, 815})), + ?match(ok, mnesia:dirty_write({Tab, 2, 815})), + + A ! fun () -> + mnesia:backup_checkpoint(CPName, add_table_a) + end, + ?match_receive({A,ok}), + B ! fun () -> + mnesia:backup_checkpoint(CPName, add_table_b) + end, + ?match_receive({B,ok}), + + Mod = mnesia_backup, %% Assume local files + + List_a = view(add_table_a, Mod), + List_b = view(add_table_b, Mod), + + ?match(List_a, List_b), + + ?match(ok,file:delete(add_table_a)), + ?match(ok, file:delete(add_table_b)), + ?verify_mnesia(Nodes, []). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +backup_consistency(suite) -> + [ + interupted_install_fallback, + interupted_uninstall_fallback, + mnesia_down_during_backup_causes_switch, + mnesia_down_during_backup_causes_abort, + schema_transactions_during_backup + ]. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +interupted_install_fallback(doc) -> + ["Verify that a interrupted install_fallback really", + "is performed on all nodes or none"]; + +interupted_install_fallback(suite) -> + [ + inst_fallback_process_dies, + fatal_when_inconsistency + ]. + +inst_fallback_process_dies(suite) -> + []; +inst_fallback_process_dies(Config) when is_list(Config) -> + ?is_debug_compiled, + + Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), + {success, [A,_B,_C]} = ?start_activities(Nodes), + + TestPid = self(), + DebugId = {mnesia_bup, fallback_receiver_loop, pre_swap}, + DebugFun = + fun(PrevContext, _EvalContext) -> + ?verbose("fallback_receiver_loop - pre_swap pid ~p #~p~n", + [self(),PrevContext]), + TestPid ! {self(),fallback_preswap}, + case receive_messages([fallback_continue]) of + [{TestPid,fallback_continue}] -> + ?deactivate_debug_fun(DebugId), + PrevContext+1 + end + end, + ?activate_debug_fun(DebugId, DebugFun, 1), + + Tab = install_table, + Def = [{attributes, [key, value]}, {disc_copies, Nodes}], + + ?match({atomic, ok}, mnesia:create_table(Tab, Def)), + + ?match(ok, mnesia:dirty_write({Tab, 1, 4711})), + ?match(ok, mnesia:dirty_write({Tab, 2, 42})), + ?match(ok, mnesia:dirty_write({Tab, 3, 256})), + + {ok, CPName, _NodeList} = + mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, + {ram_overrides_dump,true}]), + + ?match(ok, mnesia:backup_checkpoint(CPName, install_backup)), + + A ! fun() -> mnesia:install_fallback(install_backup) end, + [{AnsPid,fallback_preswap}] = receive_messages([fallback_preswap]), + exit(A, kill), + AnsPid ! {self(), fallback_continue}, + ?match_receive({'EXIT', A, killed}), + timer:sleep(2000), %% Wait till fallback is installed everywhere + + mnesia_test_lib:kill_mnesia(Nodes), + ?verbose("~n---->Mnesia is stopped everywhere<-----~n", []), + ?match([], mnesia_test_lib:start_mnesia(Nodes,[Tab])), + + check_data(Nodes, Tab), + ?match(ok, file:delete(install_backup)), + ?verify_mnesia(Nodes, []). + +check_data([N1 | R], Tab) -> + ?match([{Tab, 1, 4711}], rpc:call(N1, mnesia, dirty_read, [{Tab, 1}])), + ?match([{Tab, 2, 42}], rpc:call(N1, mnesia, dirty_read, [{Tab, 2}])), + ?match([{Tab, 3, 256}], rpc:call(N1, mnesia, dirty_read, [{Tab, 3}])), + check_data(R, Tab); +check_data([], _Tab) -> + ok. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +fatal_when_inconsistency(suite) -> + []; +fatal_when_inconsistency(Config) when is_list(Config) -> + ?is_debug_compiled, + + [Node1, Node2, Node3] = Nodes = + ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]), + {success, [A,_B,_C]} = ?start_activities(Nodes), + + TestPid = self(), + DebugId = {mnesia_bup, fallback_receiver_loop, pre_swap}, + DebugFun = + fun(PrevContext, _EvalContext) -> + ?verbose("fallback_receiver_loop - pre_swap pid ~p #~p~n", + [self(),PrevContext]), + TestPid ! {self(),fallback_preswap}, + case receive_messages([fallback_continue]) of + [{TestPid,fallback_continue}] -> + ?deactivate_debug_fun(DebugId), + PrevContext+1 + end + end, + ?activate_debug_fun(DebugId, DebugFun, 1), + + Tab = install_table, + Def = [{attributes, [key, value]}, {disc_copies, Nodes}], + + ?match({atomic, ok}, mnesia:create_table(Tab, Def)), + + ?match(ok, mnesia:dirty_write({Tab, 1, 4711})), + ?match(ok, mnesia:dirty_write({Tab, 2, 42})), + ?match(ok, mnesia:dirty_write({Tab, 3, 256})), + + {ok, CPName, _NodeList} = + mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, + {ram_overrides_dump,true}]), + + ?match(ok, mnesia:backup_checkpoint(CPName, install_backup)), + ?match(ok, mnesia:dirty_write({Tab, 2, 42424242})), + + A ! fun() -> + mnesia:install_fallback(install_backup) + end, + + [{AnsPid,fallback_preswap}] = receive_messages([fallback_preswap]), + exit(AnsPid, kill), %% Kill install-fallback on local node will + AnsPid ! {self(), fallback_continue}, + ?deactivate_debug_fun(DebugId), + + ?match_receive({A,{error,{"Cannot install fallback", + {'EXIT',AnsPid,killed}}}}), + mnesia_test_lib:kill_mnesia(Nodes), + ?verbose("EXPECTING FATAL from 2 nodes WITH CORE DUMP~n", []), + + ?match([], mnesia_test_lib:start_mnesia([Node1],[])), + is_running(Node1, yes), + ?match([{Node2, mnesia, _}], mnesia_test_lib:start_mnesia([Node2],[])), + is_running(Node2, no), + ?match([{Node3, mnesia, _}], mnesia_test_lib:start_mnesia([Node3],[])), + is_running(Node3, no), + mnesia_test_lib:kill_mnesia(Nodes), + + ?match(ok, mnesia:install_fallback(install_backup)), + mnesia_test_lib:start_mnesia(Nodes,[Tab]), + + check_data(Nodes, Tab), + + ?match(ok,file:delete(install_backup)), + ?verify_mnesia(Nodes, []). + +is_running(Node, Shouldbe) -> + timer:sleep(1000), + Running = rpc:call(Node, mnesia, system_info, [is_running]), + case Running of + Shouldbe -> ok; + _ -> is_running(Node, Shouldbe) + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +interupted_uninstall_fallback(doc) -> + ["Verify that a interrupted uninstall_fallback really", + "is performed on all nodes or none"]; +interupted_uninstall_fallback(suite) -> + [ + after_delete + ]. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +after_delete(doc) -> + ["interrupt the uninstall after deletion of ", + "fallback files - there shall be no fallback"]; +after_delete(suite) -> []; +after_delete(Config) when is_list(Config) -> + do_uninstall(Config, post_delete). + +%%%%%%%%%%%%%%%%%%%%%%%%% + +do_uninstall(Config,DebugPoint) -> + ?is_debug_compiled, + + Nodes = ?acquire_nodes(3, Config), + %%?verbose("NODES = ~p ~n",[Nodes]), + + {success, [P1,P2,P3]} = ?start_activities(Nodes), + + NP1 = node(P1), + NP2 = node(P2), + + {A,B,C} = case node() of + NP1 -> + %%?verbose("first case ~n"), + {P3,P2,P1}; + NP2 -> + %%?verbose("second case ~n"), + {P3, P1, P2}; + _ -> + { P1, P2, P3} + end, + + Node1 = node(A), + Node2 = node(B), + Node3 = node(C), + + ?verbose(" A pid:~p node:~p ~n",[A,Node1]), + ?verbose(" B pid:~p node:~p ~n",[B,Node2]), + ?verbose(" C pid:~p node:~p ~n",[C,Node3]), + + + TestPid = self(), + %%?verbose("TestPid : ~p~n",[TestPid]), + DebugId = {mnesia_bup, uninstall_fallback2, DebugPoint}, + DebugFun = fun(PrevContext, _EvalContext) -> + ?verbose("uninstall_fallback pid ~p #~p~n" + ,[self(),PrevContext]), + TestPid ! {self(),uninstall_predelete}, + case receive_messages([uninstall_continue]) of + [{TestPid,uninstall_continue}] -> + ?deactivate_debug_fun(DebugId), + %%?verbose("uninstall_fallback continues~n"), + PrevContext+1 + end + end, + ?remote_activate_debug_fun(Node1,DebugId, DebugFun, 1), + + Tab = install_table, + Def = [{attributes, [key, value]}, + {ram_copies, Nodes}], %% necessary to test different types ??? + + ?match({atomic, ok}, mnesia:create_table(Tab, Def)), + + ?match(ok, mnesia:dirty_write({Tab, 1, 4711})), + ?match(ok, mnesia:dirty_write({Tab, 2, 42})), + ?match(ok, mnesia:dirty_write({Tab, 3, 256})), + + {ok, CPName, _NodeList} = + mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, + {ram_overrides_dump,true}]), + + ?match(ok, mnesia:backup_checkpoint(CPName,install_backup)), + timer:sleep(timer:seconds(1)), + + A ! fun () -> + mnesia:install_fallback(install_backup) + end, + ?match_receive({A,ok}), + + A ! fun () -> + mnesia:uninstall_fallback() + end, + %% + %% catch the debug entry in mnesia and kill one Mnesia node + %% + + + [{AnsPid,uninstall_predelete}] = receive_messages([uninstall_predelete]), + + ?verbose("AnsPid : ~p~n",[AnsPid]), + + mnesia_test_lib:kill_mnesia([Node2]), + timer:sleep(timer:seconds(1)), + + AnsPid ! {self(),uninstall_continue}, + + ?match_receive({A,ok}), + + mnesia_test_lib:kill_mnesia(Nodes) , + mnesia_test_lib:start_mnesia(Nodes,[Tab]), + + A ! fun () -> + R1 = mnesia:dirty_read({Tab,1}), + R2 = mnesia:dirty_read({Tab,2}), + R3 = mnesia:dirty_read({Tab,3}), + {R1,R2,R3} + end, + ?match_receive({ A, {[],[],[]} }), + + B ! fun () -> + R1 = mnesia:dirty_read({Tab,1}), + R2 = mnesia:dirty_read({Tab,2}), + R3 = mnesia:dirty_read({Tab,3}), + {R1,R2,R3} + end, + ?match_receive({ B, {[],[],[]} }), + + C ! fun () -> + R1 = mnesia:dirty_read({Tab,1}), + R2 = mnesia:dirty_read({Tab,2}), + R3 = mnesia:dirty_read({Tab,3}), + {R1,R2,R3} + end, + ?match_receive({ C, {[],[],[]} }), + + ?match(ok,file:delete(install_backup)), + ?verify_mnesia(Nodes, []). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +mnesia_down_during_backup_causes_switch(doc) -> + ["Verify that an ongoing backup is not disturbed", + "even if the node hosting the replica that currently", + "is being backup'ed is stopped. The backup utility", + "is expected to switch over to another replica and", + "fulfill the backup."]; +mnesia_down_during_backup_causes_switch(suite) -> + [ + cause_switch_before, + cause_switch_after + ]. + +%%%%%%%%%%%%%%% + +cause_switch_before(doc) -> + ["interrupt the backup before iterating the retainer"]; +cause_switch_before(suite) -> []; +cause_switch_before(Config) when is_list(Config) -> + do_something_during_backup(cause_switch,pre,Config). + +%%%%%%%%%%%%%%% + +cause_switch_after(doc) -> + ["interrupt the backup after iterating the retainer"]; +cause_switch_after(suite) -> []; +cause_switch_after(Config) when is_list(Config) -> + do_something_during_backup(cause_switch,post,Config). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +mnesia_down_during_backup_causes_abort(doc) -> + ["Verify that an ongoing backup is aborted nicely", + "without leaving any backup file if the last replica", + "of a table becomes unavailable due to a node down", + "or some crash."]; +mnesia_down_during_backup_causes_abort(suite) -> + [ + cause_abort_before, + cause_abort_after + ]. + +%%%%%%%%%%%%%%%%%% + +cause_abort_before(doc) -> + ["interrupt the backup before iterating the retainer"]; + +cause_abort_before(suite) -> []; +cause_abort_before(Config) when is_list(Config) -> + do_something_during_backup(cause_abort,pre,Config). + +%%%%%%%%%%%%%%%%%% + +cause_abort_after(doc) -> + ["interrupt the backup after iterating the retainer"]; + +cause_abort_after(suite) -> []; +cause_abort_after(Config) when is_list(Config) -> + do_something_during_backup(cause_abort,post,Config). + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +schema_transactions_during_backup(doc) -> + ["Verify that an schema transactions does not", + "affect an ongoing backup."]; +schema_transactions_during_backup(suite) -> + [ + change_schema_before, + change_schema_after + ]. + +%%%%%%%%%%%%% + +change_schema_before(doc) -> + ["interrupt the backup before iterating the retainer"]; +change_schema_before(suite) -> []; +change_schema_before(Config) when is_list(Config) -> + do_something_during_backup(change_schema,pre,Config). + +%%%%%%%%%%%%%%%% + +change_schema_after(doc) -> + ["interrupt the backup after iterating the retainer"]; +change_schema_after(suite) -> []; +change_schema_after(Config) when is_list(Config) -> + do_something_during_backup(change_schema,post,Config). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +do_something_during_backup(Action,DebugPoint,Config) -> + ?is_debug_compiled, + + Nodes = ?acquire_nodes(3, Config), + + {success, [A,B,C]} = ?start_activities(Nodes), + + Node1 = node(A), + Node2 = node(B), + Node3 = node(C), + + TestPid = self(), + %%?verbose("TestPid : ~p~n",[TestPid]), + + Tab = interrupt_table, + Bak = interrupt_backup, + Def = [{attributes, [key, value]}, + {ram_copies, [Node2,Node3]}], + %% necessary to test different types ??? + + ?match({atomic, ok}, mnesia:create_table(Tab, Def)), + + + + DebugId = {mnesia_log, tab_copier, DebugPoint}, + DebugFun = fun(PrevContext, EvalContext) -> + ?verbose("interrupt backup pid ~p #~p ~n context ~p ~n" + ,[self(),PrevContext,EvalContext]), + TestPid ! {self(),interrupt_backup_pre}, + global:set_lock({{lock_for_backup, Tab}, self()}, + Nodes, + infinity), + + %%?verbose("interrupt backup - continues ~n"), + ?deactivate_debug_fun(DebugId), + PrevContext+1 + end, + ?remote_activate_debug_fun(Node1,DebugId, DebugFun, 1), + + ?match(ok, mnesia:dirty_write({Tab, 1, 4711})), + ?match(ok, mnesia:dirty_write({Tab, 2, 42})), + ?match(ok, mnesia:dirty_write({Tab, 3, 256})), + + {ok, CPName, _NodeList} = + mnesia:activate_checkpoint([{max, mnesia:system_info(tables)}, + {ram_overrides_dump,true}]), + + A ! fun () -> + %%?verbose("node: ~p pid: ~p ~n",[node(),self()]), + mnesia:table_info(Tab,where_to_read) + end, + + ReadNode_a = receive { A, ReadNode_a_tmp } -> ReadNode_a_tmp end, + ?verbose("ReadNode ~p ~n",[ReadNode_a]), + + global:set_lock({{lock_for_backup, Tab}, self()}, Nodes, infinity), + + A ! fun () -> %% A shall perform the backup, so the test proc is + %% able to do further actions in between + mnesia:backup_checkpoint(CPName, Bak) + end, + + %% catch the debug function of mnesia, stop the backup process + %% kill the node ReadNode_a and continue the backup process + %% As there is a second replica of the table, the backup shall continue + + case receive_messages([interrupt_backup_pre]) of + [{_AnsPid,interrupt_backup_pre}] -> ok + end, + + case Action of + cause_switch -> + mnesia_test_lib:kill_mnesia([ReadNode_a]), + timer:sleep(timer:seconds(1)); + cause_abort -> + mnesia_test_lib:kill_mnesia([Node2,Node3]), + timer:sleep(timer:seconds(1)); + change_schema -> + Tab2 = second_interrupt_table, + Def2 = [{attributes, [key, value]}, + {ram_copies, Nodes}], + + ?match({atomic, ok}, mnesia:create_table(Tab2, Def2)) + end, + + %% AnsPid ! {self(),interrupt_backup_continue}, + global:del_lock({{lock_for_backup, Tab}, self()}, Nodes), + + case Action of + cause_abort -> + + %% answer of A when finishing the backup + ?match_receive({A,{error, _}}), + + ?match({error,{"Cannot install fallback",_}}, + mnesia:install_fallback(Bak)); + _ -> %% cause_switch, change_schema + + ?match_receive({A,ok}), %% answer of A when finishing the backup + + %% send a fun to that node where mnesia is still running + WritePid = case ReadNode_a of + Node2 -> C; %% node(C) == Node3 + Node3 -> B + end, + WritePid ! fun () -> + ?match(ok, mnesia:dirty_write({Tab, 1, 815})), + ?match(ok, mnesia:dirty_write({Tab, 2, 816})), + ok + end, + ?match_receive({ WritePid, ok }), + ?match(ok, mnesia:install_fallback(Bak)) + end, + + %% Stop and then start mnesia and check table consistency + %%?verbose("Restarting Mnesia~n", []), + mnesia_test_lib:kill_mnesia(Nodes), + mnesia_test_lib:start_mnesia(Nodes,[Tab]), + + case Action of + cause_switch -> + %% the backup should exist + cross_check_tables([A,B,C],Tab,{[{Tab,1,4711}], + [{Tab,2,42}], + [{Tab,3,256}] }), + ?match(ok,file:delete(Bak)); + cause_abort -> + %% the backup should NOT exist + cross_check_tables([A,B,C],Tab,{[],[],[]}), + %% file does not exist + ?match({error, _},file:delete(Bak)); + change_schema -> + %% the backup should exist + cross_check_tables([A,B,C],Tab,{[{Tab,1,4711}], + [{Tab,2,42}], + [{Tab,3,256}] }), + ?match(ok,file:delete(Bak)) + end, + ?verify_mnesia(Nodes, []). + +%% check the contents of the table +cross_check_tables([],_tab,_elements) -> ok; +cross_check_tables([Pid|Rest],Tab,{Val1,Val2,Val3}) -> + Pid ! fun () -> + R1 = mnesia:dirty_read({Tab,1}), + R2 = mnesia:dirty_read({Tab,2}), + R3 = mnesia:dirty_read({Tab,3}), + {R1,R2,R3} + end, + ?match_receive({ Pid, {Val1, Val2, Val3 } }), + cross_check_tables(Rest,Tab,{Val1,Val2,Val3} ). |