%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 1997-2018. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
%% %CopyrightEnd%
%%
%%
-module(mnesia_consistency_test).
-author('[email protected]').
-export([init_per_testcase/2, end_per_testcase/2,
init_per_group/2, end_per_group/2,
all/0, groups/0]).
-export([consistency_after_change_table_copy_type/1,
consistency_after_rename_of_node/1,
consistency_after_restart_1_ram/1,
consistency_after_restart_1_disc/1,
consistency_after_restart_1_disc_only/1,
consistency_after_restart_2_ram/1,
consistency_after_restart_2_disc/1,
consistency_after_restart_2_disc_only/1,
consistency_after_dump_tables_1_ram/1,
consistency_after_dump_tables_2_ram/1,
consistency_after_add_replica_2_ram/1,
consistency_after_add_replica_2_disc/1,
consistency_after_add_replica_2_disc_only/1,
consistency_after_add_replica_3_ram/1,
consistency_after_add_replica_3_disc/1,
consistency_after_add_replica_3_disc_only/1,
consistency_after_del_replica_2_ram/1,
consistency_after_del_replica_2_disc/1,
consistency_after_del_replica_2_disc_only/1,
consistency_after_del_replica_3_ram/1,
consistency_after_del_replica_3_disc/1,
consistency_after_del_replica_3_disc_only/1,
consistency_after_move_replica_2_ram/1,
consistency_after_move_replica_2_disc/1,
consistency_after_move_replica_2_disc_only/1,
consistency_after_move_replica_3_ram/1,
consistency_after_move_replica_3_disc/1,
consistency_after_move_replica_3_disc_only/1,
consistency_after_transform_table_ram/1,
consistency_after_transform_table_disc/1,
consistency_after_transform_table_disc_only/1,
consistency_after_fallback_2_ram/1,
consistency_after_fallback_2_disc/1,
consistency_after_fallback_2_disc_only/1,
consistency_after_fallback_3_ram/1,
consistency_after_fallback_3_disc/1,
consistency_after_fallback_3_disc_only/1,
consistency_after_restore_clear_ram/1,
consistency_after_restore_clear_disc/1,
consistency_after_restore_clear_disc_only/1,
consistency_after_restore_recreate_ram/1,
consistency_after_restore_recreate_disc/1,
consistency_after_restore_recreate_disc_only/1,
updates_during_checkpoint_activation_1_ram/1,
updates_during_checkpoint_activation_1_disc/1,
updates_during_checkpoint_activation_1_disc_only/1,
updates_during_checkpoint_activation_2_ram/1,
updates_during_checkpoint_activation_2_disc/1,
updates_during_checkpoint_activation_2_disc_only/1,
updates_during_checkpoint_activation_3_ram/1,
updates_during_checkpoint_activation_3_disc/1,
updates_during_checkpoint_activation_3_disc_only/1,
updates_during_checkpoint_iteration_2_ram/1,
updates_during_checkpoint_iteration_2_disc/1,
updates_during_checkpoint_iteration_2_disc_only/1,
load_table_with_activated_checkpoint_ram/1,
load_table_with_activated_checkpoint_disc/1,
load_table_with_activated_checkpoint_disc_only/1,
add_table_copy_to_table_checkpoint_ram/1,
add_table_copy_to_table_checkpoint_disc/1,
add_table_copy_to_table_checkpoint_disc_only/1,
inst_fallback_process_dies/1, fatal_when_inconsistency/1,
after_delete/1,cause_switch_before/1, cause_switch_after/1,
cause_abort_before/1, cause_abort_after/1,
change_schema_before/1, change_schema_after/1]).
-export([change_tab/3]).
-include("mnesia_test_lib.hrl").
init_per_testcase(Func, Conf) ->
mnesia_test_lib:init_per_testcase(Func, Conf).
end_per_testcase(Func, Conf) ->
mnesia_test_lib:end_per_testcase(Func, Conf).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
all() ->
[{group, consistency_after_restart},
{group, consistency_after_dump_tables},
{group, consistency_after_add_replica},
{group, consistency_after_del_replica},
{group, consistency_after_move_replica},
{group, consistency_after_transform_table},
consistency_after_change_table_copy_type,
{group, consistency_after_restore},
consistency_after_rename_of_node,
{group, checkpoint_retainer_consistency},
{group, backup_consistency}].
groups() ->
[{consistency_after_restart, [],
[consistency_after_restart_1_ram,
consistency_after_restart_1_disc,
consistency_after_restart_1_disc_only,
consistency_after_restart_2_ram,
consistency_after_restart_2_disc,
consistency_after_restart_2_disc_only]},
{consistency_after_dump_tables, [],
[consistency_after_dump_tables_1_ram,
consistency_after_dump_tables_2_ram]},
{consistency_after_add_replica, [],
[consistency_after_add_replica_2_ram,
consistency_after_add_replica_2_disc,
consistency_after_add_replica_2_disc_only,
consistency_after_add_replica_3_ram,
consistency_after_add_replica_3_disc,
consistency_after_add_replica_3_disc_only]},
{consistency_after_del_replica, [],
[consistency_after_del_replica_2_ram,
consistency_after_del_replica_2_disc,
consistency_after_del_replica_2_disc_only,
consistency_after_del_replica_3_ram,
consistency_after_del_replica_3_disc,
consistency_after_del_replica_3_disc_only]},
{consistency_after_move_replica, [],
[consistency_after_move_replica_2_ram,
consistency_after_move_replica_2_disc,
consistency_after_move_replica_2_disc_only,
consistency_after_move_replica_3_ram,
consistency_after_move_replica_3_disc,
consistency_after_move_replica_3_disc_only]},
{consistency_after_transform_table, [],
[consistency_after_transform_table_ram,
consistency_after_transform_table_disc,
consistency_after_transform_table_disc_only]},
{consistency_after_fallback, [],
[consistency_after_fallback_2_ram,
consistency_after_fallback_2_disc,
consistency_after_fallback_2_disc_only,
consistency_after_fallback_3_ram,
consistency_after_fallback_3_disc,
consistency_after_fallback_3_disc_only]},
{consistency_after_restore, [],
[consistency_after_restore_clear_ram,
consistency_after_restore_clear_disc,
consistency_after_restore_clear_disc_only,
consistency_after_restore_recreate_ram,
consistency_after_restore_recreate_disc,
consistency_after_restore_recreate_disc_only]},
{checkpoint_retainer_consistency, [],
[{group, updates_during_checkpoint_activation},
{group, updates_during_checkpoint_iteration},
{group, load_table_with_activated_checkpoint},
{group, add_table_copy_to_table_checkpoint},
{group, consistency_after_fallback}
]},
{updates_during_checkpoint_activation, [],
[updates_during_checkpoint_activation_1_ram,
updates_during_checkpoint_activation_1_disc,
updates_during_checkpoint_activation_1_disc_only,
updates_during_checkpoint_activation_2_ram,
updates_during_checkpoint_activation_2_disc,
updates_during_checkpoint_activation_2_disc_only,
updates_during_checkpoint_activation_3_ram,
updates_during_checkpoint_activation_3_disc,
updates_during_checkpoint_activation_3_disc_only]},
{updates_during_checkpoint_iteration, [],
[updates_during_checkpoint_iteration_2_ram,
updates_during_checkpoint_iteration_2_disc,
updates_during_checkpoint_iteration_2_disc_only]},
{load_table_with_activated_checkpoint, [],
[load_table_with_activated_checkpoint_ram,
load_table_with_activated_checkpoint_disc,
load_table_with_activated_checkpoint_disc_only]},
{add_table_copy_to_table_checkpoint, [],
[add_table_copy_to_table_checkpoint_ram,
add_table_copy_to_table_checkpoint_disc,
add_table_copy_to_table_checkpoint_disc_only]},
{backup_consistency, [],
[{group, interupted_install_fallback},
{group, interupted_uninstall_fallback},
{group, mnesia_down_during_backup_causes_switch},
{group, mnesia_down_during_backup_causes_abort},
{group, schema_transactions_during_backup}]},
{interupted_install_fallback, [],
[inst_fallback_process_dies, fatal_when_inconsistency]},
{interupted_uninstall_fallback, [], [after_delete]},
{mnesia_down_during_backup_causes_switch, [],
[cause_switch_before, cause_switch_after]},
{mnesia_down_during_backup_causes_abort, [],
[cause_abort_before, cause_abort_after]},
{schema_transactions_during_backup, [],
[change_schema_before, change_schema_after]}].
init_per_group(_GroupName, Config) ->
Config.
end_per_group(_GroupName, Config) ->
Config.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% stolen from mnesia_tpcb.erl:
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% Account record, total size must be at least 100 bytes
-define(ACCOUNT_FILLER,
{123456789012345678901234567890123456789012345678901234567890,
123456789012345678901234567890123456789012345678901234567890,
123456789012345678901234567890123456789012345678901234}).
-record(account,
{
id = 0, %% Unique account id
branch_id = 0, %% Branch where the account is held
balance = 0, %% Account balance
filler = ?ACCOUNT_FILLER %% Gap filler to ensure size >= 100 bytes
}).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% Branch record, total size must be at least 100 bytes
-define(BRANCH_FILLER,
{123456789012345678901234567890123456789012345678901234567890,
123456789012345678901234567890123456789012345678901234567890,
123456789012345678901234567890123456789012345678901234567890}).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% Teller record, total size must be at least 100 bytes
-define(TELLER_FILLER,
{123456789012345678901234567890123456789012345678901234567890,
123456789012345678901234567890123456789012345678901234567890,
1234567890123456789012345678901234567890123456789012345678}).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% History record, total size must be at least 50 bytes
-define(HISTORY_FILLER, 1234567890).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-record(tab_config,
{
db_nodes = [node()],
replica_nodes = [node()],
replica_type = ram_copies,
use_running_mnesia = false,
n_branches = 1,
n_tellers_per_branch = 10, %% Must be 10
n_accounts_per_branch = 100000, %% Must be 100000
branch_filler = ?BRANCH_FILLER,
account_filler = ?ACCOUNT_FILLER,
teller_filler = ?TELLER_FILLER
}).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% stolen from mnesia_tpcb.erl:
list2rec(List, Fields, DefaultTuple) ->
[Name|Defaults] = tuple_to_list(DefaultTuple),
List2 = list2rec(List, Fields, Defaults, []),
list_to_tuple([Name] ++ List2).
list2rec(_List, [], [], Acc) ->
Acc;
list2rec(List, [F|Fields], [D|Defaults], Acc) ->
{Val, List2} =
case lists:keysearch(F, 1, List) of
false ->
{D, List};
{value, {F, NewVal}} ->
{NewVal, lists:keydelete(F, 1, List)}
end,
list2rec(List2, Fields, Defaults, Acc ++ [Val]).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
tpcb_config(ReplicaType, _NodeConfig, Nodes, NoDriverNodes) ->
[{n_branches, 10},
{n_drivers_per_node, 10},
{replica_nodes, Nodes},
{driver_nodes, Nodes -- NoDriverNodes},
{use_running_mnesia, true},
{report_interval, infinity},
{n_accounts_per_branch, 100},
{replica_type, ReplicaType},
{reuse_history_id, true}].
%% Stolen from mnesia_tpcb:dist
tpcb_config_dist(ReplicaType, _NodeConfig, Nodes, _Config) ->
[{db_nodes, Nodes},
{driver_nodes, Nodes},
{replica_nodes, Nodes},
{n_drivers_per_node, 10},
{n_branches, 1},
{use_running_mnesia, true},
{n_accounts_per_branch, 10},
{replica_type, ReplicaType},
{stop_after, timer:minutes(15)},
{report_interval, timer:seconds(10)},
{reuse_history_id, true}].
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% stolen from mnesia_recovery_test.erl:
receive_messages([]) -> [];
receive_messages(ListOfMsgs) ->
receive
{Pid, Msg} ->
case lists:member(Msg, ListOfMsgs) of
false ->
?warning("I (~p) have received unexpected msg~n ~p ~n",
[self(),{Pid, Msg}]),
receive_messages(ListOfMsgs);
true ->
?verbose("I (~p) got msg ~p from ~p ~n", [self(),Msg, Pid]),
[{Pid, Msg} | receive_messages(ListOfMsgs -- [Msg])]
end;
Else -> ?warning("Recevied unexpected Msg~n ~p ~n", [Else])
after timer:minutes(3) ->
?error("Timeout in receive msgs while waiting for ~p~n",
[ListOfMsgs])
end.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
consistency_after_restart_1_ram(suite) -> [];
consistency_after_restart_1_ram(Config) when is_list(Config) ->
consistency_after_restart(ram_copies, 2, Config).
consistency_after_restart_1_disc(suite) -> [];
consistency_after_restart_1_disc(Config) when is_list(Config) ->
consistency_after_restart(disc_copies, 2, Config).
consistency_after_restart_1_disc_only(suite) -> [];
consistency_after_restart_1_disc_only(Config) when is_list(Config) ->
consistency_after_restart(disc_only_copies, 2, Config).
consistency_after_restart_2_ram(suite) -> [];
consistency_after_restart_2_ram(Config) when is_list(Config) ->
consistency_after_restart(ram_copies, 3, Config).
consistency_after_restart_2_disc(suite) -> [];
consistency_after_restart_2_disc(Config) when is_list(Config) ->
consistency_after_restart(disc_copies, 3, Config).
consistency_after_restart_2_disc_only(suite) -> [];
consistency_after_restart_2_disc_only(Config) when is_list(Config) ->
consistency_after_restart(disc_only_copies, 3, Config).
consistency_after_restart(ReplicaType, NodeConfig, Config) ->
[Node1 | _] = Nodes = ?acquire_nodes(NodeConfig, Config),
{success, [A]} = ?start_activities([Node1]),
?log("consistency_after_restart with ~p on ~p~n",
[ReplicaType, Nodes]),
TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, [Node1]),
mnesia_tpcb:init(TpcbConfig),
A ! fun () -> mnesia_tpcb:run(TpcbConfig) end,
timer:sleep(timer:seconds(10)),
mnesia_test_lib:kill_mnesia([Node1]),
%% Start and wait for tables to be loaded on all nodes
timer:sleep(timer:seconds(3)),
?match([], mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller, history])),
mnesia_tpcb:stop(),
?match(ok, mnesia_tpcb:verify_tabs()),
?verify_mnesia(Nodes, []).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
consistency_after_dump_tables_1_ram(suite) -> [];
consistency_after_dump_tables_1_ram(Config) when is_list(Config) ->
consistency_after_dump_tables(ram_copies, 1, Config).
consistency_after_dump_tables_2_ram(suite) -> [];
consistency_after_dump_tables_2_ram(Config) when is_list(Config) ->
consistency_after_dump_tables(ram_copies, 2, Config).
consistency_after_dump_tables(ReplicaType, NodeConfig, Config) ->
[Node1 | _] = Nodes = ?acquire_nodes(NodeConfig, Config),
{success, [A]} = ?start_activities([Node1]),
?log("consistency_after_dump_tables with ~p on ~p~n",
[ReplicaType, Nodes]),
TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []),
mnesia_tpcb:init(TpcbConfig),
A ! fun() -> mnesia_tpcb:run(TpcbConfig) end,
timer:sleep(timer:seconds(10)),
?match({atomic, ok}, rpc:call(Node1, mnesia, dump_tables,
[[branch, teller, account, history]])),
mnesia_tpcb:stop(),
?match(ok, mnesia_tpcb:verify_tabs()),
mnesia_test_lib:kill_mnesia(Nodes),
timer:sleep(timer:seconds(1)),
?match([], mnesia_test_lib:start_mnesia(Nodes,[account, branch,
teller, history])),
mnesia_tpcb:stop(),
?match(ok, mnesia_tpcb:verify_tabs()),
?verify_mnesia(Nodes, []).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
consistency_after_add_replica_2_ram(suite) -> [];
consistency_after_add_replica_2_ram(Config) when is_list(Config) ->
consistency_after_add_replica(ram_copies, 2, Config).
consistency_after_add_replica_2_disc(suite) -> [];
consistency_after_add_replica_2_disc(Config) when is_list(Config) ->
consistency_after_add_replica(disc_copies, 2, Config).
consistency_after_add_replica_2_disc_only(suite) -> [];
consistency_after_add_replica_2_disc_only(Config) when is_list(Config) ->
consistency_after_add_replica(disc_only_copies, 2, Config).
consistency_after_add_replica_3_ram(suite) -> [];
consistency_after_add_replica_3_ram(Config) when is_list(Config) ->
consistency_after_add_replica(ram_copies, 3, Config).
consistency_after_add_replica_3_disc(suite) -> [];
consistency_after_add_replica_3_disc(Config) when is_list(Config) ->
consistency_after_add_replica(disc_copies, 3, Config).
consistency_after_add_replica_3_disc_only(suite) -> [];
consistency_after_add_replica_3_disc_only(Config) when is_list(Config) ->
consistency_after_add_replica(disc_only_copies, 3, Config).
consistency_after_add_replica(ReplicaType, NodeConfig, Config) ->
Nodes0 = ?acquire_nodes(NodeConfig, Config),
AddNode = lists:last(Nodes0),
Nodes = Nodes0 -- [AddNode],
Node1 = hd(Nodes),
{success, [A]} = ?start_activities([Node1]),
?log("consistency_after_add_replica with ~p on ~p~n",
[ReplicaType, Nodes0]),
TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []),
mnesia_tpcb:init(TpcbConfig),
A ! fun () -> mnesia_tpcb:run(TpcbConfig) end,
timer:sleep(timer:seconds(10)),
?match({atomic, ok}, mnesia:add_table_copy(account, AddNode, ReplicaType)),
mnesia_tpcb:stop(),
?match(ok, mnesia_tpcb:verify_tabs()),
?verify_mnesia(Nodes0, []).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
consistency_after_del_replica_2_ram(suite) -> [];
consistency_after_del_replica_2_ram(Config) when is_list(Config) ->
consistency_after_del_replica(ram_copies, 2, Config).
consistency_after_del_replica_2_disc(suite) -> [];
consistency_after_del_replica_2_disc(Config) when is_list(Config) ->
consistency_after_del_replica(disc_copies, 2, Config).
consistency_after_del_replica_2_disc_only(suite) -> [];
consistency_after_del_replica_2_disc_only(Config) when is_list(Config) ->
consistency_after_del_replica(disc_only_copies, 2, Config).
consistency_after_del_replica_3_ram(suite) -> [];
consistency_after_del_replica_3_ram(Config) when is_list(Config) ->
consistency_after_del_replica(ram_copies, 3, Config).
consistency_after_del_replica_3_disc(suite) -> [];
consistency_after_del_replica_3_disc(Config) when is_list(Config) ->
consistency_after_del_replica(disc_copies, 3, Config).
consistency_after_del_replica_3_disc_only(suite) -> [];
consistency_after_del_replica_3_disc_only(Config) when is_list(Config) ->
consistency_after_del_replica(disc_only_copies, 3, Config).
consistency_after_del_replica(ReplicaType, NodeConfig, Config) ->
Nodes = ?acquire_nodes(NodeConfig, Config),
Node1 = hd(Nodes),
Node2 = lists:last(Nodes),
{success, [A]} = ?start_activities([Node1]),
?log("consistency_after_del_replica with ~p on ~p~n",
[ReplicaType, Nodes]),
TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []),
mnesia_tpcb:init(TpcbConfig),
A ! fun () -> mnesia_tpcb:run(TpcbConfig) end,
timer:sleep(timer:seconds(10)),
?match({atomic, ok}, mnesia:del_table_copy(account, Node2)),
mnesia_tpcb:stop(),
?match(ok, mnesia_tpcb:verify_tabs()),
?verify_mnesia(Nodes, []).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
consistency_after_move_replica_2_ram(suite) -> [];
consistency_after_move_replica_2_ram(Config) when is_list(Config) ->
consistency_after_move_replica(ram_copies, 2, Config).
consistency_after_move_replica_2_disc(suite) -> [];
consistency_after_move_replica_2_disc(Config) when is_list(Config) ->
consistency_after_move_replica(disc_copies, 2, Config).
consistency_after_move_replica_2_disc_only(suite) -> [];
consistency_after_move_replica_2_disc_only(Config) when is_list(Config) ->
consistency_after_move_replica(disc_only_copies, 2, Config).
consistency_after_move_replica_3_ram(suite) -> [];
consistency_after_move_replica_3_ram(Config) when is_list(Config) ->
consistency_after_move_replica(ram_copies, 3, Config).
consistency_after_move_replica_3_disc(suite) -> [];
consistency_after_move_replica_3_disc(Config) when is_list(Config) ->
consistency_after_move_replica(disc_copies, 3, Config).
consistency_after_move_replica_3_disc_only(suite) -> [];
consistency_after_move_replica_3_disc_only(Config) when is_list(Config) ->
consistency_after_move_replica(disc_only_copies, 3, Config).
consistency_after_move_replica(ReplicaType, NodeConfig, Config) ->
Nodes = ?acquire_nodes(NodeConfig, Config ++ [{tc_timeout, timer:minutes(10)}]),
Node1 = hd(Nodes),
Node2 = lists:last(Nodes),
{success, [A]} = ?start_activities([Node1]),
?log("consistency_after_move_replica with ~p on ~p~n",
[ReplicaType, Nodes]),
TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes -- [Node2], []),
mnesia_tpcb:init(TpcbConfig),
A ! fun () -> mnesia_tpcb:run(TpcbConfig) end,
timer:sleep(timer:seconds(10)),
?match({atomic, ok}, mnesia:move_table_copy(account, Node1, Node2)),
?log("First move completed from node ~p to ~p ~n", [Node1, Node2]),
?match({atomic, ok}, mnesia:move_table_copy(account, Node2, Node1)),
mnesia_tpcb:stop(),
?match(ok, mnesia_tpcb:verify_tabs()),
?verify_mnesia(Nodes, []).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
consistency_after_transform_table_ram(suite) -> [];
consistency_after_transform_table_ram(Config) when is_list(Config) ->
consistency_after_transform_table(ram_copies, Config).
consistency_after_transform_table_disc(suite) -> [];
consistency_after_transform_table_disc(Config) when is_list(Config) ->
consistency_after_transform_table(disc_copies, Config).
consistency_after_transform_table_disc_only(suite) -> [];
consistency_after_transform_table_disc_only(Config) when is_list(Config) ->
consistency_after_transform_table(disc_only_copies, Config).
consistency_after_transform_table(Type, Config) ->
Nodes = [N1, N2,_N3] = ?acquire_nodes(3, Config),
?match({atomic, ok}, mnesia:create_table(tab1, [{index, [3]}, {Type, [N1]}])),
?match({atomic, ok}, mnesia:create_table(tab2, [{index, [3]}, {Type, [N1,N2]}])),
?match({atomic, ok}, mnesia:create_table(tab3, [{index, [3]}, {Type, Nodes}])),
?match({atomic, ok}, mnesia:create_table(empty, [{index, [3]},{Type, Nodes}])),
Tabs = lists:sort([tab1, tab2, tab3, empty]),
[[mnesia:dirty_write({Tab, N, N}) || N <- lists:seq(1,10)] ||
Tab <- Tabs -- [empty, tab4]],
mnesia:dump_log(),
Ok = lists:duplicate(4, {atomic, ok}),
?match(Ok, [mnesia:transform_table(Tab, fun({T, N, N}) -> {T, N, N, ok} end,
[k,a,n]) || Tab <- Tabs]),
[?match([k,a,n], mnesia:table_info(Tab, attributes)) || Tab <- Tabs],
Filter = fun(Tab) -> mnesia:foldl(fun(A, Acc) when size(A) == 3 -> [A|Acc];
(A, Acc) when size(A) == 4 -> Acc
end, [], Tab)
end,
?match([[],[],[],[]], [element(2,mnesia:transaction(Filter, [Tab])) || Tab <- Tabs]),
mnesia_test_lib:kill_mnesia(Nodes),
mnesia_test_lib:start_mnesia(Nodes, Tabs),
?match([Tabs, Tabs, Tabs],
[lists:sort(rpc:call(Node, mnesia,system_info, [tables]) -- [schema]) || Node <- Nodes]),
?match([[],[],[],[]], [element(2,mnesia:transaction(Filter, [Tab])) || Tab <- Tabs]),
[?match([k,a,n], mnesia:table_info(Tab, attributes)) || Tab <- Tabs],
?verify_mnesia(Nodes, []).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
consistency_after_change_table_copy_type(doc) ->
["Check that the database is consistent after change of copy type.",
" While applications are updating the involved tables. "].
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
consistency_after_fallback_2_ram(suite) -> [];
consistency_after_fallback_2_ram(Config) when is_list(Config) ->
consistency_after_fallback(ram_copies, 2, Config).
consistency_after_fallback_2_disc(suite) -> [];
consistency_after_fallback_2_disc(Config) when is_list(Config) ->
consistency_after_fallback(disc_copies, 2, Config).
consistency_after_fallback_2_disc_only(suite) -> [];
consistency_after_fallback_2_disc_only(Config) when is_list(Config) ->
consistency_after_fallback(disc_only_copies, 2, Config).
consistency_after_fallback_3_ram(suite) -> [];
consistency_after_fallback_3_ram(Config) when is_list(Config) ->
consistency_after_fallback(ram_copies, 3, Config).
consistency_after_fallback_3_disc(suite) -> [];
consistency_after_fallback_3_disc(Config) when is_list(Config) ->
consistency_after_fallback(disc_copies, 3, Config).
consistency_after_fallback_3_disc_only(suite) -> [];
consistency_after_fallback_3_disc_only(Config) when is_list(Config) ->
consistency_after_fallback(disc_only_copies, 3, Config).
consistency_after_fallback(ReplicaType, NodeConfig, Config) ->
put(mnesia_test_verbose, true),
%%?verbose("Starting consistency_after_fallback2 at ~p~n", [self()]),
Delay = 5,
Nodes = ?acquire_nodes(NodeConfig, [{tc_timeout, timer:minutes(10)} | Config]),
Node1 = hd(Nodes),
%%?verbose("Mnesia info: ~p~n", [mnesia:info()]),
{success, [A]} = ?start_activities([Node1]),
?log("consistency_after_fallback with ~p on ~p~n",
[ReplicaType, Nodes]),
TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes, []),
mnesia_tpcb:init(TpcbConfig),
A ! fun () -> mnesia_tpcb:run(TpcbConfig) end,
timer:sleep(timer:seconds(Delay)),
%% Make a backup
?verbose("Doing backup~n", []),
?match(ok, mnesia:backup(consistency_after_fallback2)),
%% Install the backup as a fallback
?verbose("Doing fallback~n", []),
?match(ok, mnesia:install_fallback(consistency_after_fallback2)),
timer:sleep(timer:seconds(Delay)),
%% Stop tpcb
?verbose("Stopping TPC-B~n", []),
mnesia_tpcb:stop(),
?match(ok, mnesia_tpcb:verify_tabs()),
%% Stop and then start mnesia and check table consistency
?verbose("Kill Mnesia~n", []),
mnesia_test_lib:kill_mnesia(Nodes),
?verbose("Start Mnesia~n", []),
mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller,history]),
?verbose("Verify tabs~n", []),
?match(ok, mnesia_tpcb:verify_tabs()),
if
ReplicaType == ram_copies ->
%% Test that change_table_copy work i.e. no account.dcd file exists.
?match({atomic, ok}, mnesia:change_table_copy_type(account, node(), disc_copies));
true ->
ignore
end,
file:delete(consistency_after_fallback2),
?verify_mnesia(Nodes, []).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
consistency_after_restore_clear_ram(suite) -> [];
consistency_after_restore_clear_ram(Config) when is_list(Config) ->
consistency_after_restore(ram_copies, clear_tables, Config).
consistency_after_restore_clear_disc(suite) -> [];
consistency_after_restore_clear_disc(Config) when is_list(Config) ->
consistency_after_restore(disc_copies, clear_tables, Config).
consistency_after_restore_clear_disc_only(suite) -> [];
consistency_after_restore_clear_disc_only(Config) when is_list(Config) ->
consistency_after_restore(disc_only_copies, clear_tables, Config).
consistency_after_restore_recreate_ram(suite) -> [];
consistency_after_restore_recreate_ram(Config) when is_list(Config) ->
consistency_after_restore(ram_copies, recreate_tables, Config).
consistency_after_restore_recreate_disc(suite) -> [];
consistency_after_restore_recreate_disc(Config) when is_list(Config) ->
consistency_after_restore(disc_copies, recreate_tables, Config).
consistency_after_restore_recreate_disc_only(suite) -> [];
consistency_after_restore_recreate_disc_only(Config) when is_list(Config) ->
consistency_after_restore(disc_only_copies, recreate_tables, Config).
consistency_after_restore(ReplicaType, Op, Config) ->
Delay = 1,
Nodes = ?acquire_nodes(3, [{tc_timeout, timer:minutes(10)} | Config]),
[Node1, Node2, _Node3] = Nodes,
File = "cons_backup_restore",
?log("consistency_after_restore with ~p on ~p~n",
[ReplicaType, Nodes]),
Tabs = [carA, carB, carC, carD],
?match({atomic, ok}, mnesia:create_table(carA, [{ReplicaType, Nodes}])),
?match({atomic, ok}, mnesia:create_table(carB, [{ReplicaType, Nodes -- [Node1]}])),
?match({atomic, ok}, mnesia:create_table(carC, [{ReplicaType, Nodes -- [Node2]}])),
?match({atomic, ok}, mnesia:create_table(carD, [{ReplicaType, [Node2]}])),
NList = lists:seq(0, 20),
[lists:foreach(fun(E) -> ok = mnesia:dirty_write({Tab, E, 1}) end, NList) ||
Tab <- Tabs],
{ok, Name, _} = mnesia:activate_checkpoint([{max, [schema | Tabs]},
{ram_overrides_dump, true}]),
?verbose("Doing backup~n", []),
?match(ok, mnesia:backup_checkpoint(Name, File)),
?match(ok, mnesia:deactivate_checkpoint(Name)),
[lists:foreach(fun(E) -> ok = mnesia:dirty_write({Tab, E, 2}) end, NList) ||
Tab <- Tabs],
Pids1 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carA, Op]), carA} || _ <- lists:seq(1, 5)],
Pids2 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carB, Op]), carB} || _ <- lists:seq(1, 5)],
Pids3 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carC, Op]), carC} || _ <- lists:seq(1, 5)],
Pids4 = [{'EXIT', spawn_link(?MODULE, change_tab, [self(), carD, Op]), carD} || _ <- lists:seq(1, 5)],
AllPids = Pids1 ++ Pids2 ++ Pids3 ++ Pids4,
Restore = fun(F, Args) ->
case mnesia:restore(F, Args) of
{atomic, List} -> lists:sort(List);
Else -> Else
end
end,
timer:sleep(timer:seconds(Delay)), %% Let changers grab locks
?verbose("Doing restore~n", []),
?match(Tabs, Restore(File, [{default_op, Op}])),
Collect = fun(Msg, Acc) ->
receive Msg -> Acc
after 10000 -> [Msg|Acc]
end
end,
Failed1 = lists:foldl(Collect, [], AllPids),
Failed = lists:foldl(Collect, [], Failed1),
case Failed of
[] -> ok;
_ ->
?match([], Failed),
io:format("TIME: ~p sec~n", [erlang:system_time(seconds) band 16#FF]),
Dbg = fun({_, Pid, Tab}) ->
io:format("Tab ~p: ~p~n",[Tab, process_info(Pid, current_stacktrace)]),
[io:format(" ~p~n", [Rec]) || Rec <- mnesia:dirty_match_object({Tab, '_', '_'})]
end,
[Dbg(Msg) || Msg <- Failed],
io:format(" Held: ~p~n", [mnesia_locker:get_held_locks()]),
io:format("Queue: ~p~n", [mnesia_locker:get_lock_queue()])
end,
case ?match(ok, restore_verify_tabs(Tabs)) of
{success, ok} ->
file:delete(File);
_ ->
{T, M, S} = time(),
File2 = ?flat_format("consistency_error~w~w~w.BUP", [T, M, S]),
file:rename(File, File2)
end,
?verify_mnesia(Nodes, []).
change_tab(Father, Tab, Test) ->
Key = rand:uniform(20),
Update = fun() ->
Time = erlang:system_time(seconds) band 16#FF,
case put(time, Time) of
Time -> ok;
_ -> io:format("~p ~p ~p sec~n", [self(), Tab, Time])
end,
case mnesia:read({Tab, Key}) of
[{Tab, Key, 1}] -> quit;
[{Tab, Key, _N}] -> mnesia:write({Tab, Key, 3})
end
end,
case mnesia:transaction(Update) of
{atomic, quit} ->
exit(Tab);
{aborted, {no_exists, Tab}} when Test == recreate_tables -> %% I'll allow this
change_tab(Father, Tab, Test);
{atomic, ok} ->
change_tab(Father, Tab, Test)
end.
restore_verify_tabs([Tab | R]) ->
?match({atomic, ok},
mnesia:transaction(fun() -> mnesia:foldl(fun({_, _, 1}, ok) ->
ok;
(Else, Acc) ->
[Else|Acc]
end, ok, Tab)
end)),
restore_verify_tabs(R);
restore_verify_tabs([]) ->
ok.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
consistency_after_rename_of_node(doc) ->
["Skipped because it is an unimportant case."].
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
updates_during_checkpoint_activation_1_ram(suite) -> [];
updates_during_checkpoint_activation_1_ram(Config) when is_list(Config) ->
updates_during_checkpoint_activation(ram_copies, 1, Config).
updates_during_checkpoint_activation_1_disc(suite) -> [];
updates_during_checkpoint_activation_1_disc(Config) when is_list(Config) ->
updates_during_checkpoint_activation(disc_copies, 1, Config).
updates_during_checkpoint_activation_1_disc_only(suite) -> [];
updates_during_checkpoint_activation_1_disc_only(Config) when is_list(Config) ->
updates_during_checkpoint_activation(disc_only_copies, 1, Config).
updates_during_checkpoint_activation_2_ram(suite) -> [];
updates_during_checkpoint_activation_2_ram(Config) when is_list(Config) ->
updates_during_checkpoint_activation(ram_copies, 2, Config).
updates_during_checkpoint_activation_2_disc(suite) -> [];
updates_during_checkpoint_activation_2_disc(Config) when is_list(Config) ->
updates_during_checkpoint_activation(disc_copies, 2, Config).
updates_during_checkpoint_activation_2_disc_only(suite) -> [];
updates_during_checkpoint_activation_2_disc_only(Config) when is_list(Config) ->
updates_during_checkpoint_activation(disc_only_copies, 2, Config).
updates_during_checkpoint_activation_3_ram(suite) -> [];
updates_during_checkpoint_activation_3_ram(Config) when is_list(Config) ->
updates_during_checkpoint_activation(ram_copies, 3, Config).
updates_during_checkpoint_activation_3_disc(suite) -> [];
updates_during_checkpoint_activation_3_disc(Config) when is_list(Config) ->
updates_during_checkpoint_activation(disc_copies, 3, Config).
updates_during_checkpoint_activation_3_disc_only(suite) -> [];
updates_during_checkpoint_activation_3_disc_only(Config) when is_list(Config) ->
updates_during_checkpoint_activation(disc_only_copies, 3, Config).
updates_during_checkpoint_activation(ReplicaType,NodeConfig,Config) ->
%%?verbose("updates_during_checkpoint_activation2 at ~p~n", [self()]),
Delay = 5,
Nodes = ?acquire_nodes(NodeConfig, Config),
Node1 = hd(Nodes),
%%?verbose("Mnesia info: ~p~n", [mnesia:info()]),
{success, [A]} = ?start_activities([Node1]),
?log("consistency_after_fallback with ~p on ~p~n",
[ReplicaType, Nodes]),
TpcbConfig = tpcb_config_dist(ReplicaType, NodeConfig, Nodes, Config),
%%TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes),
mnesia_tpcb:init(TpcbConfig),
A ! fun () -> mnesia_tpcb:run(TpcbConfig) end,
timer:sleep(timer:seconds(Delay)),
{ok, CPName, _NodeList} =
mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
{ram_overrides_dump, true}]),
timer:sleep(timer:seconds(Delay)),
%% Stop tpcb
?verbose("Stopping TPC-B~n", []),
mnesia_tpcb:stop(),
?match(ok, mnesia_tpcb:verify_tabs()),
?match(ok, mnesia:backup_checkpoint(CPName,
updates_during_checkpoint_activation2)),
timer:sleep(timer:seconds(Delay)),
?match(ok, mnesia:install_fallback(updates_during_checkpoint_activation2)),
%% Stop and then start mnesia and check table consistency
%%?verbose("Restarting Mnesia~n", []),
mnesia_test_lib:kill_mnesia(Nodes),
file:delete(updates_during_checkpoint_activation2),
mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller, history]),
?match(ok, mnesia_tpcb:verify_tabs()),
?verify_mnesia(Nodes, []).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
updates_during_checkpoint_iteration_2_ram(suite) -> [];
updates_during_checkpoint_iteration_2_ram(Config) when is_list(Config) ->
updates_during_checkpoint_iteration(ram_copies, 2, Config).
updates_during_checkpoint_iteration_2_disc(suite) -> [];
updates_during_checkpoint_iteration_2_disc(Config) when is_list(Config) ->
updates_during_checkpoint_iteration(disc_copies, 2, Config).
updates_during_checkpoint_iteration_2_disc_only(suite) -> [];
updates_during_checkpoint_iteration_2_disc_only(Config) when is_list(Config) ->
updates_during_checkpoint_iteration(disc_only_copies, 2, Config).
updates_during_checkpoint_iteration(ReplicaType,NodeConfig,Config) ->
%?verbose("updates_during_checkpoint_iteration2 at ~p~n", [self()]),
Delay = 5,
Nodes = ?acquire_nodes(NodeConfig, Config),
Node1 = hd(Nodes),
%?verbose("Mnesia info: ~p~n", [mnesia:info()]),
File = updates_during_checkpoint_iteration2,
{success, [A]} = ?start_activities([Node1]),
?log("updates_during_checkpoint_iteration with ~p on ~p~n",
[ReplicaType, Nodes]),
TpcbConfig = tpcb_config_dist(ReplicaType, NodeConfig, Nodes, Config),
%%TpcbConfig = tpcb_config(ReplicaType, NodeConfig, Nodes),
TpcbConfigRec = list2rec(TpcbConfig,
record_info(fields,tab_config),
#tab_config{}),
mnesia_tpcb:init(TpcbConfig),
?match(ok, mnesia_tpcb:verify_tabs()),
{ok, CPName, _NodeList} =
mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
{ram_overrides_dump,true}]),
A ! fun () -> mnesia:backup_checkpoint(CPName, File) end,
do_changes_during_backup(TpcbConfigRec),
?match_receive({A,ok}),
timer:sleep(timer:seconds(Delay)),
?match(ok, mnesia:install_fallback(File)),
timer:sleep(timer:seconds(Delay)),
?match({error,{"Bad balance",_,_}}, mnesia_tpcb:verify_tabs()),
mnesia_test_lib:kill_mnesia(Nodes),
mnesia_test_lib:start_mnesia(Nodes,[account,branch,teller, history]),
?match(ok, mnesia_tpcb:verify_tabs()),
?match(ok, file:delete(File)),
?verify_mnesia(Nodes, []).
do_changes_during_backup(TpcbConfig) ->
loop_branches(TpcbConfig#tab_config.n_branches,
TpcbConfig#tab_config.n_accounts_per_branch).
loop_branches(N_br,N_acc) when N_br >= 1 ->
loop_accounts(N_br,N_acc),
loop_branches(N_br-1,N_acc);
loop_branches(_,_) -> done.
loop_accounts(N_br, N_acc) when N_acc >= 1 ->
A = #account{id=N_acc, branch_id=N_br, balance = 4711},
ok = mnesia:dirty_write(A),
loop_accounts(N_br, N_acc-1);
loop_accounts(_,_) -> done.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
load_table_with_activated_checkpoint_ram(suite) -> [];
load_table_with_activated_checkpoint_ram(Config) when is_list(Config) ->
load_table_with_activated_checkpoint(ram_copies, Config).
load_table_with_activated_checkpoint_disc(suite) -> [];
load_table_with_activated_checkpoint_disc(Config) when is_list(Config) ->
load_table_with_activated_checkpoint(disc_copies, Config).
load_table_with_activated_checkpoint_disc_only(suite) -> [];
load_table_with_activated_checkpoint_disc_only(Config) when is_list(Config) ->
load_table_with_activated_checkpoint(disc_only_copies, Config).
load_table_with_activated_checkpoint(Type, Config) ->
Nodes = ?acquire_nodes(2, Config),
Node1 = hd(Nodes),
Tab = load_test,
Def = [{attributes, [key, value]},
{Type, Nodes}], %% ??? important that RAM ???
?match({atomic, ok}, mnesia:create_table(Tab, Def)),
?match(ok, mnesia:dirty_write({Tab, 1, 4711})),
?match(ok, mnesia:dirty_write({Tab, 2, 42})),
?match(ok, mnesia:dirty_write({Tab, 3, 256})),
timer:sleep(timer:seconds(1)),
{ok, CPName, _NodeList} =
mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
{ram_overrides_dump,true}]),
mnesia_test_lib:stop_mnesia([Node1]),
mnesia_test_lib:start_mnesia([Node1],[Tab]),
%%--- check, whether the checkpiont is attached to both replicas
{success, [A,B]} = ?start_activities(Nodes),
A ! fun () ->
mnesia:table_info(Tab,checkpoints)
end,
?match_receive({A,[CPName]}),
B ! fun () ->
mnesia:table_info(Tab,checkpoints)
end,
?match_receive({B,[CPName]}),
%%--- check, whether both retainers are consistent
?match(ok, mnesia:dirty_write({Tab, 1, 815})),
A ! fun () ->
mnesia:backup_checkpoint(CPName, load_table_a)
end,
?match_receive({A,ok}),
B ! fun () ->
mnesia:backup_checkpoint(CPName, load_table_b)
end,
?match_receive({B,ok}),
Mod = mnesia_backup, %% Assume local files
List_a = view(load_table_a, Mod),
List_b = view(load_table_b, Mod),
?match(List_a, List_b),
?match(ok,file:delete(load_table_a)),
?match(ok,file:delete(load_table_b)),
?verify_mnesia(Nodes, []).
view(Source, Mod) ->
View = fun(Item, Acc) ->
?verbose("tab - item : ~p ~n",[Item]),
case Item of
{schema, Tab, Cs} -> %% Remove cookie information
NewCs = lists:keyreplace(cookie, 1, Cs,
{cookie, skip_cookie}),
Item2 = {schema, Tab, NewCs},
{[Item], [Item2|Acc]};
_ ->
{[Item], [Item|Acc]}
end
end,
{ok,TabList} =
mnesia:traverse_backup(Source, Mod, dummy, read_only, View, []),
lists:sort(TabList).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
add_table_copy_to_table_checkpoint_ram(suite) -> [];
add_table_copy_to_table_checkpoint_ram(Config) when is_list(Config) ->
add_table_copy_to_table_with_activated_checkpoint(ram_copies, Config).
add_table_copy_to_table_checkpoint_disc(suite) -> [];
add_table_copy_to_table_checkpoint_disc(Config) when is_list(Config) ->
add_table_copy_to_table_with_activated_checkpoint(disc_copies, Config).
add_table_copy_to_table_checkpoint_disc_only(suite) -> [];
add_table_copy_to_table_checkpoint_disc_only(Config) when is_list(Config) ->
add_table_copy_to_table_with_activated_checkpoint(disc_only_copies, Config).
add_table_copy_to_table_with_activated_checkpoint(Type,Config) ->
Nodes = ?acquire_nodes(2, Config),
%?verbose("NODES = ~p ~n",[Nodes]),
[Node1,Node2] = Nodes,
Tab = add_test,
Def = [{attributes, [key, value]},
{Type, [Node1]}], %% ??? important that RAM ???
?match({atomic, ok}, mnesia:create_table(Tab, Def)),
?match(ok, mnesia:dirty_write({Tab, 1, 4711})),
?match(ok, mnesia:dirty_write({Tab, 2, 42})),
?match(ok, mnesia:dirty_write({Tab, 3, 256})),
{ok, CPName, _NodeList} =
mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
{ram_overrides_dump,true}]),
?match({atomic,ok},mnesia:add_table_copy(Tab,Node2,ram_copies)),
%%--- check, whether the checkpiont is attached to both replicas
{success, [A,B]} = ?start_activities(Nodes),
A ! fun () ->
mnesia:table_info(Tab,checkpoints)
end,
?match_receive({A,[CPName]}),
B ! fun () ->
mnesia:table_info(Tab,checkpoints)
end,
?match_receive({B,[CPName]}),
%%--- check, whether both retainers are consistent
?match(ok, mnesia:dirty_write({Tab, 1, 815})),
?match(ok, mnesia:dirty_write({Tab, 2, 815})),
A ! fun () ->
mnesia:backup_checkpoint(CPName, add_table_a)
end,
?match_receive({A,ok}),
B ! fun () ->
mnesia:backup_checkpoint(CPName, add_table_b)
end,
?match_receive({B,ok}),
Mod = mnesia_backup, %% Assume local files
List_a = view(add_table_a, Mod),
List_b = view(add_table_b, Mod),
?match(List_a, List_b),
?match(ok,file:delete(add_table_a)),
?match(ok, file:delete(add_table_b)),
?verify_mnesia(Nodes, []).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
inst_fallback_process_dies(suite) ->
[];
inst_fallback_process_dies(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
{success, [A,_B,_C]} = ?start_activities(Nodes),
TestPid = self(),
DebugId = {mnesia_bup, fallback_receiver_loop, pre_swap},
DebugFun =
fun(PrevContext, _EvalContext) ->
?verbose("fallback_receiver_loop - pre_swap pid ~p #~p~n",
[self(),PrevContext]),
TestPid ! {self(),fallback_preswap},
case receive_messages([fallback_continue]) of
[{TestPid,fallback_continue}] ->
?deactivate_debug_fun(DebugId),
PrevContext+1
end
end,
?activate_debug_fun(DebugId, DebugFun, 1),
Tab = install_table,
Def = [{attributes, [key, value]}, {disc_copies, Nodes}],
?match({atomic, ok}, mnesia:create_table(Tab, Def)),
?match(ok, mnesia:dirty_write({Tab, 1, 4711})),
?match(ok, mnesia:dirty_write({Tab, 2, 42})),
?match(ok, mnesia:dirty_write({Tab, 3, 256})),
{ok, CPName, _NodeList} =
mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
{ram_overrides_dump,true}]),
?match(ok, mnesia:backup_checkpoint(CPName, install_backup)),
A ! fun() -> mnesia:install_fallback(install_backup) end,
[{AnsPid,fallback_preswap}] = receive_messages([fallback_preswap]),
exit(A, kill),
AnsPid ! {self(), fallback_continue},
?match_receive({'EXIT', A, killed}),
timer:sleep(2000), %% Wait till fallback is installed everywhere
mnesia_test_lib:kill_mnesia(Nodes),
?verbose("~n---->Mnesia is stopped everywhere<-----~n", []),
?match([], mnesia_test_lib:start_mnesia(Nodes,[Tab])),
check_data(Nodes, Tab),
?match(ok, file:delete(install_backup)),
?verify_mnesia(Nodes, []).
check_data([N1 | R], Tab) ->
?match([{Tab, 1, 4711}], rpc:call(N1, mnesia, dirty_read, [{Tab, 1}])),
?match([{Tab, 2, 42}], rpc:call(N1, mnesia, dirty_read, [{Tab, 2}])),
?match([{Tab, 3, 256}], rpc:call(N1, mnesia, dirty_read, [{Tab, 3}])),
check_data(R, Tab);
check_data([], _Tab) ->
ok.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
fatal_when_inconsistency(suite) ->
[];
fatal_when_inconsistency(Config) when is_list(Config) ->
?is_debug_compiled,
[Node1, Node2, Node3] = Nodes =
?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
{success, [A,_B,_C]} = ?start_activities(Nodes),
TestPid = self(),
DebugId = {mnesia_bup, fallback_receiver_loop, pre_swap},
DebugFun =
fun(PrevContext, _EvalContext) ->
?verbose("fallback_receiver_loop - pre_swap pid ~p #~p~n",
[self(),PrevContext]),
TestPid ! {self(),fallback_preswap},
case receive_messages([fallback_continue]) of
[{TestPid,fallback_continue}] ->
?deactivate_debug_fun(DebugId),
PrevContext+1
end
end,
?activate_debug_fun(DebugId, DebugFun, 1),
Tab = install_table,
Def = [{attributes, [key, value]}, {disc_copies, Nodes}],
?match({atomic, ok}, mnesia:create_table(Tab, Def)),
?match(ok, mnesia:dirty_write({Tab, 1, 4711})),
?match(ok, mnesia:dirty_write({Tab, 2, 42})),
?match(ok, mnesia:dirty_write({Tab, 3, 256})),
{ok, CPName, _NodeList} =
mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
{ram_overrides_dump,true}]),
?match(ok, mnesia:backup_checkpoint(CPName, install_backup)),
?match(ok, mnesia:dirty_write({Tab, 2, 42424242})),
A ! fun() ->
mnesia:install_fallback(install_backup)
end,
[{AnsPid,fallback_preswap}] = receive_messages([fallback_preswap]),
exit(AnsPid, kill), %% Kill install-fallback on local node will
AnsPid ! {self(), fallback_continue},
?deactivate_debug_fun(DebugId),
?match_receive({A,{error,{"Cannot install fallback",
{'EXIT',AnsPid,killed}}}}),
mnesia_test_lib:kill_mnesia(Nodes),
?verbose("EXPECTING FATAL from 2 nodes WITH CORE DUMP~n", []),
?match([], mnesia_test_lib:start_mnesia([Node1],[])),
is_running(Node1, yes),
?match([{Node2, mnesia, _}], mnesia_test_lib:start_mnesia([Node2],[])),
is_running(Node2, no),
?match([{Node3, mnesia, _}], mnesia_test_lib:start_mnesia([Node3],[])),
is_running(Node3, no),
mnesia_test_lib:kill_mnesia(Nodes),
?match(ok, mnesia:install_fallback(install_backup)),
mnesia_test_lib:start_mnesia(Nodes,[Tab]),
check_data(Nodes, Tab),
?match(ok,file:delete(install_backup)),
?verify_mnesia(Nodes, []).
is_running(Node, Shouldbe) ->
timer:sleep(1000),
Running = rpc:call(Node, mnesia, system_info, [is_running]),
case Running of
Shouldbe -> ok;
_ -> is_running(Node, Shouldbe)
end.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
after_delete(doc) ->
["interrupt the uninstall after deletion of ",
"fallback files - there shall be no fallback"];
after_delete(suite) -> [];
after_delete(Config) when is_list(Config) ->
do_uninstall(Config, post_delete).
%%%%%%%%%%%%%%%%%%%%%%%%%
do_uninstall(Config,DebugPoint) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config),
%%?verbose("NODES = ~p ~n",[Nodes]),
{success, [P1,P2,P3]} = ?start_activities(Nodes),
NP1 = node(P1),
NP2 = node(P2),
{A,B,C} = case node() of
NP1 ->
%%?verbose("first case ~n"),
{P3,P2,P1};
NP2 ->
%%?verbose("second case ~n"),
{P3, P1, P2};
_ ->
{ P1, P2, P3}
end,
Node1 = node(A),
Node2 = node(B),
Node3 = node(C),
?verbose(" A pid:~p node:~p ~n",[A,Node1]),
?verbose(" B pid:~p node:~p ~n",[B,Node2]),
?verbose(" C pid:~p node:~p ~n",[C,Node3]),
TestPid = self(),
%%?verbose("TestPid : ~p~n",[TestPid]),
DebugId = {mnesia_bup, uninstall_fallback2, DebugPoint},
DebugFun = fun(PrevContext, _EvalContext) ->
?verbose("uninstall_fallback pid ~p #~p~n"
,[self(),PrevContext]),
TestPid ! {self(),uninstall_predelete},
case receive_messages([uninstall_continue]) of
[{TestPid,uninstall_continue}] ->
?deactivate_debug_fun(DebugId),
%%?verbose("uninstall_fallback continues~n"),
PrevContext+1
end
end,
?remote_activate_debug_fun(Node1,DebugId, DebugFun, 1),
Tab = install_table,
Def = [{attributes, [key, value]},
{ram_copies, Nodes}], %% necessary to test different types ???
?match({atomic, ok}, mnesia:create_table(Tab, Def)),
?match(ok, mnesia:dirty_write({Tab, 1, 4711})),
?match(ok, mnesia:dirty_write({Tab, 2, 42})),
?match(ok, mnesia:dirty_write({Tab, 3, 256})),
{ok, CPName, _NodeList} =
mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
{ram_overrides_dump,true}]),
?match(ok, mnesia:backup_checkpoint(CPName,install_backup)),
timer:sleep(timer:seconds(1)),
A ! fun () ->
mnesia:install_fallback(install_backup)
end,
?match_receive({A,ok}),
A ! fun () ->
mnesia:uninstall_fallback()
end,
%%
%% catch the debug entry in mnesia and kill one Mnesia node
%%
[{AnsPid,uninstall_predelete}] = receive_messages([uninstall_predelete]),
?verbose("AnsPid : ~p~n",[AnsPid]),
mnesia_test_lib:kill_mnesia([Node2]),
timer:sleep(timer:seconds(1)),
AnsPid ! {self(),uninstall_continue},
?match_receive({A,ok}),
mnesia_test_lib:kill_mnesia(Nodes) ,
mnesia_test_lib:start_mnesia(Nodes,[Tab]),
A ! fun () ->
R1 = mnesia:dirty_read({Tab,1}),
R2 = mnesia:dirty_read({Tab,2}),
R3 = mnesia:dirty_read({Tab,3}),
{R1,R2,R3}
end,
?match_receive({ A, {[],[],[]} }),
B ! fun () ->
R1 = mnesia:dirty_read({Tab,1}),
R2 = mnesia:dirty_read({Tab,2}),
R3 = mnesia:dirty_read({Tab,3}),
{R1,R2,R3}
end,
?match_receive({ B, {[],[],[]} }),
C ! fun () ->
R1 = mnesia:dirty_read({Tab,1}),
R2 = mnesia:dirty_read({Tab,2}),
R3 = mnesia:dirty_read({Tab,3}),
{R1,R2,R3}
end,
?match_receive({ C, {[],[],[]} }),
?match(ok,file:delete(install_backup)),
?verify_mnesia(Nodes, []).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%
cause_switch_before(doc) ->
["interrupt the backup before iterating the retainer"];
cause_switch_before(suite) -> [];
cause_switch_before(Config) when is_list(Config) ->
do_something_during_backup(cause_switch,pre,Config).
%%%%%%%%%%%%%%%
cause_switch_after(doc) ->
["interrupt the backup after iterating the retainer"];
cause_switch_after(suite) -> [];
cause_switch_after(Config) when is_list(Config) ->
do_something_during_backup(cause_switch,post,Config).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%
cause_abort_before(doc) ->
["interrupt the backup before iterating the retainer"];
cause_abort_before(suite) -> [];
cause_abort_before(Config) when is_list(Config) ->
do_something_during_backup(cause_abort,pre,Config).
%%%%%%%%%%%%%%%%%%
cause_abort_after(doc) ->
["interrupt the backup after iterating the retainer"];
cause_abort_after(suite) -> [];
cause_abort_after(Config) when is_list(Config) ->
do_something_during_backup(cause_abort,post,Config).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%
change_schema_before(doc) ->
["interrupt the backup before iterating the retainer"];
change_schema_before(suite) -> [];
change_schema_before(Config) when is_list(Config) ->
do_something_during_backup(change_schema,pre,Config).
%%%%%%%%%%%%%%%%
change_schema_after(doc) ->
["interrupt the backup after iterating the retainer"];
change_schema_after(suite) -> [];
change_schema_after(Config) when is_list(Config) ->
do_something_during_backup(change_schema,post,Config).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
do_something_during_backup(Action,DebugPoint,Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config),
{success, [A,B,C]} = ?start_activities(Nodes),
Node1 = node(A),
Node2 = node(B),
Node3 = node(C),
TestPid = self(),
%%?verbose("TestPid : ~p~n",[TestPid]),
Tab = interrupt_table,
Bak = interrupt_backup,
Def = [{attributes, [key, value]},
{ram_copies, [Node2,Node3]}],
%% necessary to test different types ???
?match({atomic, ok}, mnesia:create_table(Tab, Def)),
DebugId = {mnesia_log, tab_copier, DebugPoint},
DebugFun = fun(PrevContext, EvalContext) ->
?verbose("interrupt backup pid ~p #~p ~n context ~p ~n"
,[self(),PrevContext,EvalContext]),
TestPid ! {self(),interrupt_backup_pre},
global:set_lock({{lock_for_backup, Tab}, self()},
Nodes,
infinity),
%%?verbose("interrupt backup - continues ~n"),
?deactivate_debug_fun(DebugId),
PrevContext+1
end,
?remote_activate_debug_fun(Node1,DebugId, DebugFun, 1),
?match(ok, mnesia:dirty_write({Tab, 1, 4711})),
?match(ok, mnesia:dirty_write({Tab, 2, 42})),
?match(ok, mnesia:dirty_write({Tab, 3, 256})),
{ok, CPName, _NodeList} =
mnesia:activate_checkpoint([{max, mnesia:system_info(tables)},
{ram_overrides_dump,true}]),
A ! fun () ->
%%?verbose("node: ~p pid: ~p ~n",[node(),self()]),
mnesia:table_info(Tab,where_to_read)
end,
ReadNode_a = receive { A, ReadNode_a_tmp } -> ReadNode_a_tmp end,
?verbose("ReadNode ~p ~n",[ReadNode_a]),
global:set_lock({{lock_for_backup, Tab}, self()}, Nodes, infinity),
A ! fun () -> %% A shall perform the backup, so the test proc is
%% able to do further actions in between
mnesia:backup_checkpoint(CPName, Bak)
end,
%% catch the debug function of mnesia, stop the backup process
%% kill the node ReadNode_a and continue the backup process
%% As there is a second replica of the table, the backup shall continue
case receive_messages([interrupt_backup_pre]) of
[{_AnsPid,interrupt_backup_pre}] -> ok
end,
case Action of
cause_switch ->
mnesia_test_lib:kill_mnesia([ReadNode_a]),
timer:sleep(timer:seconds(1));
cause_abort ->
mnesia_test_lib:kill_mnesia([Node2,Node3]),
timer:sleep(timer:seconds(1));
change_schema ->
Tab2 = second_interrupt_table,
Def2 = [{attributes, [key, value]},
{ram_copies, Nodes}],
?match({atomic, ok}, mnesia:create_table(Tab2, Def2))
end,
%% AnsPid ! {self(),interrupt_backup_continue},
global:del_lock({{lock_for_backup, Tab}, self()}, Nodes),
case Action of
cause_abort ->
%% answer of A when finishing the backup
?match_receive({A,{error, _}}),
?match({error,{"Cannot install fallback",_}},
mnesia:install_fallback(Bak));
_ -> %% cause_switch, change_schema
?match_receive({A,ok}), %% answer of A when finishing the backup
%% send a fun to that node where mnesia is still running
WritePid = case ReadNode_a of
Node2 -> C; %% node(C) == Node3
Node3 -> B
end,
WritePid ! fun () ->
?match(ok, mnesia:dirty_write({Tab, 1, 815})),
?match(ok, mnesia:dirty_write({Tab, 2, 816})),
ok
end,
?match_receive({ WritePid, ok }),
?match(ok, mnesia:install_fallback(Bak))
end,
%% Stop and then start mnesia and check table consistency
%%?verbose("Restarting Mnesia~n", []),
mnesia_test_lib:kill_mnesia(Nodes),
mnesia_test_lib:start_mnesia(Nodes,[Tab]),
case Action of
cause_switch ->
%% the backup should exist
cross_check_tables([A,B,C],Tab,{[{Tab,1,4711}],
[{Tab,2,42}],
[{Tab,3,256}] }),
?match(ok,file:delete(Bak));
cause_abort ->
%% the backup should NOT exist
cross_check_tables([A,B,C],Tab,{[],[],[]}),
%% file does not exist
?match({error, _},file:delete(Bak));
change_schema ->
%% the backup should exist
cross_check_tables([A,B,C],Tab,{[{Tab,1,4711}],
[{Tab,2,42}],
[{Tab,3,256}] }),
?match(ok,file:delete(Bak))
end,
?verify_mnesia(Nodes, []).
%% check the contents of the table
cross_check_tables([],_tab,_elements) -> ok;
cross_check_tables([Pid|Rest],Tab,{Val1,Val2,Val3}) ->
Pid ! fun () ->
R1 = mnesia:dirty_read({Tab,1}),
R2 = mnesia:dirty_read({Tab,2}),
R3 = mnesia:dirty_read({Tab,3}),
{R1,R2,R3}
end,
?match_receive({ Pid, {Val1, Val2, Val3 } }),
cross_check_tables(Rest,Tab,{Val1,Val2,Val3} ).
