%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 1996-2010. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
%% compliance with the License. You should have received a copy of the
%% Erlang Public License along with this software. If not, it can be
%% retrieved online at http://www.erlang.org/.
%%
%% Software distributed under the License is distributed on an "AS IS"
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
%% the License for the specific language governing rights and limitations
%% under the License.
%%
%% %CopyrightEnd%
%%
%%
-module(mnesia_recovery_test).
-author('[email protected]').
-compile([export_all]).
-include("mnesia_test_lib.hrl").
-include_lib("kernel/include/file.hrl").
init_per_testcase(Func, Conf) ->
mnesia_test_lib:init_per_testcase(Func, Conf).
end_per_testcase(Func, Conf) ->
mnesia_test_lib:end_per_testcase(Func, Conf).
-define(receive_messages(Msgs), receive_messages(Msgs, ?FILE, ?LINE)).
% First Some debug logging
-define(dgb, true).
-ifdef(dgb).
-define(dl(X, Y), ?verbose("**TRACING: " ++ X ++ "**~n", Y)).
-else.
-define(dl(X, Y), ok).
-endif.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
all(doc) ->
["Verify recoverability",
"Verify that the effects of committed transactions are preserved",
"after recovery from system failures. It must be possible to",
"restore the tables to a consistent state on a node, from (any kind",
"of) replica on other nodes as well as from local disk on the failed",
"node. The system must also recover from instantaneous",
"interruption causing disk files to not be completely synchronized."];
all(suite) ->
[
mnesia_down,
explicit_stop,
coord_dies,
schema_trans,
async_dirty,
sync_dirty,
sym_trans,
asym_trans,
after_full_disc_partition,
after_corrupt_files,
disc_less,
garb_decision,
system_upgrade
].
schema_trans(suite) ->
[{mnesia_schema_recovery_test, all}].
tpcb_config(ReplicaType, _NodeConfig, Nodes) ->
[{n_branches, 5},
{n_drivers_per_node, 5},
{replica_nodes, Nodes},
{driver_nodes, Nodes},
{use_running_mnesia, true},
{report_interval, infinity},
{n_accounts_per_branch, 20},
{replica_type, ReplicaType}].
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
mnesia_down(doc) ->
[" Various tests about recovery when mnesia goes down on one or several nodes."];
mnesia_down(suite) ->
[
mnesia_down_during_startup,
master_node_tests,
read_during_down,
with_checkpoint,
delete_during_start
].
master_node_tests(doc) ->
["Verify that mnesia loads the correct data after it has been down, regarding master node settings."];
master_node_tests(suite) ->
[
no_master_2,
no_master_3,
one_master_2,
one_master_3,
two_master_2,
two_master_3,
all_master_2,
all_master_3
].
no_master_2(suite) -> [];
no_master_2(Config) when is_list(Config) -> mnesia_down_2(no, Config).
no_master_3(suite) -> [];
no_master_3(Config) when is_list(Config) -> mnesia_down_3(no, Config).
one_master_2(suite) -> [];
one_master_2(Config) when is_list(Config) -> mnesia_down_2(one, Config).
one_master_3(suite) -> [];
one_master_3(Config) when is_list(Config) -> mnesia_down_3(one, Config).
two_master_2(suite) -> [];
two_master_2(Config) when is_list(Config) -> mnesia_down_2(two, Config).
two_master_3(suite) -> [];
two_master_3(Config) when is_list(Config) -> mnesia_down_3(two, Config).
all_master_2(suite) -> [];
all_master_2(Config) when is_list(Config) -> mnesia_down_2(all, Config).
all_master_3(suite) -> [];
all_master_3(Config) when is_list(Config) -> mnesia_down_3(all, Config).
mnesia_down_2(Masters, Config) ->
Nodes = [N1, N2] = ?acquire_nodes(2, Config),
?match({atomic, ok}, mnesia:create_table(tab1, [{ram_copies, Nodes}])),
?match({atomic, ok}, mnesia:create_table(tab2, [{disc_copies, Nodes}])),
?match({atomic, ok}, mnesia:create_table(tab3, [{disc_only_copies, Nodes}])),
?match({atomic, ok}, mnesia:create_table(tab4, [{ram_copies, [N1]}])),
?match({atomic, ok}, mnesia:create_table(tab5, [{ram_copies, [N2]}])),
?match({atomic, ok}, mnesia:create_table(tab6, [{disc_copies, [N1]}])),
?match({atomic, ok}, mnesia:create_table(tab7, [{disc_copies, [N2]}])),
?match({atomic, ok}, mnesia:create_table(tab8, [{disc_only_copies, [N1]}])),
?match({atomic, ok}, mnesia:create_table(tab9, [{disc_only_copies, [N2]}])),
?match({atomic, ok}, mnesia:create_table(tab10, [{ram_copies, [N1]}, {disc_copies, [N2]}])),
?match({atomic, ok}, mnesia:create_table(tab11, [{ram_copies, [N2]}, {disc_copies, [N1]}])),
?match({atomic, ok}, mnesia:create_table(tab12, [{ram_copies, [N1]}, {disc_only_copies, [N2]}])),
?match({atomic, ok}, mnesia:create_table(tab13, [{ram_copies, [N2]}, {disc_only_copies, [N1]}])),
?match({atomic, ok}, mnesia:create_table(tab14, [{disc_only_copies, [N1]}, {disc_copies, [N2]}])),
?match({atomic, ok}, mnesia:create_table(tab15, [{disc_only_copies, [N2]}, {disc_copies, [N1]}])),
Tabs = [tab1, tab2, tab3, tab4, tab5, tab6, tab7, tab8,
tab9, tab10, tab11, tab12, tab13, tab14, tab15],
[?match(ok, rpc:call(Node, mnesia, wait_for_tables, [Tabs, 10000])) || Node <- Nodes],
[insert_data(Tab, 20) || Tab <- Tabs],
VTabs =
case Masters of
no ->
Tabs -- [tab4, tab5]; % ram copies
one ->
?match(ok, rpc:call(N1, mnesia, set_master_nodes, [[N1]])),
Tabs -- [tab1, tab4, tab5, tab10, tab12]; % ram_copies
two ->
?match(ok, rpc:call(N1, mnesia, set_master_nodes, [Nodes])),
Tabs -- [tab4, tab5];
all ->
[?match(ok, rpc:call(Node, mnesia, set_master_nodes, [[Node]])) || Node <- Nodes],
Tabs -- [tab1, tab4, tab5, tab10, tab11, tab12, tab13]
end,
mnesia_test_lib:kill_mnesia([N1]),
?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
?match([], mnesia_test_lib:kill_mnesia([N2])),
?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
[?match(ok, rpc:call(N1, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs],
[?match(ok, rpc:call(N2, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs],
?verify_mnesia(Nodes, []).
mnesia_down_3(Masters, Config) ->
Nodes = [N1, N2, N3] = ?acquire_nodes(3, Config),
?match({atomic, ok}, mnesia:create_table(tab1, [{ram_copies, Nodes}])),
?match({atomic, ok}, mnesia:create_table(tab2, [{disc_copies, Nodes}])),
?match({atomic, ok}, mnesia:create_table(tab3, [{disc_only_copies, Nodes}])),
?match({atomic, ok}, mnesia:create_table(tab4, [{ram_copies, [N1]}])),
?match({atomic, ok}, mnesia:create_table(tab5, [{ram_copies, [N2]}])),
?match({atomic, ok}, mnesia:create_table(tab16, [{ram_copies, [N3]}])),
?match({atomic, ok}, mnesia:create_table(tab6, [{disc_copies, [N1]}])),
?match({atomic, ok}, mnesia:create_table(tab7, [{disc_copies, [N2]}])),
?match({atomic, ok}, mnesia:create_table(tab17, [{disc_copies, [N3]}])),
?match({atomic, ok}, mnesia:create_table(tab8, [{disc_only_copies, [N1]}])),
?match({atomic, ok}, mnesia:create_table(tab9, [{disc_only_copies, [N2]}])),
?match({atomic, ok}, mnesia:create_table(tab18, [{disc_only_copies, [N3]}])),
?match({atomic, ok}, mnesia:create_table(tab10, [{ram_copies, [N1]}, {disc_copies, [N2, N3]}])),
?match({atomic, ok}, mnesia:create_table(tab11, [{ram_copies, [N2]}, {disc_copies, [N3, N1]}])),
?match({atomic, ok}, mnesia:create_table(tab19, [{ram_copies, [N3]}, {disc_copies, [N1, N2]}])),
?match({atomic, ok}, mnesia:create_table(tab12, [{ram_copies, [N1]}, {disc_only_copies, [N2, N3]}])),
?match({atomic, ok}, mnesia:create_table(tab13, [{ram_copies, [N2]}, {disc_only_copies, [N3, N1]}])),
?match({atomic, ok}, mnesia:create_table(tab20, [{ram_copies, [N3]}, {disc_only_copies, [N1, N2]}])),
?match({atomic, ok}, mnesia:create_table(tab14, [{disc_only_copies, [N1]}, {disc_copies, [N2, N3]}])),
?match({atomic, ok}, mnesia:create_table(tab15, [{disc_only_copies, [N2]}, {disc_copies, [N3, N1]}])),
?match({atomic, ok}, mnesia:create_table(tab21, [{disc_only_copies, [N3]}, {disc_copies, [N1, N2]}])),
Tabs = [tab1, tab2, tab3, tab4, tab5, tab6, tab7, tab8,
tab9, tab10, tab11, tab12, tab13, tab14, tab15,
tab16, tab17, tab18, tab19, tab20, tab21],
[?match(ok, rpc:call(Node, mnesia, wait_for_tables, [Tabs, 10000])) || Node <- Nodes],
[insert_data(Tab, 20) || Tab <- Tabs],
VTabs =
case Masters of
no ->
Tabs -- [tab4, tab5, tab16]; % ram copies
one ->
?match(ok, rpc:call(N1, mnesia, set_master_nodes, [[N1]])),
Tabs -- [tab1, tab4, tab5, tab16, tab10, tab12]; % ram copies
two ->
?match(ok, rpc:call(N1, mnesia, set_master_nodes, [Nodes])),
Tabs -- [tab4, tab5, tab16]; % ram copies
all ->
[?match(ok, rpc:call(Node, mnesia, set_master_nodes, [[Node]])) || Node <- Nodes],
Tabs -- [tab1, tab4, tab5, tab16, tab10,
tab11, tab19, tab12, tab13, tab20] % ram copies
end,
mnesia_test_lib:kill_mnesia([N1]),
?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
?match([], mnesia_test_lib:kill_mnesia([N2])),
?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
?match([], mnesia_test_lib:kill_mnesia([N3])),
?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
?match([], mnesia_test_lib:kill_mnesia([N2, N1])),
?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
?match([], mnesia_test_lib:kill_mnesia([N2, N3])),
?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
?match([], mnesia_test_lib:kill_mnesia([N1, N3])),
?match([], mnesia_test_lib:start_mnesia(Nodes, Tabs)),
[?match(ok, rpc:call(N1, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs],
[?match(ok, rpc:call(N2, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs],
[?match(ok, rpc:call(N3, ?MODULE, verify_data, [Tab, 20])) || Tab <- VTabs],
?verify_mnesia(Nodes, []).
read_during_down(doc) ->
["Verify that read operation can continue to read when mnesia goes down"];
read_during_down(suite) ->
[
dirty_read_during_down,
trans_read_during_down
].
dirty_read_during_down(suite) ->
[];
dirty_read_during_down(Config) when is_list(Config) ->
read_during_down(dirty, Config).
trans_read_during_down(suite) ->
[];
trans_read_during_down(Config) when is_list(Config) ->
read_during_down(trans, Config).
read_during_down(Op, Config) when is_list(Config) ->
Ns = [N1|TNs] = ?acquire_nodes(3, Config),
Tabs = [ram, disc, disco],
?match({atomic, ok}, mnesia:create_table(ram, [{ram_copies, TNs}])),
?match({atomic, ok}, mnesia:create_table(disc, [{disc_copies, TNs}])),
?match({atomic, ok}, mnesia:create_table(disco, [{disc_only_copies, TNs}])),
%% Create some work for mnesia_controller when a node goes down
[{atomic, ok} = mnesia:create_table(list_to_atom("temp" ++ integer_to_list(N)),
[{ram_copies, Ns}]) || N <- lists:seq(1, 50)],
Write = fun(Tab) -> mnesia:write({Tab, key, val}) end,
?match([ok,ok,ok],
[mnesia:sync_dirty(Write, [Tab]) || Tab <- Tabs]),
Readers = [spawn_link(N1, ?MODULE, reader, [Tab, Op]) || Tab <- Tabs],
[_|_] = W2R= [mnesia:table_info(Tab, where_to_read) || Tab <- Tabs],
?log("W2R ~p~n", [W2R]),
loop_and_kill_mnesia(10, hd(W2R), Tabs),
[Pid ! self() || Pid <- Readers],
?match([ok, ok, ok], [receive ok -> ok after 1000 -> {Pid, mnesia_lib:dist_coredump()} end || Pid <- Readers]),
?verify_mnesia(Ns, []).
reader(Tab, OP) ->
Res = case OP of
dirty ->
catch mnesia:dirty_read({Tab, key});
trans ->
Read = fun() -> mnesia:read({Tab, key}) end,
{_, Temp} = mnesia:transaction(Read),
Temp
end,
case Res of
[{Tab, key, val}] -> ok;
Else ->
?error("Expected ~p Got ~p ~n", [[{Tab, key, val}], Else]),
erlang:error(test_failed)
end,
receive Pid ->
Pid ! ok
after 50 ->
reader(Tab, OP)
end.
loop_and_kill_mnesia(0, _Node, _Tabs) -> ok;
loop_and_kill_mnesia(N, Node, Tabs) ->
mnesia_test_lib:kill_mnesia([Node]),
timer:sleep(100),
?match([], mnesia_test_lib:start_mnesia([Node], Tabs)),
[KN | _] = W2R= [mnesia:table_info(Tab, where_to_read) || Tab <- Tabs],
?match([KN, KN,KN], W2R),
timer:sleep(100),
loop_and_kill_mnesia(N-1, KN, Tabs).
mnesia_down_during_startup(doc) ->
["Verify that mnesia can come back up again in a consistent state",
"after it has gone down during startup (with different store and",
"when it goes down in different situations"];
mnesia_down_during_startup(suite) ->
[
mnesia_down_during_startup_disk_ram,
mnesia_down_during_startup_init_ram,
mnesia_down_during_startup_init_disc,
mnesia_down_during_startup_init_disc_only,
mnesia_down_during_startup_tm_ram,
mnesia_down_during_startup_tm_disc,
mnesia_down_during_startup_tm_disc_only
].
mnesia_down_during_startup_disk_ram(suite) -> [];
mnesia_down_during_startup_disk_ram(Config) when is_list(Config)->
[Node1, Node2] = ?acquire_nodes(2, Config ++
[{tc_timeout, timer:minutes(2)}]),
Tab = down_during_startup,
Def = [{ram_copies, [Node2]}, {disc_copies, [Node1]}],
?match({atomic, ok}, mnesia:create_table(Tab, Def)),
?match(ok, mnesia:dirty_write({Tab, 876234, test_ok})),
timer:sleep(500),
mnesia_test_lib:kill_mnesia([Node1, Node2]),
timer:sleep(500),
mnesia_test_lib:start_mnesia([Node1, Node2], [Tab]),
mnesia_test_lib:kill_mnesia([Node1]),
timer:sleep(500),
?match([], mnesia_test_lib:start_mnesia([Node1], [Tab])),
?match([{Tab, 876234, test_ok}], mnesia:dirty_read({Tab,876234})),
?verify_mnesia([Node1, Node2], []).
mnesia_down_during_startup_init_ram(suite) -> [];
mnesia_down_during_startup_init_ram(Config) when is_list(Config) ->
?is_debug_compiled,
DP = {mnesia_loader, do_get_network_copy},
Type = ram_copies,
mnesia_down_during_startup2(Config, Type, DP, self()).
mnesia_down_during_startup_init_disc(suite) -> [];
mnesia_down_during_startup_init_disc(Config) when is_list(Config) ->
?is_debug_compiled,
DP = {mnesia_loader, do_get_network_copy},
Type = disc_copies,
mnesia_down_during_startup2(Config, Type, DP, self()).
mnesia_down_during_startup_init_disc_only(suite) -> [];
mnesia_down_during_startup_init_disc_only(Config) when is_list(Config) ->
?is_debug_compiled,
DP = {mnesia_loader, do_get_network_copy},
Type = disc_only_copies,
mnesia_down_during_startup2(Config, Type, DP, self()).
mnesia_down_during_startup_tm_ram(suite) -> [];
mnesia_down_during_startup_tm_ram(Config) when is_list(Config) ->
?is_debug_compiled,
DP = {mnesia_tm, init},
Type = ram_copies,
mnesia_down_during_startup2(Config, Type, DP, self()).
mnesia_down_during_startup_tm_disc(suite) -> [];
mnesia_down_during_startup_tm_disc(Config) when is_list(Config) ->
?is_debug_compiled,
DP = {mnesia_tm, init},
Type = disc_copies,
mnesia_down_during_startup2(Config, Type, DP, self()).
mnesia_down_during_startup_tm_disc_only(suite) -> [];
mnesia_down_during_startup_tm_disc_only(Config) when is_list(Config) ->
?is_debug_compiled,
DP = {mnesia_tm, init},
Type = disc_only_copies,
mnesia_down_during_startup2(Config, Type, DP, self()).
mnesia_down_during_startup2(Config, ReplicaType, Debug_Point, _Father) ->
?log("TC~n mnesia_down_during_startup with type ~w and stops at ~w~n",
[ReplicaType, Debug_Point]),
Tpcb_tabs = [history,teller,account,branch],
Nodes = ?acquire_nodes(2, Config),
Node1 = hd(Nodes),
{success, [A]} = ?start_activities([Node1]),
TpcbConfig = tpcb_config(ReplicaType, 2, Nodes),
mnesia_tpcb:init(TpcbConfig),
A ! fun () -> mnesia_tpcb:run(TpcbConfig) end,
?match_receive(timeout),
timer:sleep(timer:seconds(10)), % Let tpcb run for a while
mnesia_tpcb:stop(),
?match(ok, mnesia_tpcb:verify_tabs()),
mnesia_test_lib:kill_mnesia([Node1]),
timer:sleep(timer:seconds(2)),
Self = self(),
TestFun = fun(_MnesiaEnv, _EvalEnv) ->
?deactivate_debug_fun(Debug_Point),
Self ! fun_done,
spawn(mnesia_test_lib, kill_mnesia, [[Node1]])
end,
?activate_debug_fun(Debug_Point, TestFun, []), % Kill when debug has been reached
mnesia:start(),
Res = receive fun_done -> ok after timer:minutes(3) -> timeout end, % Wait till it's killed
?match(ok, Res),
?match(ok, timer:sleep(timer:seconds(2))), % Wait a while, at least till it dies;
?match([], mnesia_test_lib:start_mnesia([Node1], Tpcb_tabs)),
?match(ok, mnesia_tpcb:verify_tabs()), % Verify it
?verify_mnesia(Nodes, []).
with_checkpoint(doc) ->
["Restart mnesia with checkpoint"];
with_checkpoint(suite) ->
[with_checkpoint_same, with_checkpoint_other].
with_checkpoint_same(suite) -> [];
with_checkpoint_same(Config) when is_list(Config) ->
with_checkpoint(Config, same).
with_checkpoint_other(suite) -> [];
with_checkpoint_other(Config) when is_list(Config) ->
with_checkpoint(Config, other).
with_checkpoint(Config, Type) when is_list(Config) ->
Nodes = [Node1, Node2] = ?acquire_nodes(2, Config),
Kill = case Type of
same -> %% Node1 is the one used for creating the checkpoint
Node1; %% and which we bring down
other ->
Node2 %% Here we bring node2 down..
end,
?match({atomic, ok}, mnesia:create_table(ram, [{ram_copies, Nodes}])),
?match({atomic, ok}, mnesia:create_table(disc, [{disc_copies, Nodes}])),
?match({atomic, ok}, mnesia:create_table(disco, [{disc_only_copies, Nodes}])),
Tabs = [ram, disc, disco],
?match({ok, sune, _}, mnesia:activate_checkpoint([{name, sune},
{max, mnesia:system_info(tables)},
{ram_overrides_dump, true}])),
?match([], check_retainers(sune, Nodes)),
?match(ok, mnesia:deactivate_checkpoint(sune)),
?match([], check_chkp(Nodes)),
timer:sleep(500), %% Just to help debugging the io:formats now comes in the
%% correct order... :-)
?match({ok, sune, _}, mnesia:activate_checkpoint([{name, sune},
{max, mnesia:system_info(tables)},
{ram_overrides_dump, true}])),
[[mnesia:dirty_write({Tab,Key,Key}) || Key <- lists:seq(1,10)] || Tab <- Tabs],
mnesia_test_lib:kill_mnesia([Kill]),
timer:sleep(100),
mnesia_test_lib:start_mnesia([Kill], Tabs),
io:format("Mnesia on ~p started~n", [Kill]),
?match([], check_retainers(sune, Nodes)),
?match(ok, mnesia:deactivate_checkpoint(sune)),
?match([], check_chkp(Nodes)),
case Kill of
Node1 ->
ignore;
Node2 ->
mnesia_test_lib:kill_mnesia([Kill]),
timer:sleep(500), %% Just to help debugging
?match({ok, sune, _}, mnesia:activate_checkpoint([{name, sune},
{max, mnesia:system_info(tables)},
{ram_overrides_dump, true}])),
[[mnesia:dirty_write({Tab,Key,Key+2}) || Key <- lists:seq(1,10)] ||
Tab <- Tabs],
mnesia_test_lib:start_mnesia([Kill], Tabs),
io:format("Mnesia on ~p started ~n", [Kill]),
?match([], check_retainers(sune, Nodes)),
?match(ok, mnesia:deactivate_checkpoint(sune)),
?match([], check_chkp(Nodes)),
ok
end,
?verify_mnesia(Nodes, []).
check_chkp(Nodes) ->
{Good, Bad} = rpc:multicall(Nodes, ?MODULE, check, []),
lists:flatten(Good ++ Bad).
check() ->
[PCP] = ets:match_object(mnesia_gvar, {pending_checkpoint_pids, '_'}),
[PC] = ets:match_object(mnesia_gvar, {pending_checkpoints, '_'}),
[CPN] = ets:match_object(mnesia_gvar, {checkpoints, '_'}),
F = lists:filter(fun({_, []}) -> false; (_W) -> true end,
[PCP,PC,CPN]),
CPP = ets:match_object(mnesia_gvar, {{checkpoint, '_'}, '_'}),
Rt = ets:match_object(mnesia_gvar, {{'_', {retainer, '_'}}, '_'}),
F ++ CPP ++ Rt.
check_retainers(CHP, Nodes) ->
{[R1,R2], []} = rpc:multicall(Nodes, ?MODULE, get_all_retainers, [CHP]),
(R1 -- R2) ++ (R2 -- R1).
get_all_retainers(CHP) ->
Tabs = mnesia:system_info(local_tables),
Iter = fun(Tab) ->
{ok, Res} =
mnesia_checkpoint:iterate(CHP, Tab, fun(R, A) -> [R|A] end, [],
retainer, checkpoint),
%% io:format("Retainer content ~w ~n", [Res]),
Res
end,
Elements = [Iter(Tab) || Tab <- Tabs],
lists:sort(lists:flatten(Elements)).
delete_during_start(doc) ->
["Test that tables can be delete during start, hopefully with tables"
" in the loader queue or soon to be"];
delete_during_start(suite) -> [];
delete_during_start(Config) when is_list(Config) ->
[N1, N2, N3] = Nodes = ?acquire_nodes(3, Config),
Tabs = [list_to_atom("tab" ++ integer_to_list(I)) || I <- lists:seq(1, 30)],
?match({atomic, ok}, mnesia:change_table_copy_type(schema, N2, ram_copies)),
?match({atomic, ok}, mnesia:change_table_copy_type(schema, N3, ram_copies)),
[?match({atomic, ok},mnesia:create_table(Tab, [{ram_copies,Nodes}])) || Tab <- Tabs],
lists:foldl(fun(Tab, I) ->
?match({atomic, ok},
mnesia:change_table_load_order(Tab,I)),
I+1
end, 1, Tabs),
mnesia_test_lib:kill_mnesia([N2,N3]),
%% timer:sleep(500),
?match({[ok,ok],[]}, rpc:multicall([N2,N3], mnesia,start,
[[{extra_db_nodes,[N1]}]])),
[Tab1,Tab2,Tab3|_] = Tabs,
?match({atomic, ok}, mnesia:delete_table(Tab1)),
?match({atomic, ok}, mnesia:delete_table(Tab2)),
?log("W4T ~p~n", [rpc:multicall([N2,N3], mnesia, wait_for_tables, [[Tab1,Tab2,Tab3],1])]),
Remain = Tabs--[Tab1,Tab2],
?match(ok, rpc:call(N2, mnesia, wait_for_tables, [Remain,10000])),
?match(ok, rpc:call(N3, mnesia, wait_for_tables, [Remain,10000])),
?match(ok, rpc:call(N2, ?MODULE, verify_where2read, [Remain])),
?match(ok, rpc:call(N3, ?MODULE, verify_where2read, [Remain])),
?verify_mnesia(Nodes, []).
verify_where2read([Tab|Tabs]) ->
true = (node() == mnesia:table_info(Tab,where_to_read)),
verify_where2read(Tabs);
verify_where2read([]) -> ok.
%%-------------------------------------------------------------------------------------------
explicit_stop(doc) ->
["Stop Mnesia in different situations"];
explicit_stop(suite) ->
[explicit_stop_during_snmp].
%% This is a bad implementation, but at least gives a indication if something is wrong
explicit_stop_during_snmp(suite) -> [];
explicit_stop_during_snmp(Config) when is_list(Config) ->
Nodes = ?acquire_nodes(2, Config),
[Node1, Node2] = Nodes,
Tab = snmp_tab,
Def = [{attributes, [key, value]},
{snmp, [{key, integer}]},
{mnesia_test_lib:storage_type(disc_copies, Config),
[Node1, Node2]}],
?match({atomic, ok}, mnesia:create_table(Tab, Def)),
?match({atomic, ok}, mnesia:transaction(fun() -> mnesia:write({Tab, 1, 1}) end)),
Do_trans_Pid1 = spawn_link(Node2, ?MODULE, do_trans_loop, [Tab, self()]),
Do_trans_Pid2 = spawn_link(?MODULE, do_trans_loop, [Tab, self()]),
Start_stop_Pid = spawn_link(?MODULE, start_stop, [Node1, 10, self()]),
receive
test_done ->
ok
after timer:minutes(5) ->
?error("test case time out~n", [])
end,
?verify_mnesia(Nodes, []),
exit(Do_trans_Pid1, kill),
exit(Do_trans_Pid2, kill),
exit(Start_stop_Pid, kill),
ok.
do_trans_loop(Tab, Father) ->
%% Do not trap exit
do_trans_loop2(Tab, Father).
do_trans_loop2(Tab, Father) ->
Trans =
fun() ->
[{Tab, 1, Val}] = mnesia:read({Tab, 1}),
mnesia:write({Tab, 1, Val + 1})
end,
case mnesia:transaction(Trans) of
{atomic, ok} ->
timer:sleep(200),
do_trans_loop2(Tab, Father);
{aborted, {node_not_running, N}} when N == node() ->
timer:sleep(200),
do_trans_loop2(Tab, Father);
{aborted, {no_exists, Tab}} ->
timer:sleep(200),
do_trans_loop2(Tab, Father);
Else ->
?error("Transaction failed: ~p ~n", [Else]),
Father ! test_done,
exit(shutdown)
end.
start_stop(_Node1, 0, Father) ->
Father ! test_done,
exit(shutdown);
start_stop(Node1, N, Father) when N > 0->
timer:sleep(timer:seconds(5)),
?match(stopped, rpc:call(Node1, mnesia, stop, [])),
timer:sleep(timer:seconds(2)),
?match([], mnesia_test_lib:start_mnesia([Node1])),
start_stop(Node1, N-1, Father).
coord_dies(suite) -> [];
coord_dies(doc) -> [""];
coord_dies(Config) when is_list(Config) ->
Nodes = [N1, N2] = ?acquire_nodes(2, Config),
?match({atomic, ok}, mnesia:create_table(tab1, [{ram_copies, Nodes}])),
?match({atomic, ok}, mnesia:create_table(tab2, [{ram_copies, [N1]}])),
?match({atomic, ok}, mnesia:create_table(tab3, [{ram_copies, [N2]}])),
Tester = self(),
U1 = fun(Tab) ->
[{Tab,key,Val}] = mnesia:read(Tab,key,write),
mnesia:write({Tab,key, Val+1}),
Tester ! {self(),continue},
receive
continue -> exit(crash)
end
end,
U2 = fun(Tab) ->
[{Tab,key,Val}] = mnesia:read(Tab,key,write),
mnesia:write({Tab,key, Val+1}),
mnesia:transaction(U1, [Tab])
end,
[mnesia:dirty_write(Tab,{Tab,key,0}) || Tab <- [tab1,tab2,tab3]],
Pid1 = spawn(fun() -> mnesia:transaction(U2, [tab1]) end),
Pid2 = spawn(fun() -> mnesia:transaction(U2, [tab2]) end),
Pid3 = spawn(fun() -> mnesia:transaction(U2, [tab3]) end),
[receive {Pid,continue} -> ok end || Pid <- [Pid1,Pid2,Pid3]],
Pid1 ! continue, Pid2 ! continue, Pid3 ! continue,
?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab1,key}) end)),
?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab2,key}) end)),
?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab3,key}) end)),
Pid4 = spawn(fun() -> mnesia:transaction(U2, [tab1]) end),
Pid5 = spawn(fun() -> mnesia:transaction(U2, [tab2]) end),
Pid6 = spawn(fun() -> mnesia:transaction(U2, [tab3]) end),
erlang:monitor(process, Pid4),erlang:monitor(process, Pid5),erlang:monitor(process, Pid6),
[receive {Pid,continue} -> ok end || Pid <- [Pid4,Pid5,Pid6]],
exit(Pid4,crash),
?match_receive({'DOWN',_,_,Pid4, _}),
?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab1,key}) end)),
exit(Pid5,crash),
?match_receive({'DOWN',_,_,Pid5, _}),
?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab2,key}) end)),
exit(Pid6,crash),
?match_receive({'DOWN',_,_,Pid6, _}),
?match({atomic,[{_,key,1}]}, mnesia:transaction(fun() -> mnesia:read({tab3,key}) end)),
?verify_mnesia(Nodes, []).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
sym_trans(doc) ->
["Recovery of symmetrical transactions in a couple of different",
"situations; when coordinator or participant or node dies"];
sym_trans(suite) ->
[sym_trans_before_commit_kill_coord_node, %% coordinator node dies
sym_trans_before_commit_kill_coord_pid, %% coordinator process dies
sym_trans_before_commit_kill_part_after_ask, %% participating node dies
sym_trans_before_commit_kill_part_before_ask,
sym_trans_after_commit_kill_coord_node,
sym_trans_after_commit_kill_coord_pid,
sym_trans_after_commit_kill_part_after_ask,
sym_trans_after_commit_kill_part_do_commit_pre,
sym_trans_after_commit_kill_part_do_commit_post].
%kill_after_debug_point(Config, TestCase, {Debug_node, Debug_Point}, TransFun, Tab)
sym_trans_before_commit_kill_coord_node(suite) -> [];
sym_trans_before_commit_kill_coord_node(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sym_trans_before_commit_kill_coord,
Def = [{attributes, [key, value]}, {ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_sym}},
do_sym_trans, [{Tab, Def}], Nodes).
sym_trans_before_commit_kill_coord_pid(suite) -> [];
sym_trans_before_commit_kill_coord_pid(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sym_trans_before_commit_kill_coord,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_sym}},
do_sym_trans, [{Tab, Def}], Nodes).
sym_trans_before_commit_kill_part_after_ask(suite) -> [];
sym_trans_before_commit_kill_part_after_ask(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sym_trans_before_commit_kill_part_after_ask,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
kill_after_debug_point(Part1, {Coord, {mnesia_tm, multi_commit_sym}},
do_sym_trans, [{Tab, Def}], Nodes).
sym_trans_before_commit_kill_part_before_ask(suite) -> [];
sym_trans_before_commit_kill_part_before_ask(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sym_trans_before_commit_kill_part_before_ask,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
kill_after_debug_point(Part1, {Part1, {mnesia_tm, doit_ask_commit}},
do_sym_trans, [{Tab, Def}], Nodes).
sym_trans_after_commit_kill_coord_node(suite) -> [];
sym_trans_after_commit_kill_coord_node(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sym_trans_after_commit_kill_coord,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_sym, post}},
do_sym_trans, [{Tab, Def}], Nodes).
sym_trans_after_commit_kill_coord_pid(suite) -> [];
sym_trans_after_commit_kill_coord_pid(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sym_trans_after_commit_kill_coord,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_sym, post}},
do_sym_trans, [{Tab,Def}], Nodes).
sym_trans_after_commit_kill_part_after_ask(suite) -> [];
sym_trans_after_commit_kill_part_after_ask(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sym_trans_after_commit_kill_part_after_ask,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
kill_after_debug_point(Part1, {Coord, {mnesia_tm, multi_commit_sym, post}},
do_sym_trans, [{Tab, Def}], Nodes).
sym_trans_after_commit_kill_part_do_commit_pre(suite) -> [];
sym_trans_after_commit_kill_part_do_commit_pre(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sym_trans_after_commit_kill_part_do_commit_pre,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
TransFun = do_sym_trans,
kill_after_debug_point(Part1, {Part1, {mnesia_tm, do_commit, pre}},
TransFun, [{Tab, Def}], Nodes).
sym_trans_after_commit_kill_part_do_commit_post(suite) -> [];
sym_trans_after_commit_kill_part_do_commit_post(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sym_trans_after_commit_kill_part_do_commit_post,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
TransFun = do_sym_trans,
kill_after_debug_point(Part1, {Part1, {mnesia_tm, do_commit, post}},
TransFun, [{Tab, Def}], Nodes).
do_sym_trans([Tab], _Fahter) ->
?dl("Starting SYM_TRANS with active debug fun ", []),
Trans = fun() ->
[{_,_,Val}] = mnesia:read({Tab, 1}),
mnesia:write({Tab, 1, Val+1})
end,
Res = mnesia:transaction(Trans),
case Res of
{atomic, ok} -> ok;
{aborted, _Reason} -> ok;
Else -> ?error("Wrong output from mensia:transaction(FUN):~n ~p~n",
[Else])
end,
?dl("SYM_TRANSACTION done: ~p (deactiv dbgfun) ", [Res]),
ok.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
sync_dirty(doc) ->
["Verify recovery of synchronously operations in a couple of different",
"situations"];
sync_dirty(suite) ->
[sync_dirty_pre_kill_part,
sync_dirty_pre_kill_coord_node,
sync_dirty_pre_kill_coord_pid,
sync_dirty_post_kill_part,
sync_dirty_post_kill_coord_node,
sync_dirty_post_kill_coord_pid
].
sync_dirty_pre_kill_part(suite) -> [];
sync_dirty_pre_kill_part(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sync_dirty_pre,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
TransFun = do_sync_dirty,
kill_after_debug_point(Part1, {Part1, {mnesia_tm, sync_dirty, pre}},
TransFun, [{Tab, Def}], Nodes).
sync_dirty_pre_kill_coord_node(suite) -> [];
sync_dirty_pre_kill_coord_node(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sync_dirty_pre,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
TransFun = do_sync_dirty,
kill_after_debug_point(Coord, {Part1, {mnesia_tm, sync_dirty, pre}},
TransFun, [{Tab, Def}], Nodes).
sync_dirty_pre_kill_coord_pid(suite) -> [];
sync_dirty_pre_kill_coord_pid(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sync_dirty_pre,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
TransFun = do_sync_dirty,
kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, sync_dirty, pre}},
TransFun, [{Tab, Def}], Nodes).
sync_dirty_post_kill_part(suite) -> [];
sync_dirty_post_kill_part(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sync_dirty_post,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
TransFun = do_sync_dirty,
kill_after_debug_point(Part1, {Part1, {mnesia_tm, sync_dirty, post}},
TransFun, [{Tab, Def}], Nodes).
sync_dirty_post_kill_coord_node(suite) -> [];
sync_dirty_post_kill_coord_node(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sync_dirty_post,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
TransFun = do_sync_dirty,
kill_after_debug_point(Coord, {Part1, {mnesia_tm, sync_dirty, post}},
TransFun, [{Tab, Def}], Nodes).
sync_dirty_post_kill_coord_pid(suite) -> [];
sync_dirty_post_kill_coord_pid(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = sync_dirty_post,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
TransFun = do_sync_dirty,
kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, sync_dirty, post}},
TransFun, [{Tab, Def}], Nodes).
do_sync_dirty([Tab], _Father) ->
?dl("Starting SYNC_DIRTY", []),
SYNC = fun() ->
[{_,_,Val}] = mnesia:read({Tab, 1}),
mnesia:write({Tab, 1, Val+1})
end,
{_, Res} = ?match(ok, mnesia:sync_dirty(SYNC)),
?dl("SYNC_DIRTY done: ~p ", [Res]),
ok.
async_dirty(doc) ->
["Verify recovery of asynchronously dirty operations in a couple of different",
"situations"];
async_dirty(suite) ->
[async_dirty_pre_kill_part,
async_dirty_pre_kill_coord_node,
async_dirty_pre_kill_coord_pid,
async_dirty_post_kill_part,
async_dirty_post_kill_coord_node,
async_dirty_post_kill_coord_pid].
async_dirty_pre_kill_part(suite) -> [];
async_dirty_pre_kill_part(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = async_dirty_pre,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
TransFun = do_async_dirty,
kill_after_debug_point(Part1, {Part1, {mnesia_tm, async_dirty, pre}},
TransFun, [{Tab, Def}], Nodes).
async_dirty_pre_kill_coord_node(suite) -> [];
async_dirty_pre_kill_coord_node(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = async_dirty_pre,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
TransFun = do_async_dirty,
kill_after_debug_point(Coord, {Part1, {mnesia_tm, async_dirty, pre}},
TransFun, [{Tab, Def}], Nodes).
async_dirty_pre_kill_coord_pid(suite) -> [];
async_dirty_pre_kill_coord_pid(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = async_dirty_pre,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
TransFun = do_async_dirty,
kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, async_dirty, pre}},
TransFun, [{Tab, Def}], Nodes).
async_dirty_post_kill_part(suite) -> [];
async_dirty_post_kill_part(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = async_dirty_post,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
TransFun = do_async_dirty,
kill_after_debug_point(Part1, {Part1, {mnesia_tm, async_dirty, post}},
TransFun, [{Tab, Def}], Nodes).
async_dirty_post_kill_coord_node(suite) -> [];
async_dirty_post_kill_coord_node(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = async_dirty_post,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
TransFun = do_async_dirty,
kill_after_debug_point(Coord, {Part1, {mnesia_tm, async_dirty, post}},
TransFun, [{Tab, Def}], Nodes).
async_dirty_post_kill_coord_pid(suite) -> [];
async_dirty_post_kill_coord_pid(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab = async_dirty_post,
Def = [{attributes, [key, value]},{ram_copies, [Part2]},{disc_copies, [Coord, Part1]}],
TransFun = do_async_dirty,
kill_after_debug_point(coord_pid, {Part1, {mnesia_tm, async_dirty, post}},
TransFun, [{Tab, Def}], Nodes).
do_async_dirty([Tab], _Fahter) ->
?dl("Starting ASYNC", []),
ASYNC = fun() ->
[{_,_,Val}] = mnesia:read({Tab, 1}),
mnesia:write({Tab, 1, Val+1})
end,
{_, Res} = ?match(ok, mnesia:async_dirty(ASYNC)),
?dl("ASYNC done: ~p ", [Res]),
ok.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
asym_trans(doc) ->
["Recovery of asymmetrical transactions in a couple of different",
"situations, currently the error cases are not covered, i.e. ",
"not tested are the situations when we kill mnesia or a process",
"during a recovery"];
asym_trans(suite) ->
[
asym_trans_kill_part_ask,
asym_trans_kill_part_commit_vote,
asym_trans_kill_part_pre_commit,
asym_trans_kill_part_log_commit,
asym_trans_kill_part_do_commit,
asym_trans_kill_coord_got_votes,
asym_trans_kill_coord_pid_got_votes,
asym_trans_kill_coord_log_commit_rec,
asym_trans_kill_coord_pid_log_commit_rec,
asym_trans_kill_coord_log_commit_dec,
asym_trans_kill_coord_pid_log_commit_dec,
asym_trans_kill_coord_rec_acc_pre_commit_log_commit,
asym_trans_kill_coord_pid_rec_acc_pre_commit_log_commit,
asym_trans_kill_coord_rec_acc_pre_commit_done_commit,
asym_trans_kill_coord_pid_rec_acc_pre_commit_done_commit
].
asym_trans_kill_part_ask(suite) -> [];
asym_trans_kill_part_ask(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(Part1, {Part1, {mnesia_tm, doit_ask_commit}},
TransFun, [Tab1, Tab2], Nodes).
asym_trans_kill_part_commit_vote(suite) -> [];
asym_trans_kill_part_commit_vote(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, vote_yes}},
TransFun, [Tab1, Tab2], Nodes).
asym_trans_kill_part_pre_commit(suite) -> [];
asym_trans_kill_part_pre_commit(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, pre_commit}},
TransFun, [Tab1, Tab2], Nodes).
asym_trans_kill_part_log_commit(suite) -> [];
asym_trans_kill_part_log_commit(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, log_commit}},
TransFun, [Tab1, Tab2], Nodes).
asym_trans_kill_part_do_commit(suite) -> [];
asym_trans_kill_part_do_commit(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(Part1, {Part1, {mnesia_tm, commit_participant, do_commit}},
TransFun, [Tab1, Tab2], Nodes).
asym_trans_kill_coord_got_votes(suite) -> [];
asym_trans_kill_coord_got_votes(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_asym_got_votes}},
TransFun, [Tab1, Tab2], Nodes).
asym_trans_kill_coord_pid_got_votes(suite) -> [];
asym_trans_kill_coord_pid_got_votes(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_asym_got_votes}},
TransFun, [Tab1, Tab2], Nodes).
asym_trans_kill_coord_log_commit_rec(suite) -> [];
asym_trans_kill_coord_log_commit_rec(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_asym_log_commit_rec}},
TransFun, [Tab1, Tab2], Nodes).
asym_trans_kill_coord_pid_log_commit_rec(suite) -> [];
asym_trans_kill_coord_pid_log_commit_rec(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_asym_log_commit_rec}},
TransFun, [Tab1, Tab2], Nodes).
asym_trans_kill_coord_log_commit_dec(suite) -> [];
asym_trans_kill_coord_log_commit_dec(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(Coord, {Coord, {mnesia_tm, multi_commit_asym_log_commit_dec}},
TransFun, [Tab1, Tab2], Nodes).
asym_trans_kill_coord_pid_log_commit_dec(suite) -> [];
asym_trans_kill_coord_pid_log_commit_dec(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, multi_commit_asym_log_commit_dec}},
TransFun, [Tab1, Tab2], Nodes).
asym_trans_kill_coord_rec_acc_pre_commit_log_commit(suite) -> [];
asym_trans_kill_coord_rec_acc_pre_commit_log_commit(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(Coord, {Coord, {mnesia_tm, rec_acc_pre_commit_log_commit}},
TransFun, [Tab1, Tab2], Nodes).
asym_trans_kill_coord_pid_rec_acc_pre_commit_log_commit(suite) -> [];
asym_trans_kill_coord_pid_rec_acc_pre_commit_log_commit(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, rec_acc_pre_commit_log_commit}},
TransFun, [Tab1, Tab2], Nodes).
asym_trans_kill_coord_rec_acc_pre_commit_done_commit(suite) -> [];
asym_trans_kill_coord_rec_acc_pre_commit_done_commit(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(Coord, {Coord, {mnesia_tm, rec_acc_pre_commit_done_commit}},
TransFun, [Tab1, Tab2], Nodes).
asym_trans_kill_coord_pid_rec_acc_pre_commit_done_commit(suite) -> [];
asym_trans_kill_coord_pid_rec_acc_pre_commit_done_commit(Config) when is_list(Config) ->
?is_debug_compiled,
Nodes = ?acquire_nodes(3, Config ++ [{tc_timeout, timer:minutes(2)}]),
[Coord, Part1, Part2] = Nodes,
Tab1 = {asym1, [{ram_copies, [Part2]}, {disc_copies, [Coord]}]},
Tab2 = {asym2, [{ram_copies, [Coord]}, {disc_copies, [Part1]}]},
TransFun = do_asym_trans,
kill_after_debug_point(coord_pid, {Coord, {mnesia_tm, rec_acc_pre_commit_done_commit}},
TransFun, [Tab1, Tab2], Nodes).
do_asym_trans([Tab1, Tab2 | _R], Garbhandler) ->
?dl("Starting asym trans ", []),
ASym_Trans = fun() ->
TidTs = {_Mod, Tid, _Store} =
mnesia:get_activity_id(),
?verbose("===> asym_trans: ~w~n", [TidTs]),
Garbhandler ! {trans_id, Tid},
[{_, _, Val1}] = mnesia:read({Tab1, 1}),
[{_, _, Val2}] = mnesia:read({Tab2, 1}),
mnesia:write({Tab1, 1, Val1+1}),
mnesia:write({Tab2, 1, Val2+1})
end,
Res = mnesia:transaction(ASym_Trans),
case Res of
{atomic, ok} -> ok;
{aborted, _Reason} -> ok;
_Else -> ?error("Wrong output from mensia:transaction(FUN):~n ~p~n", [Res])
end,
?dl("Asym trans finished with: ~p ", [Res]).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
kill_after_debug_point(Kill, {DebugNode, Debug_Point}, TransFun, TabsAndDefs, Nodes) ->
[Coord | _rest] = Nodes,
Create = fun({Tab, Def}) -> ?match({atomic, ok}, mnesia:create_table(Tab, Def)) end,
lists:foreach(Create, TabsAndDefs),
Tabs = [T || {T, _} <- TabsAndDefs],
Write = fun(Tab) -> ?match(ok, mnesia:dirty_write({Tab, 1, 100})) end,
lists:foreach(Write, Tabs),
Self = self(),
SyncFun = fun(_Env1, _Env2) -> % Just Sync with test prog
Self ! {self(), fun_in_position},
?dl("SyncFun, sending fun_in_position ", []),
receive continue ->
?dl("SyncFun received continue ",[]),
ok
after timer:seconds(60) ->
?error("Timeout in sync_fun on ~p~n", [node()])
end
end,
Garb_handler = spawn_link(?MODULE, garb_handler, [[]]),
?remote_activate_debug_fun(DebugNode, Debug_Point, SyncFun, []),
?dl("fun_in_position activated at ~p with ~p", [DebugNode, Debug_Point]),
%% Spawn and do the transaction
Pid = spawn(Coord, ?MODULE, TransFun, [Tabs, Garb_handler]),
%% Wait till all the Nodes are in correct position
[{StoppedPid,_}] = ?receive_messages([fun_in_position]),
?dl("Received fun_in_position; Removing the debug funs ~p", [DebugNode]),
?remote_deactivate_debug_fun(DebugNode, Debug_Point),
case Kill of
coord_pid ->
?dl("Intentionally killing pid ~p ", [Pid]),
exit(Pid, normal);
Node ->
mnesia_test_lib:kill_mnesia([Node])
end,
StoppedPid ! continue, %% Send continue, it may still be alive
%% Start and check that the databases are consistent
?dl("Done, Restarting and verifying result ",[]),
case Kill of
coord_pid -> ok;
_ -> % Ok, mnesia on some node was killed restart it
timer:sleep(timer:seconds(3)), %% Just let it have the time to die
?match(ok, rpc:call(Kill, mnesia, start, [[]])),
?match(ok, rpc:call(Kill, mnesia, wait_for_tables, [Tabs, 60000]))
end,
Trans_res = verify_tabs(Tabs, Nodes),
case TransFun of
do_asym_trans ->
%% Verifies that decisions are garbed, only valid for asym_tran
Garb_handler ! {get_tids, self()},
Tid_list = receive
{tids, List} ->
?dl("Fun rec ~w", [List]),
List
end,
garb_of_decisions(Kill, Nodes, Tid_list, Trans_res);
_ ->
ignore
end,
?verify_mnesia(Nodes, []).
garb_of_decisions(Kill, Nodes, Tid_list, Trans_res) ->
[Coord, Part1, Part2] = Nodes,
%% Check that decision log is empty on all nodes after the trans is finished
verify_garb_decision_log(Nodes, Tid_list),
case Trans_res of
aborted ->
%% Check that aborted trans have not been restarted!!
?match(1, length(Tid_list)),
%% Check the transient decision logs
%% A transaction should only be aborted in an early stage of
%% the trans before the any Node have logged anything
verify_garb_transient_logs(Nodes, Tid_list, aborted),
%% And only when the coordinator are have died
%% Else he would have restarted the transaction
?match(Kill, Coord);
updated ->
case length(Tid_list) of
1 ->
%% If there was only one transaction, it should be logged as
%% comitted on every node!
[Tid1] = Tid_list,
verify_garb_transient_logs(Nodes, [Tid1], committed);
2 ->
%% If there is two transaction id, then the first
%% TID should have been aborted and the transaction
%% restarted with a new TID
[Tid1, Tid2] = Tid_list,
verify_garb_transient_logs(Nodes, [Tid1], aborted),
%% If mnesia is killed on a node i.e Coord and Part1 than they
%% won't know about the restarted trans! The rest of the nodes
%% should know that the trans was committed
case Kill of
coord_pid ->
verify_garb_transient_logs(Nodes, [Tid2], committed);
Coord ->
verify_garb_transient_logs([Part1, Part2], [Tid2], committed),
verify_garb_transient_logs([Coord], [Tid2], not_found);
Part1 ->
verify_garb_transient_logs([Coord, Part2], [Tid2], committed),
verify_garb_transient_logs([Part1], [Tid2], not_found)
end
end
end.
verify_garb_decision_log([], _Tids) -> ok;
verify_garb_decision_log([Node|R], Tids) ->
Check = fun(Tid) -> %% Node, Tid used in debugging!
?match({{not_found, _}, Node, Tid},
{outcome(Tid, [mnesia_decision]), Node, Tid})
end,
rpc:call(Node, lists, foreach, [Check, Tids]),
verify_garb_decision_log(R, Tids).
verify_garb_transient_logs([], _Tids, _) -> ok;
verify_garb_transient_logs([Node|R], Tids, Exp_Res) ->
Check = fun(Tid) ->
LatestTab = mnesia_lib:val(latest_transient_decision),
PrevTabs = mnesia_lib:val(previous_transient_decisions),
case outcome(Tid, [LatestTab | PrevTabs]) of
{found, {_, [{_,_Tid, Exp_Res}]}} -> ok;
{not_found, _} when Exp_Res == not_found -> ok;
{not_found, _} when Exp_Res == aborted -> ok;
Else -> ?error("Expected ~p in trans ~p on ~p got ~p~n",
[Exp_Res, Tid, Node, Else])
end
end,
rpc:call(Node, lists, foreach, [Check, Tids]),
verify_garb_transient_logs(R, Tids, Exp_Res).
outcome(Tid, Tabs) ->
outcome(Tid, Tabs, Tabs).
outcome(Tid, [Tab | Tabs], AllTabs) ->
case catch ets:lookup(Tab, Tid) of
{'EXIT', _} ->
outcome(Tid, Tabs, AllTabs);
[] ->
outcome(Tid, Tabs, AllTabs);
Val ->
{found, {Tab, Val}}
end;
outcome(_Tid, [], AllTabs) ->
{not_found, AllTabs}.
verify_tabs([Tab|R], Nodes) ->
[_Coord, Part1, Part2 | _rest] = Nodes,
Read = fun() -> mnesia:read({Tab, 1}) end,
{success, A} = ?match({atomic, _}, mnesia:transaction(Read)),
?match(A, rpc:call(Part1, mnesia, transaction, [Read])),
?match(A, rpc:call(Part2, mnesia, transaction, [Read])),
{atomic, [{Tab, 1, Res}]} = A,
verify_tabs(R, Nodes, Res).
verify_tabs([], _Nodes, Res) ->
case Res of
100 -> aborted;
101 -> updated
end;
verify_tabs([Tab | Rest], Nodes, Res) ->
[Coord, Part1, Part2 | _rest] = Nodes,
Read = fun() -> mnesia:read({Tab, 1}) end,
Exp = {atomic, [{Tab, 1, Res}]},
?match(Exp, rpc:call(Coord, mnesia, transaction, [Read])),
?match(Exp, rpc:call(Part1, mnesia, transaction, [Read])),
?match(Exp, rpc:call(Part2, mnesia, transaction, [Read])),
verify_tabs(Rest, Nodes, Res).
%% Gather TIDS and send them to requesting process and exit!
garb_handler(List) ->
receive
{trans_id, ID} -> garb_handler([ID|List]);
{get_tids, Pid} -> Pid ! {tids, lists:reverse(List)}
end.
%%%%%%%%%%%%%%%%%%%%%%%
receive_messages([], _File, _Line) -> [];
receive_messages(ListOfMsgs, File, Line) ->
receive
{Pid, Msg} ->
case lists:member(Msg, ListOfMsgs) of
false ->
mnesia_test_lib:log("<>WARNING<>~n"
"Received unexpected msg~n ~p ~n"
"While waiting for ~p~n",
[{Pid, Msg}, ListOfMsgs], File, Line),
receive_messages(ListOfMsgs, File, Line);
true ->
?dl("Got msg ~p from ~p ", [Msg, node(Pid)]),
[{Pid, Msg} | receive_messages(ListOfMsgs -- [Msg], File, Line)]
end;
Else -> mnesia_test_lib:log("<>WARNING<>~n"
"Recevied unexpected or bad formatted msg~n ~p ~n"
"While waiting for ~p~n",
[Else, ListOfMsgs], File, Line),
receive_messages(ListOfMsgs, File, Line)
after timer:minutes(2) ->
?error("Timeout in receive msgs while waiting for ~p~n",
[ListOfMsgs])
end.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
after_full_disc_partition(doc) ->
["Verify that the database does not get corrupt",
"when Mnesia encounters a full disc partition"].
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% interrupted_fallback_start
%% is implemented in consistency interupted_install_fallback!
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
after_corrupt_files(doc) ->
["Verify that mnesia (and dets) can handle corrupt files"];
after_corrupt_files(suite) -> % cope with unsynced disks
[after_corrupt_files_decision_log_head,
after_corrupt_files_decision_log_tail,
after_corrupt_files_latest_log_head,
after_corrupt_files_latest_log_tail,
after_corrupt_files_table_dat_head,
after_corrupt_files_table_dat_tail,
after_corrupt_files_schema_dat_head,
after_corrupt_files_schema_dat_tail
].
after_corrupt_files_decision_log_head(suite) -> [];
after_corrupt_files_decision_log_head(Config) when is_list(Config) ->
after_corrupt_files(Config, "DECISION.LOG", head, repair).
after_corrupt_files_decision_log_tail(suite) -> [];
after_corrupt_files_decision_log_tail(Config) when is_list(Config) ->
after_corrupt_files(Config, "DECISION.LOG", tail, repair).
after_corrupt_files_latest_log_head(suite) -> [];
after_corrupt_files_latest_log_head(Config) when is_list(Config) ->
after_corrupt_files(Config, "LATEST.LOG", head, repair).
after_corrupt_files_latest_log_tail(suite) -> [];
after_corrupt_files_latest_log_tail(Config) when is_list(Config) ->
after_corrupt_files(Config, "LATEST.LOG", tail, repair).
after_corrupt_files_table_dat_head(suite) -> [];
after_corrupt_files_table_dat_head(Config) when is_list(Config) ->
after_corrupt_files(Config, "rec_files.DAT", head, crash).
after_corrupt_files_table_dat_tail(suite) -> [];
after_corrupt_files_table_dat_tail(Config) when is_list(Config) ->
after_corrupt_files(Config, "rec_files.DAT", tail, repair).
after_corrupt_files_schema_dat_head(suite) -> [];
after_corrupt_files_schema_dat_head(Config) when is_list(Config) ->
after_corrupt_files(Config, "schema.DAT", head, crash).
after_corrupt_files_schema_dat_tail(suite) -> [];
after_corrupt_files_schema_dat_tail(Config) when is_list(Config) ->
after_corrupt_files(Config, "schema.DAT", tail, crash).
%%% BUGBUG: We should also write testcase's for autorepair=false i.e.
%%% not the standard case!
after_corrupt_files(Config, File, Where, Behaviour) ->
[Node] = ?acquire_nodes(1, Config ++ [{tc_timeout, timer:minutes(2)}]),
Tab = rec_files,
Def = [{disc_only_copies, [Node]}],
?match({atomic, ok}, mnesia:create_table(Tab, Def)),
insert_data(Tab, 100),
Dir = mnesia:system_info(directory),
mnesia_test_lib:kill_mnesia([Node]),
timer:sleep(timer:seconds(10)), % Let dets finish whatever it does
DirFile = Dir ++ "/" ++ File,
{ok, Fd} = file:open(DirFile, read_write),
{ok, FileInfo} = file:read_file_info(DirFile),
case Where of
head ->
?match({ok, _NewP}, file:position(Fd, {bof, 1})),
?match(ok, file:write(Fd, [255, 255, 255, 255, 255, 255, 255, 255, 254])),
ok;
tail ->
Size = FileInfo#file_info.size,
Half = Size div 2,
?dl(" Size = ~p Half = ~p ", [Size, Half]),
?match({ok, _NewP}, file:position(Fd, {bof, Half})),
?match(ok, file:truncate(Fd)),
ok
end,
?match(ok, file:close(Fd)),
?warning("++++++SOME OF THE after_corrupt* TEST CASES WILL INTENTIONALLY CRASH MNESIA+++++++~n", []),
Pid = spawn_link(?MODULE, mymnesia_start, [self()]),
receive
{Pid, ok} ->
?match(ok, mnesia:wait_for_tables([schema, Tab], 10000)),
?match(ok, verify_data(Tab, 100)),
case mnesia_monitor:get_env(auto_repair) of
false ->
?error("Mnesia should have crashed in ~p ~p ~n",
[File, Where]);
true ->
ok
end,
?verify_mnesia([Node], []);
{Pid, {error, ED}} ->
case {mnesia_monitor:get_env(auto_repair), Behaviour} of
{true, repair} ->
?error("Mnesia crashed with ~p: in ~p ~p ~n",
[ED, File, Where]);
_ -> %% Every other can crash!
ok
end,
?verify_mnesia([], [Node]);
Msg ->
?error("~p ~p: Got ~p during start of Mnesia~n",
[File, Where, Msg])
end.
mymnesia_start(Tester) ->
Res = mnesia:start(),
unlink(Tester),
Tester ! {self(), Res}.
verify_data(_, 0) -> ok;
verify_data(Tab, N) ->
Actual = mnesia:dirty_read({Tab, N}),
Expected = [{Tab, N, N}],
if
Expected == Actual ->
verify_data(Tab, N - 1);
true ->
mnesia:schema(Tab),
{not_equal, node(), Expected, Actual}
end.
insert_data(_Tab, 0) -> ok;
insert_data(Tab, N) ->
ok = mnesia:sync_dirty(fun() -> mnesia:write({Tab, N, N}) end),
insert_data(Tab, N-1).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
disc_less(doc) ->
["Here is a simple test case of a simple recovery of a disc less node. "
"However a lot more test cases involving disc less nodes should "
"be written"];
disc_less(suite) -> [];
disc_less(Config) when is_list(Config) ->
[Node1, Node2, Node3] = Nodes = ?acquire_nodes(3, Config),
case mnesia_test_lib:diskless(Config) of
true -> skip;
false ->
?match({atomic, ok}, mnesia:change_table_copy_type(schema, Node3, ram_copies))
end,
Tab1 = disc_less1,
Tab2 = disc_less2,
Tab3 = disc_less3,
Def1 = [{ram_copies, [Node3]}, {disc_copies, [Node1, Node2]}],
Def2 = [{ram_copies, [Node3]}, {disc_copies, [Node1]}],
Def3 = [{ram_copies, [Node3]}, {disc_copies, [Node2]}],
?match({atomic, ok}, mnesia:create_table(Tab1, Def1)),
?match({atomic, ok}, mnesia:create_table(Tab2, Def2)),
?match({atomic, ok}, mnesia:create_table(Tab3, Def3)),
insert_data(Tab1, 100),
insert_data(Tab2, 100),
insert_data(Tab3, 100),
mnesia_test_lib:kill_mnesia([Node1, Node2]),
timer:sleep(500),
mnesia_test_lib:kill_mnesia([Node3]),
?match(ok, rpc:call(Node1, mnesia, start, [])),
?match(ok, rpc:call(Node2, mnesia, start, [])),
timer:sleep(500),
?match(ok, rpc:call(Node3, mnesia, start, [[{extra_db_nodes, [Node1, Node2]}]])),
?match(ok, rpc:call(Node3, mnesia, wait_for_tables, [[Tab1, Tab2, Tab3], 20000])),
?match(ok, rpc:call(Node3, ?MODULE, verify_data, [Tab1, 100])),
?match(ok, rpc:call(Node3, ?MODULE, verify_data, [Tab2, 100])),
?match(ok, rpc:call(Node3, ?MODULE, verify_data, [Tab3, 100])),
?match(ok, rpc:call(Node2, ?MODULE, verify_data, [Tab1, 100])),
?match(ok, rpc:call(Node2, ?MODULE, verify_data, [Tab2, 100])),
?match(ok, rpc:call(Node2, ?MODULE, verify_data, [Tab3, 100])),
?match(ok, rpc:call(Node1, ?MODULE, verify_data, [Tab1, 100])),
?match(ok, rpc:call(Node1, ?MODULE, verify_data, [Tab2, 100])),
?match(ok, rpc:call(Node1, ?MODULE, verify_data, [Tab3, 100])),
?verify_mnesia(Nodes, []).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
system_upgrade(doc) ->
["Test on-line and off-line upgrade of the Mnesia application"].
garb_decision(doc) ->
["Test that decisions are garbed correctly."];
garb_decision(suite) -> [];
garb_decision(Config) when is_list(Config) ->
[Node1, Node2, Node3] = Nodes = ?acquire_nodes(3, Config),
check_garb(Nodes),
?match({atomic, ok},mnesia:create_table(a, [{disc_copies, Nodes}])),
check_garb(Nodes),
?match({atomic, ok},mnesia:create_table(b, [{ram_copies, Nodes}])),
check_garb(Nodes),
?match({atomic, ok},mnesia:create_table(c, [{ram_copies, [Node1, Node3]},
{disc_copies, [Node2]}])),
check_garb(Nodes),
?match({atomic, ok},mnesia:create_table(d, [{disc_copies, [Node1, Node3]},
{ram_copies, [Node2]}])),
check_garb(Nodes),
W = fun(Tab) -> mnesia:write({Tab,1,1}) end,
A = fun(Tab) -> mnesia:write({Tab,1,1}), exit(1) end,
?match({atomic, ok}, mnesia:transaction(W,[a])),
check_garb(Nodes),
?match({atomic, ok}, mnesia:transaction(W,[b])),
check_garb(Nodes),
?match({atomic, ok}, mnesia:transaction(W,[c])),
check_garb(Nodes),
?match({atomic, ok}, mnesia:transaction(W,[d])),
check_garb(Nodes),
?match({aborted,1}, mnesia:transaction(A,[a])),
check_garb(Nodes),
?match({aborted,1}, mnesia:transaction(A,[b])),
check_garb(Nodes),
?match({aborted,1}, mnesia:transaction(A,[c])),
check_garb(Nodes),
?match({aborted,1}, mnesia:transaction(A,[d])),
check_garb(Nodes),
rpc:call(Node2, mnesia, lkill, []),
?match({atomic, ok}, mnesia:transaction(W,[a])),
?match({atomic, ok}, mnesia:transaction(W,[b])),
?match({atomic, ok}, mnesia:transaction(W,[c])),
?match({atomic, ok}, mnesia:transaction(W,[d])),
check_garb(Nodes),
?match([], mnesia_test_lib:start_mnesia([Node2])),
check_garb(Nodes),
timer:sleep(2000),
check_garb(Nodes),
%%%%%% Check transient_decision logs %%%%%
?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync
[{atomic, ok} = mnesia:transaction(W,[a]) || _ <- lists:seq(1,30)],
?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync
TD0 = mnesia_lib:val(latest_transient_decision),
?match(0, ets:info(TD0, size)),
{atomic, ok} = mnesia:transaction(W,[a]),
?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync
?match(TD0, mnesia_lib:val(latest_transient_decision)),
[{atomic, ok} = mnesia:transaction(W,[a]) || _ <- lists:seq(1,30)],
?match(dumped, mnesia:dump_log()), sys:get_status(mnesia_recover), % sync
?match(false, TD0 =:= mnesia_lib:val(latest_transient_decision)),
?match(true, lists:member(TD0, mnesia_lib:val(previous_transient_decisions))),
?verify_mnesia(Nodes, []).
check_garb(Nodes) ->
rpc:multicall(Nodes, sys, get_status, [mnesia_recover]),
?match({_, []},rpc:multicall(Nodes, erlang, apply, [fun check_garb/0, []])).
check_garb() ->
try
Ds = ets:tab2list(mnesia_decision),
Check = fun({trans_tid,serial, _}) -> false;
({mnesia_down,_,_,_}) -> false;
(_Else) -> true
end,
Node = node(),
?match({Node, []}, {node(), lists:filter(Check, Ds)})
catch _:_ -> ok
end,
ok.
