Release mnesia tests

1 files changed, 787 insertions, 0 deletions

diff --git a/lib/mnesia/test/mnesia_schema_recovery_test.erl b/lib/mnesia/test/mnesia_schema_recovery_test.erl
new file mode 100644
index 0000000000..387238ae6b
--- /dev/null
+++ b/lib/mnesia/test/mnesia_schema_recovery_test.erl
@@ -0,0 +1,787 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 1998-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%
+-module(mnesia_schema_recovery_test).
+-author('[email protected]').
+-compile([export_all]).
+-include("mnesia_test_lib.hrl").
+
+init_per_testcase(Func, Conf) ->
+    mnesia_test_lib:init_per_testcase(Func, Conf).
+
+fin_per_testcase(Func, Conf) ->
+    mnesia_test_lib:fin_per_testcase(Func, Conf).
+
+-define(receive_messages(Msgs), receive_messages(Msgs, ?FILE, ?LINE)).
+
+% First Some debug logging 
+-define(dgb, true).
+-ifdef(dgb).
+-define(dl(X, Y), ?verbose("**TRACING: " ++ X ++ "**~n", Y)).
+-else. 
+-define(dl(X, Y), ok).
+-endif.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+all(doc) ->
+    ["Verify recoverabiliy of schema transactions.",
+     " Verify that a schema transaction",
+     " can be completed when it has been logged correctly and Mnesia",
+     " crashed before the log has been dumped. Then the transaction ",
+     " should be handled during the log dump at startup"
+    ];
+all(suite) ->
+    [interrupted_before_log_dump,
+     interrupted_after_log_dump].
+
+interrupted_before_log_dump(suite) ->
+    [interrupted_before_create_ram,
+     interrupted_before_create_disc,
+     interrupted_before_create_disc_only,
+     interrupted_before_create_nostore,
+     interrupted_before_delete_ram,
+     interrupted_before_delete_disc,
+     interrupted_before_delete_disc_only,
+     interrupted_before_add_ram,
+     interrupted_before_add_disc,
+     interrupted_before_add_disc_only,
+     interrupted_before_add_kill_copier,
+     interrupted_before_move_ram,
+     interrupted_before_move_disc,
+     interrupted_before_move_disc_only,
+     interrupted_before_move_kill_copier,
+     interrupted_before_delcopy_ram,
+     interrupted_before_delcopy_disc,
+     interrupted_before_delcopy_disc_only,
+     interrupted_before_delcopy_kill_copier,
+     interrupted_before_addindex_ram,
+     interrupted_before_addindex_disc,
+     interrupted_before_addindex_disc_only,
+     interrupted_before_delindex_ram,
+     interrupted_before_delindex_disc,
+     interrupted_before_delindex_disc_only,
+     interrupted_before_change_type_ram2disc,
+     interrupted_before_change_type_ram2disc_only,
+     interrupted_before_change_type_disc2ram,
+     interrupted_before_change_type_disc2disc_only,
+     interrupted_before_change_type_disc_only2ram,
+     interrupted_before_change_type_disc_only2disc,
+     interrupted_before_change_type_other_node,
+     interrupted_before_change_schema_type    %% Change schema table copy type!!
+    ].
+
+interrupted_after_log_dump(suite) ->
+    [interrupted_after_create_ram,
+     interrupted_after_create_disc,
+     interrupted_after_create_disc_only,
+     interrupted_after_create_nostore,
+     interrupted_after_delete_ram,
+     interrupted_after_delete_disc,
+     interrupted_after_delete_disc_only,
+     interrupted_after_add_ram,
+     interrupted_after_add_disc,
+     interrupted_after_add_disc_only,
+     interrupted_after_add_kill_copier,
+     interrupted_after_move_ram,
+     interrupted_after_move_disc,
+     interrupted_after_move_disc_only,
+     interrupted_after_move_kill_copier,
+     interrupted_after_delcopy_ram,
+     interrupted_after_delcopy_disc,
+     interrupted_after_delcopy_disc_only,
+     interrupted_after_delcopy_kill_copier,
+     interrupted_after_addindex_ram,
+     interrupted_after_addindex_disc,
+     interrupted_after_addindex_disc_only,
+     interrupted_after_delindex_ram,
+     interrupted_after_delindex_disc,
+     interrupted_after_delindex_disc_only,
+     interrupted_after_change_type_ram2disc,
+     interrupted_after_change_type_ram2disc_only,
+     interrupted_after_change_type_disc2ram,
+     interrupted_after_change_type_disc2disc_only,
+     interrupted_after_change_type_disc_only2ram,
+     interrupted_after_change_type_disc_only2disc,
+     interrupted_after_change_type_other_node,
+     interrupted_after_change_schema_type     %% Change schema table copy type!!
+
+%     interrupted_before_change_access_mode,
+%     interrupted_before_transform,
+%     interrupted_before_restore,
+    ].
+
+interrupted_before_create_ram(suite) -> [];
+interrupted_before_create_ram(Config) when is_list(Config) ->
+    KillAt = {mnesia_dumper, dump_schema_op},
+    interrupted_create(Config, ram_copies, all, KillAt).
+
+interrupted_before_create_disc(suite) -> [];
+interrupted_before_create_disc(Config) when is_list(Config) ->
+    KillAt = {mnesia_dumper, dump_schema_op},
+    interrupted_create(Config, disc_copies, all, KillAt).
+
+interrupted_before_create_disc_only(suite) -> [];
+interrupted_before_create_disc_only(Config) when is_list(Config) ->
+    KillAt = {mnesia_dumper, dump_schema_op},
+    interrupted_create(Config, disc_only_copies, all, KillAt).
+
+interrupted_before_create_nostore(suite) -> [];
+interrupted_before_create_nostore(Config) when is_list(Config) ->
+    KillAt = {mnesia_dumper, dump_schema_op},
+    interrupted_create(Config, ram_copies, one, KillAt).
+
+interrupted_after_create_ram(suite) -> [];
+interrupted_after_create_ram(Config) when is_list(Config) ->
+    KillAt = {mnesia_dumper, post_dump},
+    interrupted_create(Config, ram_copies, all, KillAt).
+
+interrupted_after_create_disc(suite) -> [];
+interrupted_after_create_disc(Config) when is_list(Config) ->
+    KillAt = {mnesia_dumper, post_dump},
+    interrupted_create(Config, disc_copies, all, KillAt).
+
+interrupted_after_create_disc_only(suite) -> [];
+interrupted_after_create_disc_only(Config) when is_list(Config) ->
+    KillAt = {mnesia_dumper, post_dump},
+    interrupted_create(Config, disc_only_copies, all, KillAt).
+
+interrupted_after_create_nostore(suite) -> [];
+interrupted_after_create_nostore(Config) when is_list(Config) ->
+    KillAt = {mnesia_dumper, post_dump},
+    interrupted_create(Config, ram_copies, one, KillAt).
+
+%%% After dump don't need debug point
+interrupted_create(Config, Type, _Where, {mnesia_dumper, post_dump}) ->
+    [Node1] = Nodes = ?acquire_nodes(1, [{tc_timeout, timer:seconds(30)} | Config]),
+    ?match({atomic, ok},mnesia:create_table(itrpt, [{Type, Nodes}])),
+    ?match({atomic, ok},mnesia:create_table(test, [{disc_copies,[Node1]}])),
+    ?match(ok, mnesia:dirty_write({itrpt, before, 1})),
+    ?match(ok, mnesia:dirty_write({test, found_in_log, 1})),
+    ?match(stopped, mnesia:stop()),
+    ?match([], mnesia_test_lib:start_mnesia([Node1], [itrpt,test])),  
+    %% Verify 
+    ?match([{test, found_in_log, 1}], mnesia:dirty_read({test, found_in_log})),
+    case Type of
+	ram_copies -> 	      
+	    ?match([], mnesia:dirty_read({itrpt, before}));
+	_ ->
+	    ?match([{itrpt, before, 1}], mnesia:dirty_read({itrpt, before}))
+    end,
+    ?verify_mnesia(Nodes, []);
+interrupted_create(Config, Type, Where, KillAt) ->
+    ?is_debug_compiled,
+    [Node1, Node2] = Nodes = ?acquire_nodes(2, [{tc_timeout, timer:seconds(30)} | Config]),
+    {success, [A]} = ?start_activities([Node2]),
+    setup_dbgpoint(KillAt, Node2),
+
+    if       %% CREATE TABLE
+	Where == all ->    % tables on both nodes
+	    A ! fun() -> mnesia:create_table(itrpt, [{Type, Nodes}]) end;
+	true ->            % no table on the killed node
+	    A ! fun() -> mnesia:create_table(itrpt, [{Type, [Node1]}]) end
+    end,
+    
+    kill_at_debug(),
+    ?match([], mnesia_test_lib:start_mnesia([Node2], [itrpt])),  
+    %% Verify 
+    ?match(ok, mnesia:dirty_write({itrpt, before, 1})),
+    verify_tab(Node1, Node2),
+    ?verify_mnesia(Nodes, []).
+
+interrupted_before_delete_ram(suite) -> [];
+interrupted_before_delete_ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_delete(Config, ram_copies, Debug_Point).
+interrupted_before_delete_disc(suite) -> [];
+interrupted_before_delete_disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_delete(Config, disc_copies, Debug_Point).
+interrupted_before_delete_disc_only(suite) -> [];
+interrupted_before_delete_disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_delete(Config, disc_only_copies, Debug_Point).
+
+interrupted_after_delete_ram(suite) -> [];
+interrupted_after_delete_ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_delete(Config, ram_copies, Debug_Point).
+interrupted_after_delete_disc(suite) -> [];
+interrupted_after_delete_disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_delete(Config, disc_copies, Debug_Point).
+interrupted_after_delete_disc_only(suite) -> [];
+interrupted_after_delete_disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_delete(Config, disc_only_copies, Debug_Point).
+
+interrupted_delete(Config, Type, KillAt) ->
+    ?is_debug_compiled,
+    [Node1, Node2] = Nodes = ?acquire_nodes(2, [{tc_timeout, timer:seconds(30)} | Config]),
+    Tab = itrpt,
+    ?match({atomic, ok}, mnesia:create_table(Tab, [{Type, [Node2]}])),
+    ?match(ok, mnesia:dirty_write({Tab, before, 1})),
+    {_Alive, Kill} = {Node1, Node2},
+    {success, [A]} = ?start_activities([Kill]),
+    
+    setup_dbgpoint(KillAt, Kill),
+    A ! fun() -> mnesia:delete_table(Tab) end,
+    
+    kill_at_debug(),
+    ?match([], mnesia_test_lib:start_mnesia([Node2], [])),
+    Bad = {badrpc, {'EXIT', {aborted,{no_exists, Tab, all}}}},
+    ?match(Bad, rpc:call(Node1, mnesia, table_info, [Tab, all])),
+    ?match(Bad, rpc:call(Node2, mnesia, table_info, [Tab, all])),
+    ?verify_mnesia(Nodes, []).
+
+interrupted_before_add_ram(suite) -> [];
+interrupted_before_add_ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_add(Config, ram_copies, kill_reciever, Debug_Point).
+interrupted_before_add_disc(suite) -> [];
+interrupted_before_add_disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_add(Config, disc_copies, kill_reciever, Debug_Point).
+interrupted_before_add_disc_only(suite) -> [];
+interrupted_before_add_disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_add(Config, disc_only_copies, kill_reciever, Debug_Point).
+interrupted_before_add_kill_copier(suite) -> [];
+interrupted_before_add_kill_copier(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_add(Config, ram_copies, kill_copier, Debug_Point).
+
+interrupted_after_add_ram(suite) -> [];
+interrupted_after_add_ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_add(Config, ram_copies, kill_reciever, Debug_Point).
+interrupted_after_add_disc(suite) -> [];
+interrupted_after_add_disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_add(Config, disc_copies, kill_reciever, Debug_Point).
+interrupted_after_add_disc_only(suite) -> [];
+interrupted_after_add_disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_add(Config, disc_only_copies, kill_reciever, Debug_Point).
+interrupted_after_add_kill_copier(suite) -> [];
+interrupted_after_add_kill_copier(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_add(Config, ram_copies, kill_copier, Debug_Point).
+
+%%% After dump don't need debug point
+interrupted_add(Config, Type, _Where, {mnesia_dumper, post_dump}) ->
+    [Node1, Node2] = Nodes =
+	?acquire_nodes(2, [{tc_timeout, timer:seconds(30)} | Config]),
+    Tab = itrpt,
+    ?match({atomic, ok}, mnesia:create_table(Tab, [{Type, [Node2]}, {local_content,true}])),
+    ?match({atomic, ok},mnesia:create_table(test, [{disc_copies,[Node1]}])),
+    ?match({atomic, ok}, mnesia:add_table_copy(Tab, Node1, Type)),
+    ?match(ok, mnesia:dirty_write({itrpt, before, 1})),
+    ?match(ok, mnesia:dirty_write({test, found_in_log, 1})),
+    ?match(stopped, mnesia:stop()),
+    ?match([], mnesia_test_lib:start_mnesia([Node1], [itrpt,test])),
+    %% Verify 
+    ?match([{test, found_in_log, 1}], mnesia:dirty_read({test, found_in_log})),
+    case Type of
+	ram_copies -> 	      
+	    ?match([], mnesia:dirty_read({itrpt, before}));
+	_ ->
+	    ?match([{itrpt, before, 1}], mnesia:dirty_read({itrpt, before}))
+    end,
+    ?verify_mnesia(Nodes, []);
+interrupted_add(Config, Type, Who, KillAt) ->
+    ?is_debug_compiled,
+    [Node1, Node2] = Nodes =
+	?acquire_nodes(2, [{tc_timeout, timer:seconds(30)} | Config]),
+    {_Alive, Kill} = 
+	if Who == kill_reciever -> 
+		{Node1, Node2};
+	   true -> 
+		{Node2, Node1}
+	end,    
+    {success, [A]} = ?start_activities([Kill]),
+    Tab = itrpt,
+    ?match({atomic, ok}, mnesia:create_table(Tab, [{Type, [Node1]}])),
+    ?match(ok, mnesia:dirty_write({Tab, before, 1})),
+    
+    setup_dbgpoint(KillAt, Kill),
+    
+    A ! fun() -> mnesia:add_table_copy(Tab, Node2, Type) end,
+    kill_at_debug(),
+    ?match([], mnesia_test_lib:start_mnesia([Kill], [itrpt])),    
+    verify_tab(Node1, Node2),
+    ?verify_mnesia(Nodes, []).
+
+interrupted_before_move_ram(suite) -> [];
+interrupted_before_move_ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_move(Config, ram_copies, kill_reciever, Debug_Point).
+interrupted_before_move_disc(suite) -> [];
+interrupted_before_move_disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_move(Config, disc_copies, kill_reciever, Debug_Point).
+interrupted_before_move_disc_only(suite) -> [];
+interrupted_before_move_disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_move(Config, disc_only_copies, kill_reciever, Debug_Point).
+interrupted_before_move_kill_copier(suite) -> [];
+interrupted_before_move_kill_copier(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_move(Config, ram_copies, kill_copier, Debug_Point).
+
+interrupted_after_move_ram(suite) -> [];
+interrupted_after_move_ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_move(Config, ram_copies, kill_reciever, Debug_Point).
+interrupted_after_move_disc(suite) -> [];
+interrupted_after_move_disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_move(Config, disc_copies, kill_reciever, Debug_Point).
+interrupted_after_move_disc_only(suite) -> [];
+interrupted_after_move_disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_move(Config, disc_only_copies, kill_reciever, Debug_Point).
+interrupted_after_move_kill_copier(suite) -> [];
+interrupted_after_move_kill_copier(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_move(Config, ram_copies, kill_copier, Debug_Point).
+
+%%% After dump don't need debug point
+interrupted_move(Config, Type, _Where, {mnesia_dumper, post_dump}) ->
+    [Node1, Node2] = Nodes =
+	?acquire_nodes(2, [{tc_timeout, timer:seconds(30)} | Config]),
+    Tab = itrpt,
+    ?match({atomic, ok},mnesia:create_table(test, [{disc_copies,[Node1]}])),
+    ?match({atomic, ok}, mnesia:create_table(Tab, [{Type, [Node1]}])),
+    ?match(ok, mnesia:dirty_write({itrpt, before, 1})),
+    ?match({atomic, ok}, mnesia:move_table_copy(Tab, Node1, Node2)),
+    ?match(ok, mnesia:dirty_write({itrpt, aFter, 1})),
+    ?match(ok, mnesia:dirty_write({test, found_in_log, 1})),
+    ?match(stopped, mnesia:stop()),
+    ?match([], mnesia_test_lib:start_mnesia([Node1], [itrpt,test])),
+    %% Verify 
+    ?match([{test, found_in_log, 1}], mnesia:dirty_read({test, found_in_log})),
+    ?match([{itrpt, before, 1}], mnesia:dirty_read({itrpt, before})),
+    ?match([{itrpt, aFter, 1}], mnesia:dirty_read({itrpt, aFter})),
+    ?verify_mnesia(Nodes, []);
+interrupted_move(Config, Type, Who, KillAt) ->
+    ?is_debug_compiled,
+    [Node1, Node2] = Nodes = 
+	?acquire_nodes(2, [{tc_timeout, timer:seconds(30)} | Config]),
+    Tab = itrpt,
+    ?match({atomic, ok}, mnesia:create_table(Tab, [{Type, [Node1]}])),
+    ?match(ok, mnesia:dirty_write({Tab, before, 1})),
+
+    {_Alive, Kill} = 
+	if Who == kill_reciever -> 
+		if Type == ram_copies -> 
+			{atomic, ok} = mnesia:dump_tables([Tab]);
+		   true -> 
+			ignore
+		end,
+		{Node1, Node2};
+	   true -> 
+		{Node2, Node1}
+	end,
+    
+    {success, [A]} = ?start_activities([Kill]),
+    
+    setup_dbgpoint(KillAt, Kill),
+    A ! fun() -> mnesia:move_table_copy(Tab, Node1, Node2) end,
+    kill_at_debug(),
+    ?match([], mnesia_test_lib:start_mnesia([Kill], [itrpt])),    
+    verify_tab(Node1, Node2),
+    ?verify_mnesia(Nodes, []).
+
+interrupted_before_delcopy_ram(suite) -> [];
+interrupted_before_delcopy_ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_delcopy(Config, ram_copies, kill_reciever, Debug_Point).
+interrupted_before_delcopy_disc(suite) -> [];
+interrupted_before_delcopy_disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_delcopy(Config, disc_copies, kill_reciever, Debug_Point).
+interrupted_before_delcopy_disc_only(suite) -> [];
+interrupted_before_delcopy_disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_delcopy(Config, disc_only_copies, kill_reciever, Debug_Point).
+interrupted_before_delcopy_kill_copier(suite) -> [];
+interrupted_before_delcopy_kill_copier(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_delcopy(Config, ram_copies, kill_copier, Debug_Point).
+
+interrupted_after_delcopy_ram(suite) -> [];
+interrupted_after_delcopy_ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_delcopy(Config, ram_copies, kill_reciever, Debug_Point).
+interrupted_after_delcopy_disc(suite) -> [];
+interrupted_after_delcopy_disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_delcopy(Config, disc_copies, kill_reciever, Debug_Point).
+interrupted_after_delcopy_disc_only(suite) -> [];
+interrupted_after_delcopy_disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_delcopy(Config, disc_only_copies, kill_reciever, Debug_Point).
+interrupted_after_delcopy_kill_copier(suite) -> [];
+interrupted_after_delcopy_kill_copier(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_delcopy(Config, ram_copies, kill_copier, Debug_Point).
+
+
+%%% After dump don't need debug point
+interrupted_delcopy(Config, Type, _Where, {mnesia_dumper, post_dump}) ->
+    [Node1, Node2] = Nodes =
+	?acquire_nodes(2, [{tc_timeout, timer:seconds(30)} | Config]),
+    Tab = itrpt,
+    ?match({atomic, ok},mnesia:create_table(test, [{disc_copies,[Node1]}])),
+    ?match({atomic, ok}, mnesia:create_table(Tab, [{Type, [Node1,Node2]}])),
+    ?match({atomic, ok}, mnesia:del_table_copy(Tab, Node1)),
+    ?match(ok, mnesia:dirty_write({test, found_in_log, 1})),
+    ?match(stopped, mnesia:stop()),
+    ?match([], mnesia_test_lib:start_mnesia([Node1], [test])),
+    %% Verify 
+    ?match([{test, found_in_log, 1}], mnesia:dirty_read({test, found_in_log})),
+    ?match([Node2], mnesia:table_info(itrpt,Type)),
+    ?verify_mnesia(Nodes, []);
+interrupted_delcopy(Config, Type, Who, KillAt) ->
+    ?is_debug_compiled,
+    [Node1, Node2] = Nodes = 
+	?acquire_nodes(2, [{tc_timeout, timer:seconds(30)} | Config]),
+    Tab = itrpt,
+    ?match({atomic, ok}, mnesia:create_table(Tab, [{Type, [Node1, Node2]}])),
+    ?match(ok, mnesia:dirty_write({Tab, before, 1})),
+
+    {_Alive, Kill} = 
+	if Who == kill_reciever -> 
+		{Node1, Node2};
+	   true -> 
+		if 
+		    Type == ram_copies -> 
+			{atomic, ok} = mnesia:dump_tables([Tab]);
+		    true -> 
+			ignore
+		end,
+		{Node2, Node1}
+	end,
+
+    {success, [A]} = ?start_activities([Kill]),
+    setup_dbgpoint(KillAt, Kill),
+    A ! fun() -> mnesia:del_table_copy(Tab, Node2) end,
+    kill_at_debug(),
+    ?match([], mnesia_test_lib:start_mnesia([Kill], [itrpt])),    
+    verify_tab(Node1, Node2),
+    ?verify_mnesia(Nodes, []).
+
+interrupted_before_addindex_ram(suite) -> [];
+interrupted_before_addindex_ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_addindex(Config, ram_copies, Debug_Point).
+interrupted_before_addindex_disc(suite) -> [];
+interrupted_before_addindex_disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_addindex(Config, disc_copies, Debug_Point).
+interrupted_before_addindex_disc_only(suite) -> [];
+interrupted_before_addindex_disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_addindex(Config, disc_only_copies, Debug_Point).
+
+interrupted_after_addindex_ram(suite) -> [];
+interrupted_after_addindex_ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},
+    interrupted_addindex(Config, ram_copies, Debug_Point).
+interrupted_after_addindex_disc(suite) -> [];
+interrupted_after_addindex_disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_addindex(Config, disc_copies, Debug_Point).
+interrupted_after_addindex_disc_only(suite) -> [];
+interrupted_after_addindex_disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_addindex(Config, disc_only_copies, Debug_Point).
+
+
+%%% After dump don't need debug point
+interrupted_addindex(Config, Type, {mnesia_dumper, post_dump}) ->
+    [Node1] = Nodes = ?acquire_nodes(1, [{tc_timeout, timer:seconds(30)} | Config]),
+    Tab = itrpt,
+    ?match({atomic,ok},mnesia:create_table(Tab, [{Type, Nodes}])),
+    ?match({atomic,ok},mnesia:create_table(test, [{disc_copies,[Node1]}])),
+    ?match({atomic,ok}, mnesia:add_table_index(Tab, val)),
+    ?match(ok, mnesia:dirty_write({itrpt, before, 1})),
+    ?match(ok, mnesia:dirty_write({test, found_in_log, 1})),
+    ?match(stopped, mnesia:stop()),
+    ?match([], mnesia_test_lib:start_mnesia([Node1], [itrpt,test])),  
+    %% Verify 
+    ?match([{test, found_in_log, 1}], mnesia:dirty_read({test, found_in_log})),
+    case Type of
+	ram_copies -> 	      
+	    ?match([], mnesia:dirty_index_read(itrpt, 1, val));
+	_ ->
+	    ?match([{itrpt, before, 1}], mnesia:dirty_index_read(itrpt, 1, val))
+    end,
+    ?verify_mnesia(Nodes, []);
+interrupted_addindex(Config, Type, KillAt) ->
+    ?is_debug_compiled,
+    [Node1, Node2] = Nodes = ?acquire_nodes(2, [{tc_timeout, timer:seconds(30)} | Config]),
+    Tab = itrpt,
+    ?match({atomic, ok}, mnesia:create_table(Tab, [{Type, [Node1]}])),
+    ?match(ok, mnesia:dirty_write({Tab, before, 1})),
+    {_Alive, Kill} = {Node1, Node2},
+    {success, [A]} = ?start_activities([Kill]),
+    
+    setup_dbgpoint(KillAt, Kill),
+    A ! fun() -> mnesia:add_table_index(Tab, val) end,
+    kill_at_debug(),
+    ?match([], mnesia_test_lib:start_mnesia([Node2], [])),
+
+    verify_tab(Node1, Node2),
+    ?match([{Tab, b, a}, {Tab, a, a}], 
+	   rpc:call(Node1, mnesia, dirty_index_read, [itrpt, a, val])),
+    ?match([{Tab, b, a}, {Tab, a, a}], 
+	   rpc:call(Node2, mnesia, dirty_index_read, [itrpt, a, val])),
+    ?verify_mnesia(Nodes, []).
+
+interrupted_before_delindex_ram(suite) -> [];
+interrupted_before_delindex_ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_delindex(Config, ram_copies, Debug_Point).
+interrupted_before_delindex_disc(suite) -> [];
+interrupted_before_delindex_disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_delindex(Config, disc_copies, Debug_Point).
+interrupted_before_delindex_disc_only(suite) -> [];
+interrupted_before_delindex_disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_delindex(Config, disc_only_copies, Debug_Point).
+
+interrupted_after_delindex_ram(suite) -> [];
+interrupted_after_delindex_ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_delindex(Config, ram_copies, Debug_Point).
+interrupted_after_delindex_disc(suite) -> [];
+interrupted_after_delindex_disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_delindex(Config, disc_copies, Debug_Point).
+interrupted_after_delindex_disc_only(suite) -> [];
+interrupted_after_delindex_disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_delindex(Config, disc_only_copies, Debug_Point).
+
+%%% After dump don't need debug point
+interrupted_delindex(Config, Type, {mnesia_dumper, post_dump}) ->
+    [Node1] = Nodes = ?acquire_nodes(1, [{tc_timeout, timer:seconds(30)} | Config]),
+    Tab = itrpt,
+    ?match({atomic,ok},mnesia:create_table(Tab, [{Type, Nodes},{index,[val]}])),
+    ?match({atomic,ok},mnesia:create_table(test, [{disc_copies,[Node1]}])),
+    ?match({atomic,ok}, mnesia:del_table_index(Tab, val)),
+    ?match(ok, mnesia:dirty_write({itrpt, before, 1})),
+    ?match(ok, mnesia:dirty_write({test, found_in_log, 1})),
+    ?match(stopped, mnesia:stop()),
+    ?match([], mnesia_test_lib:start_mnesia([Node1], [itrpt,test])),  
+    %% Verify 
+    ?match([{test, found_in_log, 1}], mnesia:dirty_read({test, found_in_log})),
+    ?match({'EXIT',{aborted,{badarg,_}}}, mnesia:dirty_index_read(itrpt, 1, val)),
+    ?verify_mnesia(Nodes, []);
+
+interrupted_delindex(Config, Type, KillAt) ->
+    ?is_debug_compiled,
+    [Node1, Node2] = Nodes = ?acquire_nodes(2, [{tc_timeout, timer:seconds(30)} | Config]),
+    Tab = itrpt,
+    ?match({atomic, ok}, mnesia:create_table(Tab, [{index, [val]}, 
+						   {Type, [Node1]}])),
+    ?match(ok, mnesia:dirty_write({Tab, before, 1})),
+    {_Alive, Kill} = {Node1, Node2},
+    {success, [A]} = ?start_activities([Kill]),
+    setup_dbgpoint(KillAt, Kill),
+    A ! fun() -> mnesia:del_table_index(Tab, val) end,
+    kill_at_debug(),
+    ?match([], mnesia_test_lib:start_mnesia([Node2], [])),  
+    verify_tab(Node1, Node2),
+    ?match({badrpc, _}, rpc:call(Node1, mnesia, dirty_index_read, [itrpt, a, val])),
+    ?match({badrpc, _}, rpc:call(Node2, mnesia, dirty_index_read, [itrpt, a, val])),
+    ?match([], rpc:call(Node1, mnesia, table_info, [Tab, index])),
+    ?match([], rpc:call(Node2, mnesia, table_info, [Tab, index])),
+    ?verify_mnesia(Nodes, []).
+
+interrupted_before_change_type_ram2disc(suite) -> [];
+interrupted_before_change_type_ram2disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_change_type(Config, ram_copies, disc_copies, changer, Debug_Point).
+interrupted_before_change_type_ram2disc_only(suite) -> [];
+interrupted_before_change_type_ram2disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_change_type(Config, ram_copies, disc_only_copies, changer, Debug_Point).    
+interrupted_before_change_type_disc2ram(suite) -> [];
+interrupted_before_change_type_disc2ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_change_type(Config, disc_copies, ram_copies, changer, Debug_Point).
+interrupted_before_change_type_disc2disc_only(suite) -> [];
+interrupted_before_change_type_disc2disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_change_type(Config, disc_copies, disc_only_copies, changer, Debug_Point).
+interrupted_before_change_type_disc_only2ram(suite) -> [];
+interrupted_before_change_type_disc_only2ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_change_type(Config, disc_only_copies, ram_copies, changer, Debug_Point).
+interrupted_before_change_type_disc_only2disc(suite) -> [];
+interrupted_before_change_type_disc_only2disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_change_type(Config, disc_only_copies, disc_copies, changer, Debug_Point).
+interrupted_before_change_type_other_node(suite) -> [];
+interrupted_before_change_type_other_node(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, dump_schema_op},    
+    interrupted_change_type(Config, ram_copies, disc_copies, the_other_one, Debug_Point).
+
+interrupted_after_change_type_ram2disc(suite) -> [];
+interrupted_after_change_type_ram2disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_change_type(Config, ram_copies, disc_copies, changer, Debug_Point).
+interrupted_after_change_type_ram2disc_only(suite) -> [];
+interrupted_after_change_type_ram2disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_change_type(Config, ram_copies, disc_only_copies, changer, Debug_Point).    
+interrupted_after_change_type_disc2ram(suite) -> [];
+interrupted_after_change_type_disc2ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_change_type(Config, disc_copies, ram_copies, changer, Debug_Point).
+interrupted_after_change_type_disc2disc_only(suite) -> [];
+interrupted_after_change_type_disc2disc_only(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_change_type(Config, disc_copies, disc_only_copies, changer, Debug_Point).
+interrupted_after_change_type_disc_only2ram(suite) -> [];
+interrupted_after_change_type_disc_only2ram(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_change_type(Config, disc_only_copies, ram_copies, changer, Debug_Point).
+interrupted_after_change_type_disc_only2disc(suite) -> [];
+interrupted_after_change_type_disc_only2disc(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_change_type(Config, disc_only_copies, disc_copies, changer, Debug_Point).
+interrupted_after_change_type_other_node(suite) -> [];
+interrupted_after_change_type_other_node(Config) when is_list(Config) ->
+    Debug_Point = {mnesia_dumper, post_dump},    
+    interrupted_change_type(Config, ram_copies, disc_copies, the_other_one, Debug_Point).
+
+interrupted_change_type(Config, FromType, ToType, Who, KillAt) ->
+    ?is_debug_compiled,
+    [Node1, Node2] = Nodes = ?acquire_nodes(2, [{tc_timeout, timer:seconds(30)} | Config]),
+    Tab = itrpt,
+    ?match({atomic, ok}, mnesia:create_table(Tab, [{FromType, [Node2, Node1]}])),
+    ?match(ok, mnesia:dirty_write({Tab, before, 1})),
+
+    {_Alive, Kill} = 
+	if Who == changer -> {Node1, Node2};
+	   true ->           {Node2, Node1}
+	end,
+    
+    {success, [A]} = ?start_activities([Kill]),
+    setup_dbgpoint(KillAt, Kill),
+    A ! fun() -> mnesia:change_table_copy_type(Tab, Node2, ToType) end,
+    kill_at_debug(),
+    ?match([], mnesia_test_lib:start_mnesia(Nodes, [itrpt])),        
+    verify_tab(Node1, Node2),
+    ?match(FromType, rpc:call(Node1, mnesia, table_info, [Tab, storage_type])),
+    ?match(ToType, rpc:call(Node2, mnesia, table_info, [Tab, storage_type])),
+    ?verify_mnesia(Nodes, []).
+
+interrupted_before_change_schema_type(suite) ->     [];
+interrupted_before_change_schema_type(Config) when is_list(Config) ->
+    KillAt = {mnesia_dumper, dump_schema_op},
+    interrupted_change_schema_type(Config, KillAt).
+
+interrupted_after_change_schema_type(suite) ->     [];
+interrupted_after_change_schema_type(Config) when is_list(Config) ->
+    KillAt = {mnesia_dumper, post_dump},
+    interrupted_change_schema_type(Config, KillAt).
+
+-define(cleanup(N, Config),
+	mnesia_test_lib:prepare_test_case([{reload_appls, [mnesia]}],
+					  N, Config, ?FILE, ?LINE)).
+    
+interrupted_change_schema_type(Config, KillAt) ->
+    ?is_debug_compiled,
+    [Node1, Node2] = Nodes = ?acquire_nodes(2, [{tc_timeout, timer:seconds(30)} | Config]),
+
+    Tab = itrpt,
+    ?match({atomic, ok}, mnesia:create_table(Tab, [{ram_copies, [Node2, Node1]}])),
+    ?match(ok, mnesia:dirty_write({Tab, before, 1})),
+    
+    {success, [A]} = ?start_activities([Node2]),
+    setup_dbgpoint(KillAt, Node2),	
+    
+    A ! fun() -> mnesia:change_table_copy_type(schema, Node2, ram_copies) end,
+    kill_at_debug(),    
+    ?match(ok, rpc:call(Node2, mnesia, start, [[{extra_db_nodes, [Node1, Node2]}]])),
+    ?match(ok, rpc:call(Node2, mnesia, wait_for_tables, [[itrpt, schema], 2000])),
+    ?match(disc_copies, rpc:call(Node1, mnesia, table_info, [schema, storage_type])),
+    ?match(ram_copies, rpc:call(Node2, mnesia, table_info,  [schema, storage_type])),
+    
+    %% Go back to disc_copies !!    
+    {success, [B]} = ?start_activities([Node2]),
+    setup_dbgpoint(KillAt, Node2),	
+    B ! fun() -> mnesia:change_table_copy_type(schema, Node2, disc_copies) end,
+    kill_at_debug(),
+
+    ?match(ok, rpc:call(Node2, mnesia, start, [[{extra_db_nodes, [Node1, Node2]}]])),
+    ?match(ok, rpc:call(Node2, mnesia, wait_for_tables, [[itrpt, schema], 2000])),
+    ?match(disc_copies, rpc:call(Node1, mnesia, table_info, [schema, storage_type])),
+    ?match(disc_copies, rpc:call(Node2, mnesia, table_info,  [schema, storage_type])),
+
+    ?verify_mnesia(Nodes, []),
+    ?cleanup(2, Config).
+
+%%% Helpers
+verify_tab(Node1, Node2) ->
+    ?match({atomic, ok}, 
+	   rpc:call(Node1, mnesia, transaction, [fun() -> mnesia:dirty_write({itrpt, a, a}) end])),
+    ?match({atomic, ok},
+	   rpc:call(Node2, mnesia, transaction, [fun() -> mnesia:dirty_write({itrpt, b, a}) end])),
+    ?match([{itrpt,a,a}], rpc:call(Node1, mnesia, dirty_read, [{itrpt, a}])), 
+    ?match([{itrpt,a,a}], rpc:call(Node2, mnesia, dirty_read, [{itrpt, a}])),
+    ?match([{itrpt,b,a}], rpc:call(Node1, mnesia, dirty_read, [{itrpt, b}])), 
+    ?match([{itrpt,b,a}], rpc:call(Node2, mnesia, dirty_read, [{itrpt, b}])),
+    ?match([{itrpt,before,1}], rpc:call(Node1, mnesia, dirty_read, [{itrpt, before}])),
+    ?match([{itrpt,before,1}], rpc:call(Node2, mnesia, dirty_read, [{itrpt, before}])).
+
+setup_dbgpoint(DbgPoint, Where) -> 
+    Self = self(),
+    TestFun = fun(_, [InitBy]) ->
+		      case InitBy of
+			  schema_prepare ->
+			      ignore;
+			  schema_begin ->
+			      ignore;
+			  _Other ->
+			      ?deactivate_debug_fun(DbgPoint),
+			      unlink(Self),
+			      Self ! {fun_done, node()},
+			      timer:sleep(infinity)
+		      end
+	      end, 
+    %% Kill when debug has been reached
+    ?remote_activate_debug_fun(Where, DbgPoint, TestFun, []).
+
+kill_at_debug() ->   
+    %% Wait till it's killed
+    receive 
+	{fun_done, Node} -> 
+	    ?match([], mnesia_test_lib:kill_mnesia([Node]))
+    after 
+	timer:minutes(1) -> ?error("Timeout in kill_at_debug", [])
+    end.
+