mnesia_ext: Backup handling

2 files changed, 175 insertions, 89 deletions

diff --git a/lib/mnesia/src/mnesia_bup.erl b/lib/mnesia/src/mnesia_bup.erl
index 2c756b1c75..3e55deb958 100644
--- a/lib/mnesia/src/mnesia_bup.erl
+++ b/lib/mnesia/src/mnesia_bup.erl
@@ -47,7 +47,7 @@
          uninstall_fallback_master/2,
          local_uninstall_fallback/2,
          do_traverse_backup/7,
-         trav_apply/4
+         trav_apply/5
         ]).
 
 -include("mnesia.hrl").
@@ -82,7 +82,8 @@ iterate(Mod, Fun, Opaque, Acc) ->
     R = #restore{bup_module = Mod, bup_data = Opaque},
     try read_schema_section(R) of
 	{R2, {Header, Schema, Rest}} ->
-	    try iter(R2, Header, Schema, Fun, Acc, Rest) of
+            Ext = get_ext_types(Schema),
+	    try iter(R2, Header, Schema, Ext, Fun, Acc, Rest) of
 		{ok, R3, Res} ->
 		    close_read(R3),
 		    {ok, Res}
@@ -97,17 +98,32 @@ iterate(Mod, Fun, Opaque, Acc) ->
 	    Err
     end.
 
-iter(R, Header, Schema, Fun, Acc, []) ->
+get_ext_types(Schema) ->
+    try
+        List = lookup_schema(schema, Schema),
+        case lists:keyfind(user_properties, 1, List) of
+            {_, Props} ->
+                proplists:get_value(
+                  mnesia_backend_types, Props, []);
+            false ->
+                []
+        end
+    catch
+        throw:{error, {"Cannot lookup",_}} ->
+            []
+    end.
+
+iter(R, Header, Schema, Ext, Fun, Acc, []) ->
     case safe_apply(R, read, [R#restore.bup_data]) of
         {R2, []} ->
-            Res = Fun([], Header, Schema, Acc),
+            Res = Fun([], Header, Schema, Ext, Acc),
             {ok, R2, Res};
         {R2, BupItems} ->
-            iter(R2, Header, Schema, Fun, Acc, BupItems)
+            iter(R2, Header, Schema, Ext, Fun, Acc, BupItems)
     end;
-iter(R, Header, Schema, Fun, Acc, BupItems) ->
-    Acc2 = Fun(BupItems, Header, Schema, Acc),
-    iter(R, Header, Schema, Fun, Acc2, []).
+iter(R, Header, Schema, Ext, Fun, Acc, BupItems) ->
+    Acc2 = Fun(BupItems, Header, Schema, Ext, Acc),
+    iter(R, Header, Schema, Ext, Fun, Acc2, []).
 
 -spec safe_apply(#restore{}, atom(), list()) -> tuple().
 safe_apply(R, write, [_, Items]) when Items =:= [] ->
@@ -493,15 +509,15 @@ install_fallback_master(ClientPid, FA) ->
     State = {start, FA},
     Opaque = FA#fallback_args.opaque,
     Mod = FA#fallback_args.module,
-    Res = iterate(Mod, fun restore_recs/4, Opaque, State),
+    Res = iterate(Mod, fun restore_recs/5, Opaque, State),
     unlink(ClientPid),
     ClientPid ! {self(), Res},
     exit(shutdown).
 
-restore_recs(_, _, _, stop) ->
+restore_recs(_, _, _, _, stop) ->
     throw({error, "restore_recs already stopped"});
 
-restore_recs(Recs, Header, Schema, {start, FA}) ->
+restore_recs(Recs, Header, Schema, Ext, {start, FA}) ->
     %% No records in backup
     Schema2 = convert_schema(Header#log_header.log_version, Schema),
     CreateList = lookup_schema(schema, Schema2),
@@ -512,19 +528,19 @@ restore_recs(Recs, Header, Schema, {start, FA}) ->
             Args = [self(), FA],
             Pids = [spawn_link(N, ?MODULE, fallback_receiver, Args) || N <- Ns],
             send_fallback(Pids, {start, Header, Schema2}),
-            Res = restore_recs(Recs, Header, Schema2, Pids),
+            Res = restore_recs(Recs, Header, Schema2, Ext, Pids),
             global:del_lock({{mnesia_table_lock, schema}, self()}, Ns),
             Res
     catch _:Reason ->
             throw({error, {"Bad schema in restore_recs", Reason}})
     end;
 
-restore_recs([], _Header, _Schema, Pids) ->
+restore_recs([], _Header, _Schema, _Ext, Pids) ->
     send_fallback(Pids, swap),
     send_fallback(Pids, stop),
     stop;
 
-restore_recs(Recs, _, _, Pids) ->
+restore_recs(Recs, _, _, _, Pids) ->
     send_fallback(Pids, {records, Recs}),
     Pids.
 
@@ -723,7 +739,7 @@ do_fallback_start(true, false) ->
     BupFile = fallback_bup(),
     Mod = mnesia_backup,
     LocalTabs = ?ets_new_table(mnesia_local_tables, [set, public, {keypos, 2}]),
-    case iterate(Mod, fun restore_tables/4, BupFile, {start, LocalTabs}) of
+    case iterate(Mod, fun restore_tables/5, BupFile, {start, LocalTabs}) of
         {ok, _Res} ->
             ?SAFE(dets:close(schema)),
             TmpSchema = mnesia_lib:tab2tmp(schema),
@@ -744,23 +760,24 @@ do_fallback_start(true, false) ->
             {error, {"Cannot start from fallback", Reason}}
     end.
 
-restore_tables(All=[Rec | Recs], Header, Schema, State={local, LocalTabs, LT}) ->
+restore_tables(All=[Rec | Recs], Header, Schema, Ext,
+               State={local, LocalTabs, LT}) ->
     Tab = element(1, Rec),
     if
         Tab =:= LT#local_tab.name ->
             Key = element(2, Rec),
             (LT#local_tab.add)(Tab, Key, Rec, LT),
-            restore_tables(Recs, Header, Schema, State);
+            restore_tables(Recs, Header, Schema, Ext, State);
         true ->
             NewState = {new, LocalTabs},
-            restore_tables(All, Header, Schema, NewState)
+            restore_tables(All, Header, Schema, Ext, NewState)
     end;
-restore_tables(All=[Rec | Recs], Header, Schema, {new, LocalTabs}) ->
+restore_tables(All=[Rec | Recs], Header, Schema, Ext, {new, LocalTabs}) ->
     Tab = element(1, Rec),
     case ?ets_lookup(LocalTabs, Tab) of
         [] ->
             State = {not_local, LocalTabs, Tab},
-            restore_tables(Recs, Header, Schema, State);
+            restore_tables(Recs, Header, Schema, Ext, State);
         [LT] when is_record(LT, local_tab) ->
 	    State = {local, LocalTabs, LT},
 	    case LT#local_tab.opened of
@@ -769,38 +786,39 @@ restore_tables(All=[Rec | Recs], Header, Schema, {new, LocalTabs}) ->
 		    (LT#local_tab.open)(Tab, LT),
 		    ?ets_insert(LocalTabs,LT#local_tab{opened=true})
 	    end,
-            restore_tables(All, Header, Schema, State)
+            restore_tables(All, Header, Schema, Ext, State)
     end;
-restore_tables(All=[Rec | Recs], Header, Schema, S = {not_local, LocalTabs, PrevTab}) ->
+restore_tables(All=[Rec | Recs], Header, Schema, Ext,
+               S = {not_local, LocalTabs, PrevTab}) ->
     Tab = element(1, Rec),
     if
         Tab =:= PrevTab ->
-            restore_tables(Recs, Header, Schema, S);
+            restore_tables(Recs, Header, Schema, Ext, S);
         true ->
             State = {new, LocalTabs},
-            restore_tables(All, Header, Schema, State)
+            restore_tables(All, Header, Schema, Ext, State)
     end;
-restore_tables(Recs, Header, Schema, {start, LocalTabs}) ->
+restore_tables(Recs, Header, Schema, Ext, {start, LocalTabs}) ->
     Dir = mnesia_lib:dir(),
     OldDir = filename:join([Dir, "OLD_DIR"]),
     mnesia_schema:purge_dir(OldDir, []),
     mnesia_schema:purge_dir(Dir, [fallback_name()]),
-    init_dat_files(Schema, LocalTabs),
+    init_dat_files(Schema, Ext, LocalTabs),
     State = {new, LocalTabs},
-    restore_tables(Recs, Header, Schema, State);
-restore_tables([], _Header, _Schema, State) ->
+    restore_tables(Recs, Header, Schema, Ext, State);
+restore_tables([], _Header, _Schema, _Ext, State) ->
     State.
 
 %% Creates all neccessary dat files and inserts
 %% the table definitions in the schema table
 %%
 %% Returns a list of local_tab tuples for all local tables
-init_dat_files(Schema, LocalTabs) ->
+init_dat_files(Schema, Ext, LocalTabs) ->
     TmpFile = mnesia_lib:tab2tmp(schema),
     Args = [{file, TmpFile}, {keypos, 2}, {type, set}],
     case dets:open_file(schema, Args) of % Assume schema lock
         {ok, _} ->
-            create_dat_files(Schema, LocalTabs),
+            create_dat_files(Schema, Ext, LocalTabs),
             ok = dets:close(schema),
             LocalTab = #local_tab{name         = schema,
                                   storage_type = disc_copies,
@@ -815,10 +833,10 @@ init_dat_files(Schema, LocalTabs) ->
             throw({error, {"Cannot open file", schema, Args, Reason}})
     end.
 
-create_dat_files([{schema, schema, TabDef} | Tail], LocalTabs) ->
+create_dat_files([{schema, schema, TabDef} | Tail], Ext, LocalTabs) ->
     ok = dets:insert(schema, {schema, schema, TabDef}),
-    create_dat_files(Tail, LocalTabs);
-create_dat_files([{schema, Tab, TabDef} | Tail], LocalTabs) ->
+    create_dat_files(Tail, Ext, LocalTabs);
+create_dat_files([{schema, Tab, TabDef} | Tail], Ext, LocalTabs) ->
     TmpFile = mnesia_lib:tab2tmp(Tab),
     DatFile = mnesia_lib:tab2dat(Tab),
     DclFile = mnesia_lib:tab2dcl(Tab),
@@ -831,56 +849,21 @@ create_dat_files([{schema, Tab, TabDef} | Tail], LocalTabs) ->
 
     mnesia_lib:dets_sync_close(Tab),
     file:delete(TmpFile),
-    Cs = mnesia_schema:list2cs(TabDef),
+    Cs = mnesia_schema:list2cs(TabDef, Ext),
     ok = dets:insert(schema, {schema, Tab, TabDef}),
     RecName = Cs#cstruct.record_name,
     Storage = mnesia_lib:cs_to_storage_type(node(), Cs),
+    delete_ext(Storage, Tab),
+    Semantics = mnesia_lib:semantics(Storage, storage),
     if
-	Storage =:= unknown ->
+	Semantics =:= undefined ->
             ok = dets:delete(schema, {schema, Tab}),
-            create_dat_files(Tail, LocalTabs);
-        Storage =:= disc_only_copies ->
-            Args = [{file, TmpFile}, {keypos, 2},
-                    {type, mnesia_lib:disk_type(Tab, Cs#cstruct.type)}],
-            Open = fun(T, LT) when T =:= LT#local_tab.name ->
-                           case mnesia_lib:dets_sync_open(T, Args) of
-                               {ok, _} ->
-                                   ok;
-                               {error, Reason} ->
-                                   throw({error, {"Cannot open file", T, Args, Reason}})
-                           end
-                   end,
-            Add = fun(T, Key, Rec, LT) when T =:= LT#local_tab.name ->
-                          case Rec of
-                              {_T, Key} ->
-                                  ok = dets:delete(T, Key);
-                              (Rec) when T =:= RecName ->
-                                  ok = dets:insert(Tab, Rec);
-                              (Rec) ->
-                                  Rec2 = setelement(1, Rec, RecName),
-                                  ok = dets:insert(T, Rec2)
-                          end
-                  end,
-            Close = fun(T, LT) when T =:= LT#local_tab.name ->
-                            mnesia_lib:dets_sync_close(T)
-                    end,
-	    Swap = fun(T, LT) when T =:= LT#local_tab.name ->
-			   Expunge(),
-			   case LT#local_tab.opened of
-			       true ->
-				   Close(T,LT);
-			       false ->
-				   Open(T,LT),
-				   Close(T,LT)
-			   end,
-			   case file:rename(TmpFile, DatFile) of
-			       ok ->
-				   ok;
-			       {error, Reason} ->
-				   mnesia_lib:fatal("Cannot rename file ~p -> ~p: ~p~n",
-						    [TmpFile, DatFile, Reason])
-			   end
-		   end,
+            create_dat_files(Tail, Ext, LocalTabs);
+        Semantics =:= disc_only_copies ->
+	    Open = disc_only_open_fun(Storage, Cs),
+	    Add = disc_only_add_fun(Storage, Cs),
+	    Close = disc_only_close_fun(Storage),
+	    Swap = disc_only_swap_fun(Storage, Expunge, Open, Close),
             LocalTab = #local_tab{name         = Tab,
                                   storage_type = Storage,
                                   open         = Open,
@@ -890,8 +873,8 @@ create_dat_files([{schema, Tab, TabDef} | Tail], LocalTabs) ->
                                   record_name  = RecName,
                                   opened       = false},
             ?ets_insert(LocalTabs, LocalTab),
-	    create_dat_files(Tail, LocalTabs);
-        Storage =:= ram_copies; Storage =:= disc_copies ->
+	    create_dat_files(Tail, Ext, LocalTabs);
+        Semantics =:= ram_copies; Storage =:= disc_copies ->
 	    Open = fun(T, LT) when T =:= LT#local_tab.name ->
 			   mnesia_log:open_log({?MODULE, T},
 					       mnesia_log:dcl_log_header(),
@@ -952,18 +935,97 @@ create_dat_files([{schema, Tab, TabDef} | Tail], LocalTabs) ->
 				  opened       = false
 				 },
             ?ets_insert(LocalTabs, LocalTab),
-            create_dat_files(Tail, LocalTabs)
+            create_dat_files(Tail, Ext, LocalTabs);
+	true ->
+	    error({unknown_semantics, [{semantics, Semantics},
+				       {tabdef, TabDef},
+				       {ext, Ext}]})
     end;
-create_dat_files([{schema, Tab} | Tail], LocalTabs) ->
+create_dat_files([{schema, Tab} | Tail], Ext, LocalTabs) ->
     ?ets_delete(LocalTabs, Tab),
     ok = dets:delete(schema, {schema, Tab}),
     TmpFile = mnesia_lib:tab2tmp(Tab),
     mnesia_lib:dets_sync_close(Tab),
     file:delete(TmpFile),
-    create_dat_files(Tail, LocalTabs);
-create_dat_files([], _LocalTabs) ->
+    create_dat_files(Tail, Ext, LocalTabs);
+create_dat_files([], _Ext, _LocalTabs) ->
     ok.
 
+delete_ext({ext, Alias, Mod}, Tab) ->
+    Mod:close_table(Alias, Tab),
+    Mod:delete_table(Alias, Tab),
+    ok;
+delete_ext(_, _) ->
+    ok.
+
+
+disc_only_open_fun(disc_only_copies, #cstruct{name = Tab} =Cs) ->
+    TmpFile = mnesia_lib:tab2tmp(Tab),
+    Args = [{file, TmpFile}, {keypos, 2},
+	    {type, mnesia_lib:disk_type(Tab, Cs#cstruct.type)}],
+    fun(T, LT) when T =:= LT#local_tab.name ->
+	    case mnesia_lib:dets_sync_open(T, Args) of
+		{ok, _} ->
+		    ok;
+		{error, Reason} ->
+		    throw({error, {"Cannot open file", T, Args, Reason}})
+	    end
+    end;
+disc_only_open_fun({ext,Alias,Mod}, Cs) ->
+    fun(T, LT) when T =:= LT#local_tab.name ->
+	    ok = Mod:load_table(Alias, T, restore, mnesia_schema:cs2list(Cs))
+    end.
+
+disc_only_add_fun(Storage, #cstruct{name = Tab,
+				    record_name = RecName}) ->
+    fun(T, Key, Rec, #local_tab{name = T}) when T =:= Tab->
+	    case Rec of
+		{_T, Key} ->
+		    ok = mnesia_lib:db_erase(Storage, T, Key);
+		(Rec) when T =:= RecName ->
+		    ok = mnesia_lib:db_put(Storage, T, Rec);
+		(Rec) ->
+		    ok = mnesia_lib:db_put(Storage, T,
+					   setelement(1, Rec, RecName))
+	    end
+    end.
+
+disc_only_close_fun(disc_only_copies) ->
+    fun(T, LT) when T =:= LT#local_tab.name ->
+	    mnesia_lib:dets_sync_close(T)
+    end;
+disc_only_close_fun({ext, Alias, Mod}) ->
+    fun(T, _LT) ->
+	    Mod:sync_close_table(Alias, T)
+    end.
+
+
+disc_only_swap_fun(disc_only_copies, Expunge, Open, Close) ->
+    fun(T, LT) when T =:= LT#local_tab.name ->
+            TmpFile = mnesia_lib:tab2tmp(T),
+            DatFile = mnesia_lib:tab2dat(T),
+	    Expunge(),
+	    case LT#local_tab.opened of
+		true ->
+		    Close(T,LT);
+		false ->
+		    Open(T,LT),
+		    Close(T,LT)
+	    end,
+	    case file:rename(TmpFile, DatFile) of
+		ok ->
+		    ok;
+		{error, Reason} ->
+		    mnesia_lib:fatal("Cannot rename file ~p -> ~p: ~p~n",
+				     [TmpFile, DatFile, Reason])
+	    end
+    end;
+disc_only_swap_fun({ext, _Alias, _Mod}, _Expunge, _Open, Close) ->
+    fun(T, #local_tab{name = T} = LT) ->
+	    Close(T, LT)
+    end.
+
+
 uninstall_fallback() ->
     uninstall_fallback([{scope, global}]).
 
@@ -1140,7 +1202,7 @@ do_traverse_backup(ClientPid, Source, SourceMod, Target, TargetMod, Fun, Acc) ->
         end,
     A = {start, Fun, Acc, TargetMod, Iter},
     Res =
-        case iterate(SourceMod, fun trav_apply/4, Source, A) of
+        case iterate(SourceMod, fun trav_apply/5, Source, A) of
             {ok, {iter, _, Acc2, _, Iter2}} when TargetMod =/= read_only ->
                 try
 		    do_apply(TargetMod, commit_write, [Iter2], Iter2),
@@ -1159,7 +1221,7 @@ do_traverse_backup(ClientPid, Source, SourceMod, Target, TargetMod, Fun, Acc) ->
     unlink(ClientPid),
     ClientPid ! {iter_done, self(), Res}.
 
-trav_apply(Recs, _Header, _Schema, {iter, Fun, Acc, Mod, Iter}) ->
+trav_apply(Recs, _Header, _Schema, _Ext, {iter, Fun, Acc, Mod, Iter}) ->
     {NewRecs, Acc2} = filter_foldl(Fun, Acc, Recs),
     if
         Mod =/= read_only, NewRecs =/= [] ->
@@ -1168,7 +1230,7 @@ trav_apply(Recs, _Header, _Schema, {iter, Fun, Acc, Mod, Iter}) ->
         true ->
             {iter, Fun, Acc2, Mod, Iter}
     end;
-trav_apply(Recs, Header, Schema, {start, Fun, Acc, Mod, Iter}) ->
+trav_apply(Recs, Header, Schema, Ext, {start, Fun, Acc, Mod, Iter}) ->
     Iter2 =
         if
             Mod =/= read_only ->
@@ -1176,8 +1238,9 @@ trav_apply(Recs, Header, Schema, {start, Fun, Acc, Mod, Iter}) ->
             true ->
                 Iter
         end,
-    TravAcc = trav_apply(Schema, Header, Schema, {iter, Fun, Acc, Mod, Iter2}),
-    trav_apply(Recs, Header, Schema, TravAcc).
+    TravAcc = trav_apply(Schema, Header, Schema, Ext,
+                         {iter, Fun, Acc, Mod, Iter2}),
+    trav_apply(Recs, Header, Schema, Ext, TravAcc).
 
 filter_foldl(Fun, Acc, [Head|Tail]) ->
     case Fun(Head, Acc) of
diff --git a/lib/mnesia/src/mnesia_checkpoint.erl b/lib/mnesia/src/mnesia_checkpoint.erl
index 1407bc0557..e284005ac6 100644
--- a/lib/mnesia/src/mnesia_checkpoint.erl
+++ b/lib/mnesia/src/mnesia_checkpoint.erl
@@ -675,6 +675,16 @@ tab2retainer({Tab, Name}) ->
     FlatName = lists:flatten(io_lib:write(Name)),
     mnesia_lib:dir(lists:concat([?MODULE, "_", Tab, "_", FlatName, ".RET"])).
 
+retainer_create(_Cp, R, Tab, Name, Ext = {ext, Alias, Mod}) ->
+    T = {Tab, retainer, Name},
+    P = mnesia_schema:cs2list(val({Tab, cstruct})),
+    Mod:delete_table(Alias, T),
+    ok = Mod:create_table(Alias, T, P),
+    Cs = val({Tab, cstruct}),
+    Mod:load_table(Alias, T, {retainer, create_table},
+		   mnesia_schema:cs2list(Cs)),
+    dbg_out("Checkpoint retainer created ~p ~p~n", [Name, Tab]),
+    R#retainer{store = {Ext, T}, really_retain = true};
 retainer_create(_Cp, R, Tab, Name, disc_only_copies) ->
     Fname = tab2retainer({Tab, Name}),
     file:delete(Fname),
@@ -734,15 +744,23 @@ traverse_dcd({Cont, Recs}, Log, Fun) ->     %% trashed data??
 traverse_dcd(eof, _Log, _Fun) ->
     ok.
 
+retainer_get({{ext, Alias, Mod}, Store}, Key) ->
+    Mod:lookup(Alias, Store, Key);
 retainer_get({ets, Store}, Key) -> ?ets_lookup(Store, Key);
 retainer_get({dets, Store}, Key) -> dets:lookup(Store, Key).
 
+retainer_put({{ext, Alias, Mod}, Store}, Val) ->
+    Mod:insert(Alias, Store, Val);
 retainer_put({ets, Store}, Val) -> ?ets_insert(Store, Val);
 retainer_put({dets, Store}, Val) -> dets:insert(Store, Val).
 
+retainer_first({{ext, Alias, Mod}, Store}) ->
+    Mod:first(Alias, Store);
 retainer_first({ets, Store}) -> ?ets_first(Store);
 retainer_first({dets, Store}) -> dets:first(Store).
  
+retainer_next({{ext, Alias, Mod}, Store}, Key) ->
+    Mod:next(Alias, Store, Key);
 retainer_next({ets, Store}, Key) -> ?ets_next(Store, Key);
 retainer_next({dets, Store}, Key) -> dets:next(Store, Key).
 
@@ -761,11 +779,16 @@ retainer_next({dets, Store}, Key) -> dets:next(Store, Key).
 
 retainer_fixtable(Tab, Bool) when is_atom(Tab) ->
     mnesia_lib:db_fixtable(val({Tab, storage_type}), Tab, Bool);
+retainer_fixtable({Ext = {ext, _, _}, Tab}, Bool) ->
+    mnesia_lib:db_fixtable(Ext, Tab, Bool);
 retainer_fixtable({ets, Tab}, Bool) ->
     mnesia_lib:db_fixtable(ram_copies, Tab, Bool);
 retainer_fixtable({dets, Tab}, Bool) ->
     mnesia_lib:db_fixtable(disc_only_copies, Tab, Bool).
 
+retainer_delete({{ext, Alias, Mod}, Store}) ->
+    Mod:close_table(Alias, Store),
+    Mod:delete_table(Alias, Store);
 retainer_delete({ets, Store}) ->
     ?ets_delete_table(Store);
 retainer_delete({dets, Store}) ->