Merge branch 'dgud/mnesia/ext-backend/PR-858/OTP-13058'

* dgud/mnesia/ext-backend/PR-858/OTP-13058:
  mnesia_ext: Add basic backend extension tests
  mnesia_ext: reuse snmp field for ext updates
  mnesia_ext: Create table/data containers from mnesia monitor not temporary processes
  mnesia_ext: Implement ext copies index
  mnesia_ext: Load table ext
  mnesia_ext: Dumper and schema changes
  mnesia_ext: Refactor mnesia_schema.erl
  mnesia_ext: Ext support in fragmented tables
  mnesia_ext: Backup handling
  mnesia_ext: Create schema functionality
  mnesia_ext: Add ext copies and db_fold to low level api
  mnesia_ext: Refactor record_validation code
  mnesia_ext: Add create_external and increase protocol version to monitor
  mnesia_ext: Add ext copies to records
  mnesia_ext: Add supervisor and behaviour modules

1 files changed, 103 insertions, 41 deletions

diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl
index e3d9bb1484..9586adbf93 100644
--- a/lib/mnesia/src/mnesia.erl
+++ b/lib/mnesia/src/mnesia.erl
@@ -82,7 +82,8 @@
 	 system_info/0,                      % Not for public use
 
 	 %% Database mgt
-	 create_schema/1, delete_schema/1,
+	 create_schema/1, create_schema/2, delete_schema/1,
+         add_backend_type/2,
 	 backup/1, backup/2, traverse_backup/4, traverse_backup/6,
 	 install_fallback/1, install_fallback/2,
 	 uninstall_fallback/0, uninstall_fallback/1,
@@ -197,6 +198,9 @@ e_has_var(X, Pos) ->
 %% Start and stop
 
 start() ->
+    start([]).
+
+start_() ->
     {Time , Res} =  timer:tc(application, start, [?APPLICATION, temporary]),
 
     Secs = Time div 1000000,
@@ -232,7 +236,7 @@ patched_start([{Env, Val} | Tail]) when is_atom(Env) ->
 patched_start([Head | _]) ->
     {error, {bad_type, Head}};
 patched_start([]) ->
-    start().
+    start_().
 
 stop() ->
     case application:stop(?APPLICATION) of
@@ -297,6 +301,7 @@ ms() ->
 
      %% Keep these last in the list, so
      %% mnesia_sup kills these last
+     mnesia_ext_sup,
      mnesia_monitor,
      mnesia_event
     ].
@@ -556,22 +561,16 @@ write(_Tid, _Ts, Tab, Val, LockKind) ->
     abort({bad_type, Tab, Val, LockKind}).
 
 write_to_store(Tab, Store, Oid, Val) ->
-    case ?catch_val({Tab, record_validation}) of
-	{RecName, Arity, Type}
-	  when tuple_size(Val) == Arity, RecName == element(1, Val) ->
-	    case Type of
-		bag ->
-		    ?ets_insert(Store, {Oid, Val, write});
-		_  ->
-		    ?ets_delete(Store, Oid),
-		    ?ets_insert(Store, {Oid, Val, write})
-	    end,
-	    ok;
-	{'EXIT', _} ->
-	    abort({no_exists, Tab});
-	_ ->
-	    abort({bad_type, Val})
-    end.
+    {_, _, Type} = mnesia_lib:validate_record(Tab, Val),
+    Oid = {Tab, element(2, Val)},
+    case Type of
+	bag ->
+	    ?ets_insert(Store, {Oid, Val, write});
+	_  ->
+	    ?ets_delete(Store, Oid),
+	    ?ets_insert(Store, {Oid, Val, write})
+    end,
+    ok.
 
 delete({Tab, Key}) ->
     delete(Tab, Key, write);
@@ -1548,16 +1547,9 @@ dirty_write(Tab, Val) ->
 
 do_dirty_write(SyncMode, Tab, Val)
   when is_atom(Tab), Tab /= schema, is_tuple(Val), tuple_size(Val) > 2 ->
-    case ?catch_val({Tab, record_validation}) of
-	{RecName, Arity, _Type}
-	when tuple_size(Val) == Arity, RecName == element(1, Val) ->
-	    Oid = {Tab, element(2, Val)},
-	    mnesia_tm:dirty(SyncMode, {Oid, Val, write});
-	{'EXIT', _} ->
-	    abort({no_exists, Tab});
-	_ ->
-	    abort({bad_type, Val})
-    end;
+    {_, _, _} = mnesia_lib:validate_record(Tab, Val),
+    Oid = {Tab, element(2, Val)},
+    mnesia_tm:dirty(SyncMode, {Oid, Val, write});
 do_dirty_write(_SyncMode, Tab, Val) ->
     abort({bad_type, Tab, Val}).
 
@@ -1609,8 +1601,8 @@ dirty_update_counter(Tab, Key, Incr) ->
 
 do_dirty_update_counter(SyncMode, Tab, Key, Incr)
   when is_atom(Tab), Tab /= schema, is_integer(Incr) ->
-    case ?catch_val({Tab, record_validation}) of
-	{RecName, 3, set} ->
+    case mnesia_lib:validate_key(Tab, Key) of
+	{RecName, 3, Type} when Type == set; Type == ordered_set ->
 	    Oid = {Tab, Key},
 	    mnesia_tm:dirty(SyncMode, {Oid, {RecName, Incr}, update_counter});
 	_ ->
@@ -1910,6 +1902,8 @@ raw_table_info(Tab, Item) ->
 		info_reply(?ets_info(Tab, Item), Tab, Item);
 	    disc_only_copies ->
 		info_reply(dets:info(Tab, Item), Tab, Item);
+            {ext, Alias, Mod} ->
+                info_reply(catch Mod:info(Alias, Tab, Item), Tab, Item);
 	    unknown ->
 		bad_info_reply(Tab, Item)
 	end
@@ -2022,15 +2016,26 @@ display_tab_info() ->
     MasterTabs = mnesia_recover:get_master_node_tables(),
     io:format("master node tables = ~p~n", [lists:sort(MasterTabs)]),
 
+    case get_backend_types() of
+	[] -> ok;
+	Ts -> list_backend_types(Ts, "backend types      = ")
+    end,
+
+    case get_index_plugins() of
+	[] -> ok;
+	Ps -> list_index_plugins(Ps, "index plugins      = ")
+    end,
+
     Tabs = system_info(tables),
 
-    {Unknown, Ram, Disc, DiscOnly} =
-	lists:foldl(fun storage_count/2, {[], [], [], []}, Tabs),
+    {Unknown, Ram, Disc, DiscOnly, Ext} =
+	lists:foldl(fun storage_count/2, {[], [], [], [], []}, Tabs),
 
     io:format("remote             = ~p~n", [lists:sort(Unknown)]),
     io:format("ram_copies         = ~p~n", [lists:sort(Ram)]),
     io:format("disc_copies        = ~p~n", [lists:sort(Disc)]),
     io:format("disc_only_copies   = ~p~n", [lists:sort(DiscOnly)]),
+    [io:format("~-19s= ~p~n", [atom_to_list(A), Ts]) || {A,Ts} <- Ext],
 
     Rfoldl = fun(T, Acc) ->
 		     Rpat =
@@ -2038,7 +2043,7 @@ display_tab_info() ->
 			     read_only ->
 				 lists:sort([{A, read_only} || A <- val({T, active_replicas})]);
 			     read_write ->
-				 table_info(T, where_to_commit)
+				 [fix_wtc(W) || W <- table_info(T, where_to_commit)]
 			 end,
 		     case lists:keysearch(Rpat, 1, Acc) of
 			 {value, {_Rpat, Rtabs}} ->
@@ -2051,12 +2056,60 @@ display_tab_info() ->
     Rdisp = fun({Rpat, Rtabs}) -> io:format("~p = ~p~n", [Rpat, Rtabs]) end,
     lists:foreach(Rdisp, lists:sort(Repl)).
 
-storage_count(T, {U, R, D, DO}) ->
+get_backend_types() ->
+    case ?catch_val({schema, user_property, mnesia_backend_types}) of
+	{'EXIT', _} ->
+	    [];
+	{mnesia_backend_types, Ts} ->
+	    lists:sort(Ts)
+    end.
+
+get_index_plugins() ->
+    case ?catch_val({schema, user_property, mnesia_index_plugins}) of
+	{'EXIT', _} ->
+	    [];
+	{mnesia_index_plugins, Ps} ->
+	    lists:sort(Ps)
+    end.
+
+
+list_backend_types([{A,M} | T] = Ts, Legend) ->
+    Indent = [$\s || _ <- Legend],
+    W = integer_to_list(
+	  lists:foldl(fun({Alias,_}, Wa) ->
+			      erlang:max(Wa, length(atom_to_list(Alias)))
+		      end, 0, Ts)),
+    io:fwrite(Legend ++ "~-" ++ W ++ "s - ~s~n",
+	      [atom_to_list(A), atom_to_list(M)]),
+    [io:fwrite(Indent ++ "~-" ++ W ++ "s - ~s~n",
+	       [atom_to_list(A1), atom_to_list(M1)])
+     || {A1,M1} <- T].
+
+list_index_plugins([{N,M,F} | T] = Ps, Legend) ->
+    Indent = [$\s || _ <- Legend],
+    W = integer_to_list(
+	  lists:foldl(fun({N1,_,_}, Wa) ->
+			      erlang:max(Wa, length(pp_ix_name(N1)))
+		      end, 0, Ps)),
+    io:fwrite(Legend ++ "~-" ++ W ++ "s - ~s:~s~n",
+	      [pp_ix_name(N), atom_to_list(M), atom_to_list(F)]),
+    [io:fwrite(Indent ++ "~-" ++ W ++ "s - ~s:~s~n",
+	       [pp_ix_name(N1), atom_to_list(M1), atom_to_list(F1)])
+     || {N1,M1,F1} <- T].
+
+pp_ix_name(N) ->
+    lists:flatten(io_lib:fwrite("~w", [N])).
+
+fix_wtc({N, {ext,A,_}}) -> {N, A};
+fix_wtc({N,A}) when is_atom(A) -> {N, A}.
+
+storage_count(T, {U, R, D, DO, Ext}) ->
     case table_info(T, storage_type) of
-	unknown -> {[T | U], R, D, DO};
-	ram_copies -> {U, [T | R], D, DO};
-	disc_copies -> {U, R, [T | D], DO};
-	disc_only_copies -> {U, R, D, [T | DO]}
+	unknown -> {[T | U], R, D, DO, Ext};
+	ram_copies -> {U, [T | R], D, DO, Ext};
+	disc_copies -> {U, R, [T | D], DO, Ext};
+	disc_only_copies -> {U, R, D, [T | DO], Ext};
+        {ext, A, _} -> {U, R, D, DO, orddict:append(A, T, Ext)}
     end.
 
 system_info(Item) ->
@@ -2071,9 +2124,10 @@ system_info2(all) ->
 system_info2(db_nodes) ->
     DiscNs = ?catch_val({schema, disc_copies}),
     RamNs = ?catch_val({schema, ram_copies}),
+    ExtNs = ?catch_val({schema, external_copies}),
     if
-	is_list(DiscNs), is_list(RamNs) ->
-	    DiscNs ++ RamNs;
+	is_list(DiscNs), is_list(RamNs), is_list(ExtNs) ->
+	    DiscNs ++ RamNs ++ ExtNs;
 	true ->
 	    case mnesia_schema:read_nodes() of
 		{ok, Nodes} -> Nodes;
@@ -2177,6 +2231,7 @@ system_info2(access_module) -> mnesia_monitor:get_env(access_module);
 system_info2(auto_repair) -> mnesia_monitor:get_env(auto_repair);
 system_info2(is_running) -> mnesia_lib:is_running();
 system_info2(backup_module) -> mnesia_monitor:get_env(backup_module);
+system_info2(backend_types) -> mnesia_schema:backend_types();
 system_info2(event_module) -> mnesia_monitor:get_env(event_module);
 system_info2(debug) -> mnesia_monitor:get_env(debug);
 system_info2(dump_log_load_regulation) -> mnesia_monitor:get_env(dump_log_load_regulation);
@@ -2213,6 +2268,7 @@ system_info_items(yes) ->
     [
      access_module,
      auto_repair,
+     backend_types,
      backup_module,
      checkpoints,
      db_nodes,
@@ -2306,11 +2362,17 @@ load_mnesia_or_abort() ->
 %% Database mgt
 
 create_schema(Ns) ->
-    mnesia_bup:create_schema(Ns).
+    create_schema(Ns, []).
+
+create_schema(Ns, Properties) ->
+    mnesia_bup:create_schema(Ns, Properties).
 
 delete_schema(Ns) ->
     mnesia_schema:delete_schema(Ns).
 
+add_backend_type(Alias, Module) ->
+    mnesia_schema:add_backend_type(Alias, Module).
+
 backup(Opaque) ->
     mnesia_log:backup(Opaque).