diff options
author | Sverker Eriksson <[email protected]> | 2017-08-30 20:55:08 +0200 |
---|---|---|
committer | Sverker Eriksson <[email protected]> | 2017-08-30 20:55:08 +0200 |
commit | 7c67bbddb53c364086f66260701bc54a61c9659c (patch) | |
tree | 92ab0d4b91d5e2f6e7a3f9d61ea25089e8a71fe0 /lib/mnesia/src | |
parent | 97dc5e7f396129222419811c173edc7fa767b0f8 (diff) | |
parent | 3b7a6ffddc819bf305353a593904cea9e932e7dc (diff) | |
download | otp-7c67bbddb53c364086f66260701bc54a61c9659c.tar.gz otp-7c67bbddb53c364086f66260701bc54a61c9659c.tar.bz2 otp-7c67bbddb53c364086f66260701bc54a61c9659c.zip |
Merge tag 'OTP-19.0' into sverker/19/binary_to_atom-utf8-crash/ERL-474/OTP-14590
Diffstat (limited to 'lib/mnesia/src')
35 files changed, 3917 insertions, 2206 deletions
diff --git a/lib/mnesia/src/Makefile b/lib/mnesia/src/Makefile index ac38fa05ef..08a00e6aba 100644 --- a/lib/mnesia/src/Makefile +++ b/lib/mnesia/src/Makefile @@ -1,18 +1,19 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2013. All Rights Reserved. +# Copyright Ericsson AB 1996-2016. All Rights Reserved. # -# The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved online at http://www.erlang.org/. -# -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # %CopyrightEnd% # @@ -42,6 +43,7 @@ RELSYSDIR = $(RELEASE_PATH)/lib/mnesia-$(VSN) # ---------------------------------------------------- MODULES= \ mnesia \ + mnesia_backend_type \ mnesia_backup \ mnesia_bup \ mnesia_checkpoint \ @@ -49,6 +51,7 @@ MODULES= \ mnesia_controller \ mnesia_dumper\ mnesia_event \ + mnesia_ext_sup \ mnesia_frag \ mnesia_frag_hash \ mnesia_frag_old_hash \ diff --git a/lib/mnesia/src/mnesia.app.src b/lib/mnesia/src/mnesia.app.src index 3715488ec2..006ad4bac1 100644 --- a/lib/mnesia/src/mnesia.app.src +++ b/lib/mnesia/src/mnesia.app.src @@ -3,6 +3,7 @@ {vsn, "%VSN%"}, {modules, [ mnesia, + mnesia_backend_type, mnesia_backup, mnesia_bup, mnesia_checkpoint, @@ -10,6 +11,7 @@ mnesia_controller, mnesia_dumper, mnesia_event, + mnesia_ext_sup, mnesia_frag, mnesia_frag_hash, mnesia_frag_old_hash, @@ -47,6 +49,7 @@ mnesia_tm ]}, {applications, [kernel, stdlib]}, - {mod, {mnesia_sup, []}}]}. + {mod, {mnesia_sup, []}}, + {runtime_dependencies, ["stdlib-2.0","kernel-3.0","erts-7.0"]}]}. diff --git a/lib/mnesia/src/mnesia.appup.src b/lib/mnesia/src/mnesia.appup.src index 355aafb215..c245299740 100644 --- a/lib/mnesia/src/mnesia.appup.src +++ b/lib/mnesia/src/mnesia.appup.src @@ -1,22 +1,16 @@ %% -*- erlang -*- {"%VSN%", [ - {"4.7.1", [{restart_application, mnesia}]}, - {"4.7", [{restart_application, mnesia}]}, - {"4.6", [{restart_application, mnesia}]}, - {"4.5.1", [{restart_application, mnesia}]}, - {"4.5", [{restart_application, mnesia}]}, + {<<"4\\.1[0-9].*">>, [{restart_application, mnesia}]}, + {<<"4\\.[5-9].*">>, [{restart_application, mnesia}]}, {"4.4.19", [{restart_application, mnesia}]}, {"4.4.18", [{restart_application, mnesia}]}, {"4.4.17", [{restart_application, mnesia}]}, {"4.4.16", [{restart_application, mnesia}]} ], [ - {"4.7.1", [{restart_application, mnesia}]}, - {"4.7", [{restart_application, mnesia}]}, - {"4.6", [{restart_application, mnesia}]}, - {"4.5.1", [{restart_application, mnesia}]}, - {"4.5", [{restart_application, mnesia}]}, + {<<"4\\.1[0-9].*">>, [{restart_application, mnesia}]}, + {<<"4\\.[5-9].*">>, [{restart_application, mnesia}]}, {"4.4.19", [{restart_application, mnesia}]}, {"4.4.18", [{restart_application, mnesia}]}, {"4.4.17", [{restart_application, mnesia}]}, diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index 70466d10d7..9586adbf93 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -81,7 +82,8 @@ system_info/0, % Not for public use %% Database mgt - create_schema/1, delete_schema/1, + create_schema/1, create_schema/2, delete_schema/1, + add_backend_type/2, backup/1, backup/2, traverse_backup/4, traverse_backup/6, install_fallback/1, install_fallback/2, uninstall_fallback/0, uninstall_fallback/1, @@ -104,7 +106,8 @@ set_master_nodes/1, set_master_nodes/2, %% Misc admin - dump_log/0, subscribe/1, unsubscribe/1, report_event/1, + dump_log/0, sync_log/0, + subscribe/1, unsubscribe/1, report_event/1, %% Snmp snmp_open_table/2, snmp_close_table/1, @@ -144,7 +147,7 @@ %% Local function in order to avoid external function call val(Var) -> case ?catch_val(Var) of - {'EXIT', Reason} -> mnesia_lib:other_val(Var, Reason); + {'EXIT', _} -> mnesia_lib:other_val(Var); Value -> Value end. @@ -195,6 +198,9 @@ e_has_var(X, Pos) -> %% Start and stop start() -> + start([]). + +start_() -> {Time , Res} = timer:tc(application, start, [?APPLICATION, temporary]), Secs = Time div 1000000, @@ -230,7 +236,7 @@ patched_start([{Env, Val} | Tail]) when is_atom(Env) -> patched_start([Head | _]) -> {error, {bad_type, Head}}; patched_start([]) -> - start(). + start_(). stop() -> case application:stop(?APPLICATION) of @@ -295,6 +301,7 @@ ms() -> %% Keep these last in the list, so %% mnesia_sup kills these last + mnesia_ext_sup, mnesia_monitor, mnesia_event ]. @@ -305,6 +312,8 @@ ms() -> -spec abort(_) -> no_return(). +abort(Reason = {aborted, _}) -> + exit(Reason); abort(Reason) -> exit({aborted, Reason}). @@ -552,22 +561,16 @@ write(_Tid, _Ts, Tab, Val, LockKind) -> abort({bad_type, Tab, Val, LockKind}). write_to_store(Tab, Store, Oid, Val) -> - case ?catch_val({Tab, record_validation}) of - {RecName, Arity, Type} - when tuple_size(Val) == Arity, RecName == element(1, Val) -> - case Type of - bag -> - ?ets_insert(Store, {Oid, Val, write}); - _ -> - ?ets_delete(Store, Oid), - ?ets_insert(Store, {Oid, Val, write}) - end, - ok; - {'EXIT', _} -> - abort({no_exists, Tab}); - _ -> - abort({bad_type, Val}) - end. + {_, _, Type} = mnesia_lib:validate_record(Tab, Val), + Oid = {Tab, element(2, Val)}, + case Type of + bag -> + ?ets_insert(Store, {Oid, Val, write}); + _ -> + ?ets_delete(Store, Oid), + ?ets_insert(Store, {Oid, Val, write}) + end, + ok. delete({Tab, Key}) -> delete(Tab, Key, write); @@ -806,7 +809,7 @@ next(Tid,Ts,Tab,Key) tid -> lock_table(Tid, Ts, Tab, read), do_fixtable(Tab,Ts), - New = (catch dirty_next(Tab,Key)), + New = ?CATCH(dirty_next(Tab,Key)), stored_keys(Tab,New,Key,Ts,next, val({Tab, setorbag})); _Protocol -> @@ -832,7 +835,7 @@ prev(Tid,Ts,Tab,Key) tid -> lock_table(Tid, Ts, Tab, read), do_fixtable(Tab,Ts), - New = (catch dirty_prev(Tab,Key)), + New = ?CATCH(dirty_prev(Tab,Key)), stored_keys(Tab,New,Key,Ts,prev, val({Tab, setorbag})); _Protocol -> @@ -964,7 +967,7 @@ foldl(Fun, Acc, Tab, LockKind) when is_function(Fun) -> foldl(ActivityId, Opaque, Fun, Acc, Tab, LockKind) -> {Type, Prev} = init_iteration(ActivityId, Opaque, Tab, LockKind), - Res = (catch do_foldl(ActivityId, Opaque, Tab, dirty_first(Tab), Fun, Acc, Type, Prev)), + Res = ?CATCH(do_foldl(ActivityId, Opaque, Tab, dirty_first(Tab), Fun, Acc, Type, Prev)), close_iteration(Res, Tab). do_foldl(A, O, Tab, '$end_of_table', Fun, RAcc, _Type, Stored) -> @@ -1010,7 +1013,7 @@ foldr(ActivityId, Opaque, Fun, Acc, Tab, LockKind) -> true -> %% Order doesn't matter for set and bag TempPrev %% Keep the order so we can use ordsets:del_element end, - Res = (catch do_foldr(ActivityId, Opaque, Tab, dirty_last(Tab), Fun, Acc, Type, Prev)), + Res = ?CATCH(do_foldr(ActivityId, Opaque, Tab, dirty_last(Tab), Fun, Acc, Type, Prev)), close_iteration(Res, Tab). do_foldr(A, O, Tab, '$end_of_table', Fun, RAcc, _Type, Stored) -> @@ -1139,10 +1142,12 @@ match_object(_Tid, _Ts, Tab, Pat, _LockKind) -> add_written_match(S, Pat, Tab, Objs) -> Ops = find_ops(S, Tab, Pat), - add_match(Ops, Objs, val({Tab, setorbag})). + FixedRes = add_match(Ops, Objs, val({Tab, setorbag})), + MS = ets:match_spec_compile([{Pat, [], ['$_']}]), + ets:match_spec_run(FixedRes, MS). find_ops(S, Tab, Pat) -> - GetWritten = [{{{Tab, '_'}, Pat, write}, [], ['$_']}, + GetWritten = [{{{Tab, '_'}, '_', write}, [], ['$_']}, {{{Tab, '_'}, '_', delete}, [], ['$_']}, {{{Tab, '_'}, Pat, delete_object}, [], ['$_']}], ets:select(S, GetWritten). @@ -1542,16 +1547,9 @@ dirty_write(Tab, Val) -> do_dirty_write(SyncMode, Tab, Val) when is_atom(Tab), Tab /= schema, is_tuple(Val), tuple_size(Val) > 2 -> - case ?catch_val({Tab, record_validation}) of - {RecName, Arity, _Type} - when tuple_size(Val) == Arity, RecName == element(1, Val) -> - Oid = {Tab, element(2, Val)}, - mnesia_tm:dirty(SyncMode, {Oid, Val, write}); - {'EXIT', _} -> - abort({no_exists, Tab}); - _ -> - abort({bad_type, Val}) - end; + {_, _, _} = mnesia_lib:validate_record(Tab, Val), + Oid = {Tab, element(2, Val)}, + mnesia_tm:dirty(SyncMode, {Oid, Val, write}); do_dirty_write(_SyncMode, Tab, Val) -> abort({bad_type, Tab, Val}). @@ -1603,8 +1601,8 @@ dirty_update_counter(Tab, Key, Incr) -> do_dirty_update_counter(SyncMode, Tab, Key, Incr) when is_atom(Tab), Tab /= schema, is_integer(Incr) -> - case ?catch_val({Tab, record_validation}) of - {RecName, 3, set} -> + case mnesia_lib:validate_key(Tab, Key) of + {RecName, 3, Type} when Type == set; Type == ordered_set -> Oid = {Tab, Key}, mnesia_tm:dirty(SyncMode, {Oid, {RecName, Incr}, update_counter}); _ -> @@ -1623,13 +1621,7 @@ dirty_read(Oid) -> dirty_read(Tab, Key) when is_atom(Tab), Tab /= schema -> -%% case catch ?ets_lookup(Tab, Key) of -%% {'EXIT', _} -> - %% Bad luck, we have to perform a real lookup - dirty_rpc(Tab, mnesia_lib, db_get, [Tab, Key]); -%% Val -> -%% Val -%% end; + dirty_rpc(Tab, mnesia_lib, db_get, [Tab, Key]); dirty_read(Tab, _Key) -> abort({bad_type, Tab}). @@ -1808,7 +1800,7 @@ do_dirty_rpc(Tab, Node, M, F, Args) -> {badrpc, Reason} -> timer:sleep(20), %% Do not be too eager, and can't use yield on SMP %% Sync with mnesia_monitor - try sys:get_status(mnesia_monitor) catch _:_ -> ok end, + _ = try sys:get_status(mnesia_monitor) catch _:_ -> ok end, case mnesia_controller:call({check_w2r, Node, Tab}) of % Sync NewNode when NewNode =:= Node -> ErrorTag = mnesia_lib:dirty_rpc_error_tag(Reason), @@ -1902,21 +1894,23 @@ any_table_info(Tab, _Item) -> abort({bad_type, Tab}). raw_table_info(Tab, Item) -> - case ?catch_val({Tab, storage_type}) of - ram_copies -> - info_reply(catch ?ets_info(Tab, Item), Tab, Item); - disc_copies -> - info_reply(catch ?ets_info(Tab, Item), Tab, Item); - disc_only_copies -> - info_reply(catch dets:info(Tab, Item), Tab, Item); - unknown -> - bad_info_reply(Tab, Item); - {'EXIT', _} -> + try + case ?ets_lookup_element(mnesia_gvar, {Tab, storage_type}, 2) of + ram_copies -> + info_reply(?ets_info(Tab, Item), Tab, Item); + disc_copies -> + info_reply(?ets_info(Tab, Item), Tab, Item); + disc_only_copies -> + info_reply(dets:info(Tab, Item), Tab, Item); + {ext, Alias, Mod} -> + info_reply(catch Mod:info(Alias, Tab, Item), Tab, Item); + unknown -> + bad_info_reply(Tab, Item) + end + catch error:_ -> bad_info_reply(Tab, Item) end. -info_reply({'EXIT', _Reason}, Tab, Item) -> - bad_info_reply(Tab, Item); info_reply({error, _Reason}, Tab, Item) -> bad_info_reply(Tab, Item); info_reply(Val, _Tab, _Item) -> @@ -2022,15 +2016,26 @@ display_tab_info() -> MasterTabs = mnesia_recover:get_master_node_tables(), io:format("master node tables = ~p~n", [lists:sort(MasterTabs)]), + case get_backend_types() of + [] -> ok; + Ts -> list_backend_types(Ts, "backend types = ") + end, + + case get_index_plugins() of + [] -> ok; + Ps -> list_index_plugins(Ps, "index plugins = ") + end, + Tabs = system_info(tables), - {Unknown, Ram, Disc, DiscOnly} = - lists:foldl(fun storage_count/2, {[], [], [], []}, Tabs), + {Unknown, Ram, Disc, DiscOnly, Ext} = + lists:foldl(fun storage_count/2, {[], [], [], [], []}, Tabs), io:format("remote = ~p~n", [lists:sort(Unknown)]), io:format("ram_copies = ~p~n", [lists:sort(Ram)]), io:format("disc_copies = ~p~n", [lists:sort(Disc)]), io:format("disc_only_copies = ~p~n", [lists:sort(DiscOnly)]), + [io:format("~-19s= ~p~n", [atom_to_list(A), Ts]) || {A,Ts} <- Ext], Rfoldl = fun(T, Acc) -> Rpat = @@ -2038,7 +2043,7 @@ display_tab_info() -> read_only -> lists:sort([{A, read_only} || A <- val({T, active_replicas})]); read_write -> - table_info(T, where_to_commit) + [fix_wtc(W) || W <- table_info(T, where_to_commit)] end, case lists:keysearch(Rpat, 1, Acc) of {value, {_Rpat, Rtabs}} -> @@ -2051,18 +2056,65 @@ display_tab_info() -> Rdisp = fun({Rpat, Rtabs}) -> io:format("~p = ~p~n", [Rpat, Rtabs]) end, lists:foreach(Rdisp, lists:sort(Repl)). -storage_count(T, {U, R, D, DO}) -> +get_backend_types() -> + case ?catch_val({schema, user_property, mnesia_backend_types}) of + {'EXIT', _} -> + []; + {mnesia_backend_types, Ts} -> + lists:sort(Ts) + end. + +get_index_plugins() -> + case ?catch_val({schema, user_property, mnesia_index_plugins}) of + {'EXIT', _} -> + []; + {mnesia_index_plugins, Ps} -> + lists:sort(Ps) + end. + + +list_backend_types([{A,M} | T] = Ts, Legend) -> + Indent = [$\s || _ <- Legend], + W = integer_to_list( + lists:foldl(fun({Alias,_}, Wa) -> + erlang:max(Wa, length(atom_to_list(Alias))) + end, 0, Ts)), + io:fwrite(Legend ++ "~-" ++ W ++ "s - ~s~n", + [atom_to_list(A), atom_to_list(M)]), + [io:fwrite(Indent ++ "~-" ++ W ++ "s - ~s~n", + [atom_to_list(A1), atom_to_list(M1)]) + || {A1,M1} <- T]. + +list_index_plugins([{N,M,F} | T] = Ps, Legend) -> + Indent = [$\s || _ <- Legend], + W = integer_to_list( + lists:foldl(fun({N1,_,_}, Wa) -> + erlang:max(Wa, length(pp_ix_name(N1))) + end, 0, Ps)), + io:fwrite(Legend ++ "~-" ++ W ++ "s - ~s:~s~n", + [pp_ix_name(N), atom_to_list(M), atom_to_list(F)]), + [io:fwrite(Indent ++ "~-" ++ W ++ "s - ~s:~s~n", + [pp_ix_name(N1), atom_to_list(M1), atom_to_list(F1)]) + || {N1,M1,F1} <- T]. + +pp_ix_name(N) -> + lists:flatten(io_lib:fwrite("~w", [N])). + +fix_wtc({N, {ext,A,_}}) -> {N, A}; +fix_wtc({N,A}) when is_atom(A) -> {N, A}. + +storage_count(T, {U, R, D, DO, Ext}) -> case table_info(T, storage_type) of - unknown -> {[T | U], R, D, DO}; - ram_copies -> {U, [T | R], D, DO}; - disc_copies -> {U, R, [T | D], DO}; - disc_only_copies -> {U, R, D, [T | DO]} + unknown -> {[T | U], R, D, DO, Ext}; + ram_copies -> {U, [T | R], D, DO, Ext}; + disc_copies -> {U, R, [T | D], DO, Ext}; + disc_only_copies -> {U, R, D, [T | DO], Ext}; + {ext, A, _} -> {U, R, D, DO, orddict:append(A, T, Ext)} end. system_info(Item) -> - case catch system_info2(Item) of - {'EXIT',Error} -> abort(Error); - Other -> Other + try system_info2(Item) + catch _:Error -> abort(Error) end. system_info2(all) -> @@ -2072,9 +2124,10 @@ system_info2(all) -> system_info2(db_nodes) -> DiscNs = ?catch_val({schema, disc_copies}), RamNs = ?catch_val({schema, ram_copies}), + ExtNs = ?catch_val({schema, external_copies}), if - is_list(DiscNs), is_list(RamNs) -> - DiscNs ++ RamNs; + is_list(DiscNs), is_list(RamNs), is_list(ExtNs) -> + DiscNs ++ RamNs ++ ExtNs; true -> case mnesia_schema:read_nodes() of {ok, Nodes} -> Nodes; @@ -2168,7 +2221,7 @@ system_info2(version) -> Version; false -> %% Ensure that it does not match - {mnesia_not_loaded, node(), now()} + {mnesia_not_loaded, node(), erlang:timestamp()} end; Version -> Version @@ -2178,6 +2231,7 @@ system_info2(access_module) -> mnesia_monitor:get_env(access_module); system_info2(auto_repair) -> mnesia_monitor:get_env(auto_repair); system_info2(is_running) -> mnesia_lib:is_running(); system_info2(backup_module) -> mnesia_monitor:get_env(backup_module); +system_info2(backend_types) -> mnesia_schema:backend_types(); system_info2(event_module) -> mnesia_monitor:get_env(event_module); system_info2(debug) -> mnesia_monitor:get_env(debug); system_info2(dump_log_load_regulation) -> mnesia_monitor:get_env(dump_log_load_regulation); @@ -2214,6 +2268,7 @@ system_info_items(yes) -> [ access_module, auto_repair, + backend_types, backup_module, checkpoints, db_nodes, @@ -2307,11 +2362,17 @@ load_mnesia_or_abort() -> %% Database mgt create_schema(Ns) -> - mnesia_bup:create_schema(Ns). + create_schema(Ns, []). + +create_schema(Ns, Properties) -> + mnesia_bup:create_schema(Ns, Properties). delete_schema(Ns) -> mnesia_schema:delete_schema(Ns). +add_backend_type(Alias, Module) -> + mnesia_schema:add_backend_type(Alias, Module). + backup(Opaque) -> mnesia_log:backup(Opaque). @@ -2378,11 +2439,10 @@ del_table_index(Tab, Ix) -> mnesia_schema:del_table_index(Tab, Ix). transform_table(Tab, Fun, NewA) -> - case catch val({Tab, record_name}) of - {'EXIT', Reason} -> - mnesia:abort(Reason); - OldRN -> - mnesia_schema:transform_table(Tab, Fun, NewA, OldRN) + try val({Tab, record_name}) of + OldRN -> mnesia_schema:transform_table(Tab, Fun, NewA, OldRN) + catch exit:Reason -> + mnesia:abort(Reason) end. transform_table(Tab, Fun, NewA, NewRN) -> @@ -2554,6 +2614,9 @@ set_master_nodes(Tab, Nodes) -> dump_log() -> mnesia_controller:sync_dump_log(user). +sync_log() -> + mnesia_monitor:sync_log(latest_log). + subscribe(What) -> mnesia_subscr:subscribe(self(), What). @@ -2790,7 +2853,7 @@ pre_qlc(Opts, Tab) -> end. post_qlc(Tab) -> - case catch get(mnesia_activity_state) of + case get(mnesia_activity_state) of {_,#tid{},_} -> ok; _ -> case ?catch_val({Tab, setorbag}) of diff --git a/lib/mnesia/src/mnesia.hrl b/lib/mnesia/src/mnesia.hrl index 2855792646..0716dd87c8 100644 --- a/lib/mnesia/src/mnesia.hrl +++ b/lib/mnesia/src/mnesia.hrl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -35,11 +36,16 @@ -define(ets_last(Tab), ets:last(Tab)). -define(ets_prev(Tab, Key), ets:prev(Tab, Key)). -define(ets_slot(Tab, Pos), ets:slot(Tab, Pos)). --define(ets_new_table(Tab, Props), ets:new(Tab, Props)). +-define(ets_new_table(Tab, Props), _ = ets:new(Tab, Props)). -define(ets_delete_table(Tab), ets:delete(Tab)). -define(ets_fixtable(Tab, Bool), ets:fixtable(Tab, Bool)). --define(catch_val(Var), (catch ?ets_lookup_element(mnesia_gvar, Var, 2))). + +-define(SAFE(OP), try (OP) catch error:_ -> ok end). +-define(CATCH(OP), try (OP) catch _:_Reason -> {'EXIT', _Reason} end). + +-define(catch_val(Var), (try ?ets_lookup_element(mnesia_gvar, Var, 2) + catch error:_ -> {'EXIT', {badarg, []}} end)). %% It's important that counter is first, since we compare tid's @@ -53,13 +59,16 @@ up_stores = [], %% list of upper layer stores for nested trans level = 1}). %% transaction level --define(unique_cookie, {erlang:now(), node()}). +-define(unique_cookie, {{erlang:monotonic_time() + erlang:time_offset(), + erlang:unique_integer(),1}, + node()}). -record(cstruct, {name, % Atom type = set, % set | bag ram_copies = [], % [Node] disc_copies = [], % [Node] disc_only_copies = [], % [Node] + external_copies = [], % [{{Alias,Mod},[Node]}] load_order = 0, % Integer access_mode = read_write, % read_write | read_only majority = false, % true | false @@ -95,7 +104,7 @@ ram_copies = [], disc_copies = [], disc_only_copies = [], - snmp = [], + ext = [], schema_ops = [] }). diff --git a/lib/mnesia/src/mnesia_backend_type.erl b/lib/mnesia/src/mnesia_backend_type.erl new file mode 100644 index 0000000000..4791b82a01 --- /dev/null +++ b/lib/mnesia/src/mnesia_backend_type.erl @@ -0,0 +1,115 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2015. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% Behaviour definition for mnesia backend types. +%% + +%%%header_doc_include + +-module(mnesia_backend_type). + +-export([behaviour_info/1]). + +%%%header_doc_include + +%%%impl_doc_include + +%% Note that mnesia considers all callbacks mandatory!! +%% +behaviour_info(callbacks) -> + [ + {add_aliases, 1}, % (Aliases) -> ok + {check_definition, 4}, % (TypeAlias, Tab, Nodes, Properties) -> ok + {close_table, 2}, % (TypeAlias, Tab) -> ok + {create_table, 3}, % (TypeAlias, Tab, Properties) -> ok + {delete, 3}, % (TypeAlias, Tab, Key) -> true | ok + {delete_table, 2}, % (TypeAlias, Tab) -> ok + {first, 2}, % (TypeAlias, Tab) -> Key::Term | '$end_of_table' + {fixtable, 3}, % (TypeAlias, Tab, Bool) -> ok | true + {last, 2}, % (TypeAlias, Tab) -> Key::Term | '$end_of_table' + {index_is_consistent,3}, % (TypeAlias, IxTag, Bool) -> ok + {init_backend, 0}, % () -> ok + {info, 3}, % (TypeAlias, Tab, Item) -> Term + {insert, 3}, % (TypeAlias, Tab, Object) -> ok + {lookup, 3}, % (TypeAlias, Tab, Key) -> [Objects] + {is_index_consistent,2}, % (TypeAlias, IxTag) -> Bool + {load_table, 4}, % (TypeAlias, Tab, Reason, CsList) -> ok + {match_delete, 3}, % (TypeAlias, Tab, Pattern) -> ok + {next, 3}, % (TypeAlias, Tab, Key) -> Key::Term | '$end_of_table' + {prev, 3}, % (TypeAlias, Tab, Key) -> Key::Term | '$end_of_table' + {receiver_first_message, 4}, % (Sender, FirstMsg, Alias, Tab) -> {Size, State} + {receive_data, 5}, % (Data, Alias, Name, Sender, State) -> {more, State} | {{more, Msg}, State} + {receive_done, 4}, % (Alias, Tab, Sender, State) -> ok + {real_suffixes, 0}, % () -> [FileSuffix] + {remove_aliases, 1}, % (Aliases) -> ok + {repair_continuation, 2}, % (Continuation, MatchSpec) -> Continuation + {select, 1}, % (Continuation) -> {[Match], Continuation'} | '$end_of_table' + {select, 3}, % (TypeAlias, Tab, Pattern) -> {[Match], Continuation'} | '$end_of_table' + {select, 4}, % (TypeAlias, Tab, MatchSpec, Limit) {[Match], Continuation'} | '$end_of_table' + {sender_init, 4}, % (TypeAlias, Tab, LoadReason, Pid) -> + % {standard, Init(), Chunk()} | {Init(), Chunk()} + {semantics, 2}, % (TypeAlias, storage | types | index_fun | index_types) -> + % ram_copies | disc_copies, set | ordered_set | bag, fun(), ordered | bag + {slot, 3}, % (TypeAlias, Tab, Pos) -> '$end_of_table' | Objects | {error, Reason} + {sync_close_table, 2}, % (TypeAlias, Tab) -> ok + {tmp_suffixes, 0}, % () -> [FileSuffix] + {update_counter, 4}, % (TypeAlias, Tab, Counter, Val) -> NewVal + {validate_key, 6}, % (TypeAlias, Tab, RecName, Arity, Type, Key) -> {RecName, Arity, Type} + {validate_record, 6} % (TypeAlias, Tab, RecName, Arity, Type, Obj) -> {RecName, Arity, Type} + ]. + +%%%impl_doc_include + +%% -type tab() :: atom(). +%% -type alias() :: atom(). +%% -type rec_name() :: atom(). +%% -type type() :: set | bag | ordered_set. +%% -type proplist() :: [{atom(), any()}]. +%% -type key() :: any(). +%% -type db_object() :: tuple(). + +%% -type matchspec() :: ets:match_spec(). +%% -type limit() :: integer() | infinity. + +%% -type cont_fun() :: any(). +%% -type cont() :: '$end_of_table' | cont_fun(). + +%% -callback check_definition(alias(), tab(), [node()], proplist()) -> ok. +%% -callback create_table(alias(), tab(), proplist()) -> tab(). +%% -callback load_table(alias(), tab(), any()) -> ok. +%% -callback delete_table(alias(), tab()) -> ok. +%% -callback first(alias(), tab()) -> key(). +%% -callback last(alias(), tab()) -> key(). +%% -callback next(alias(), tab(), key()) -> key(). +%% -callback prev(alias(), tab(), key()) -> key(). +%% -callback insert(alias(), tab(), db_object()) -> ok. +%% -callback lookup(alias(), tab(), key()) -> [db_object()]. +%% -callback delete(alias(), tab(), key()) -> ok. +%% -callback update_counter(alias(), tab(), key(), integer()) -> integer(). +%% -callback select(cont()) -> {list(), cont()}. +%% -callback select(alias(), tab(), matchspec()) -> list() | '$end_of_table'. +%% -callback select(alias(), tab(), matchspec(), limit()) -> {list(), cont()}. +%% -callback slot(alias(), tab(), integer()) -> key(). +%% -callback validate_key(alias(), tab(), rec_name(), arity(), type(), key()) -> +%% {rec_name(), arity(), type()}. +%% -callback validate_record(alias(),tab(),rec_name(),arity(),type(),db_obj()) -> +%% {rec_name(), arity(), type()}. +%% -callback repair_continuation(cont(), matchspec()) -> cont(). diff --git a/lib/mnesia/src/mnesia_backup.erl b/lib/mnesia/src/mnesia_backup.erl index 736f2ed9bf..719e010c17 100644 --- a/lib/mnesia/src/mnesia_backup.erl +++ b/lib/mnesia/src/mnesia_backup.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. -%% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/mnesia/src/mnesia_bup.erl b/lib/mnesia/src/mnesia_bup.erl index fd87be1759..3e55deb958 100644 --- a/lib/mnesia/src/mnesia_bup.erl +++ b/lib/mnesia/src/mnesia_bup.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -27,6 +28,7 @@ fallback_exists/0, tm_fallback_start/1, create_schema/1, + create_schema/2, install_fallback/1, install_fallback/2, uninstall_fallback/0, @@ -45,7 +47,7 @@ uninstall_fallback_master/2, local_uninstall_fallback/2, do_traverse_backup/7, - trav_apply/4 + trav_apply/5 ]). -include("mnesia.hrl"). @@ -78,45 +80,58 @@ %% BunchOfRecords will be [] when the iteration is done. iterate(Mod, Fun, Opaque, Acc) -> R = #restore{bup_module = Mod, bup_data = Opaque}, - case catch read_schema_section(R) of - {error, Reason} -> - {error, Reason}; - {R2, {Header, Schema, Rest}} -> - case catch iter(R2, Header, Schema, Fun, Acc, Rest) of - {ok, R3, Res} -> - catch safe_apply(R3, close_read, [R3#restore.bup_data]), - {ok, Res}; - {error, Reason} -> - catch safe_apply(R2, close_read, [R2#restore.bup_data]), - {error, Reason}; - {'EXIT', Pid, Reason} -> - catch safe_apply(R2, close_read, [R2#restore.bup_data]), - {error, {'EXIT', Pid, Reason}}; - {'EXIT', Reason} -> - catch safe_apply(R2, close_read, [R2#restore.bup_data]), - {error, {'EXIT', Reason}} - end + try read_schema_section(R) of + {R2, {Header, Schema, Rest}} -> + Ext = get_ext_types(Schema), + try iter(R2, Header, Schema, Ext, Fun, Acc, Rest) of + {ok, R3, Res} -> + close_read(R3), + {ok, Res} + catch throw:Err -> + close_read(R2), + Err; + _:Reason -> + close_read(R2), + {error, {Reason, erlang:get_stacktrace()}} + end + catch throw:{error,_} = Err -> + Err end. -iter(R, Header, Schema, Fun, Acc, []) -> +get_ext_types(Schema) -> + try + List = lookup_schema(schema, Schema), + case lists:keyfind(user_properties, 1, List) of + {_, Props} -> + proplists:get_value( + mnesia_backend_types, Props, []); + false -> + [] + end + catch + throw:{error, {"Cannot lookup",_}} -> + [] + end. + +iter(R, Header, Schema, Ext, Fun, Acc, []) -> case safe_apply(R, read, [R#restore.bup_data]) of {R2, []} -> - Res = Fun([], Header, Schema, Acc), + Res = Fun([], Header, Schema, Ext, Acc), {ok, R2, Res}; {R2, BupItems} -> - iter(R2, Header, Schema, Fun, Acc, BupItems) + iter(R2, Header, Schema, Ext, Fun, Acc, BupItems) end; -iter(R, Header, Schema, Fun, Acc, BupItems) -> - Acc2 = Fun(BupItems, Header, Schema, Acc), - iter(R, Header, Schema, Fun, Acc2, []). +iter(R, Header, Schema, Ext, Fun, Acc, BupItems) -> + Acc2 = Fun(BupItems, Header, Schema, Ext, Acc), + iter(R, Header, Schema, Ext, Fun, Acc2, []). -spec safe_apply(#restore{}, atom(), list()) -> tuple(). safe_apply(R, write, [_, Items]) when Items =:= [] -> R; safe_apply(R, What, Args) -> - Abort = fun(Re) -> abort_restore(R, What, Args, Re) end, + Abort = abort_restore_fun(R, What, Args), Mod = R#restore.bup_module, - case catch apply(Mod, What, Args) of + try apply(Mod, What, Args) of {ok, Opaque, Items} when What =:= read -> {R#restore{bup_data = Opaque}, Items}; {ok, Opaque} when What =/= read-> @@ -125,16 +140,23 @@ safe_apply(R, What, Args) -> Abort(Re); Re -> Abort(Re) + catch _:Re -> + Abort(Re) end. -abort_restore(R, What, Args, Reason) -> - Mod = R#restore.bup_module, - Opaque = R#restore.bup_data, +-spec abort_restore_fun(_, _, _) -> fun((_) -> no_return()). +abort_restore_fun(R, What, Args) -> + fun(Re) -> abort_restore(R, What, Args, Re) end. + +abort_restore(R = #restore{bup_module=Mod}, What, Args, Reason) -> dbg_out("Restore aborted. ~p:~p~p -> ~p~n", [Mod, What, Args, Reason]), - catch apply(Mod, close_read, [Opaque]), + close_read(R), throw({error, Reason}). +close_read(#restore{bup_module=Mod, bup_data=Opaque}) -> + ?SAFE(Mod:close_read(Opaque)). + fallback_to_schema() -> Fname = fallback_bup(), fallback_to_schema(Fname). @@ -145,45 +167,41 @@ fallback_to_schema(Fname) -> {error, Reason} -> {error, Reason}; Schema -> - case catch lookup_schema(schema, Schema) of - {error, _} -> - {error, "No schema in fallback"}; - List -> - {ok, fallback, List} + try lookup_schema(schema, Schema) of + List -> {ok, fallback, List} + catch throw:_ -> + {error, "No schema in fallback"} end end. %% Opens Opaque reads schema and then close read_schema(Mod, Opaque) -> R = #restore{bup_module = Mod, bup_data = Opaque}, - case catch read_schema_section(R) of - {error, Reason} -> - {error, Reason}; + try read_schema_section(R) of {R2, {_Header, Schema, _}} -> - catch safe_apply(R2, close_read, [R2#restore.bup_data]), - Schema + close_read(R2), + Schema + catch throw:{error,_} = Error -> + Error end. %% Open backup media and extract schema %% rewind backup media and leave it open %% Returns {R, {Header, Schema}} read_schema_section(R) -> - case catch do_read_schema_section(R) of - {'EXIT', Reason} -> - catch safe_apply(R, close_read, [R#restore.bup_data]), - {error, {'EXIT', Reason}}; - {error, Reason} -> - catch safe_apply(R, close_read, [R#restore.bup_data]), - {error, Reason}; - {R2, {H, Schema, Rest}} -> - Schema2 = convert_schema(H#log_header.log_version, Schema), - {R2, {H, Schema2, Rest}} - end. + {R2, {H, Schema, Rest}} = do_read_schema_section(R), + Schema2 = convert_schema(H#log_header.log_version, Schema), + {R2, {H, Schema2, Rest}}. do_read_schema_section(R) -> R2 = safe_apply(R, open_read, [R#restore.bup_data]), - {R3, RawSchema} = safe_apply(R2, read, [R2#restore.bup_data]), - do_read_schema_section(R3, verify_header(RawSchema), []). + try + {R3, RawSchema} = safe_apply(R2, read, [R2#restore.bup_data]), + do_read_schema_section(R3, verify_header(RawSchema), []) + catch T:E -> + close_read(R2), + erlang:raise(T,E,erlang:get_stacktrace()) + end. do_read_schema_section(R, {ok, B, C, []}, Acc) -> case safe_apply(R, read, [R#restore.bup_data]) of @@ -201,7 +219,7 @@ do_read_schema_section(R, {ok, B, _C, Rest}, Acc) -> {R, {B, Acc, Rest}}; do_read_schema_section(_R, {error, Reason}, _Acc) -> - {error, Reason}. + throw({error, Reason}). verify_header([H | RawSchema]) when is_record(H, log_header) -> Current = mnesia_log:backup_log_header(), @@ -218,7 +236,7 @@ verify_header([H | RawSchema]) when is_record(H, log_header) -> {error, {"Bad kind of header. Cannot be used as backup.", H}} end; verify_header(RawSchema) -> - {error, {"Missing header. Cannot be used as backup.", catch hd(RawSchema)}}. + {error, {"Missing header. Cannot be used as backup.", ?CATCH(hd(RawSchema))}}. refresh_cookie(Schema, NewCookie) -> case lists:keysearch(schema, 2, Schema) of @@ -261,7 +279,7 @@ convert_0_1(Schema) -> Cs = mnesia_schema:list2cs(List), convert_0_1(Schema2, [], Cs); false -> - List = mnesia_schema:get_initial_schema(disc_copies, [node()]), + List = mnesia_schema:get_initial_schema(disc_copies, [node()], []), Cs = mnesia_schema:list2cs(List), convert_0_1(Schema, [], Cs) end. @@ -309,16 +327,19 @@ schema2bup({schema, Tab, TableDef}) -> %% Create schema on the given nodes %% Requires that old schemas has been deleted %% Returns ok | {error, Reason} -create_schema([]) -> - create_schema([node()]); -create_schema(Ns) when is_list(Ns) -> +create_schema(Nodes) -> + create_schema(Nodes, []). + +create_schema([], Props) -> + create_schema([node()], Props); +create_schema(Ns, Props) when is_list(Ns), is_list(Props) -> case is_set(Ns) of true -> - create_schema(Ns, mnesia_schema:ensure_no_schema(Ns)); + create_schema(Ns, mnesia_schema:ensure_no_schema(Ns), Props); false -> {error, {combine_error, Ns}} end; -create_schema(Ns) -> +create_schema(Ns, _Props) -> {error, {badarg, Ns}}. is_set(List) when is_list(List) -> @@ -326,7 +347,7 @@ is_set(List) when is_list(List) -> is_set(_) -> false. -create_schema(Ns, ok) -> +create_schema(Ns, ok, Props) -> %% Ensure that we access the intended Mnesia %% directory. This function may not be called %% during startup since it will cause the @@ -345,7 +366,7 @@ create_schema(Ns, ok) -> Str = mk_str(), File = mnesia_lib:dir(Str), file:delete(File), - case catch make_initial_backup(Ns, File, Mod) of + try make_initial_backup(Ns, File, Mod, Props) of {ok, _Res} -> case do_install_fallback(File, Mod) of ok -> @@ -353,8 +374,8 @@ create_schema(Ns, ok) -> ok; {error, Reason} -> {error, Reason} - end; - {error, Reason} -> + end + catch throw:{error, Reason} -> {error, Reason} end end @@ -362,17 +383,20 @@ create_schema(Ns, ok) -> {error, Reason} -> {error, Reason} end; -create_schema(_Ns, {error, Reason}) -> +create_schema(_Ns, {error, Reason}, _) -> {error, Reason}; -create_schema(_Ns, Reason) -> +create_schema(_Ns, Reason, _) -> {error, Reason}. mk_str() -> - Now = [integer_to_list(I) || I <- tuple_to_list(now())], + Now = integer_to_list(erlang:unique_integer([positive])), lists:concat([node()] ++ Now ++ ".TMP"). make_initial_backup(Ns, Opaque, Mod) -> - Orig = mnesia_schema:get_initial_schema(disc_copies, Ns), + make_initial_backup(Ns, Opaque, Mod, []). + +make_initial_backup(Ns, Opaque, Mod, Props) -> + Orig = mnesia_schema:get_initial_schema(disc_copies, Ns, Props), Modded = proplists:delete(storage_properties, proplists:delete(majority, Orig)), Schema = [{schema, schema, Modded}], O2 = do_apply(Mod, open_write, [Opaque], Opaque), @@ -384,10 +408,11 @@ make_initial_backup(Ns, Opaque, Mod) -> do_apply(_, write, [_, Items], Opaque) when Items =:= [] -> Opaque; do_apply(Mod, What, Args, _Opaque) -> - case catch apply(Mod, What, Args) of + try apply(Mod, What, Args) of {ok, Opaque2} -> Opaque2; - {error, Reason} -> throw({error, Reason}); - {'EXIT', Reason} -> throw({error, {'EXIT', Reason}}) + {error, Reason} -> throw({error, Reason}) + catch _:Reason -> + throw({error, {'EXIT', Reason}}) end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -425,11 +450,11 @@ do_install_fallback(_Opaque, Args) -> {error, {badarg, Args}}. check_fallback_args([Arg | Tail], FA) -> - case catch check_fallback_arg_type(Arg, FA) of - {'EXIT', _Reason} -> - {error, {badarg, Arg}}; + try check_fallback_arg_type(Arg, FA) of FA2 -> check_fallback_args(Tail, FA2) + catch error:_ -> + {error, {badarg, Arg}} end; check_fallback_args([], FA) -> {ok, FA}. @@ -484,38 +509,38 @@ install_fallback_master(ClientPid, FA) -> State = {start, FA}, Opaque = FA#fallback_args.opaque, Mod = FA#fallback_args.module, - Res = (catch iterate(Mod, fun restore_recs/4, Opaque, State)), + Res = iterate(Mod, fun restore_recs/5, Opaque, State), unlink(ClientPid), ClientPid ! {self(), Res}, exit(shutdown). -restore_recs(_, _, _, stop) -> +restore_recs(_, _, _, _, stop) -> throw({error, "restore_recs already stopped"}); -restore_recs(Recs, Header, Schema, {start, FA}) -> +restore_recs(Recs, Header, Schema, Ext, {start, FA}) -> %% No records in backup Schema2 = convert_schema(Header#log_header.log_version, Schema), CreateList = lookup_schema(schema, Schema2), - case catch mnesia_schema:list2cs(CreateList) of - {'EXIT', Reason} -> - throw({error, {"Bad schema in restore_recs", Reason}}); + try mnesia_schema:list2cs(CreateList) of Cs -> Ns = get_fallback_nodes(FA, Cs#cstruct.disc_copies), global:set_lock({{mnesia_table_lock, schema}, self()}, Ns, infinity), Args = [self(), FA], Pids = [spawn_link(N, ?MODULE, fallback_receiver, Args) || N <- Ns], send_fallback(Pids, {start, Header, Schema2}), - Res = restore_recs(Recs, Header, Schema2, Pids), + Res = restore_recs(Recs, Header, Schema2, Ext, Pids), global:del_lock({{mnesia_table_lock, schema}, self()}, Ns), Res + catch _:Reason -> + throw({error, {"Bad schema in restore_recs", Reason}}) end; -restore_recs([], _Header, _Schema, Pids) -> +restore_recs([], _Header, _Schema, _Ext, Pids) -> send_fallback(Pids, swap), send_fallback(Pids, stop), stop; -restore_recs(Recs, _, _, Pids) -> +restore_recs(Recs, _, _, _, Pids) -> send_fallback(Pids, {records, Recs}), Pids. @@ -579,45 +604,46 @@ fallback_tmp_name() -> "FALLBACK.TMP". fallback_receiver(Master, FA) -> process_flag(trap_exit, true), - case catch register(mnesia_fallback, self()) of - {'EXIT', _} -> - Reason = {already_exists, node()}, - local_fallback_error(Master, Reason); - true -> - FA2 = check_fallback_dir(Master, FA), - Bup = FA2#fallback_args.fallback_bup, - case mnesia_lib:exists(Bup) of - true -> - Reason2 = {already_exists, node()}, - local_fallback_error(Master, Reason2); - false -> - Mod = mnesia_backup, - Tmp = FA2#fallback_args.fallback_tmp, - R = #restore{mode = replace, - bup_module = Mod, - bup_data = Tmp}, - file:delete(Tmp), - case catch fallback_receiver_loop(Master, R, FA2, schema) of - {error, Reason} -> - local_fallback_error(Master, Reason); - Other -> - exit(Other) - end - end - end. + Res = try + register(mnesia_fallback, self()), + FA2 = check_fallback_dir(FA), + Bup = FA2#fallback_args.fallback_bup, + false = mnesia_lib:exists(Bup), + Mod = mnesia_backup, + Tmp = FA2#fallback_args.fallback_tmp, + R = #restore{mode = replace, + bup_module = Mod, + bup_data = Tmp}, + file:delete(Tmp), + fallback_receiver_loop(Master, R, FA2, schema) + catch + error:_ -> + Reason = {already_exists, node()}, + local_fallback_error(Master, Reason); + throw:{error, Reason} -> + local_fallback_error(Master, Reason) + end, + exit(Res). local_fallback_error(Master, Reason) -> Master ! {self(), {error, Reason}}, unlink(Master), exit(Reason). + check_fallback_dir(Master, FA) -> + try check_fallback_dir(FA) + catch throw:{error,Reason} -> + local_fallback_error(Master, Reason) + end. + +check_fallback_dir(FA) -> case mnesia:system_info(schema_location) of ram -> Reason = {has_no_disc, node()}, - local_fallback_error(Master, Reason); + throw({error, Reason}); _ -> - Dir = check_fallback_dir_arg(Master, FA), + Dir = check_fallback_dir_arg(FA), Bup = filename:join([Dir, fallback_name()]), Tmp = filename:join([Dir, fallback_tmp_name()]), FA#fallback_args{fallback_bup = Bup, @@ -625,22 +651,20 @@ check_fallback_dir(Master, FA) -> mnesia_dir = Dir} end. -check_fallback_dir_arg(Master, FA) -> +check_fallback_dir_arg(FA) -> case FA#fallback_args.use_default_dir of true -> mnesia_lib:dir(); false when FA#fallback_args.scope =:= local -> Dir = FA#fallback_args.mnesia_dir, - case catch mnesia_monitor:do_check_type(dir, Dir) of - {'EXIT', _R} -> + try mnesia_monitor:do_check_type(dir, Dir) + catch _:_ -> Reason = {badarg, {dir, Dir}, node()}, - local_fallback_error(Master, Reason); - AbsDir-> - AbsDir - end; + throw({error, Reason}) + end; false when FA#fallback_args.scope =:= global -> Reason = {combine_error, global, dir, node()}, - local_fallback_error(Master, Reason) + throw({error, Reason}) end. fallback_receiver_loop(Master, R, FA, State) -> @@ -666,7 +690,7 @@ fallback_receiver_loop(Master, R, FA, State) -> Bup = FA#fallback_args.fallback_bup, Tmp = FA#fallback_args.fallback_tmp, throw_bad_res(ok, file:rename(Tmp, Bup)), - catch mnesia_lib:set(active_fallback, true), + ?SAFE(mnesia_lib:set(active_fallback, true)), ?eval_debug_fun({?MODULE, fallback_receiver_loop, post_swap}, []), Master ! {self(), ok}, fallback_receiver_loop(Master, R, FA, stop); @@ -697,7 +721,7 @@ throw_bad_res(_Expected, Actual) -> throw({error, Actual}). tm_fallback_start(IgnoreFallback) -> mnesia_schema:lock_schema(), Res = do_fallback_start(fallback_exists(), IgnoreFallback), - mnesia_schema: unlock_schema(), + mnesia_schema:unlock_schema(), case Res of ok -> ok; {error, Reason} -> exit(Reason) @@ -715,9 +739,9 @@ do_fallback_start(true, false) -> BupFile = fallback_bup(), Mod = mnesia_backup, LocalTabs = ?ets_new_table(mnesia_local_tables, [set, public, {keypos, 2}]), - case catch iterate(Mod, fun restore_tables/4, BupFile, {start, LocalTabs}) of + case iterate(Mod, fun restore_tables/5, BupFile, {start, LocalTabs}) of {ok, _Res} -> - catch dets:close(schema), + ?SAFE(dets:close(schema)), TmpSchema = mnesia_lib:tab2tmp(schema), DatSchema = mnesia_lib:tab2dat(schema), AllLT = ?ets_match_object(LocalTabs, '_'), @@ -733,28 +757,27 @@ do_fallback_start(true, false) -> {error, {"Cannot start from fallback. Rename error.", Reason}} end; {error, Reason} -> - {error, {"Cannot start from fallback", Reason}}; - {'EXIT', Reason} -> {error, {"Cannot start from fallback", Reason}} end. -restore_tables(All=[Rec | Recs], Header, Schema, State={local, LocalTabs, LT}) -> +restore_tables(All=[Rec | Recs], Header, Schema, Ext, + State={local, LocalTabs, LT}) -> Tab = element(1, Rec), if Tab =:= LT#local_tab.name -> Key = element(2, Rec), (LT#local_tab.add)(Tab, Key, Rec, LT), - restore_tables(Recs, Header, Schema, State); + restore_tables(Recs, Header, Schema, Ext, State); true -> NewState = {new, LocalTabs}, - restore_tables(All, Header, Schema, NewState) + restore_tables(All, Header, Schema, Ext, NewState) end; -restore_tables(All=[Rec | Recs], Header, Schema, {new, LocalTabs}) -> +restore_tables(All=[Rec | Recs], Header, Schema, Ext, {new, LocalTabs}) -> Tab = element(1, Rec), case ?ets_lookup(LocalTabs, Tab) of [] -> State = {not_local, LocalTabs, Tab}, - restore_tables(Recs, Header, Schema, State); + restore_tables(Recs, Header, Schema, Ext, State); [LT] when is_record(LT, local_tab) -> State = {local, LocalTabs, LT}, case LT#local_tab.opened of @@ -763,38 +786,39 @@ restore_tables(All=[Rec | Recs], Header, Schema, {new, LocalTabs}) -> (LT#local_tab.open)(Tab, LT), ?ets_insert(LocalTabs,LT#local_tab{opened=true}) end, - restore_tables(All, Header, Schema, State) + restore_tables(All, Header, Schema, Ext, State) end; -restore_tables(All=[Rec | Recs], Header, Schema, S = {not_local, LocalTabs, PrevTab}) -> +restore_tables(All=[Rec | Recs], Header, Schema, Ext, + S = {not_local, LocalTabs, PrevTab}) -> Tab = element(1, Rec), if Tab =:= PrevTab -> - restore_tables(Recs, Header, Schema, S); + restore_tables(Recs, Header, Schema, Ext, S); true -> State = {new, LocalTabs}, - restore_tables(All, Header, Schema, State) + restore_tables(All, Header, Schema, Ext, State) end; -restore_tables(Recs, Header, Schema, {start, LocalTabs}) -> +restore_tables(Recs, Header, Schema, Ext, {start, LocalTabs}) -> Dir = mnesia_lib:dir(), OldDir = filename:join([Dir, "OLD_DIR"]), mnesia_schema:purge_dir(OldDir, []), mnesia_schema:purge_dir(Dir, [fallback_name()]), - init_dat_files(Schema, LocalTabs), + init_dat_files(Schema, Ext, LocalTabs), State = {new, LocalTabs}, - restore_tables(Recs, Header, Schema, State); -restore_tables([], _Header, _Schema, State) -> + restore_tables(Recs, Header, Schema, Ext, State); +restore_tables([], _Header, _Schema, _Ext, State) -> State. %% Creates all neccessary dat files and inserts %% the table definitions in the schema table %% %% Returns a list of local_tab tuples for all local tables -init_dat_files(Schema, LocalTabs) -> +init_dat_files(Schema, Ext, LocalTabs) -> TmpFile = mnesia_lib:tab2tmp(schema), Args = [{file, TmpFile}, {keypos, 2}, {type, set}], case dets:open_file(schema, Args) of % Assume schema lock {ok, _} -> - create_dat_files(Schema, LocalTabs), + create_dat_files(Schema, Ext, LocalTabs), ok = dets:close(schema), LocalTab = #local_tab{name = schema, storage_type = disc_copies, @@ -809,10 +833,10 @@ init_dat_files(Schema, LocalTabs) -> throw({error, {"Cannot open file", schema, Args, Reason}}) end. -create_dat_files([{schema, schema, TabDef} | Tail], LocalTabs) -> +create_dat_files([{schema, schema, TabDef} | Tail], Ext, LocalTabs) -> ok = dets:insert(schema, {schema, schema, TabDef}), - create_dat_files(Tail, LocalTabs); -create_dat_files([{schema, Tab, TabDef} | Tail], LocalTabs) -> + create_dat_files(Tail, Ext, LocalTabs); +create_dat_files([{schema, Tab, TabDef} | Tail], Ext, LocalTabs) -> TmpFile = mnesia_lib:tab2tmp(Tab), DatFile = mnesia_lib:tab2dat(Tab), DclFile = mnesia_lib:tab2dcl(Tab), @@ -825,56 +849,21 @@ create_dat_files([{schema, Tab, TabDef} | Tail], LocalTabs) -> mnesia_lib:dets_sync_close(Tab), file:delete(TmpFile), - Cs = mnesia_schema:list2cs(TabDef), + Cs = mnesia_schema:list2cs(TabDef, Ext), ok = dets:insert(schema, {schema, Tab, TabDef}), RecName = Cs#cstruct.record_name, Storage = mnesia_lib:cs_to_storage_type(node(), Cs), + delete_ext(Storage, Tab), + Semantics = mnesia_lib:semantics(Storage, storage), if - Storage =:= unknown -> + Semantics =:= undefined -> ok = dets:delete(schema, {schema, Tab}), - create_dat_files(Tail, LocalTabs); - Storage =:= disc_only_copies -> - Args = [{file, TmpFile}, {keypos, 2}, - {type, mnesia_lib:disk_type(Tab, Cs#cstruct.type)}], - Open = fun(T, LT) when T =:= LT#local_tab.name -> - case mnesia_lib:dets_sync_open(T, Args) of - {ok, _} -> - ok; - {error, Reason} -> - throw({error, {"Cannot open file", T, Args, Reason}}) - end - end, - Add = fun(T, Key, Rec, LT) when T =:= LT#local_tab.name -> - case Rec of - {_T, Key} -> - ok = dets:delete(T, Key); - (Rec) when T =:= RecName -> - ok = dets:insert(Tab, Rec); - (Rec) -> - Rec2 = setelement(1, Rec, RecName), - ok = dets:insert(T, Rec2) - end - end, - Close = fun(T, LT) when T =:= LT#local_tab.name -> - mnesia_lib:dets_sync_close(T) - end, - Swap = fun(T, LT) when T =:= LT#local_tab.name -> - Expunge(), - case LT#local_tab.opened of - true -> - Close(T,LT); - false -> - Open(T,LT), - Close(T,LT) - end, - case file:rename(TmpFile, DatFile) of - ok -> - ok; - {error, Reason} -> - mnesia_lib:fatal("Cannot rename file ~p -> ~p: ~p~n", - [TmpFile, DatFile, Reason]) - end - end, + create_dat_files(Tail, Ext, LocalTabs); + Semantics =:= disc_only_copies -> + Open = disc_only_open_fun(Storage, Cs), + Add = disc_only_add_fun(Storage, Cs), + Close = disc_only_close_fun(Storage), + Swap = disc_only_swap_fun(Storage, Expunge, Open, Close), LocalTab = #local_tab{name = Tab, storage_type = Storage, open = Open, @@ -884,8 +873,8 @@ create_dat_files([{schema, Tab, TabDef} | Tail], LocalTabs) -> record_name = RecName, opened = false}, ?ets_insert(LocalTabs, LocalTab), - create_dat_files(Tail, LocalTabs); - Storage =:= ram_copies; Storage =:= disc_copies -> + create_dat_files(Tail, Ext, LocalTabs); + Semantics =:= ram_copies; Storage =:= disc_copies -> Open = fun(T, LT) when T =:= LT#local_tab.name -> mnesia_log:open_log({?MODULE, T}, mnesia_log:dcl_log_header(), @@ -946,18 +935,97 @@ create_dat_files([{schema, Tab, TabDef} | Tail], LocalTabs) -> opened = false }, ?ets_insert(LocalTabs, LocalTab), - create_dat_files(Tail, LocalTabs) + create_dat_files(Tail, Ext, LocalTabs); + true -> + error({unknown_semantics, [{semantics, Semantics}, + {tabdef, TabDef}, + {ext, Ext}]}) end; -create_dat_files([{schema, Tab} | Tail], LocalTabs) -> +create_dat_files([{schema, Tab} | Tail], Ext, LocalTabs) -> ?ets_delete(LocalTabs, Tab), ok = dets:delete(schema, {schema, Tab}), TmpFile = mnesia_lib:tab2tmp(Tab), mnesia_lib:dets_sync_close(Tab), file:delete(TmpFile), - create_dat_files(Tail, LocalTabs); -create_dat_files([], _LocalTabs) -> + create_dat_files(Tail, Ext, LocalTabs); +create_dat_files([], _Ext, _LocalTabs) -> ok. +delete_ext({ext, Alias, Mod}, Tab) -> + Mod:close_table(Alias, Tab), + Mod:delete_table(Alias, Tab), + ok; +delete_ext(_, _) -> + ok. + + +disc_only_open_fun(disc_only_copies, #cstruct{name = Tab} =Cs) -> + TmpFile = mnesia_lib:tab2tmp(Tab), + Args = [{file, TmpFile}, {keypos, 2}, + {type, mnesia_lib:disk_type(Tab, Cs#cstruct.type)}], + fun(T, LT) when T =:= LT#local_tab.name -> + case mnesia_lib:dets_sync_open(T, Args) of + {ok, _} -> + ok; + {error, Reason} -> + throw({error, {"Cannot open file", T, Args, Reason}}) + end + end; +disc_only_open_fun({ext,Alias,Mod}, Cs) -> + fun(T, LT) when T =:= LT#local_tab.name -> + ok = Mod:load_table(Alias, T, restore, mnesia_schema:cs2list(Cs)) + end. + +disc_only_add_fun(Storage, #cstruct{name = Tab, + record_name = RecName}) -> + fun(T, Key, Rec, #local_tab{name = T}) when T =:= Tab-> + case Rec of + {_T, Key} -> + ok = mnesia_lib:db_erase(Storage, T, Key); + (Rec) when T =:= RecName -> + ok = mnesia_lib:db_put(Storage, T, Rec); + (Rec) -> + ok = mnesia_lib:db_put(Storage, T, + setelement(1, Rec, RecName)) + end + end. + +disc_only_close_fun(disc_only_copies) -> + fun(T, LT) when T =:= LT#local_tab.name -> + mnesia_lib:dets_sync_close(T) + end; +disc_only_close_fun({ext, Alias, Mod}) -> + fun(T, _LT) -> + Mod:sync_close_table(Alias, T) + end. + + +disc_only_swap_fun(disc_only_copies, Expunge, Open, Close) -> + fun(T, LT) when T =:= LT#local_tab.name -> + TmpFile = mnesia_lib:tab2tmp(T), + DatFile = mnesia_lib:tab2dat(T), + Expunge(), + case LT#local_tab.opened of + true -> + Close(T,LT); + false -> + Open(T,LT), + Close(T,LT) + end, + case file:rename(TmpFile, DatFile) of + ok -> + ok; + {error, Reason} -> + mnesia_lib:fatal("Cannot rename file ~p -> ~p: ~p~n", + [TmpFile, DatFile, Reason]) + end + end; +disc_only_swap_fun({ext, _Alias, _Mod}, _Expunge, _Open, Close) -> + fun(T, #local_tab{name = T} = LT) -> + Close(T, LT) + end. + + uninstall_fallback() -> uninstall_fallback([{scope, global}]). @@ -996,10 +1064,10 @@ uninstall_fallback_master(ClientPid, FA) -> case fallback_to_schema(Bup) of {ok, fallback, List} -> Cs = mnesia_schema:list2cs(List), - case catch get_fallback_nodes(FA, Cs#cstruct.disc_copies) of + try get_fallback_nodes(FA, Cs#cstruct.disc_copies) of Ns when is_list(Ns) -> - do_uninstall(ClientPid, Ns, FA); - {error, Reason} -> + do_uninstall(ClientPid, Ns, FA) + catch throw:{error, Reason} -> local_fallback_error(ClientPid, Reason) end; {error, Reason} -> @@ -1042,21 +1110,17 @@ local_uninstall_fallback(Master, FA) -> %% Don't trap exit register(mnesia_fallback, self()), % May exit - FA2 = check_fallback_dir(Master, FA), % May exit + FA2 = check_fallback_dir(Master, FA), % May exit Master ! {self(), started}, receive {Master, do_uninstall} -> ?eval_debug_fun({?MODULE, uninstall_fallback2, pre_delete}, []), - catch mnesia_lib:set(active_fallback, false), + ?SAFE(mnesia_lib:set(active_fallback, false)), Tmp = FA2#fallback_args.fallback_tmp, Bup = FA2#fallback_args.fallback_bup, file:delete(Tmp), - Res = - case fallback_exists(Bup) of - true -> file:delete(Bup); - false -> ok - end, + Res = file:delete(Bup), ?eval_debug_fun({?MODULE, uninstall_fallback2, post_delete}, []), Master ! {self(), Res}, unlink(Master), @@ -1075,10 +1139,8 @@ rec_uninstall(ClientPid, [Pid | Pids], AccRes) -> {Pid, BadRes} -> rec_uninstall(ClientPid, Pids, BadRes) end; -rec_uninstall(ClientPid, [], Res) -> - ClientPid ! {self(), Res}, - unlink(ClientPid), - exit(normal). +rec_uninstall(_, [], Res) -> + Res. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Backup traversal @@ -1129,30 +1191,29 @@ do_traverse_backup(ClientPid, Source, SourceMod, Target, TargetMod, Fun, Acc) -> Iter = if TargetMod =/= read_only -> - case catch do_apply(TargetMod, open_write, [Target], Target) of - {error, Error} -> + try do_apply(TargetMod, open_write, [Target], Target) + catch throw:{error, Error} -> unlink(ClientPid), ClientPid ! {iter_done, self(), {error, Error}}, - exit(Error); - Else -> Else + exit(Error) end; true -> ignore end, A = {start, Fun, Acc, TargetMod, Iter}, Res = - case iterate(SourceMod, fun trav_apply/4, Source, A) of + case iterate(SourceMod, fun trav_apply/5, Source, A) of {ok, {iter, _, Acc2, _, Iter2}} when TargetMod =/= read_only -> - case catch do_apply(TargetMod, commit_write, [Iter2], Iter2) of - {error, Reason} -> - {error, Reason}; - _ -> - {ok, Acc2} + try + do_apply(TargetMod, commit_write, [Iter2], Iter2), + {ok, Acc2} + catch throw:{error, Reason} -> + {error, Reason} end; {ok, {iter, _, Acc2, _, _}} -> {ok, Acc2}; {error, Reason} when TargetMod =/= read_only-> - catch do_apply(TargetMod, abort_write, [Iter], Iter), + ?CATCH(do_apply(TargetMod, abort_write, [Iter], Iter)), {error, {"Backup traversal failed", Reason}}; {error, Reason} -> {error, {"Backup traversal failed", Reason}} @@ -1160,7 +1221,7 @@ do_traverse_backup(ClientPid, Source, SourceMod, Target, TargetMod, Fun, Acc) -> unlink(ClientPid), ClientPid ! {iter_done, self(), Res}. -trav_apply(Recs, _Header, _Schema, {iter, Fun, Acc, Mod, Iter}) -> +trav_apply(Recs, _Header, _Schema, _Ext, {iter, Fun, Acc, Mod, Iter}) -> {NewRecs, Acc2} = filter_foldl(Fun, Acc, Recs), if Mod =/= read_only, NewRecs =/= [] -> @@ -1169,7 +1230,7 @@ trav_apply(Recs, _Header, _Schema, {iter, Fun, Acc, Mod, Iter}) -> true -> {iter, Fun, Acc2, Mod, Iter} end; -trav_apply(Recs, Header, Schema, {start, Fun, Acc, Mod, Iter}) -> +trav_apply(Recs, Header, Schema, Ext, {start, Fun, Acc, Mod, Iter}) -> Iter2 = if Mod =/= read_only -> @@ -1177,8 +1238,9 @@ trav_apply(Recs, Header, Schema, {start, Fun, Acc, Mod, Iter}) -> true -> Iter end, - TravAcc = trav_apply(Schema, Header, Schema, {iter, Fun, Acc, Mod, Iter2}), - trav_apply(Recs, Header, Schema, TravAcc). + TravAcc = trav_apply(Schema, Header, Schema, Ext, + {iter, Fun, Acc, Mod, Iter2}), + trav_apply(Recs, Header, Schema, Ext, TravAcc). filter_foldl(Fun, Acc, [Head|Tail]) -> case Fun(Head, Acc) of diff --git a/lib/mnesia/src/mnesia_checkpoint.erl b/lib/mnesia/src/mnesia_checkpoint.erl index eb8fe38908..9eb939e8d3 100644 --- a/lib/mnesia/src/mnesia_checkpoint.erl +++ b/lib/mnesia/src/mnesia_checkpoint.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2009. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016 %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -31,8 +32,7 @@ tm_retain/5, tm_enter_pending/1, tm_enter_pending/3, - tm_exit_pending/1, - convert_cp_record/1 + tm_exit_pending/1 ]). %% Public interface @@ -69,12 +69,12 @@ -import(mnesia_lib, [add/2, del/2, set/2, unset/1]). -import(mnesia_lib, [dbg_out/2]). --record(checkpoint_args, {name = {now(), node()}, +-record(checkpoint_args, {name = {erlang:unique_integer([positive]), node()}, allow_remote = true, ram_overrides_dump = false, nodes = [], node = node(), - now = now(), + now, %% unused cookie = ?unique_cookie, min = [], max = [], @@ -88,25 +88,6 @@ pid }). -%% Old record definition --record(checkpoint, {name, - allow_remote, - ram_overrides_dump, - nodes, - node, - now, - min, - max, - pending_tab, - wait_for_old, - is_activated, - ignore_new, - retainers, - iterators, - supervisor, - pid - }). - -record(retainer, {cp_name, tab_name, store, writers = [], really_retain = true}). -record(iter, {tab_name, oid_tab, main_tab, retainer_tab, source, val, pid}). @@ -129,15 +110,6 @@ tm_prepare(Cp) when is_record(Cp, checkpoint_args) -> start_retainer(Cp); true -> {error, {already_exists, Name, node()}} - end; -tm_prepare(Cp) when is_record(Cp, checkpoint) -> - %% Node with old protocol sent an old checkpoint record - %% and we have to convert it - case convert_cp_record(Cp) of - {ok, NewCp} -> - tm_prepare(NewCp); - {error, Reason} -> - {error, Reason} end. tm_mnesia_down(Node) -> @@ -156,7 +128,8 @@ tm_enter_pending(Pending) -> tm_enter_pending([], Pending) -> Pending; tm_enter_pending([Tab | Tabs], Pending) -> - catch ?ets_insert(Tab, Pending), + %% io:format("Add ~p ~p ~p~n",[Tab, Pending, hd(tl(element(2, process_info(self(), current_stacktrace))))]), + ?SAFE(?ets_insert(Tab, Pending)), tm_enter_pending(Tabs, Pending). tm_exit_pending(Tid) -> @@ -371,9 +344,7 @@ activate(Args) -> end. args2cp(Args) when is_list(Args)-> - case catch lists:foldl(fun check_arg/2, #checkpoint_args{}, Args) of - {'EXIT', Reason} -> - {error, Reason}; + try lists:foldl(fun check_arg/2, #checkpoint_args{}, Args) of Cp -> case check_tables(Cp) of {error, Reason} -> @@ -381,6 +352,10 @@ args2cp(Args) when is_list(Args)-> {ok, Overriders, AllTabs} -> arrange_retainers(Cp, Overriders, AllTabs) end + catch exit:Reason -> + {error, Reason}; + error:Reason -> + {error, Reason} end; args2cp(Args) -> {error, {badarg, Args}}. @@ -390,10 +365,10 @@ check_arg({name, Name}, Cp) -> true -> exit({already_exists, Name}); false -> - case catch tab2retainer({foo, Name}) of - List when is_list(List) -> - Cp#checkpoint_args{name = Name}; - _ -> + try + [_|_] = tab2retainer({foo, Name}), + Cp#checkpoint_args{name = Name} + catch _:_ -> exit({badarg, Name}) end end; @@ -453,22 +428,22 @@ check_tables(Cp) -> arrange_retainers(Cp, Overriders, AllTabs) -> R = #retainer{cp_name = Cp#checkpoint_args.name}, - case catch [R#retainer{tab_name = Tab, - writers = select_writers(Cp, Tab)} - || Tab <- AllTabs] of - {'EXIT', Reason} -> - {error, Reason}; + try [R#retainer{tab_name = Tab, + writers = select_writers(Cp, Tab)} + || Tab <- AllTabs] of Retainers -> {ok, Cp#checkpoint_args{ram_overrides_dump = Overriders, - retainers = Retainers, - nodes = writers(Retainers)}} + retainers = Retainers, + nodes = writers(Retainers)}} + catch throw:Reason -> + {error, Reason} end. select_writers(Cp, Tab) -> case filter_remote(Cp, val({Tab, active_replicas})) of [] -> - exit({"Cannot prepare checkpoint (replica not available)", - [Tab, Cp#checkpoint_args.name]}); + throw({"Cannot prepare checkpoint (replica not available)", + [Tab, Cp#checkpoint_args.name]}); Writers -> This = node(), case {lists:member(Tab, Cp#checkpoint_args.max), @@ -518,12 +493,12 @@ check_prep([], Name, Nodes, IgnoreNew) -> collect_pending(Name, Nodes, IgnoreNew) -> case rpc:multicall(Nodes, ?MODULE, call, [Name, collect_pending]) of {Replies, []} -> - case catch ?ets_new_table(mnesia_union, [bag]) of - {'EXIT', Reason} -> %% system limit + try + UnionTab = ?ets_new_table(mnesia_union, [bag]), + compute_union(Replies, Nodes, Name, UnionTab, IgnoreNew) + catch error:Reason -> %% system limit Msg = "Cannot create an ets table pending union", - {error, {system_limit, Msg, Reason}}; - UnionTab -> - compute_union(Replies, Nodes, Name, UnionTab, IgnoreNew) + {error, {system_limit, Msg, Reason}} end; {_, BadNodes} -> deactivate(Nodes, Name), @@ -641,11 +616,7 @@ init(Cp) -> process_flag(priority, high), %% Needed dets files might starve the system Name = Cp#checkpoint_args.name, Props = [set, public, {keypos, 2}], - case catch ?ets_new_table(mnesia_pending_checkpoint, Props) of - {'EXIT', Reason} -> %% system limit - Msg = "Cannot create an ets table for pending transactions", - Error = {error, {system_limit, Name, Msg, Reason}}, - proc_lib:init_ack(Cp#checkpoint_args.supervisor, Error); + try ?ets_new_table(mnesia_pending_checkpoint, Props) of PendingTab -> Rs = [prepare_tab(Cp, R) || R <- Cp#checkpoint_args.retainers], Cp2 = Cp#checkpoint_args{retainers = Rs, @@ -658,6 +629,10 @@ init(Cp) -> dbg_out("Checkpoint ~p (~p) started~n", [Name, self()]), proc_lib:init_ack(Cp2#checkpoint_args.supervisor, {ok, self()}), retainer_loop(Cp2) + catch error:Reason -> %% system limit + Msg = "Cannot create an ets table for pending transactions", + Error = {error, {system_limit, Name, Msg, Reason}}, + proc_lib:init_ack(Cp#checkpoint_args.supervisor, Error) end. prepare_tab(Cp, R) -> @@ -700,6 +675,16 @@ tab2retainer({Tab, Name}) -> FlatName = lists:flatten(io_lib:write(Name)), mnesia_lib:dir(lists:concat([?MODULE, "_", Tab, "_", FlatName, ".RET"])). +retainer_create(_Cp, R, Tab, Name, Ext = {ext, Alias, Mod}) -> + T = {Tab, retainer, Name}, + P = mnesia_schema:cs2list(val({Tab, cstruct})), + Mod:delete_table(Alias, T), + ok = Mod:create_table(Alias, T, P), + Cs = val({Tab, cstruct}), + Mod:load_table(Alias, T, {retainer, create_table}, + mnesia_schema:cs2list(Cs)), + dbg_out("Checkpoint retainer created ~p ~p~n", [Name, Tab]), + R#retainer{store = {Ext, T}, really_retain = true}; retainer_create(_Cp, R, Tab, Name, disc_only_copies) -> Fname = tab2retainer({Tab, Name}), file:delete(Fname), @@ -759,15 +744,23 @@ traverse_dcd({Cont, Recs}, Log, Fun) -> %% trashed data?? traverse_dcd(eof, _Log, _Fun) -> ok. +retainer_get({{ext, Alias, Mod}, Store}, Key) -> + Mod:lookup(Alias, Store, Key); retainer_get({ets, Store}, Key) -> ?ets_lookup(Store, Key); retainer_get({dets, Store}, Key) -> dets:lookup(Store, Key). +retainer_put({{ext, Alias, Mod}, Store}, Val) -> + Mod:insert(Alias, Store, Val); retainer_put({ets, Store}, Val) -> ?ets_insert(Store, Val); retainer_put({dets, Store}, Val) -> dets:insert(Store, Val). +retainer_first({{ext, Alias, Mod}, Store}) -> + Mod:first(Alias, Store); retainer_first({ets, Store}) -> ?ets_first(Store); retainer_first({dets, Store}) -> dets:first(Store). +retainer_next({{ext, Alias, Mod}, Store}, Key) -> + Mod:next(Alias, Store, Key); retainer_next({ets, Store}, Key) -> ?ets_next(Store, Key); retainer_next({dets, Store}, Key) -> dets:next(Store, Key). @@ -786,11 +779,16 @@ retainer_next({dets, Store}, Key) -> dets:next(Store, Key). retainer_fixtable(Tab, Bool) when is_atom(Tab) -> mnesia_lib:db_fixtable(val({Tab, storage_type}), Tab, Bool); +retainer_fixtable({Ext = {ext, _, _}, Tab}, Bool) -> + mnesia_lib:db_fixtable(Ext, Tab, Bool); retainer_fixtable({ets, Tab}, Bool) -> mnesia_lib:db_fixtable(ram_copies, Tab, Bool); retainer_fixtable({dets, Tab}, Bool) -> mnesia_lib:db_fixtable(disc_only_copies, Tab, Bool). +retainer_delete({{ext, Alias, Mod}, Store}) -> + Mod:close_table(Alias, Store), + Mod:delete_table(Alias, Store); retainer_delete({ets, Store}) -> ?ets_delete_table(Store); retainer_delete({dets, Store}) -> @@ -798,104 +796,142 @@ retainer_delete({dets, Store}) -> Fname = tab2retainer(Store), file:delete(Fname). -retainer_loop(Cp) -> - Name = Cp#checkpoint_args.name, +retainer_loop(Cp = #checkpoint_args{is_activated=false, name=Name}) -> receive - {_From, {retain, Tid, Tab, Key, OldRecs}} - when Cp#checkpoint_args.wait_for_old == [] -> - R = val({Tab, {retainer, Name}}), - PendingTab = Cp#checkpoint_args.pending_tab, - case R#retainer.really_retain of - true when PendingTab =:= undefined -> - Store = R#retainer.store, - case retainer_get(Store, Key) of - [] -> retainer_put(Store, {Tab, Key, OldRecs}); - _ -> already_retained - end; - true -> - case ets:member(PendingTab, Tid) of - true -> ignore; - false -> - Store = R#retainer.store, - case retainer_get(Store, Key) of - [] -> retainer_put(Store, {Tab, Key, OldRecs}); - _ -> already_retained - end - end; - false -> - ignore - end, - retainer_loop(Cp); - %% Adm + {From, {activate, Pending}} -> + StillPending = mnesia_recover:still_pending(Pending), + enter_still_pending(StillPending, Cp#checkpoint_args.pending_tab), + Local = [Tid || #tid{pid=Pid} = Tid <- StillPending, node(Pid) =/= node()], + Cp2 = maybe_activate(Cp#checkpoint_args{wait_for_old = Local}), + + reply(From, Name, activated), + retainer_loop(Cp2); + + {_From, {exit_pending, Tid}} when is_list(Cp#checkpoint_args.wait_for_old) -> + StillPending = lists:delete(Tid, Cp#checkpoint_args.wait_for_old), + Cp2 = Cp#checkpoint_args{wait_for_old = StillPending}, + Cp3 = maybe_activate(Cp2), + retainer_loop(Cp3); + {From, deactivate} -> do_stop(Cp), reply(From, Name, deactivated), unlink(From), exit(shutdown); + {From, get_checkpoint} -> + reply(From, Name, Cp), + retainer_loop(Cp); + {_From, {add_retainer, R, Node}} -> + Cp2 = do_add_retainer(Cp, R, Node), + retainer_loop(Cp2); + + {From, collect_pending} -> + PendingTab = Cp#checkpoint_args.pending_tab, + del(pending_checkpoints, PendingTab), + Pending = ?ets_match_object(PendingTab, '_'), + reply(From, Name, {ok, Pending}), + retainer_loop(Cp); + + {_From, {mnesia_down, Node}} -> + Cp2 = do_del_retainers(Cp, Node), + retainer_loop(Cp2); + {'EXIT', Parent, _} when Parent == Cp#checkpoint_args.supervisor -> %% do_stop(Cp), %% assume that entire Mnesia is terminating exit(shutdown); - {_From, {mnesia_down, Node}} -> - Cp2 = do_del_retainers(Cp, Node), - retainer_loop(Cp2); + {'EXIT', From, _Reason} -> + Iters = [Iter || Iter <- Cp#checkpoint_args.iterators, + check_iter(From, Iter)], + retainer_loop(Cp#checkpoint_args{iterators = Iters}); + + {system, From, Msg} -> + dbg_out("~p got {system, ~p, ~p}~n", [?MODULE, From, Msg]), + sys:handle_system_msg(Msg, From, Cp#checkpoint_args.supervisor, + ?MODULE, [], Cp) + end; + +retainer_loop(Cp = #checkpoint_args{name=Name}) -> + receive + {_From, {retain, Tid, Tab, Key, OldRecs}} -> + R = val({Tab, {retainer, Name}}), + PendingTab = Cp#checkpoint_args.pending_tab, + case R#retainer.really_retain of + true -> + Store = R#retainer.store, + try true = ets:member(PendingTab, Tid), + %% io:format("CP: ~p ~p ~p ~p~n",[true, Tab, Key, Tid]), + case retainer_get(Store, Key) of + [] -> ignore; + _ -> ets:delete(element(2,Store), Key) + end + catch _:_ -> + %% io:format("CP: ~p ~p ~p ~p~n",[false, Tab, Key, Tid]), + case retainer_get(Store, Key) of + [] -> retainer_put(Store, {Tab, Key, OldRecs}); + _ -> already_retained + end + end; + false -> + ignore + end, + retainer_loop(Cp); + + %% Adm {From, get_checkpoint} -> reply(From, Name, Cp), retainer_loop(Cp); - {From, {add_copy, Tab, Node}} when Cp#checkpoint_args.wait_for_old == [] -> + {From, {add_copy, Tab, Node}} -> {Res, Cp2} = do_add_copy(Cp, Tab, Node), reply(From, Name, Res), retainer_loop(Cp2); - {From, {del_copy, Tab, Node}} when Cp#checkpoint_args.wait_for_old == [] -> + {From, {del_copy, Tab, Node}} -> Cp2 = do_del_copy(Cp, Tab, Node), reply(From, Name, ok), retainer_loop(Cp2); - {From, {change_copy, Tab, From, To}} when Cp#checkpoint_args.wait_for_old == [] -> + {From, {change_copy, Tab, From, To}} -> Cp2 = do_change_copy(Cp, Tab, From, To), reply(From, Name, ok), retainer_loop(Cp2); {_From, {add_retainer, R, Node}} -> Cp2 = do_add_retainer(Cp, R, Node), retainer_loop(Cp2); - {_From, {del_retainer, R, Node}} when Cp#checkpoint_args.wait_for_old == [] -> + {_From, {del_retainer, R, Node}} -> Cp2 = do_del_retainer(Cp, R, Node), retainer_loop(Cp2); %% Iteration - {From, {iter_begin, Iter}} when Cp#checkpoint_args.wait_for_old == [] -> + {From, {iter_begin, Iter}} -> Cp2 = iter_begin(Cp, From, Iter), retainer_loop(Cp2); - {From, {iter_end, Iter}} when Cp#checkpoint_args.wait_for_old == [] -> + {From, {iter_end, Iter}} -> retainer_fixtable(Iter#iter.oid_tab, false), Iters = Cp#checkpoint_args.iterators -- [Iter], reply(From, Name, ok), - retainer_loop(Cp#checkpoint_args{iterators = Iters}); - - {_From, {exit_pending, Tid}} - when is_list(Cp#checkpoint_args.wait_for_old) -> - StillPending = lists:delete(Tid, Cp#checkpoint_args.wait_for_old), - Cp2 = Cp#checkpoint_args{wait_for_old = StillPending}, - Cp3 = maybe_activate(Cp2), - retainer_loop(Cp3); + retainer_loop(Cp#checkpoint_args{iterators = Iters}); - {From, collect_pending} -> - PendingTab = Cp#checkpoint_args.pending_tab, - del(pending_checkpoints, PendingTab), - Pending = ?ets_match_object(PendingTab, '_'), - reply(From, Name, {ok, Pending}), + {_From, {exit_pending, _Tid}} -> retainer_loop(Cp); - {From, {activate, Pending}} -> - StillPending = mnesia_recover:still_pending(Pending), - enter_still_pending(StillPending, Cp#checkpoint_args.pending_tab), - Cp2 = maybe_activate(Cp#checkpoint_args{wait_for_old = StillPending}), - reply(From, Name, activated), + {From, deactivate} -> + do_stop(Cp), + reply(From, Name, deactivated), + unlink(From), + exit(shutdown); + + {_From, {mnesia_down, Node}} -> + Cp2 = do_del_retainers(Cp, Node), retainer_loop(Cp2); + {'EXIT', Parent, _} when Parent == Cp#checkpoint_args.supervisor -> + %% do_stop(Cp), + %% assume that entire Mnesia is terminating + exit(shutdown); + {'EXIT', From, _Reason} -> Iters = [Iter || Iter <- Cp#checkpoint_args.iterators, check_iter(From, Iter)], @@ -903,13 +939,17 @@ retainer_loop(Cp) -> {system, From, Msg} -> dbg_out("~p got {system, ~p, ~p}~n", [?MODULE, From, Msg]), - sys:handle_system_msg(Msg, From, no_parent, ?MODULE, [], Cp) + sys:handle_system_msg(Msg, From, Cp#checkpoint_args.supervisor, + ?MODULE, [], Cp); + Msg -> + dbg_out("~p got ~p~n", [?MODULE, Msg]) end. maybe_activate(Cp) - when Cp#checkpoint_args.wait_for_old == [], - Cp#checkpoint_args.is_activated == false -> - Cp#checkpoint_args{pending_tab = undefined, is_activated = true}; + when Cp#checkpoint_args.wait_for_old == [], + Cp#checkpoint_args.is_activated == false -> + Cp#checkpoint_args{%% pending_tab = undefined, + is_activated = true}; maybe_activate(Cp) -> Cp. @@ -1154,7 +1194,7 @@ iterate(Name, Tab, Fun, Acc, Source, Val) -> {error, Reason}; {ok, Iter, Pid} -> link(Pid), % We don't want any pending fixtable's - Res = (catch iter(Fun, Acc, Iter)), + Res = ?CATCH(iter(Fun, Acc, Iter)), unlink(Pid), call(Name, {iter_end, Iter}), case Res of @@ -1226,70 +1266,11 @@ system_terminate(_Reason, _Parent,_Debug, Cp) -> system_code_change(Cp, _Module, _OldVsn, _Extra) -> {ok, Cp}. -convert_cp_record(Cp) when is_record(Cp, checkpoint) -> - ROD = - case Cp#checkpoint.ram_overrides_dump of - true -> Cp#checkpoint.min ++ Cp#checkpoint.max; - false -> [] - end, - - {ok, #checkpoint_args{name = Cp#checkpoint.name, - allow_remote = Cp#checkpoint.name, - ram_overrides_dump = ROD, - nodes = Cp#checkpoint.nodes, - node = Cp#checkpoint.node, - now = Cp#checkpoint.now, - cookie = ?unique_cookie, - min = Cp#checkpoint.min, - max = Cp#checkpoint.max, - pending_tab = Cp#checkpoint.pending_tab, - wait_for_old = Cp#checkpoint.wait_for_old, - is_activated = Cp#checkpoint.is_activated, - ignore_new = Cp#checkpoint.ignore_new, - retainers = Cp#checkpoint.retainers, - iterators = Cp#checkpoint.iterators, - supervisor = Cp#checkpoint.supervisor, - pid = Cp#checkpoint.pid - }}; -convert_cp_record(Cp) when is_record(Cp, checkpoint_args) -> - AllTabs = Cp#checkpoint_args.min ++ Cp#checkpoint_args.max, - ROD = case Cp#checkpoint_args.ram_overrides_dump of - [] -> - false; - AllTabs -> - true; - _ -> - error - end, - if - ROD == error -> - {error, {"Old node cannot handle new checkpoint protocol", - ram_overrides_dump}}; - true -> - {ok, #checkpoint{name = Cp#checkpoint_args.name, - allow_remote = Cp#checkpoint_args.name, - ram_overrides_dump = ROD, - nodes = Cp#checkpoint_args.nodes, - node = Cp#checkpoint_args.node, - now = Cp#checkpoint_args.now, - min = Cp#checkpoint_args.min, - max = Cp#checkpoint_args.max, - pending_tab = Cp#checkpoint_args.pending_tab, - wait_for_old = Cp#checkpoint_args.wait_for_old, - is_activated = Cp#checkpoint_args.is_activated, - ignore_new = Cp#checkpoint_args.ignore_new, - retainers = Cp#checkpoint_args.retainers, - iterators = Cp#checkpoint_args.iterators, - supervisor = Cp#checkpoint_args.supervisor, - pid = Cp#checkpoint_args.pid - }} - end. - %%%%%%%%%%%%%%%%%%%%%%%%%% val(Var) -> case ?catch_val(Var) of - {'EXIT', _ReASoN_} -> mnesia_lib:other_val(Var, _ReASoN_); - _VaLuE_ -> _VaLuE_ + {'EXIT', _} -> mnesia_lib:other_val(Var); + _VaLuE_ -> _VaLuE_ end. diff --git a/lib/mnesia/src/mnesia_checkpoint_sup.erl b/lib/mnesia/src/mnesia_checkpoint_sup.erl index 2fe8df52f7..ae3f77f37e 100644 --- a/lib/mnesia/src/mnesia_checkpoint_sup.erl +++ b/lib/mnesia/src/mnesia_checkpoint_sup.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/mnesia/src/mnesia_controller.erl b/lib/mnesia/src/mnesia_controller.erl index 78f7bfa325..4791e2e290 100644 --- a/lib/mnesia/src/mnesia_controller.erl +++ b/lib/mnesia/src/mnesia_controller.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -51,6 +52,7 @@ force_load_table/1, async_dump_log/1, sync_dump_log/1, + snapshot_dcd/1, connect_nodes/1, connect_nodes/2, wait_for_schema_commit_lock/0, @@ -76,7 +78,7 @@ change_table_majority/1, del_active_replica/2, wait_for_tables/2, - get_network_copy/2, + get_network_copy/3, merge_schema/0, start_remote_sender/4, schedule_late_disc_load/2 @@ -139,7 +141,8 @@ max_loaders() -> -record(block_controller, {owner}). -record(dump_log, {initiated_by, - opt_reply_to + opt_reply_to, + operation = dump_log }). -record(net_load, {table, @@ -184,7 +187,7 @@ max_loaders() -> val(Var) -> case ?catch_val(Var) of - {'EXIT', Reason} -> mnesia_lib:other_val(Var, Reason); + {'EXIT', _} -> mnesia_lib:other_val(Var); Value -> Value end. @@ -198,7 +201,17 @@ sync_dump_log(InitBy) -> call({sync_dump_log, InitBy}). async_dump_log(InitBy) -> - ?SERVER_NAME ! {async_dump_log, InitBy}. + ?SERVER_NAME ! {async_dump_log, InitBy}, + ok. + +snapshot_dcd(Tables) when is_list(Tables) -> + case [T || T <- Tables, + mnesia_lib:storage_type_at_node(node(), T) =/= disc_copies] of + [] -> + call({snapshot_dcd, Tables}); + BadTabs -> + {error, {not_disc_copies, BadTabs}} + end. %% Wait for tables to be active %% If needed, we will wait for Mnesia to start @@ -229,9 +242,7 @@ do_wait_for_tables(Tabs, Timeout) -> end. reply_wait(Tabs) -> - case catch mnesia_lib:active_tables() of - {'EXIT', _} -> - {error, {node_not_running, node()}}; + try mnesia_lib:active_tables() of Active when is_list(Active) -> case Tabs -- Active of [] -> @@ -239,6 +250,7 @@ reply_wait(Tabs) -> BadTabs -> {timeout, BadTabs} end + catch exit:_ -> {error, {node_not_running, node()}} end. wait_for_tables_init(From, Tabs) -> @@ -249,13 +261,12 @@ wait_for_tables_init(From, Tabs) -> exit(normal). wait_for_init(From, Tabs, Init) -> - case catch link(Init) of - {'EXIT', _} -> - %% Mnesia is not started - {error, {node_not_running, node()}}; + try link(Init) of true when is_pid(Init) -> cast({sync_tabs, Tabs, self()}), rec_tabs(Tabs, Tabs, From, Init) + catch error:_ -> %% Mnesia is not started + {error, {node_not_running, node()}} end. sync_reply(Waiter, Tab) -> @@ -293,13 +304,19 @@ update(Fun) -> mnesia_down(Node) -> - case cast({mnesia_down, Node}) of - {error, _} -> mnesia_monitor:mnesia_down(?SERVER_NAME, Node); - _Pid -> ok + case whereis(?SERVER_NAME) of + undefined -> mnesia_monitor:mnesia_down(?SERVER_NAME, Node); + Pid -> gen_server:cast(Pid, {mnesia_down, Node}) end. + wait_for_schema_commit_lock() -> - link(whereis(?SERVER_NAME)), - unsafe_call(wait_for_schema_commit_lock). + try + Pid = whereis(?SERVER_NAME), + link(Pid), %% Keep the link until release_schema_commit_lock + gen_server:call(Pid, wait_for_schema_commit_lock, infinity) + catch _:_ -> + mnesia:abort({node_not_running, node()}) + end. block_controller() -> call(block_controller). @@ -312,20 +329,20 @@ release_schema_commit_lock() -> unlink(whereis(?SERVER_NAME)). %% Special for preparation of add table copy -get_network_copy(Tab, Cs) -> +get_network_copy(Tid, Tab, Cs) -> % We can't let the controller queue this one % because that may cause a deadlock between schema_operations % and initial tableloadings which both takes schema locks. % But we have to get copier_done msgs when the other side % goes down. call({add_other, self()}), - Reason = {dumper,add_table_copy}, + Reason = {dumper,{add_table_copy, Tid}}, Work = #net_load{table = Tab,reason = Reason,cstruct = Cs}, %% I'll need this cause it's linked trough the subscriber %% might be solved by using monitor in subscr instead. process_flag(trap_exit, true), Load = load_table_fun(Work), - Res = (catch Load()), + Res = ?CATCH(Load()), process_flag(trap_exit, false), call({del_other, self()}), case Res of @@ -368,6 +385,8 @@ force_load_table(Tab) when is_atom(Tab), Tab /= schema -> do_force_load_table(Tab); disc_only_copies -> do_force_load_table(Tab); + {ext, _, _} -> + do_force_load_table(Tab); unknown -> set({Tab, load_by_force}, true), cast({force_load_updated, Tab}), @@ -467,7 +486,7 @@ connect_nodes2(Father, Ns, UserFun) -> process_flag(trap_exit, true), Res = try_merge_schema(New, [], UserFun), Msg = {schema_is_merged, [], late_merge, []}, - multicall([node()|Ns], Msg), + _ = multicall([node()|Ns], Msg), After = val({current, db_nodes}), Father ! {?MODULE, self(), Res, mnesia_lib:intersect(Ns,After)}, unlink(Father), @@ -548,19 +567,13 @@ schema_is_merged() -> cast(Msg) -> case whereis(?SERVER_NAME) of - undefined ->{error, {node_not_running, node()}}; + undefined -> ok; Pid -> gen_server:cast(Pid, Msg) end. abcast(Nodes, Msg) -> gen_server:abcast(Nodes, ?SERVER_NAME, Msg). -unsafe_call(Msg) -> - case whereis(?SERVER_NAME) of - undefined -> {error, {node_not_running, node()}}; - Pid -> gen_server:call(Pid, Msg, infinity) - end. - call(Msg) -> case whereis(?SERVER_NAME) of undefined -> @@ -580,11 +593,8 @@ call(Msg) -> end. remote_call(Node, Func, Args) -> - case catch gen_server:call({?MODULE, Node}, {Func, Args, self()}, infinity) of - {'EXIT', Error} -> - {error, Error}; - Else -> - Else + try gen_server:call({?MODULE, Node}, {Func, Args, self()}, infinity) + catch exit:Error -> {error, Error} end. multicall(Nodes, Msg) -> @@ -645,6 +655,15 @@ handle_call({sync_dump_log, InitBy}, From, State) -> State2 = add_worker(Worker, State), noreply(State2); +handle_call({snapshot_dcd, Tables}, From, State) -> + Worker = #dump_log{initiated_by = user, + opt_reply_to = From, + operation = fun() -> + mnesia_dumper:snapshot_dcd(Tables) + end}, + State2 = add_worker(Worker, State), + noreply(State2); + handle_call(wait_for_schema_commit_lock, From, State) -> Worker = #schema_commit_lock{owner = From}, State2 = add_worker(Worker, State), @@ -656,7 +675,7 @@ handle_call(block_controller, From, State) -> noreply(State2); handle_call({update,Fun}, From, State) -> - Res = (catch Fun()), + Res = ?CATCH(Fun()), reply(From, Res), noreply(State); @@ -758,7 +777,7 @@ handle_call({net_load, Tab, Cs}, From, State) -> true -> Worker = #net_load{table = Tab, opt_reply_to = From, - reason = {dumper,add_table_copy}, + reason = {dumper,{add_table_copy, unknown}}, cstruct = Cs }, add_worker(Worker, State); @@ -1163,11 +1182,11 @@ handle_info(Done = #loader_done{worker_pid=WPid, table_name=Tab}, State0) -> Done#loader_done.needs_announce == true, Done#loader_done.needs_reply == true -> i_have_tab(Tab), - %% Should be {dumper,add_table_copy} only + %% Should be {dumper,{add_table_copy, _}} only reply(Done#loader_done.reply_to, Done#loader_done.reply); Done#loader_done.needs_reply == true -> - %% Should be {dumper,add_table_copy} only + %% Should be {dumper,{add_table_copy,_}} only reply(Done#loader_done.reply_to, Done#loader_done.reply); Done#loader_done.needs_announce == true, Tab == schema -> @@ -1206,7 +1225,14 @@ handle_info(Done = #loader_done{worker_pid=WPid, table_name=Tab}, State0) -> {value,{_,Worker}} = lists:keysearch(WPid,1,get_loaders(State0)), add_loader(Tab,Worker,State1); _ -> - State1 + DelState = State1#state{late_loader_queue=gb_trees:delete_any(Tab, LateQueue0)}, + case ?catch_val({Tab, storage_type}) of + ram_copies -> + cast({disc_load, Tab, ram_only}), + DelState; + _ -> + DelState + end end end, State3 = opt_start_worker(State2), @@ -1228,7 +1254,7 @@ handle_info(#sender_done{worker_pid=Pid, worker_res=Res}, State) -> end; handle_info({'EXIT', Pid, R}, State) when Pid == State#state.supervisor -> - catch set(mnesia_status, stopping), + ?SAFE(set(mnesia_status, stopping)), case State#state.dumper_pid of undefined -> dbg_out("~p was ~p~n", [?SERVER_NAME, R]), @@ -1452,9 +1478,9 @@ orphan_tables([], _, _, LocalOrphans, RemoteMasters) -> node_has_tabs([Tab | Tabs], Node, State) when Node /= node() -> State2 = - case catch update_whereabouts(Tab, Node, State) of - State1 = #state{} -> State1; - {'EXIT', R} -> %% Tab was just deleted? + try update_whereabouts(Tab, Node, State) of + State1 = #state{} -> State1 + catch exit:R -> %% Tab was just deleted? case ?catch_val({Tab, cstruct}) of {'EXIT', _} -> State; % yes _ -> erlang:error(R) @@ -1509,8 +1535,8 @@ update_whereabouts(Tab, Node, State) -> Storage == unknown -> %% No own copy, continue to read remotely add_active_replica(Tab, Node), - NodeST = mnesia_lib:storage_type_at_node(Node, Tab), - ReadST = mnesia_lib:storage_type_at_node(Read, Tab), + NodeST = mnesia_lib:semantics(mnesia_lib:storage_type_at_node(Node, Tab), storage), + ReadST = mnesia_lib:semantics(mnesia_lib:storage_type_at_node(Read, Tab), storage), if %% Avoid reading from disc_only_copies NodeST == disc_only_copies -> ignore; @@ -1564,6 +1590,7 @@ last_consistent_replica(Tab, Downs) -> Ram = Cs#cstruct.ram_copies, Disc = Cs#cstruct.disc_copies, DiscOnly = Cs#cstruct.disc_only_copies, + Ext = Cs#cstruct.external_copies, BetterCopies0 = mnesia_lib:remote_copy_holders(Cs) -- Downs, BetterCopies = BetterCopies0 -- Ram, AccessMode = Cs#cstruct.access_mode, @@ -1596,7 +1623,7 @@ last_consistent_replica(Tab, Downs) -> false; Storage == ram_copies -> if - Disc == [], DiscOnly == [] -> + Disc == [], DiscOnly == [], Ext == [] -> %% Nobody has copy on disc {true, {Tab, ram_only}}; true -> @@ -1740,22 +1767,17 @@ change_table_majority(Cs) -> update_where_to_wlock(Tab) -> WNodes = val({Tab, where_to_write}), - Majority = case catch val({Tab, majority}) of - true -> true; - _ -> false - end, + Majority = ?catch_val({Tab, majority}) == true, set({Tab, where_to_wlock}, {WNodes, Majority}). %% node To now has tab loaded, but this must be undone %% This code is rpc:call'ed from the tab_copier process %% when it has *not* released it's table lock unannounce_add_table_copy(Tab, To) -> - catch del_active_replica(Tab, To), - case catch val({Tab , where_to_read}) of - To -> - mnesia_lib:set_remote_where_to_read(Tab); - _ -> - ignore + ?SAFE(del_active_replica(Tab, To)), + try To = val({Tab , where_to_read}), + mnesia_lib:set_remote_where_to_read(Tab) + catch _:_ -> ignore end. user_sync_tab(Tab) -> @@ -1789,7 +1811,7 @@ sync_and_block_table_whereabouts(Tab, ToNode, RemoteS, AccessMode) when Tab /= s true -> Current -- [ToNode]; false -> Current end, - remote_call(ToNode, block_table, [Tab]), + _ = remote_call(ToNode, block_table, [Tab]), [remote_call(Node, add_active_replica, [Tab, ToNode, RemoteS, AccessMode]) || Node <- [ToNode | Ns]], ok. @@ -1846,6 +1868,11 @@ info([Tab | Tail]) -> dets:info(Tab, size), dets:info(Tab, file_size), "bytes on disc"); + {ext, Alias, Mod} -> + info_format(Tab, + Mod:info(Alias, Tab, size), + Mod:info(Alias, Tab, memory), + "words of mem"); _ -> info_format(Tab, ?ets_info(Tab, size), @@ -2081,7 +2108,12 @@ start_remote_sender(Node, Tab, Receiver, Storage) -> dump_and_reply(ReplyTo, Worker) -> %% No trap_exit, die intentionally instead - Res = mnesia_dumper:opt_dump_log(Worker#dump_log.initiated_by), + Res = case Worker#dump_log.operation of + dump_log -> + mnesia_dumper:opt_dump_log(Worker#dump_log.initiated_by); + F when is_function(F, 0) -> + F() + end, ReplyTo ! #dumper_done{worker_pid = self(), worker_res = Res}, unlink(ReplyTo), @@ -2124,6 +2156,10 @@ load_table_fun(#net_load{cstruct=Cs, table=Tab, reason=Reason, opt_reply_to=Repl reply_to = ReplyTo, reply = {loaded, ok} }, + AddTableCopy = case Reason of + {dumper,{add_table_copy,_}} -> true; + _ -> false + end, if ReadNode == node() -> %% Already loaded locally @@ -2133,7 +2169,7 @@ load_table_fun(#net_load{cstruct=Cs, table=Tab, reason=Reason, opt_reply_to=Repl Res = mnesia_loader:disc_load_table(Tab, load_local_content), Done#loader_done{reply = Res, needs_announce = true, needs_sync = true} end; - AccessMode == read_only, Reason /= {dumper,add_table_copy} -> + AccessMode == read_only, not AddTableCopy -> fun() -> disc_load_table(Tab, Reason, ReplyTo) end; true -> fun() -> diff --git a/lib/mnesia/src/mnesia_dumper.erl b/lib/mnesia/src/mnesia_dumper.erl index e2a0aa3bda..eb02a585a6 100644 --- a/lib/mnesia/src/mnesia_dumper.erl +++ b/lib/mnesia/src/mnesia_dumper.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -34,11 +35,15 @@ -export([ get_log_writes/0, incr_log_writes/0, + needs_dump_ets/1, raw_dump_table/2, raw_named_dump_table/2, + dump_to_logfile/2, + load_from_logfile/3, start_regulator/0, opt_dump_log/1, - update/3 + update/3, + snapshot_dcd/1 ]). %% Internal stuff @@ -85,7 +90,7 @@ adjust_log_writes(DoCast) -> %% Don't care if we lost a few writes mnesia_lib:set_counter(trans_log_writes_left, Max), Diff = Max - Left, - mnesia_lib:incr_counter(trans_log_writes, Diff), + _ = mnesia_lib:incr_counter(trans_log_writes, Diff), global:del_lock(Token, [node()]) end. @@ -99,6 +104,19 @@ opt_dump_log(InitBy) -> end, perform_dump(InitBy, Reg). +snapshot_dcd(Tables) -> + lists:foreach( + fun(Tab) -> + case mnesia_lib:storage_type_at_node(node(), Tab) of + disc_copies -> + mnesia_log:ets2dcd(Tab); + _ -> + %% Storage type was checked before queueing the op, though + skip + end + end, Tables), + dumped. + %% Scan for decisions perform_dump(InitBy, Regulator) when InitBy == scan_decisions -> ?eval_debug_fun({?MODULE, perform_dump}, [InitBy]), @@ -122,7 +140,7 @@ perform_dump(InitBy, Regulator) -> U = mnesia_monitor:get_env(dump_log_update_in_place), Cont = mnesia_log:init_log_dump(), mnesia_recover:sync(), - case catch do_perform_dump(Cont, U, InitBy, Regulator, undefined) of + try do_perform_dump(Cont, U, InitBy, Regulator, undefined) of ok -> ?eval_debug_fun({?MODULE, post_dump}, [InitBy]), case mnesia_monitor:use_dir() of @@ -133,17 +151,15 @@ perform_dump(InitBy, Regulator) -> end, mnesia_recover:allow_garb(), %% And now to the crucial point... - mnesia_log:confirm_log_dump(Diff); - {error, Reason} -> - {error, Reason}; - {'EXIT', {Desc, Reason}} -> + mnesia_log:confirm_log_dump(Diff) + catch exit:Reason when Reason =/= fatal -> case mnesia_monitor:get_env(auto_repair) of true -> - mnesia_lib:important(Desc, Reason), + mnesia_lib:important(error, Reason), %% Ignore rest of the log mnesia_log:confirm_log_dump(Diff); false -> - fatal(Desc, Reason) + fatal(error, Reason) end end; {error, Reason} -> @@ -161,24 +177,25 @@ scan_decisions(Fname, InitBy, Regulator) -> mnesia_log:open_log(Name, Header, Fname, Exists, mnesia_monitor:get_env(auto_repair), read_only), Cont = start, - Res = (catch do_perform_dump(Cont, false, InitBy, Regulator, undefined)), - mnesia_log:close_log(Name), - case Res of - ok -> ok; - {'EXIT', Reason} -> {error, Reason} + try + do_perform_dump(Cont, false, InitBy, Regulator, undefined) + catch exit:Reason when Reason =/= fatal -> + {error, Reason} + after mnesia_log:close_log(Name) end end. do_perform_dump(Cont, InPlace, InitBy, Regulator, OldVersion) -> case mnesia_log:chunk_log(Cont) of {C2, Recs} -> - case catch insert_recs(Recs, InPlace, InitBy, Regulator, OldVersion) of - {'EXIT', R} -> - Reason = {"Transaction log dump error: ~p~n", [R]}, - close_files(InPlace, {error, Reason}, InitBy), - exit(Reason); + try insert_recs(Recs, InPlace, InitBy, Regulator, OldVersion) of Version -> do_perform_dump(C2, InPlace, InitBy, Regulator, Version) + catch _:R when R =/= fatal -> + ST = erlang:get_stacktrace(), + Reason = {"Transaction log dump error: ~p~n", [{R, ST}]}, + close_files(InPlace, {error, Reason}, InitBy), + exit(Reason) end; eof -> close_files(InPlace, ok, InitBy), @@ -212,11 +229,11 @@ insert_rec(Rec, InPlace, InitBy, LogV) when is_record(Rec, commit) -> D = Rec#commit.decision, case mnesia_recover:wait_for_decision(D, InitBy) of {Tid, committed} -> - do_insert_rec(Tid, Rec, InPlace, InitBy, LogV); + do_insert_rec(Tid, mnesia_tm:new_cr_format(Rec), InPlace, InitBy, LogV); {Tid, aborted} -> case InitBy of startup -> - mnesia_schema:undo_prepare_commit(Tid, Rec); + mnesia_schema:undo_prepare_commit(Tid, mnesia_tm:new_cr_format(Rec)); _ -> ok end @@ -256,15 +273,30 @@ do_insert_rec(Tid, Rec, InPlace, InitBy, LogV) -> end end, D = Rec#commit.disc_copies, + ExtOps = commit_ext(Rec), insert_ops(Tid, disc_copies, D, InPlace, InitBy, LogV), + [insert_ops(Tid, Ext, Ops, InPlace, InitBy, LogV) || + {Ext, Ops} <- ExtOps, + storage_semantics(Ext) == disc_copies], case InitBy of startup -> DO = Rec#commit.disc_only_copies, - insert_ops(Tid, disc_only_copies, DO, InPlace, InitBy, LogV); + insert_ops(Tid, disc_only_copies, DO, InPlace, InitBy, LogV), + [insert_ops(Tid, Ext, Ops, InPlace, InitBy, LogV) || + {Ext, Ops} <- ExtOps, storage_semantics(Ext) == disc_only_copies]; _ -> ignore end. +commit_ext(#commit{ext = []}) -> []; +commit_ext(#commit{ext = Ext}) -> + case lists:keyfind(ext_copies, 1, Ext) of + {_, C} -> + lists:foldl(fun({Ext0, Op}, D) -> + orddict:append(Ext0, Op, D) + end, orddict:new(), C); + false -> [] + end. update(_Tid, [], _DumperMode) -> dumped; @@ -288,17 +320,16 @@ perform_update(Tid, SchemaOps, _DumperMode, _UseDir) -> InitBy = fast_schema_update, InPlace = mnesia_monitor:get_env(dump_log_update_in_place), - ?eval_debug_fun({?MODULE, dump_schema_op}, [InitBy]), - case catch insert_ops(Tid, schema_ops, SchemaOps, InPlace, InitBy, - mnesia_log:version()) of - {'EXIT', Reason} -> - Error = {error, {"Schema update error", Reason}}, + try insert_ops(Tid, schema_ops, SchemaOps, InPlace, InitBy, + mnesia_log:version()), + ?eval_debug_fun({?MODULE, post_dump}, [InitBy]), + close_files(InPlace, ok, InitBy), + ok + catch _:Reason when Reason =/= fatal -> + ST = erlang:get_stacktrace(), + Error = {error, {"Schema update error", {Reason, ST}}}, close_files(InPlace, Error, InitBy), - fatal("Schema update error ~p ~p", [Reason, SchemaOps]); - _ -> - ?eval_debug_fun({?MODULE, post_dump}, [InitBy]), - close_files(InPlace, ok, InitBy), - ok + fatal("Schema update error ~p ~p", [{Reason,ST}, SchemaOps]) end. insert_ops(_Tid, _Storage, [], _InPlace, _InitBy, _) -> ok; @@ -316,14 +347,15 @@ insert_ops(Tid, Storage, [Op | Ops], InPlace, InitBy, Ver) when Ver < "4.3" -> %% Normal ops disc_insert(_Tid, Storage, Tab, Key, Val, Op, InPlace, InitBy) -> - case open_files(Tab, Storage, InPlace, InitBy) of + Semantics = storage_semantics(Storage), + case open_files(Tab, Semantics, Storage, InPlace, InitBy) of true -> - case Storage of + case Semantics of disc_copies when Tab /= schema -> mnesia_log:append({?MODULE,Tab}, {{Tab, Key}, Val, Op}), ok; _ -> - dets_insert(Op,Tab,Key,Val) + dets_insert(Op,Tab,Key,Val,Storage) end; false -> ignore @@ -335,36 +367,37 @@ disc_insert(_Tid, Storage, Tab, Key, Val, Op, InPlace, InitBy) -> %% Otherwise we will get a double increment. %% This is perfect but update_counter is a dirty op. -dets_insert(Op,Tab,Key,Val) -> +dets_insert(Op,Tab,Key,Val, Storage0) -> + Storage = if Tab == schema -> disc_only_copies; + true -> Storage0 + end, case Op of write -> dets_updated(Tab,Key), - ok = dets:insert(Tab, Val); + ok = mnesia_lib:db_put(Storage, Tab, Val); delete -> dets_updated(Tab,Key), - ok = dets:delete(Tab, Key); + ok = mnesia_lib:db_erase(Storage, Tab, Key); update_counter -> case dets_incr_counter(Tab,Key) of true -> {RecName, Incr} = Val, - case catch dets:update_counter(Tab, Key, Incr) of - CounterVal when is_integer(CounterVal) -> - ok; - _ when Incr < 0 -> + try _ = mnesia_lib:db_update_counter(Storage, Tab, Key, Incr) + catch error:_ when Incr < 0 -> Zero = {RecName, Key, 0}, - ok = dets:insert(Tab, Zero); - _ -> + ok = mnesia_lib:db_put(Storage, Tab, Zero); + error:_ -> Init = {RecName, Key, Incr}, - ok = dets:insert(Tab, Init) + ok = mnesia_lib:db_put(Storage, Tab, Init) end; false -> ok end; delete_object -> dets_updated(Tab,Key), - ok = dets:delete_object(Tab, Val); + mnesia_lib:db_match_erase(Storage, Tab, Val); clear_table -> dets_cleared(Tab), - ok = dets:delete_all_objects(Tab) + ok = mnesia_lib:db_match_erase(Storage, Tab, '_') end. dets_updated(Tab,Key) -> @@ -419,23 +452,29 @@ insert(_Tid, _Storage, _Tab, _Key, [], _Op, _InPlace, _InitBy) -> ok; insert(Tid, Storage, Tab, Key, Val, Op, InPlace, InitBy) -> + Semantics = storage_semantics(Storage), Item = {{Tab, Key}, Val, Op}, case InitBy of startup -> disc_insert(Tid, Storage, Tab, Key, Val, Op, InPlace, InitBy); - _ when Storage == ram_copies -> + _ when Semantics == ram_copies -> mnesia_tm:do_update_op(Tid, Storage, Item), Snmp = mnesia_tm:prepare_snmp(Tab, Key, [Item]), mnesia_tm:do_snmp(Tid, Snmp); - _ when Storage == disc_copies -> + _ when Semantics == disc_copies -> disc_insert(Tid, Storage, Tab, Key, Val, Op, InPlace, InitBy), mnesia_tm:do_update_op(Tid, Storage, Item), Snmp = mnesia_tm:prepare_snmp(Tab, Key, [Item]), mnesia_tm:do_snmp(Tid, Snmp); - _ when Storage == disc_only_copies -> + _ when Semantics == disc_only_copies -> + mnesia_tm:do_update_op(Tid, Storage, Item), + Snmp = mnesia_tm:prepare_snmp(Tab, Key, [Item]), + mnesia_tm:do_snmp(Tid, Snmp); + + _ when element(1, Storage) == ext -> mnesia_tm:do_update_op(Tid, Storage, Item), Snmp = mnesia_tm:prepare_snmp(Tab, Key, [Item]), mnesia_tm:do_snmp(Tid, Snmp); @@ -444,6 +483,9 @@ insert(Tid, Storage, Tab, Key, Val, Op, InPlace, InitBy) -> ignore end. +disc_delete_table(Tab, {ext, Alias, Mod}) -> + Mod:close_table(Alias, Tab), + Mod:delete_table(Alias, Tab); disc_delete_table(Tab, Storage) -> case mnesia_monitor:use_dir() of true -> @@ -451,7 +493,8 @@ disc_delete_table(Tab, Storage) -> Storage == disc_only_copies; Tab == schema -> mnesia_monitor:unsafe_close_dets(Tab), Dat = mnesia_lib:tab2dat(Tab), - file:delete(Dat); + file:delete(Dat), + ok; true -> DclFile = mnesia_lib:tab2dcl(Tab), case get({?MODULE,Tab}) of @@ -466,16 +509,20 @@ disc_delete_table(Tab, Storage) -> file:delete(DcdFile), ok end, - erase({?MODULE, Tab}); + erase({?MODULE, Tab}), + ok; false -> - ignore + ok end. -disc_delete_indecies(_Tab, _Cs, Storage) when Storage /= disc_only_copies -> - ignore; -disc_delete_indecies(Tab, Cs, disc_only_copies) -> - Indecies = Cs#cstruct.index, - mnesia_index:del_transient(Tab, Indecies, disc_only_copies). +disc_delete_indecies(Tab, Cs, Storage) -> + case storage_semantics(Storage) of + disc_only_copies -> + Indecies = Cs#cstruct.index, + mnesia_index:del_transient(Tab, Indecies, Storage); + _ -> + ok + end. insert_op(Tid, Storage, {{Tab, Key}, Val, Op}, InPlace, InitBy) -> %% Propagate to disc only @@ -501,6 +548,8 @@ insert_op(Tid, _, {op, change_table_copy_type, N, FromS, ToS, TabDef}, InPlace, Cs = mnesia_schema:list2cs(TabDef), Val = mnesia_schema:insert_cstruct(Tid, Cs, true), % Update ram only {schema, Tab, _} = Val, + FromSem = storage_semantics(FromS), + ToSem = storage_semantics(ToS), case lists:member(N, val({current, db_nodes})) of true when InitBy /= startup -> mnesia_controller:add_active_replica(Tab, N, Cs); @@ -513,59 +562,118 @@ insert_op(Tid, _, {op, change_table_copy_type, N, FromS, ToS, TabDef}, InPlace, Dat = mnesia_lib:tab2dat(Tab), Dcd = mnesia_lib:tab2dcd(Tab), Dcl = mnesia_lib:tab2dcl(Tab), + Logtmp = mnesia_lib:tab2logtmp(Tab), case {FromS, ToS} of - {ram_copies, disc_copies} when Tab == schema -> - ok = ensure_rename(Dmp, Dat); - {ram_copies, disc_copies} -> - file:delete(Dcl), - ok = ensure_rename(Dmp, Dcd); - {disc_copies, ram_copies} when Tab == schema -> - mnesia_lib:set(use_dir, false), - mnesia_monitor:unsafe_close_dets(Tab), - file:delete(Dat); - {disc_copies, ram_copies} -> - file:delete(Dcl), - file:delete(Dcd); - {ram_copies, disc_only_copies} -> - ok = ensure_rename(Dmp, Dat), - true = open_files(Tab, disc_only_copies, InPlace, InitBy), - %% ram_delete_table must be done before init_indecies, - %% it uses info which is reset in init_indecies, - %% it doesn't matter, because init_indecies don't use - %% the ram replica of the table when creating the disc - %% index; Could be improved :) - mnesia_schema:ram_delete_table(Tab, FromS), - PosList = Cs#cstruct.index, - mnesia_index:init_indecies(Tab, disc_only_copies, PosList); - {disc_only_copies, ram_copies} -> - mnesia_monitor:unsafe_close_dets(Tab), - disc_delete_indecies(Tab, Cs, disc_only_copies), - case InitBy of - startup -> - ignore; + {{ext,_FromAlias,_FromMod},{ext,ToAlias,ToMod}} -> + disc_delete_table(Tab, FromS), + ok = ToMod:delete_table(ToAlias, Tab), + ok = mnesia_monitor:unsafe_create_external(Tab, ToAlias, ToMod, TabDef), + ok = ToMod:load_table(ToAlias, Tab, {dumper,change_table_copy_type}, TabDef), + ok = load_from_logfile(ToS, Tab, Logtmp), + file:delete(Logtmp), + restore_indexes(Tab, ToS, Cs); + + {_,{ext,ToAlias,ToMod}} -> + case FromSem of + ram_copies -> + mnesia_schema:ram_delete_table(Tab, FromS); _ -> - mnesia_controller:get_disc_copy(Tab) + if FromSem == disc_copies -> + mnesia_schema:ram_delete_table( + Tab, FromS); + true -> ok + end, + disc_delete_table(Tab, FromS) end, - disc_delete_table(Tab, disc_only_copies); - {disc_copies, disc_only_copies} -> - ok = ensure_rename(Dmp, Dat), - true = open_files(Tab, disc_only_copies, InPlace, InitBy), - mnesia_schema:ram_delete_table(Tab, FromS), - PosList = Cs#cstruct.index, - mnesia_index:init_indecies(Tab, disc_only_copies, PosList), - file:delete(Dcl), - file:delete(Dcd); - {disc_only_copies, disc_copies} -> - mnesia_monitor:unsafe_close_dets(Tab), - disc_delete_indecies(Tab, Cs, disc_only_copies), - case InitBy of - startup -> - ignore; - _ -> - mnesia_log:ets2dcd(Tab), - mnesia_controller:get_disc_copy(Tab), - disc_delete_table(Tab, disc_only_copies) - end + + ok = ToMod:delete_table(ToAlias, Tab), + ok = mnesia_monitor:unsafe_create_external(Tab, ToAlias, ToMod, TabDef), + ok = ToMod:load_table(ToAlias, Tab, {dumper,change_table_copy_type}, TabDef), + ok = load_from_logfile(ToS, Tab, Logtmp), + file:delete(Logtmp), + restore_indexes(Tab, ToS, Cs); + + {{ext,_FromAlias,_FromMod} = FromS, ToS} -> + disc_delete_table(Tab, FromS), + case ToS of + ram_copies -> + change_disc_to_ram( + Tab, Cs, FromS, ToS, Logtmp, InitBy); + disc_copies -> + Args = [{keypos, 2}, public, named_table, + Cs#cstruct.type], + mnesia_monitor:mktab(Tab, Args), + ok = load_from_logfile(ToS, Tab, Logtmp), + file:delete(Logtmp); + disc_only_copies -> + %% ok = ensure_rename(Dmp, Dat), + true = open_files(Tab, ToS, InPlace, InitBy), + ok = load_from_logfile(ToS, Tab, Logtmp), + file:delete(Logtmp) + end, + restore_indexes(Tab, ToS, Cs); + + _NoneAreExt -> + + case {FromSem, ToSem} of + {ram_copies, disc_copies} when Tab == schema -> + ok = ensure_rename(Dmp, Dat); + {ram_copies, disc_copies} -> + file:delete(Dcl), + ok = ensure_rename(Dmp, Dcd); + {disc_copies, ram_copies} when Tab == schema -> + mnesia_lib:set(use_dir, false), + mnesia_monitor:unsafe_close_dets(Tab), + ok = file:delete(Dat); + {disc_copies, ram_copies} -> + _ = file:delete(Dcl), + _ = file:delete(Dcd), + ok; + {ram_copies, disc_only_copies} -> + ok = ensure_rename(Dmp, Dat), + true = open_files(Tab, ToS, InPlace, InitBy), + %% ram_delete_table must be done before + %% init_indecies, it uses info which is reset + %% in init_indecies, it doesn't matter, because + %% init_indecies don't use the ram replica of + %% the table when creating the disc index; + %% Could be improved :) + mnesia_schema:ram_delete_table(Tab, FromS), + restore_indexes(Tab, ToS, Cs); + {disc_only_copies, ram_copies} when FromS == disc_only_copies -> + mnesia_monitor:unsafe_close_dets(Tab), + disc_delete_indecies(Tab, Cs, FromS), + case InitBy of + startup -> + ignore; + _ -> + mnesia_controller:get_disc_copy(Tab), + ok + end, + disc_delete_table(Tab, FromS); + {disc_only_copies, ram_copies} when element(1, FromS) == ext -> + change_disc_to_ram( + Tab, Cs, FromS, ToS, Logtmp, InitBy); + {disc_copies, disc_only_copies} -> + ok = ensure_rename(Dmp, Dat), + true = open_files(Tab, ToS, InPlace, InitBy), + mnesia_schema:ram_delete_table(Tab, FromS), + restore_indexes(Tab, ToS, Cs), + _ = file:delete(Dcl), + _ = file:delete(Dcd), + ok; + {disc_only_copies, disc_copies} -> + mnesia_monitor:unsafe_close_dets(Tab), + disc_delete_indecies(Tab, Cs, disc_only_copies), + case InitBy of + startup -> + ignore; + _ -> + mnesia_log:ets2dcd(Tab), + mnesia_controller:get_disc_copy(Tab), + disc_delete_table(Tab, disc_only_copies) + end + end end; true -> ignore @@ -590,6 +698,9 @@ insert_op(Tid, _, {op, restore_recreate, TabDef}, InPlace, InitBy) -> Tab = Cs#cstruct.name, Type = Cs#cstruct.type, Storage = mnesia_lib:cs_to_storage_type(node(), Cs), + Semantics = if Storage==unknown -> unknown; + true -> storage_semantics(Storage) + end, %% Delete all possibly existing files and tables disc_delete_table(Tab, Storage), disc_delete_indecies(Tab, Cs, Storage), @@ -608,13 +719,13 @@ insert_op(Tid, _, {op, restore_recreate, TabDef}, InPlace, InitBy) -> %% And create new ones.. if - (InitBy == startup) or (Storage == unknown) -> + (InitBy == startup) or (Semantics == unknown) -> ignore; - Storage == ram_copies -> + Semantics == ram_copies -> EtsProps = proplists:get_value(ets, StorageProps, []), Args = [{keypos, 2}, public, named_table, Type | EtsProps], mnesia_monitor:mktab(Tab, Args); - Storage == disc_copies -> + Semantics == disc_copies -> EtsProps = proplists:get_value(ets, StorageProps, []), Args = [{keypos, 2}, public, named_table, Type | EtsProps], mnesia_monitor:mktab(Tab, Args), @@ -623,7 +734,7 @@ insert_op(Tid, _, {op, restore_recreate, TabDef}, InPlace, InitBy) -> {repair, false}, {mode, read_write}], {ok, Log} = mnesia_monitor:open_log(FArg), mnesia_monitor:unsafe_close_log(Log); - Storage == disc_only_copies -> + Storage == disc_only_copies -> % note: Storage, not Semantics File = mnesia_lib:tab2dat(Tab), file:delete(File), DetsProps = proplists:get_value(dets, StorageProps, []), @@ -632,7 +743,10 @@ insert_op(Tid, _, {op, restore_recreate, TabDef}, InPlace, InitBy) -> {keypos, 2}, {repair, mnesia_monitor:get_env(auto_repair)} | DetsProps ], - mnesia_monitor:open_dets(Tab, Args) + mnesia_monitor:open_dets(Tab, Args); + element(1,Storage) == ext -> + {ext, Alias, Mod} = Storage, + Mod:create_table(Alias, Tab, []) end, insert_op(Tid, ignore, {op, create_table, TabDef}, InPlace, InitBy); @@ -642,9 +756,10 @@ insert_op(Tid, _, {op, create_table, TabDef}, InPlace, InitBy) -> Tab = Cs#cstruct.name, Storage = mnesia_lib:cs_to_storage_type(node(), Cs), StorageProps = Cs#cstruct.storage_properties, + Semantics = storage_semantics(Storage), case InitBy of startup -> - case Storage of + case Semantics of unknown -> ignore; ram_copies -> @@ -662,20 +777,10 @@ insert_op(Tid, _, {op, create_table, TabDef}, InPlace, InitBy) -> read_write), mnesia_log:unsafe_close_log(temp) end; - _ -> + disc_only_copies -> DetsProps = proplists:get_value(dets, StorageProps, []), - Args = [{file, mnesia_lib:tab2dat(Tab)}, - {type, mnesia_lib:disk_type(Tab, Cs#cstruct.type)}, - {keypos, 2}, - {repair, mnesia_monitor:get_env(auto_repair)} - | DetsProps ], - case mnesia_monitor:open_dets(Tab, Args) of - {ok, _} -> - mnesia_monitor:unsafe_close_dets(Tab); - {error, Error} -> - exit({"Failed to create dets table", Error}) - end + try_create_disc_only_copy(Storage, Tab, Cs, DetsProps) end; _ -> Copies = mnesia_lib:copy_holders(Cs), @@ -698,7 +803,7 @@ insert_op(Tid, _, {op, create_table, TabDef}, InPlace, InitBy) -> false -> mnesia_lib:set({Tab, where_to_read}, node()) end, - case Storage of + case Semantics of ram_copies -> ignore; _ -> @@ -766,7 +871,7 @@ insert_op(Tid, _, {op, clear_table, TabDef}, InPlace, InitBy) -> end, %% Need to catch this, it crashes on ram_copies if %% the op comes before table is loaded at startup. - catch insert(Tid, Storage, Tab, '_', Oid, clear_table, InPlace, InitBy) + ?CATCH(insert(Tid, Storage, Tab, '_', Oid, clear_table, InPlace, InitBy)) end; insert_op(Tid, _, {op, merge_schema, TabDef}, InPlace, InitBy) -> @@ -821,7 +926,7 @@ insert_op(Tid, _, {op, del_table_copy, Storage, Node, TabDef}, InPlace, InitBy) insert_cstruct(Tid, Cs, true, InPlace, InitBy); Tab /= schema -> mnesia_controller:del_active_replica(Tab, Node), - mnesia_lib:del({Tab, Storage}, Node), + mnesia_lib:del({Tab, storage_alias(Storage)}, Node), if Node == node() -> case Cs#cstruct.local_content of @@ -879,9 +984,10 @@ insert_op(Tid, _, {op, add_index, Pos, TabDef}, InPlace, InitBy) -> Cs = mnesia_schema:list2cs(TabDef), Tab = insert_cstruct(Tid, Cs, true, InPlace, InitBy), Storage = mnesia_lib:cs_to_storage_type(node(), Cs), + Semantics = storage_semantics(Storage), case InitBy of - startup when Storage == disc_only_copies -> - true = open_files(Tab, Storage, InPlace, InitBy), + startup when Semantics == disc_only_copies -> + true = open_files(Tab, Semantics, Storage, InPlace, InitBy), mnesia_index:init_indecies(Tab, Storage, [Pos]); startup -> ignore; @@ -902,6 +1008,8 @@ insert_op(Tid, _, {op, del_index, Pos, TabDef}, InPlace, InitBy) -> mnesia_index:del_index_table(Tab, Storage, Pos); startup -> ignore; + _ when element(1, Storage) == ext -> + mnesia_index:del_index_table(Tab, Storage, Pos); _ -> mnesia_index:del_index_table(Tab, Storage, Pos) end, @@ -941,19 +1049,68 @@ insert_op(Tid, _, {op, change_table_frag, _Change, TabDef}, InPlace, InitBy) -> Cs = mnesia_schema:list2cs(TabDef), insert_cstruct(Tid, Cs, true, InPlace, InitBy). -open_files(Tab, Storage, UpdateInPlace, InitBy) - when Storage /= unknown, Storage /= ram_copies -> + +storage_semantics({ext, Alias, Mod}) -> + Mod:semantics(Alias, storage); +storage_semantics(Storage) when is_atom(Storage) -> + Storage. + +storage_alias({ext, Alias, _}) -> + Alias; +storage_alias(Storage) when is_atom(Storage) -> + Storage. + +change_disc_to_ram(Tab, Cs, FromS, ToS, Logtmp, InitBy) -> + disc_delete_indecies(Tab, Cs, FromS), + case InitBy of + startup -> + ignore; + _ -> + %% ram table will already have been created + Tab = ets:info(Tab, name), %% assertion + load_from_logfile(ToS, Tab, Logtmp), + PosList = Cs#cstruct.index, + mnesia_index:init_indecies(Tab, ToS, PosList) + end, + disc_delete_table(Tab, FromS). + + +try_create_disc_only_copy({ext,Alias,Mod}, Tab, Cs, _) -> + Mod:create_table(Alias, Tab, mnesia_schema:cs2list(Cs)); +try_create_disc_only_copy(disc_only_copies, Tab, Cs, DetsProps) -> + Args = [{file, mnesia_lib:tab2dat(Tab)}, + {type, mnesia_lib:disk_type(Tab, Cs#cstruct.type)}, + {keypos, 2}, + {repair, mnesia_monitor:get_env(auto_repair)} + | DetsProps], + case mnesia_monitor:open_dets(Tab, Args) of + {ok, _} -> + mnesia_monitor:unsafe_close_dets(Tab); + {error, Error} -> + exit({"Failed to create dets table", Error}) + end. + +restore_indexes(Tab, ToS, Cs) -> + PosList = Cs#cstruct.index, + mnesia_index:init_indecies(Tab, ToS, PosList). + + +open_files(Tab, Storage, UpdateInPlace, InitBy) -> + open_files(Tab, storage_semantics(Storage), Storage, UpdateInPlace, InitBy). + +open_files(Tab, Semantics, Storage, UpdateInPlace, InitBy) + when Storage /= unknown, Semantics /= ram_copies -> case get({?MODULE, Tab}) of undefined -> case ?catch_val({Tab, setorbag}) of {'EXIT', _} -> false; Type -> - case Storage of - disc_copies when Tab /= schema -> + Cs = val({Tab, cstruct}), + if Semantics == disc_copies, Tab /= schema -> Bool = open_disc_copies(Tab, InitBy), Bool; - _ -> + Storage == disc_only_copies; Tab == schema -> Props = val({Tab, storage_properties}), DetsProps = proplists:get_value(dets, Props, []), Fname = prepare_open(Tab, UpdateInPlace), @@ -964,6 +1121,12 @@ open_files(Tab, Storage, UpdateInPlace, InitBy) | DetsProps], {ok, _} = mnesia_monitor:open_dets(Tab, Args), put({?MODULE, Tab}, {opened_dumper, dat}), + true; + element(1, Storage) == ext -> + {ext, Alias, Mod} = Storage, + Mod:load_table(Alias, Tab, InitBy, + mnesia_schema:cs2list(Cs)), + put({?MODULE, Tab}, {opened_dumper, ext}), true end end; @@ -972,32 +1135,14 @@ open_files(Tab, Storage, UpdateInPlace, InitBy) {opened_dumper, _} -> true end; -open_files(_Tab, _Storage, _UpdateInPlace, _InitBy) -> +open_files(_Tab, _Semantics, _Storage, _UpdateInPlace, _InitBy) -> false. open_disc_copies(Tab, InitBy) -> - DclF = mnesia_lib:tab2dcl(Tab), - DumpEts = - case file:read_file_info(DclF) of - {error, enoent} -> - false; - {ok, DclInfo} -> - DcdF = mnesia_lib:tab2dcd(Tab), - case file:read_file_info(DcdF) of - {error, Reason} -> - mnesia_lib:dbg_out("File ~p info_error ~p ~n", - [DcdF, Reason]), - true; - {ok, DcdInfo} -> - Mul = case ?catch_val(dc_dump_limit) of - {'EXIT', _} -> ?DumpToEtsMultiplier; - Val -> Val - end, - DcdInfo#file_info.size =< (DclInfo#file_info.size * Mul) - end - end, + DumpEts = needs_dump_ets(Tab), if DumpEts == false; InitBy == startup -> + DclF = mnesia_lib:tab2dcl(Tab), mnesia_log:open_log({?MODULE,Tab}, mnesia_log:dcl_log_header(), DclF, @@ -1012,6 +1157,27 @@ open_disc_copies(Tab, InitBy) -> false end. +needs_dump_ets(Tab) -> + DclF = mnesia_lib:tab2dcl(Tab), + case file:read_file_info(DclF) of + {error, enoent} -> + false; + {ok, DclInfo} -> + DcdF = mnesia_lib:tab2dcd(Tab), + case file:read_file_info(DcdF) of + {error, Reason} -> + mnesia_lib:dbg_out("File ~p info_error ~p ~n", + [DcdF, Reason]), + true; + {ok, DcdInfo} -> + Mul = case ?catch_val(dc_dump_limit) of + {'EXIT', _} -> ?DumpToEtsMultiplier; + Val -> Val + end, + DcdInfo#file_info.size =< (DclInfo#file_info.size * Mul) + end + end. + %% Always opens the dcl file for writing overriding already_dumped %% mechanismen, used for schema transactions. open_dcl(Tab) -> @@ -1037,14 +1203,13 @@ prepare_open(Tab, UpdateInPlace) -> Dat; false -> Tmp = mnesia_lib:tab2tmp(Tab), - case catch mnesia_lib:copy_file(Dat, Tmp) of - ok -> - Tmp; - Error -> + try ok = mnesia_lib:copy_file(Dat, Tmp) + catch error:Error -> fatal("Cannot copy dets file ~p to ~p: ~p~n", [Dat, Tmp, Error]) - end - end. + end, + Tmp + end. del_opened_tab(Tab) -> erase({?MODULE, Tab}). @@ -1057,12 +1222,13 @@ close_files(InPlace, Outcome, InitBy, [{{?MODULE, Tab}, already_dumped} | Tail]) close_files(InPlace, Outcome, InitBy, Tail); close_files(InPlace, Outcome, InitBy, [{{?MODULE, Tab}, {opened_dumper, Type}} | Tail]) -> erase({?MODULE, Tab}), - case val({Tab, storage_type}) of + Storage = val({Tab, storage_type}), + case storage_semantics(Storage) of disc_only_copies when InitBy /= startup -> ignore; - disc_copies when Tab /= schema -> + disc_copies when Storage /= unknown, Tab /= schema -> mnesia_log:close_log({?MODULE,Tab}); - Storage -> + _ -> do_close(InPlace, Outcome, Tab, Type, Storage) end, close_files(InPlace, Outcome, InitBy, Tail); @@ -1074,11 +1240,16 @@ close_files(_, _, _InitBy, []) -> %% If storage is unknown during close clean up files, this can happen if timing %% is right and dirty_write conflicts with schema operations. +do_close(_, _, Tab, ext, {ext,Alias,Mod}) -> + Mod:close_table(Alias, Tab); do_close(_, _, Tab, dcl, unknown) -> mnesia_log:close_log({?MODULE,Tab}), file:delete(mnesia_lib:tab2dcl(Tab)); do_close(_, _, Tab, dcl, _) -> %% To be safe, can it happen? mnesia_log:close_log({?MODULE,Tab}); +do_close(_, _, _Tab, ext, unknown) -> + %% Not sure what to do here, but let's not crash + ok; do_close(InPlace, Outcome, Tab, dat, Storage) -> mnesia_monitor:close_dets(Tab), @@ -1129,7 +1300,8 @@ temp_set_master_nodes() -> Tabs = val({schema, local_tables}), Masters = [{Tab, (val({Tab, disc_copies}) ++ val({Tab, ram_copies}) ++ - val({Tab, disc_only_copies})) -- [node()]} + val({Tab, disc_only_copies}) ++ + external_copies(Tab)) -- [node()]} || Tab <- Tabs], %% UseDir = false since we don't want to remember these %% masternodes and we are running (really soon anyway) since we want this @@ -1137,6 +1309,13 @@ temp_set_master_nodes() -> mnesia_recover:log_master_nodes(Masters, false, yes), ok. +external_copies(Tab) -> + case ?catch_val({Tab, external_copies}) of + {'EXIT',_} -> []; + Ext -> + lists:concat([Ns || {_, Ns} <- Ext]) + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Raw dump of table. Dumper must have unique access to the ets table. @@ -1166,18 +1345,16 @@ raw_named_dump_table(Tab, Ftype) -> Storage = ram_copies, mnesia_lib:db_fixtable(Storage, Tab, true), - case catch raw_dump_table(TabRef, Tab) of - {'EXIT', Reason} -> - mnesia_lib:db_fixtable(Storage, Tab, false), - mnesia_lib:dets_sync_close(Tab), - file:delete(TmpFname), - mnesia_lib:unlock_table(Tab), - exit({"Dump of table to disc failed", Reason}); - ok -> - mnesia_lib:db_fixtable(Storage, Tab, false), - mnesia_lib:dets_sync_close(Tab), - mnesia_lib:unlock_table(Tab), - ok = file:rename(TmpFname, Fname) + try + ok = raw_dump_table(TabRef, Tab), + ok = file:rename(TmpFname, Fname) + catch _:Reason -> + ?SAFE(file:delete(TmpFname)), + exit({"Dump of table to disc failed", Reason}) + after + mnesia_lib:db_fixtable(Storage, Tab, false), + mnesia_lib:dets_sync_close(Tab), + mnesia_lib:unlock_table(Tab) end; {error, Reason} -> mnesia_lib:unlock_table(Tab), @@ -1190,6 +1367,59 @@ raw_named_dump_table(Tab, Ftype) -> raw_dump_table(DetsRef, EtsRef) -> dets:from_ets(DetsRef, EtsRef). +dump_to_logfile(Storage, Tab) -> + case mnesia_monitor:use_dir() of + true -> + Logtmp = mnesia_lib:tab2logtmp(Tab), + file:delete(Logtmp), + case disk_log:open([{name, make_ref()}, + {file, Logtmp}, + {repair, false}, + {linkto, self()}]) of + {ok, Fd} -> + mnesia_lib:db_fixtable(Storage, Tab, true), + try do_dump_to_logfile(Storage, Tab, Fd) + after + mnesia_lib:db_fixtable(Storage, Tab, false) + end; + {error, _} = Error -> + Error + end; + false -> + {error, {has_no_disc, node()}} + end. + +do_dump_to_logfile(Storage, Tab, Fd) -> + Pat = [{'_',[],['$_']}], + log_terms(mnesia_lib:db_select_init(Storage, Tab, Pat, 100), Storage, Tab, Pat, Fd). + +log_terms({Objs, Cont}, Storage, Tab, Pat, Fd) -> + ok = disk_log:alog_terms(Fd, Objs), + log_terms(mnesia_lib:db_select_cont(Storage, Cont, '_'), Storage, Tab, Pat, Fd); +log_terms('$end_of_table', _, _, _, Fd) -> + disk_log:close(Fd). + +load_from_logfile(Storage, Tab, F) -> + case disk_log:open([{name, make_ref()}, + {file, F}, + {repair, true}, + {linkto, self()}]) of + {ok, Fd} -> + chunk_from_log(disk_log:chunk(Fd, start), Fd, Storage, Tab); + {repaired, Fd, _, _} -> + chunk_from_log(disk_log:chunk(Fd, start), Fd, Storage, Tab); + {error, _} = E -> + E + end. + +chunk_from_log({Cont, Terms}, Fd, Storage, Tab) -> + _ = [mnesia_lib:db_put(Storage, Tab, T) || T <- Terms], + chunk_from_log(disk_log:chunk(Fd, Cont), Fd, Storage, Tab); +chunk_from_log(eof, _, _, _) -> + ok. + + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Load regulator %% @@ -1243,6 +1473,6 @@ regulate(RegulatorPid) -> val(Var) -> case ?catch_val(Var) of - {'EXIT', Reason} -> mnesia_lib:other_val(Var, Reason); + {'EXIT', _} -> mnesia_lib:other_val(Var); Value -> Value end. diff --git a/lib/mnesia/src/mnesia_event.erl b/lib/mnesia/src/mnesia_event.erl index 35fe2d4035..7320d381ea 100644 --- a/lib/mnesia/src/mnesia_event.erl +++ b/lib/mnesia/src/mnesia_event.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -63,7 +64,7 @@ handle_event(Event, State) -> %%----------------------------------------------------------------- handle_info(Msg, State) -> - handle_any_event(Msg, State), + {ok, _} = handle_any_event(Msg, State), {ok, State}. %%----------------------------------------------------------------- @@ -235,8 +236,7 @@ report_fatal(Format, Args, BinaryCore, CoreDumped) -> end. core_file(CoreDir,BinaryCore,Format,Args) -> - %% Integers = tuple_to_list(date()) ++ tuple_to_list(time()), - Integers = tuple_to_list(now()), + Integers = tuple_to_list(erlang:timestamp()), Fun = fun(I) when I < 10 -> ["_0",I]; (I) -> ["_",I] end, diff --git a/lib/mnesia/src/mnesia_ext_sup.erl b/lib/mnesia/src/mnesia_ext_sup.erl new file mode 100644 index 0000000000..3e6c72161c --- /dev/null +++ b/lib/mnesia/src/mnesia_ext_sup.erl @@ -0,0 +1,59 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1996-2015. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% This module implements a supervisor for external (plug-in) processes + +-module(mnesia_ext_sup). +-behaviour(supervisor). + +-export([start/0, + init/1, + start_proc/4, start_proc/5, + stop_proc/1]). + +-define(SHUTDOWN, 120000). % 2 minutes + +start() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +start_proc(Name, M, F, A) -> + start_proc(Name, M, F, A, []). + +start_proc(Name, M, F, A, Opts) -> + [Restart, Shutdown, Type, Modules] = + [proplists:get_value(K, Opts, Default) + || {K, Default} <- [{restart, transient}, + {shutdown, ?SHUTDOWN}, + {type, worker}, + {modules, [M]}]], + case supervisor:start_child( + ?MODULE, {Name, {M,F,A}, Restart, Shutdown, Type, Modules}) of + {error, already_present} -> + supervisor:restart_child(?MODULE, Name); + Other -> + Other + end. + +stop_proc(Name) -> + supervisor:terminate_child(?MODULE, Name). + +init(_) -> + {ok, {{one_for_all, 10, 60}, []}}. diff --git a/lib/mnesia/src/mnesia_frag.erl b/lib/mnesia/src/mnesia_frag.erl index 4a1616e054..c6e812b36d 100644 --- a/lib/mnesia/src/mnesia_frag.erl +++ b/lib/mnesia/src/mnesia_frag.erl @@ -1,18 +1,19 @@ %%% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2011. All Rights Reserved. +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -182,6 +183,8 @@ table_info2(ActivityId, Opaque, Tab, Frag, Item) -> length(val({Tab, disc_copies})); n_disc_only_copies -> length(val({Tab, disc_only_copies})); + n_external_copies -> + length(val({Tab, external_copies})); frag_names -> frag_names(Tab); @@ -406,10 +409,11 @@ verify_numbers(FH,MatchSpec) -> VerifyFun = fun(F) when is_integer(F), F >= 1, F =< N -> false; (_F) -> true end, - case catch lists:filter(VerifyFun, FragNumbers) of - [] -> - FragNumbers; - BadFrags -> + try + Frags = lists:filter(VerifyFun, FragNumbers), + Frags == [] orelse error(Frags), + FragNumbers + catch error:BadFrags -> mnesia:abort({"match_spec_to_frag_numbers: Fragment numbers out of range", BadFrags, {range, 1, N}}) end. @@ -437,7 +441,7 @@ remote_select(ReplyTo, Ref, NameNodes, MatchSpec) -> do_remote_select(ReplyTo, Ref, [{Name, Node} | NameNodes], MatchSpec) -> if Node == node() -> - Res = (catch {ok, mnesia:dirty_select(Name, MatchSpec)}), + Res = ?CATCH({ok, mnesia:dirty_select(Name, MatchSpec)}), ReplyTo ! {remote_select, Ref, Node, Res}, do_remote_select(ReplyTo, Ref, NameNodes, MatchSpec); true -> @@ -496,13 +500,13 @@ expand_cstruct(Cs, Mode) -> %% Verify keys ValidKeys = [foreign_key, n_fragments, node_pool, n_ram_copies, n_disc_copies, n_disc_only_copies, - hash_module, hash_state], + n_external_copies, hash_module, hash_state], Keys = mnesia_schema:check_keys(Tab, Props, ValidKeys), mnesia_schema:check_duplicates(Tab, Keys), %% Pick fragmentation props ForeignKey = mnesia_schema:pick(Tab, foreign_key, Props, undefined), - {ForeignKey2, N, Pool, DefaultNR, DefaultND, DefaultNDO} = + {ForeignKey2, N, Pool, DefaultNR, DefaultND, DefaultNDO, DefaultNExt} = pick_props(Tab, Cs, ForeignKey), %% Verify node_pool @@ -516,6 +520,7 @@ expand_cstruct(Cs, Mode) -> NR = mnesia_schema:pick(Tab, n_ram_copies, Props, 0), ND = mnesia_schema:pick(Tab, n_disc_copies, Props, 0), NDO = mnesia_schema:pick(Tab, n_disc_only_copies, Props, 0), + NExt = mnesia_schema:pick(Tab, n_external_copies, Props, 0), PosInt = fun(I) when is_integer(I), I >= 0 -> true; (_I) -> false @@ -526,6 +531,8 @@ expand_cstruct(Cs, Mode) -> {bad_type, Tab, {n_disc_copies, ND}}), mnesia_schema:verify(true, PosInt(NDO), {bad_type, Tab, {n_disc_only_copies, NDO}}), + mnesia_schema:verify(true, PosInt(NExt), + {bad_type, Tab, {n_external_copies, NDO}}), %% Verify n_fragments Cs2 = verify_n_fragments(N, Cs, Mode), @@ -540,13 +547,13 @@ expand_cstruct(Cs, Mode) -> hash_module = HashMod, hash_state = HashState2}, if - NR == 0, ND == 0, NDO == 0 -> - do_expand_cstruct(Cs2, FH, N, Pool, DefaultNR, DefaultND, DefaultNDO, Mode); + NR == 0, ND == 0, NDO == 0, NExt == 0 -> + do_expand_cstruct(Cs2, FH, N, Pool, DefaultNR, DefaultND, DefaultNDO, DefaultNExt, Mode); true -> - do_expand_cstruct(Cs2, FH, N, Pool, NR, ND, NDO, Mode) + do_expand_cstruct(Cs2, FH, N, Pool, NR, ND, NDO, NExt, Mode) end. -do_expand_cstruct(Cs, FH, N, Pool, NR, ND, NDO, Mode) -> +do_expand_cstruct(Cs, FH, N, Pool, NR, ND, NDO, NExt, Mode) -> Tab = Cs#cstruct.name, LC = Cs#cstruct.local_content, @@ -560,14 +567,15 @@ do_expand_cstruct(Cs, FH, N, Pool, NR, ND, NDO, Mode) -> %% Add empty fragments CommonProps = [{base_table, Tab}], Cs2 = Cs#cstruct{frag_properties = lists:sort(CommonProps)}, - expand_frag_cstructs(N, NR, ND, NDO, Cs2, Pool, Pool, FH, Mode). + expand_frag_cstructs(N, NR, ND, NDO, NExt, Cs2, Pool, Pool, FH, Mode). verify_n_fragments(N, Cs, Mode) when is_integer(N), N >= 1 -> case Mode of create -> Cs#cstruct{ram_copies = [], disc_copies = [], - disc_only_copies = []}; + disc_only_copies = [], + external_copies = []}; activate -> Reason = {combine_error, Cs#cstruct.name, {n_fragments, N}}, mnesia_schema:verify(1, N, Reason), @@ -605,7 +613,8 @@ pick_props(Tab, Cs, {ForeignTab, Attr}) -> DefaultNR = length(val({ForeignTab, ram_copies})), DefaultND = length(val({ForeignTab, disc_copies})), DefaultNDO = length(val({ForeignTab, disc_only_copies})), - {Key, N, Pool, DefaultNR, DefaultND, DefaultNDO}; + DefaultNExt = length(val({ForeignTab, external_copies})), + {Key, N, Pool, DefaultNR, DefaultND, DefaultNDO, DefaultNExt}; pick_props(Tab, Cs, undefined) -> Props = Cs#cstruct.frag_properties, DefaultN = 1, @@ -615,22 +624,23 @@ pick_props(Tab, Cs, undefined) -> DefaultNR = 1, DefaultND = 0, DefaultNDO = 0, - {undefined, N, Pool, DefaultNR, DefaultND, DefaultNDO}; + DefaultNExt = 0, + {undefined, N, Pool, DefaultNR, DefaultND, DefaultNDO, DefaultNExt}; pick_props(Tab, _Cs, BadKey) -> mnesia:abort({bad_type, Tab, {foreign_key, BadKey}}). -expand_frag_cstructs(N, NR, ND, NDO, CommonCs, Dist, Pool, FH, Mode) +expand_frag_cstructs(N, NR, ND, NDO, NExt, CommonCs, Dist, Pool, FH, Mode) when N > 1, Mode == create -> Frag = n_to_frag_name(CommonCs#cstruct.name, N), Cs = CommonCs#cstruct{name = Frag}, - {Cs2, RevModDist, RestDist} = set_frag_nodes(NR, ND, NDO, Cs, Dist, []), + {Cs2, RevModDist, RestDist} = set_frag_nodes(NR, ND, NDO, NExt, Cs, Dist, []), ModDist = lists:reverse(RevModDist), Dist2 = rearrange_dist(Cs, ModDist, RestDist, Pool), %% Adjusts backwards, but it doesn't matter. {FH2, _FromFrags, _AdditionalWriteFrags} = adjust_before_split(FH), - CsList = expand_frag_cstructs(N - 1, NR, ND, NDO, CommonCs, Dist2, Pool, FH2, Mode), + CsList = expand_frag_cstructs(N - 1, NR, ND, NDO, NExt, CommonCs, Dist2, Pool, FH2, Mode), [Cs2 | CsList]; -expand_frag_cstructs(1, NR, ND, NDO, CommonCs, Dist, Pool, FH, Mode) -> +expand_frag_cstructs(1, NR, ND, NDO, NExt, CommonCs, Dist, Pool, FH, Mode) -> BaseProps = CommonCs#cstruct.frag_properties ++ [{foreign_key, FH#frag_state.foreign_key}, {hash_module, FH#frag_state.hash_module}, @@ -643,25 +653,29 @@ expand_frag_cstructs(1, NR, ND, NDO, CommonCs, Dist, Pool, FH, Mode) -> activate -> [BaseCs]; create -> - {BaseCs2, _, _} = set_frag_nodes(NR, ND, NDO, BaseCs, Dist, []), + {BaseCs2, _, _} = set_frag_nodes(NR, ND, NDO, NExt, BaseCs, Dist, []), [BaseCs2] end. -set_frag_nodes(NR, ND, NDO, Cs, [Head | Tail], Acc) when NR > 0 -> +set_frag_nodes(NR, ND, NDO, NExt, Cs, [Head | Tail], Acc) when NR > 0 -> Pos = #cstruct.ram_copies, {Cs2, Head2} = set_frag_node(Cs, Pos, Head), - set_frag_nodes(NR - 1, ND, NDO, Cs2, Tail, [Head2 | Acc]); -set_frag_nodes(NR, ND, NDO, Cs, [Head | Tail], Acc) when ND > 0 -> + set_frag_nodes(NR - 1, ND, NDO, NExt, Cs2, Tail, [Head2 | Acc]); +set_frag_nodes(NR, ND, NDO, NExt, Cs, [Head | Tail], Acc) when ND > 0 -> Pos = #cstruct.disc_copies, {Cs2, Head2} = set_frag_node(Cs, Pos, Head), - set_frag_nodes(NR, ND - 1, NDO, Cs2, Tail, [Head2 | Acc]); -set_frag_nodes(NR, ND, NDO, Cs, [Head | Tail], Acc) when NDO > 0 -> + set_frag_nodes(NR, ND - 1, NDO, NExt, Cs2, Tail, [Head2 | Acc]); +set_frag_nodes(NR, ND, NDO, NExt, Cs, [Head | Tail], Acc) when NDO > 0 -> Pos = #cstruct.disc_only_copies, {Cs2, Head2} = set_frag_node(Cs, Pos, Head), - set_frag_nodes(NR, ND, NDO - 1, Cs2, Tail, [Head2 | Acc]); -set_frag_nodes(0, 0, 0, Cs, RestDist, ModDist) -> + set_frag_nodes(NR, ND, NDO - 1, NExt, Cs2, Tail, [Head2 | Acc]); +set_frag_nodes(NR, ND, NDO, NExt, Cs, [Head | Tail], Acc) when NExt > 0 -> + Pos = #cstruct.external_copies, + {Cs2, Head2} = set_frag_node(Cs, Pos, Head), + set_frag_nodes(NR, ND, NDO, NExt - 1, Cs2, Tail, [Head2 | Acc]); +set_frag_nodes(0, 0, 0, 0, Cs, RestDist, ModDist) -> {Cs, ModDist, RestDist}; -set_frag_nodes(_, _, _, Cs, [], _) -> +set_frag_nodes(_, _, _, _, Cs, [], _) -> mnesia:abort({combine_error, Cs#cstruct.name, "Too few nodes in node_pool"}). set_frag_node(Cs, Pos, Head) -> @@ -838,13 +852,15 @@ make_add_frag(Tab, SortedNs) -> NR = length(Cs#cstruct.ram_copies), ND = length(Cs#cstruct.disc_copies), NDO = length(Cs#cstruct.disc_only_copies), + NExt = length(Cs#cstruct.external_copies), NewCs = Cs#cstruct{name = NewFrag, frag_properties = [{base_table, Tab}], ram_copies = [], disc_copies = [], - disc_only_copies = []}, + disc_only_copies = [], + external_copies = []}, - {NewCs2, _, _} = set_frag_nodes(NR, ND, NDO, NewCs, SortedNs, []), + {NewCs2, _, _} = set_frag_nodes(NR, ND, NDO, NExt, NewCs, SortedNs, []), [NewOp] = mnesia_schema:make_create_table(NewCs2), SplitOps = split(Tab, FH2, FromIndecies, FragNames, []), @@ -886,17 +902,19 @@ adjust_before_split(FH) -> HashMod:add_frag(HashState) end, N = FH#frag_state.n_fragments + 1, - FromFrags2 = (catch lists:sort(FromFrags)), - UnionFrags = (catch lists:merge(FromFrags2, lists:sort(AdditionalWriteFrags))), VerifyFun = fun(F) when is_integer(F), F >= 1, F =< N -> false; (_F) -> true end, - case catch lists:filter(VerifyFun, UnionFrags) of - [] -> - FH2 = FH#frag_state{n_fragments = N, - hash_state = HashState2}, - {FH2, FromFrags2, UnionFrags}; - BadFrags -> + try + FromFrags2 = lists:sort(FromFrags), + UnionFrags = lists:merge(FromFrags2, lists:sort(AdditionalWriteFrags)), + + Frags = lists:filter(VerifyFun, UnionFrags), + Frags == [] orelse error(Frags), + FH2 = FH#frag_state{n_fragments = N, + hash_state = HashState2}, + {FH2, FromFrags2, UnionFrags} + catch error:BadFrags -> mnesia:abort({"add_frag: Fragment numbers out of range", BadFrags, {range, 1, N}}) end. @@ -939,7 +957,7 @@ do_split(_FH, _OldN, _FragNames, [], Ops) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Delete a fragment from a fragmented table -%% and merge its records with an other fragment +%% and merge its records with another fragment make_multi_del_frag(Tab) -> verify_multi(Tab), @@ -981,22 +999,24 @@ adjust_before_merge(FH) -> HashMod:del_frag(HashState) end, N = FH#frag_state.n_fragments, - FromFrags2 = (catch lists:sort(FromFrags)), - UnionFrags = (catch lists:merge(FromFrags2, lists:sort(AdditionalWriteFrags))), VerifyFun = fun(F) when is_integer(F), F >= 1, F =< N -> false; (_F) -> true end, - case catch lists:filter(VerifyFun, UnionFrags) of - [] -> - case lists:member(N, FromFrags2) of - true -> - FH2 = FH#frag_state{n_fragments = N - 1, - hash_state = HashState2}, - {FH2, FromFrags2, UnionFrags}; + try + FromFrags2 = lists:sort(FromFrags), + UnionFrags = lists:merge(FromFrags2, lists:sort(AdditionalWriteFrags)), + + Frags = lists:filter(VerifyFun, UnionFrags), + [] == Frags orelse error(Frags), + case lists:member(N, FromFrags2) of + true -> + FH2 = FH#frag_state{n_fragments = N - 1, + hash_state = HashState2}, + {FH2, FromFrags2, UnionFrags}; false -> - mnesia:abort({"del_frag: Last fragment number not included", N}) - end; - BadFrags -> + mnesia:abort({"del_frag: Last fragment number not included", N}) + end + catch error:BadFrags -> mnesia:abort({"del_frag: Fragment numbers out of range", BadFrags, {range, 1, N}}) end. @@ -1141,8 +1161,8 @@ remove_node(Node, Cs) -> val(Var) -> case ?catch_val(Var) of - {'EXIT', Reason} -> mnesia_lib:other_val(Var, Reason); - Value -> Value + {'EXIT', _} -> mnesia_lib:other_val(Var); + Value -> Value end. set_frag_hash(Tab, Props) -> @@ -1313,7 +1333,8 @@ count_frag([Frag | Frags], Dist) -> Dist2 = incr_nodes(val({Frag, ram_copies}), Dist), Dist3 = incr_nodes(val({Frag, disc_copies}), Dist2), Dist4 = incr_nodes(val({Frag, disc_only_copies}), Dist3), - count_frag(Frags, Dist4); + Dist5 = incr_nodes(val({Frag, external_copies}), Dist4), + count_frag(Frags, Dist5); count_frag([], Dist) -> Dist. diff --git a/lib/mnesia/src/mnesia_frag_hash.erl b/lib/mnesia/src/mnesia_frag_hash.erl index 3dfdb87f30..ae4105382e 100644 --- a/lib/mnesia/src/mnesia_frag_hash.erl +++ b/lib/mnesia/src/mnesia_frag_hash.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2011. All Rights Reserved. +%% Copyright Ericsson AB 2002-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/mnesia/src/mnesia_frag_old_hash.erl b/lib/mnesia/src/mnesia_frag_old_hash.erl index 817bb54eb1..b246c76236 100644 --- a/lib/mnesia/src/mnesia_frag_old_hash.erl +++ b/lib/mnesia/src/mnesia_frag_old_hash.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2009. All Rights Reserved. +%% Copyright Ericsson AB 2002-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/mnesia/src/mnesia_index.erl b/lib/mnesia/src/mnesia_index.erl index 54db45e3ba..c79f790973 100644 --- a/lib/mnesia/src/mnesia_index.erl +++ b/lib/mnesia/src/mnesia_index.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -22,8 +23,8 @@ -module(mnesia_index). -export([read/5, - add_index/5, - delete_index/3, + add_index/6, + delete_index/4, del_object_index/5, clear_index/4, dirty_match_object/3, @@ -38,25 +39,21 @@ get_index_table/3, tab2filename/2, - tab2tmp_filename/2, init_index/2, init_indecies/3, del_transient/2, del_transient/3, - del_index_table/3]). + del_index_table/3, --import(mnesia_lib, [verbose/2]). + index_info/2, + ext_index_instances/1]). + +-import(mnesia_lib, [val/1, verbose/2]). -include("mnesia.hrl"). -record(index, {setorbag, pos_list}). -val(Var) -> - case ?catch_val(Var) of - {'EXIT', _ReASoN_} -> mnesia_lib:other_val(Var, _ReASoN_); - _VaLuE_ -> _VaLuE_ - end. - -%% read an object list throuh its index table +%% read an object list through its index table %% we assume that table Tab has index on attribute number Pos read(Tid, Store, Tab, IxKey, Pos) -> @@ -69,62 +66,103 @@ read(Tid, Store, Tab, IxKey, Pos) -> ResList end. -add_index(Index, Tab, Key, Obj, Old) -> - add_index2(Index#index.pos_list, Index#index.setorbag, Tab, Key, Obj, Old). - -add_index2([{Pos, Ixt} |Tail], bag, Tab, K, Obj, OldRecs) -> - db_put(Ixt, {element(Pos, Obj), K}), - add_index2(Tail, bag, Tab, K, Obj, OldRecs); -add_index2([{Pos, Ixt} |Tail], Type, Tab, K, Obj, OldRecs) -> +ext_index_instances(Tab) -> + #index{pos_list = PosL} = val({Tab, index_info}), + lists:foldr( + fun({_, {{ext,Alias,Mod}, Tag}}, Acc) -> + [{Alias, Mod, Tag}|Acc]; + (_, Acc) -> + Acc + end, [], PosL). + + +add_index(#index{pos_list = PosL, setorbag = SorB}, + Storage, Tab, Key, Obj, Old) -> + add_index2(PosL, SorB, Storage, Tab, Key, Obj, Old). + +add_index2([{{Pos,Type}, Ixt} |Tail], bag, Storage, Tab, K, Obj, OldRecs) -> + ValsF = index_vals_f(Storage, Tab, Pos), + Vals = ValsF(Obj), + put_index_vals(Type, Ixt, Vals, K), + add_index2(Tail, bag, Storage, Tab, K, Obj, OldRecs); +add_index2([{{Pos, Type}, Ixt} |Tail], SorB, Storage, Tab, K, Obj, OldRecs0) -> %% Remove old tuples in index if Tab is updated - case OldRecs of - undefined -> - Old = mnesia_lib:db_get(Tab, K), - del_ixes(Ixt, Old, Pos, K); - Old -> - del_ixes(Ixt, Old, Pos, K) - end, - db_put(Ixt, {element(Pos, Obj), K}), - add_index2(Tail, Type, Tab, K, Obj, OldRecs); -add_index2([], _, _Tab, _K, _Obj, _) -> ok. - -delete_index(Index, Tab, K) -> - delete_index2(Index#index.pos_list, Tab, K). - -delete_index2([{Pos, Ixt} | Tail], Tab, K) -> - DelObjs = mnesia_lib:db_get(Tab, K), - del_ixes(Ixt, DelObjs, Pos, K), - delete_index2(Tail, Tab, K); -delete_index2([], _Tab, _K) -> ok. - - -del_ixes(_Ixt, [], _Pos, _L) -> ok; -del_ixes(Ixt, [Obj | Tail], Pos, Key) -> - db_match_erase(Ixt, {element(Pos, Obj), Key}), - del_ixes(Ixt, Tail, Pos, Key). - -del_object_index(Index, Tab, K, Obj, Old) -> - del_object_index2(Index#index.pos_list, Index#index.setorbag, Tab, K, Obj, Old). - -del_object_index2([], _, _Tab, _K, _Obj, _Old) -> ok; -del_object_index2([{Pos, Ixt} | Tail], SoB, Tab, K, Obj, Old) -> + ValsF = index_vals_f(Storage, Tab, Pos), NewVals = ValsF(Obj), + OldRecs1 = case OldRecs0 of + undefined -> mnesia_lib:db_get(Storage, Tab, K); + _ -> OldRecs0 + end, + IdxVal = ValsF(Obj), + case [Old || Old <- OldRecs1, ValsF(Old) =/= IdxVal] of + [] when OldRecs1 =:= [] -> % Write + put_index_vals(Type, Ixt, NewVals, K), + add_index2(Tail, SorB, Storage, Tab, K, Obj, OldRecs1); + [] -> %% when OldRecs1 =/= [] Update without modifying index field + add_index2(Tail, SorB, Storage, Tab, K, Obj, OldRecs1); + OldRecs -> %% Update + put_index_vals(Type, Ixt, NewVals, K), + [del_ixes(Type, Ixt, ValsF, OldObj, K) || OldObj <- OldRecs], + add_index2(Tail, SorB, Storage, Tab, K, Obj, OldRecs1) + end; +add_index2([], _, _, _Tab, _K, _Obj, _) -> ok. + +delete_index(Index, Storage, Tab, K) -> + delete_index2(Index#index.pos_list, Storage, Tab, K). + +delete_index2([{{Pos, Type}, Ixt} | Tail], Storage, Tab, K) -> + DelObjs = mnesia_lib:db_get(Storage, Tab, K), + ValsF = index_vals_f(Storage, Tab, Pos), + [del_ixes(Type, Ixt, ValsF, Obj, K) || Obj <- DelObjs], + delete_index2(Tail, Storage, Tab, K); +delete_index2([], _Storage, _Tab, _K) -> ok. + +put_index_vals(ordered, Ixt, Vals, K) -> + [db_put(Ixt, {{V, K}}) || V <- Vals]; +put_index_vals(bag, Ixt, Vals, K) -> + [db_put(Ixt, {V, K}) || V <- Vals]. + + +del_ixes(bag, Ixt, ValsF, Obj, Key) -> + Vals = ValsF(Obj), + [db_match_erase(Ixt, {V, Key}) || V <- Vals]; +del_ixes(ordered, Ixt, ValsF, Obj, Key) -> + Vals = ValsF(Obj), + [db_erase(Ixt, {V,Key}) || V <- Vals]. + +del_object_index(#index{pos_list = PosL, setorbag = SorB}, Storage, Tab, K, Obj) -> + del_object_index2(PosL, SorB, Storage, Tab, K, Obj). + +del_object_index2([], _, _Storage, _Tab, _K, _Obj) -> ok; +del_object_index2([{{Pos, Type}, Ixt} | Tail], SoB, Storage, Tab, K, Obj) -> + ValsF = index_vals_f(Storage, Tab, Pos), case SoB of bag -> - del_object_bag(Tab, K, Obj, Pos, Ixt, Old); + del_object_bag(Type, ValsF, Tab, K, Obj, Ixt); _ -> %% If set remove the tuple in index table - del_ixes(Ixt, [Obj], Pos, K) + del_ixes(Type, Ixt, ValsF, Obj, K) + end, + del_object_index2(Tail, SoB, Storage, Tab, K, Obj). + +del_object_bag(Type, ValsF, Tab, Key, Obj, Ixt) -> + IxKeys = ValsF(Obj), + Found = [{X, ValsF(X)} || X <- mnesia_lib:db_get(Tab, Key)], + del_object_bag_(IxKeys, Found, Type, Tab, Key, Obj, Ixt). + +del_object_bag_([IxK|IxKs], Found, Type, Tab, Key, Obj, Ixt) -> + case [X || {X, Ixes} <- Found, lists:member(IxK, Ixes)] of + [Old] when Old =:= Obj -> + case Type of + bag -> + db_match_erase(Ixt, {IxK, Key}); + ordered -> + db_erase(Ixt, {{IxK, Key}}) + end; + _ -> + ok end, - del_object_index2(Tail, SoB, Tab, K, Obj, Old). - -del_object_bag(Tab, Key, Obj, Pos, Ixt, undefined) -> - IxKey = element(Pos, Obj), - Old = [X || X <- mnesia_lib:db_get(Tab, Key), element(Pos, X) =:= IxKey], - del_object_bag(Tab, Key, Obj, Pos, Ixt, Old); -%% If Tab type is bag we need remove index identifier if the object being -%% deleted was the last one -del_object_bag(_Tab, Key, Obj, Pos, Ixt, Old) when Old =:= [Obj] -> - del_ixes(Ixt, [Obj], Pos, Key); -del_object_bag(_Tab, _Key, _Obj, _Pos, _Ixt, _Old) -> ok. + del_object_bag_(IxKs, Found, Type, Tab, Key, Obj, Ixt); +del_object_bag_([], _, _, _, _, _, _) -> + ok. clear_index(Index, Tab, K, Obj) -> clear_index2(Index#index.pos_list, Tab, K, Obj). @@ -136,8 +174,9 @@ clear_index2([{_Pos, Ixt} | Tail], Tab, K, Obj) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -dirty_match_object(Tab, Pat, Pos) -> +dirty_match_object(Tab, Pat, Pos) when is_integer(Pos) -> %% Assume that we are on the node where the replica is + %% Cannot use index plugins here, as they don't map to match patterns case element(2, Pat) of '_' -> IxKey = element(Pos, Pat), @@ -159,7 +198,7 @@ realkeys(Tab, Pos, IxKey) -> Index = get_index_table(Tab, Pos), db_get(Index, IxKey). % a list on the form [{IxKey, RealKey1} , .... -dirty_select(Tab, Spec, Pos) -> +dirty_select(Tab, Spec, Pos) when is_integer(Pos) -> %% Assume that we are on the node where the replica is %% Returns the records without applying the match spec %% The actual filtering is handled by the caller @@ -169,59 +208,80 @@ dirty_select(Tab, Spec, Pos) -> lists:append([mnesia_lib:db_get(StorageType, Tab, Key) || {_,Key} <- RealKeys]). dirty_read(Tab, IxKey, Pos) -> - ResList = mnesia:dirty_rpc(Tab, ?MODULE, dirty_read2, - [Tab, IxKey, Pos]), - case val({Tab, setorbag}) of - bag -> - %% Remove all tuples which don't include Ixkey - mnesia_lib:key_search_all(IxKey, Pos, ResList); - _ -> - ResList - end. + mnesia:dirty_rpc(Tab, ?MODULE, dirty_read2, + [Tab, IxKey, Pos]). dirty_read2(Tab, IxKey, Pos) -> - Ix = get_index_table(Tab, Pos), - Keys = db_match(Ix, {IxKey, '$1'}), - r_keys(Keys, Tab, []). - -r_keys([[H]|T],Tab,Ack) -> - V = mnesia_lib:db_get(Tab, H), - r_keys(T, Tab, V ++ Ack); -r_keys([], _, Ack) -> - Ack. + #index{pos_list = PosL} = val({Tab, index_info}), + Storage = val({Tab, storage_type}), + {Type, Ixt} = pick_index(PosL, Tab, Pos), + Pat = case Type of + ordered -> [{{{IxKey, '$1'}}, [], ['$1']}]; + bag -> [{{IxKey, '$1'}, [], ['$1']}] + end, + Keys = db_select(Ixt, Pat), + ValsF = index_vals_f(Storage, Tab, Pos), + lists:reverse( + lists:foldl( + fun(K, Acc) -> + lists:foldl( + fun(Obj, Acc1) -> + case lists:member(IxKey, ValsF(Obj)) of + true -> [Obj|Acc1]; + false -> Acc1 + end + end, Acc, mnesia_lib:db_get(Storage, Tab, K)) + end, [], Keys)). + +pick_index([{{{Pfx,_},IxType}, Ixt}|_], _Tab, {_} = Pfx) -> + {IxType, Ixt}; +pick_index([{{Pos,IxType}, Ixt}|_], _Tab, Pos) -> + {IxType, Ixt}; +pick_index([_|T], Tab, Pos) -> + pick_index(T, Tab, Pos); +pick_index([], Tab, Pos) -> + mnesia:abort({no_exist, Tab, {index, Pos}}). + %%%%%%% Creation, Init and deletion routines for index tables %% We can have several indexes on the same table %% this can be a fairly costly operation if table is *very* large -tab2filename(Tab, Pos) -> +tab2filename(Tab, {A}) when is_atom(A) -> + mnesia_lib:dir(Tab) ++ "_-" ++ atom_to_list(A) ++ "-.DAT"; +tab2filename(Tab, T) when is_tuple(T) -> + tab2filename(Tab, element(1, T)); +tab2filename(Tab, Pos) when is_integer(Pos) -> mnesia_lib:dir(Tab) ++ "_" ++ integer_to_list(Pos) ++ ".DAT". -tab2tmp_filename(Tab, Pos) -> - mnesia_lib:dir(Tab) ++ "_" ++ integer_to_list(Pos) ++ ".TMP". - init_index(Tab, Storage) -> - PosList = val({Tab, index}), + Cs = val({Tab, cstruct}), + PosList = Cs#cstruct.index, init_indecies(Tab, Storage, PosList). init_indecies(Tab, Storage, PosList) -> case Storage of unknown -> ignore; + {ext, Alias, Mod} -> + init_ext_index(Tab, Storage, Alias, Mod, PosList); disc_only_copies -> - init_disc_index(Tab, PosList); + init_disc_index(Tab, Storage, PosList); ram_copies -> - make_ram_index(Tab, PosList); + make_ram_index(Tab, Storage, PosList); disc_copies -> - make_ram_index(Tab, PosList) + make_ram_index(Tab, Storage, PosList) end. %% works for both ram and disc indexes del_index_table(_, unknown, _) -> ignore; -del_index_table(Tab, Storage, Pos) -> +del_index_table(Tab, Storage, {_} = Pos) -> + delete_transient_index(Tab, Pos, Storage), + mnesia_lib:del({Tab, index}, Pos); +del_index_table(Tab, Storage, Pos) when is_integer(Pos) -> delete_transient_index(Tab, Pos, Storage), mnesia_lib:del({Tab, index}, Pos). @@ -229,18 +289,29 @@ del_transient(Tab, Storage) -> PosList = val({Tab, index}), del_transient(Tab, PosList, Storage). -del_transient(_, [], _) -> done; +del_transient(_, [], _) -> ok; del_transient(Tab, [Pos | Tail], Storage) -> delete_transient_index(Tab, Pos, Storage), del_transient(Tab, Tail, Storage). +delete_transient_index(Tab, Pos, {ext, Alias, Mod}) -> + PosInfo = case Pos of + _ when is_integer(Pos) -> + Cs = val({Tab, cstruct}), + lists:keyfind(Pos, 1, Cs#cstruct.index); + {P, T} -> {P, T} + end, + Tag = {Tab, index, PosInfo}, + Mod:close_table(Alias, Tag), + Mod:delete_table(Alias, Tag), + del_index_info(Tab, Pos), + mnesia_lib:unset({Tab, {index, Pos}}); delete_transient_index(Tab, Pos, disc_only_copies) -> Tag = {Tab, index, Pos}, mnesia_monitor:unsafe_close_dets(Tag), - file:delete(tab2filename(Tab, Pos)), + _ = file:delete(tab2filename(Tab, Pos)), del_index_info(Tab, Pos), %% Uses val(..) mnesia_lib:unset({Tab, {index, Pos}}); - delete_transient_index(Tab, Pos, _Storage) -> Ixt = val({Tab, {index, Pos}}), ?ets_delete_table(Ixt), @@ -250,12 +321,13 @@ delete_transient_index(Tab, Pos, _Storage) -> %%%%% misc functions for the index create/init/delete functions above %% assuming that the file exists. -init_disc_index(_Tab, []) -> +init_disc_index(_Tab, _Storage, []) -> done; -init_disc_index(Tab, [Pos | Tail]) when is_integer(Pos) -> +init_disc_index(Tab, disc_only_copies, [{Pos,_Pref} | Tail]) -> + PosInfo = {Pos, bag}, Fn = tab2filename(Tab, Pos), - IxTag = {Tab, index, Pos}, - file:delete(Fn), + IxTag = {Tab, index, PosInfo}, + _ = file:delete(Fn), Args = [{file, Fn}, {keypos, 1}, {type, bag}], mnesia_monitor:open_dets(IxTag, Args), Storage = disc_only_copies, @@ -267,16 +339,59 @@ init_disc_index(Tab, [Pos | Tail]) when is_integer(Pos) -> mnesia_lib:db_fixtable(Storage, Tab, true), ok = dets:init_table(IxTag, create_fun(Init, Tab, Pos)), mnesia_lib:db_fixtable(Storage, Tab, false), - mnesia_lib:set({Tab, {index, Pos}}, IxTag), - add_index_info(Tab, val({Tab, setorbag}), {Pos, {dets, IxTag}}), - init_disc_index(Tab, Tail). + mnesia_lib:set({Tab, {index, PosInfo}}, IxTag), + add_index_info(Tab, val({Tab, setorbag}), {PosInfo, {dets, IxTag}}), + init_disc_index(Tab, Storage, Tail). + +init_ext_index(_, _, _, _, []) -> + done; +init_ext_index(Tab, Storage, Alias, Mod, [{Pos,Type} | Tail]) -> + PosInfo = {Pos, Type}, + IxTag = {Tab, index, PosInfo}, + CS = val({Tab, cstruct}), + CsList = mnesia_schema:cs2list(CS), + _Res = mnesia_monitor:unsafe_create_external(IxTag, Alias, Mod, CsList), + Mod:load_table(Alias, IxTag, init_index, CsList), + case Mod:is_index_consistent(Alias, IxTag) of + false -> + Mod:index_is_consistent(Alias, IxTag, false), + Mod:match_delete(Alias, IxTag, '_'), + IxValsF = index_vals_f(Storage, Tab, Pos), + IxObjF = case Type of + bag -> fun(IxVal, Key) -> {IxVal, Key} end; + ordered -> fun(IxVal, Key) -> {{IxVal, Key}} end + end, + mnesia_lib:db_fixtable(Storage, Tab, true), + mnesia_lib:db_foldl( + Storage, + fun(Rec, Acc) -> + Key = element(2, Rec), + lists:foreach( + fun(V) -> + IxObj = IxObjF(V, Key), + Mod:insert(Alias, IxTag, IxObj) + end, IxValsF(Rec)), + Acc + end, ok, Tab, + [{'_', [], ['$_']}], 100), + Mod:index_is_consistent(Alias, IxTag, true); + true -> + ignore + end, + + mnesia_lib:set({Tab, {index, PosInfo}}, IxTag), + + add_index_info(Tab, val({Tab, setorbag}), {PosInfo, {Storage, IxTag}}), + init_ext_index(Tab, Storage, Alias, Mod, Tail). create_fun(Cont, Tab, Pos) -> + IxF = index_vals_f(disc_only_copies, Tab, Pos), fun(read) -> Data = case Cont of {start, KeysPerChunk} -> - mnesia_lib:db_init_chunk(disc_only_copies, Tab, KeysPerChunk); + mnesia_lib:db_init_chunk( + disc_only_copies, Tab, KeysPerChunk); '$end_of_table' -> '$end_of_table'; _Else -> @@ -286,56 +401,82 @@ create_fun(Cont, Tab, Pos) -> '$end_of_table' -> end_of_input; {Recs, Next} -> - IdxElems = [{element(Pos, Obj), element(2, Obj)} || Obj <- Recs], + IdxElems = lists:flatmap( + fun(Obj) -> + PrimK = element(2, Obj), + [{V, PrimK} || V <- IxF(Obj)] + end, Recs), {IdxElems, create_fun(Next, Tab, Pos)} end; (close) -> ok end. -make_ram_index(_, []) -> +make_ram_index(_, _, []) -> done; -make_ram_index(Tab, [Pos | Tail]) -> - add_ram_index(Tab, Pos), - make_ram_index(Tab, Tail). +make_ram_index(Tab, Storage, [Pos | Tail]) -> + add_ram_index(Tab, Storage, Pos), + make_ram_index(Tab, Storage, Tail). -add_ram_index(Tab, Pos) when is_integer(Pos) -> - verbose("Creating index for ~w ~n", [Tab]), +add_ram_index(Tab, Storage, {Pos, _Pref}) -> + Type = ordered, + verbose("Creating index for ~w ~p ~p~n", [Tab, Pos, Type]), SetOrBag = val({Tab, setorbag}), - IndexType = case SetOrBag of - set -> duplicate_bag; - ordered_set -> duplicate_bag; - bag -> bag - end, - Index = mnesia_monitor:mktab(mnesia_index, [IndexType, public]), + IxValsF = index_vals_f(Storage, Tab, Pos), + IxFun = fun(Val, Key) -> {{Val, Key}} end, + Index = mnesia_monitor:mktab(mnesia_index, [ordered_set, public]), Insert = fun(Rec, _Acc) -> - true = ?ets_insert(Index, {element(Pos, Rec), element(2, Rec)}) + PrimK = element(2, Rec), + true = ?ets_insert( + Index, [IxFun(V, PrimK) + || V <- IxValsF(Rec)]) end, mnesia_lib:db_fixtable(ram_copies, Tab, true), - true = ets:foldl(Insert, true, Tab), + true = mnesia_lib:db_foldl(Storage, Insert, true, Tab), mnesia_lib:db_fixtable(ram_copies, Tab, false), mnesia_lib:set({Tab, {index, Pos}}, Index), - add_index_info(Tab, SetOrBag, {Pos, {ram, Index}}); -add_ram_index(_Tab, snmp) -> + add_index_info(Tab, SetOrBag, {{Pos, Type}, {ram, Index}}); +add_ram_index(_Tab, _, snmp) -> ok. -add_index_info(Tab, Type, IxElem) -> +index_info(SetOrBag, PosList) -> + IxPlugins = mnesia_schema:index_plugins(), + ExpPosList = lists:map( + fun({{P,Type},Ixt} = PI) -> + case P of + {_} = IxN -> + {_, M, F} = + lists:keyfind(IxN, 1, IxPlugins), + {{{IxN,M,F}, Type}, Ixt}; + _ -> + PI + end + end, PosList), + #index{setorbag = SetOrBag, pos_list = ExpPosList}. + +add_index_info(Tab, SetOrBag, IxElem) -> Commit = val({Tab, commit_work}), case lists:keysearch(index, 1, Commit) of false -> - Index = #index{setorbag = Type, - pos_list = [IxElem]}, - %% Check later if mnesia_tm is sensative about the order + IndexInfo = index_info(SetOrBag, [IxElem]), + %% Check later if mnesia_tm is sensitive about the order + mnesia_lib:set({Tab, index_info}, IndexInfo), + mnesia_lib:set({Tab, index}, index_positions(IndexInfo)), mnesia_lib:set({Tab, commit_work}, - mnesia_lib:sort_commit([Index | Commit])); + mnesia_lib:sort_commit([IndexInfo | Commit])); {value, Old} -> %% We could check for consistency here Index = Old#index{pos_list = [IxElem | Old#index.pos_list]}, + mnesia_lib:set({Tab, index_info}, Index), + mnesia_lib:set({Tab, index}, index_positions(Index)), NewC = lists:keyreplace(index, 1, Commit, Index), mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)) end. +index_positions(#index{pos_list = PL}) -> + [P || {{P,_},_} <- PL]. + del_index_info(Tab, Pos) -> Commit = val({Tab, commit_work}), case lists:keysearch(index, 1, Commit) of @@ -343,13 +484,21 @@ del_index_info(Tab, Pos) -> %% Something is wrong ignore skip; {value, Old} -> - case lists:keydelete(Pos, 1, Old#index.pos_list) of + case lists:filter(fun({P,_}) -> + element(1,P)=/=Pos + end, + Old#index.pos_list) of [] -> + IndexInfo = index_info(Old#index.setorbag,[]), + mnesia_lib:set({Tab, index_info}, IndexInfo), + mnesia_lib:set({Tab, index}, index_positions(IndexInfo)), NewC = lists:keydelete(index, 1, Commit), mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)); New -> Index = Old#index{pos_list = New}, + mnesia_lib:set({Tab, index_info}, Index), + mnesia_lib:set({Tab, index}, index_positions(Index)), NewC = lists:keyreplace(index, 1, Commit, Index), mnesia_lib:set({Tab, commit_work}, mnesia_lib:sort_commit(NewC)) @@ -358,33 +507,69 @@ del_index_info(Tab, Pos) -> db_put({ram, Ixt}, V) -> true = ?ets_insert(Ixt, V); +db_put({{ext, _, _} = Ext, Ixt}, V) -> + mnesia_lib:db_put(Ext, Ixt, V); db_put({dets, Ixt}, V) -> ok = dets:insert(Ixt, V). -db_get({ram, Ixt}, K) -> - ?ets_lookup(Ixt, K); +db_get({ram, _}=Ixt, IxKey) -> + Pat = [{{{IxKey, '$1'}}, [], [{element, 1, '$_'}]}], + db_select(Ixt, Pat); +db_get({{ext,_,_} = _Storage, {_,_,{_,Type}}} = Ixt, IxKey) -> + Pat = case Type of + ordered -> [{{{IxKey, '$1'}}, [], [{element, 1, '$_'}]}]; + bag -> [{{IxKey, '_'}, [], ['$_']}] + end, + db_select(Ixt, Pat); db_get({dets, Ixt}, K) -> dets:lookup(Ixt, K). +db_erase({ram, Ixt}, K) -> + ?ets_delete(Ixt, K); +db_erase({{ext,_,_} = Ext, Ixt}, K) -> + mnesia_lib:db_erase(Ext, Ixt, K); +db_erase({dets, Ixt}, K) -> + dets:delete(Ixt, K). + db_match_erase({ram, Ixt}, Pat) -> true = ?ets_match_delete(Ixt, Pat); +db_match_erase({{ext,_,_} = Ext, Ixt}, Pat) -> + mnesia_lib:db_match_erase(Ext, Ixt, Pat); db_match_erase({dets, Ixt}, Pat) -> ok = dets:match_delete(Ixt, Pat). -db_match({ram, Ixt}, Pat) -> - ?ets_match(Ixt, Pat); -db_match({dets, Ixt}, Pat) -> - dets:match(Ixt, Pat). +db_select({ram, Ixt}, Pat) -> + ets:select(Ixt, Pat); +db_select({{ext,_,_} = Ext, Ixt}, Pat) -> + mnesia_lib:db_select(Ext, Ixt, Pat); +db_select({dets, Ixt}, Pat) -> + dets:select(Ixt, Pat). + get_index_table(Tab, Pos) -> get_index_table(Tab, val({Tab, storage_type}), Pos). -get_index_table(Tab, ram_copies, Pos) -> - {ram, val({Tab, {index, Pos}})}; -get_index_table(Tab, disc_copies, Pos) -> - {ram, val({Tab, {index, Pos}})}; -get_index_table(Tab, disc_only_copies, Pos) -> - {dets, val({Tab, {index, Pos}})}; -get_index_table(_Tab, unknown, _Pos) -> - unknown. - +get_index_table(Tab, _Storage, Pos) -> + #index{pos_list = PosL} = val({Tab, index_info}), + {_IxType, Ixt} = pick_index(PosL, Tab, Pos), + Ixt. + +index_vals_f(Storage, Tab, {_} = Pos) -> + index_vals_f(Storage, Tab, + lists:keyfind(Pos, 1, mnesia_schema:index_plugins())); +index_vals_f(_Storage, Tab, {Pos,M,F}) -> + fun(Obj) -> + M:F(Tab, Pos, Obj) + end; +index_vals_f(Storage, Tab, Pos) when is_integer(Pos) -> + case mnesia_lib:semantics(Storage, index_fun) of + undefined -> + fun(Obj) -> + [element(Pos, Obj)] + end; + F when is_function(F, 4) -> + {ext, Alias, _Mod} = Storage, + fun(Obj) -> + F(Alias, Tab, Pos, Obj) + end + end. diff --git a/lib/mnesia/src/mnesia_kernel_sup.erl b/lib/mnesia/src/mnesia_kernel_sup.erl index 08f6129fc0..c9af5c460a 100644 --- a/lib/mnesia/src/mnesia_kernel_sup.erl +++ b/lib/mnesia/src/mnesia_kernel_sup.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/mnesia/src/mnesia_late_loader.erl b/lib/mnesia/src/mnesia_late_loader.erl index d09de3ca66..e273329ffc 100644 --- a/lib/mnesia/src/mnesia_late_loader.erl +++ b/lib/mnesia/src/mnesia_late_loader.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2009. All Rights Reserved. +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -36,17 +37,19 @@ -define(SERVER_NAME, ?MODULE). +-include("mnesia.hrl"). + -record(state, {supervisor}). async_late_disc_load(_, [], _) -> ok; async_late_disc_load(Node, Tabs, Reason) -> Msg = {async_late_disc_load, Tabs, Reason}, - catch ({?SERVER_NAME, Node} ! {self(), Msg}). + ?SAFE({?SERVER_NAME, Node} ! {self(), Msg}). maybe_async_late_disc_load(_, [], _) -> ok; maybe_async_late_disc_load(Node, Tabs, Reason) -> Msg = {maybe_async_late_disc_load, Tabs, Reason}, - catch ({?SERVER_NAME, Node} ! {self(), Msg}). + ?SAFE({?SERVER_NAME, Node} ! {self(), Msg}). start() -> mnesia_monitor:start_proc(?SERVER_NAME, ?MODULE, init, [self()]). diff --git a/lib/mnesia/src/mnesia_lib.erl b/lib/mnesia/src/mnesia_lib.erl index ae6631646c..10e232c800 100644 --- a/lib/mnesia/src/mnesia_lib.erl +++ b/lib/mnesia/src/mnesia_lib.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -53,6 +54,7 @@ db_erase_tab/2, db_first/1, db_first/2, + db_foldl/3, db_foldl/4, db_foldl/6, db_last/1, db_last/2, db_fixtable/3, @@ -114,7 +116,7 @@ lock_table/1, mkcore/1, not_active_here/1, - other_val/2, + other_val/1, overload_read/0, overload_read/1, overload_set/2, @@ -134,6 +136,7 @@ set_local_content_whereabouts/1, set_remote_where_to_read/1, set_remote_where_to_read/2, + semantics/2, show/1, show/2, sort_commit/1, @@ -143,6 +146,7 @@ tab2tmp/1, tab2dcd/1, tab2dcl/1, + tab2logtmp/1, to_list/1, union/2, uniq/1, @@ -150,6 +154,8 @@ unset/1, %% update_counter/2, val/1, + validate_key/2, + validate_record/2, vcore/0, vcore/1, verbose/2, @@ -296,11 +302,7 @@ active_here(Tab) -> not_active_here(Tab) -> not active_here(Tab). -exists(Fname) -> - case file:open(Fname, [raw,read]) of - {ok, F} ->file:close(F), true; - _ -> false - end. +exists(Fname) -> filelib:is_regular(Fname). dir() -> mnesia_monitor:get_env(dir). @@ -322,15 +324,41 @@ tab2dcd(Tab) -> %% Disc copies data tab2dcl(Tab) -> %% Disc copies log dir(lists:concat([Tab, ".DCL"])). +tab2logtmp(Tab) -> %% Disc copies log + dir(lists:concat([Tab, ".LOGTMP"])). + storage_type_at_node(Node, Tab) -> search_key(Node, [{disc_copies, val({Tab, disc_copies})}, {ram_copies, val({Tab, ram_copies})}, - {disc_only_copies, val({Tab, disc_only_copies})}]). + {disc_only_copies, val({Tab, disc_only_copies})}| + wrap_external(val({Tab, external_copies}))]). cs_to_storage_type(Node, Cs) -> search_key(Node, [{disc_copies, Cs#cstruct.disc_copies}, {ram_copies, Cs#cstruct.ram_copies}, - {disc_only_copies, Cs#cstruct.disc_only_copies}]). + {disc_only_copies, Cs#cstruct.disc_only_copies} | + wrap_external(Cs#cstruct.external_copies)]). + +-define(native(T), T==ram_copies; T==disc_copies; T==disc_only_copies). + +semantics({ext,Alias,Mod}, Item) -> + Mod:semantics(Alias, Item); +semantics({Alias,Mod}, Item) -> + Mod:semantics(Alias, Item); +semantics(Type, storage) when ?native(Type) -> + Type; +semantics(Type, types) when ?native(Type) -> + [set, ordered_set, bag]; +semantics(disc_only_copies, index_types) -> + [bag]; +semantics(Type, index_types) when ?native(Type) -> + [bag, ordered]; +semantics(_, _) -> + undefined. + + +wrap_external(L) -> + [{{ext,Alias,Mod},Ns} || {{Alias,Mod},Ns} <- L]. schema_cs_to_storage_type(Node, Cs) -> case cs_to_storage_type(Node, Cs) of @@ -338,7 +366,6 @@ schema_cs_to_storage_type(Node, Cs) -> Other -> Other end. - search_key(Key, [{Val, List} | Tail]) -> case lists:member(Key, List) of true -> Val; @@ -347,6 +374,33 @@ search_key(Key, [{Val, List} | Tail]) -> search_key(_Key, []) -> unknown. +validate_key(Tab, Key) -> + case ?catch_val({Tab, record_validation}) of + {RecName, Arity, Type} -> + {RecName, Arity, Type}; + {RecName, Arity, Type, Alias, Mod} -> + %% external type + Mod:validate_key(Alias, Tab, RecName, Arity, Type, Key); + {'EXIT', _} -> + mnesia:abort({no_exists, Tab}) + end. + + +validate_record(Tab, Obj) -> + case ?catch_val({Tab, record_validation}) of + {RecName, Arity, Type} + when tuple_size(Obj) == Arity, RecName == element(1, Obj) -> + {RecName, Arity, Type}; + {RecName, Arity, Type, Alias, Mod} + when tuple_size(Obj) == Arity, RecName == element(1, Obj) -> + %% external type + Mod:validate_record(Alias, Tab, RecName, Arity, Type, Obj); + {'EXIT', _} -> + mnesia:abort({no_exists, Tab}); + _ -> + mnesia:abort({bad_type, Obj}) + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% ops, we've got some global variables here :-) @@ -382,8 +436,8 @@ search_key(_Key, []) -> val(Var) -> case ?catch_val(Var) of - {'EXIT', _ReASoN_} -> mnesia_lib:other_val(Var, _ReASoN_); - _VaLuE_ -> _VaLuE_ + {'EXIT', _} -> other_val(Var); + _VaLuE_ -> _VaLuE_ end. set(Var, Val) -> @@ -392,32 +446,31 @@ set(Var, Val) -> unset(Var) -> ?ets_delete(mnesia_gvar, Var). -other_val(Var, Other) -> +other_val(Var) -> + case other_val_1(Var) of + error -> pr_other(Var); + Val -> Val + end. + +other_val_1(Var) -> case Var of {_, where_to_read} -> nowhere; {_, where_to_write} -> []; {_, active_replicas} -> []; - _ -> - pr_other(Var, Other) + _ -> error end. --spec pr_other(_,_) -> no_return(). - -pr_other(Var, Other) -> - Why = +-spec pr_other(_) -> no_return(). +pr_other(Var) -> + Why = case is_running() of no -> {node_not_running, node()}; _ -> {no_exists, Var} end, verbose("~p (~p) val(mnesia_gvar, ~w) -> ~p ~p ~n", [self(), process_info(self(), registered_name), - Var, Other, Why]), - case Other of - {badarg, [{ets, lookup_element, _, _}|_]} -> - exit(Why); - _ -> - erlang:error(Why) - end. + Var, Why, erlang:get_stacktrace()]), + mnesia:abort(Why). %% Some functions for list valued variables add(Var, Val) -> @@ -553,10 +606,16 @@ read_counter(Name) -> ?ets_lookup_element(mnesia_stats, Name, 2). cs_to_nodes(Cs) -> + ext_nodes(Cs#cstruct.external_copies) ++ Cs#cstruct.disc_only_copies ++ Cs#cstruct.disc_copies ++ Cs#cstruct.ram_copies. +ext_nodes(Ext) -> + lists:flatmap(fun({_, Ns}) -> + Ns + end, Ext). + overload_types() -> [mnesia_tm, mnesia_dump_log]. @@ -596,7 +655,7 @@ coredump(CrashInfo) -> Core = mkcore(CrashInfo), Out = core_file(), important("Writing Mnesia core to file: ~p...~p~n", [Out, CrashInfo]), - file:write_file(Out, Core), + _ = file:write_file(Out, Core), Out. core_file() -> @@ -620,7 +679,7 @@ mkcore(CrashInfo) -> Core = [ CrashInfo, {time, {date(), time()}}, - {self, catch process_info(self())}, + {self, proc_dbg_info(self())}, {nodes, catch rpc:multicall(Nodes, ?MODULE, get_node_number, [])}, {applications, catch lists:sort(application:loaded_applications())}, {flags, catch init:get_arguments()}, @@ -697,7 +756,7 @@ relatives() -> Info = fun(Name) -> case whereis(Name) of undefined -> false; - Pid -> {true, {Name, Pid, catch process_info(Pid)}} + Pid -> {true, {Name, Pid, proc_dbg_info(Pid)}} end end, lists:zf(Info, mnesia:ms()). @@ -706,14 +765,14 @@ workers({workers, Loaders, Senders, Dumper}) -> Info = fun({Pid, {send_table, Tab, _Receiver, _St}}) -> case Pid of undefined -> false; - Pid -> {true, {Pid, Tab, catch process_info(Pid)}} + Pid -> {true, {Pid, Tab, proc_dbg_info(Pid)}} end; ({Pid, What}) when is_pid(Pid) -> - {true, {Pid, What, catch process_info(Pid)}}; + {true, {Pid, What, proc_dbg_info(Pid)}}; ({Name, Pid}) -> case Pid of undefined -> false; - Pid -> {true, {Name, Pid, catch process_info(Pid)}} + Pid -> {true, {Name, Pid, proc_dbg_info(Pid)}} end end, SInfo = lists:zf(Info, Senders), @@ -727,13 +786,21 @@ locking_procs(LockList) when is_list(LockList) -> Pid = Tid#tid.pid, case node(Pid) == node() of true -> - {true, {Pid, catch process_info(Pid)}}; + {true, {Pid, proc_dbg_info(Pid)}}; _ -> false end end, lists:zf(Info, UT). +proc_dbg_info(Pid) -> + try + [process_info(Pid, current_stacktrace)| + process_info(Pid)] + catch _:R -> + [{process_info,crashed,R}] + end. + view() -> Bin = mkcore({crashinfo, {"view only~n", []}}), vcore(Bin). @@ -744,7 +811,7 @@ view(File) -> true -> view(File, dat); false -> - case suffix([".LOG", ".BUP", ".ETS"], File) of + case suffix([".LOG", ".BUP", ".ETS", ".LOGTMP"], File) of true -> view(File, log); false -> @@ -806,9 +873,9 @@ vcore(File) -> vcore_elem({schema_file, {ok, B}}) -> Fname = "/tmp/schema.DAT", - file:write_file(Fname, B), - dets:view(Fname), - file:delete(Fname); + _ = file:write_file(Fname, B), + _ = dets:view(Fname), + _ = file:delete(Fname); vcore_elem({logfile, {ok, BinList}}) -> Fun = fun({F, Info}) -> @@ -896,7 +963,7 @@ dirty_rpc_error_tag(Reason) -> end. fatal(Format, Args) -> - catch set(mnesia_status, stopping), + ?SAFE(catch set(mnesia_status, stopping)), Core = mkcore({crashinfo, {Format, Args}}), report_fatal(Format, Args, Core), timer:sleep(10000), % Enough to write the core dump to disc? @@ -908,7 +975,7 @@ report_fatal(Format, Args) -> report_fatal(Format, Args, Core) -> report_system_event({mnesia_fatal, Format, Args, Core}), - catch exit(whereis(mnesia_monitor), fatal). + ?SAFE(exit(whereis(mnesia_monitor), fatal)). %% We sleep longer and longer the more we try %% Made some testing and came up with the following constants @@ -918,19 +985,7 @@ random_time(Retries, _Counter0) -> UpperLimit = 500, Dup = Retries * Retries, MaxIntv = trunc(UpperLimit * (1-(50/((Dup)+50)))), - - case get(random_seed) of - undefined -> - {X, Y, Z} = erlang:now(), %% time() - random:seed(X, Y, Z), - Time = Dup + random:uniform(MaxIntv), - %% dbg_out("---random_test rs ~w max ~w val ~w---~n", [Retries, MaxIntv, Time]), - Time; - _ -> - Time = Dup + random:uniform(MaxIntv), - %% dbg_out("---random_test rs ~w max ~w val ~w---~n", [Retries, MaxIntv, Time]), - Time - end. + Dup + rand:uniform(MaxIntv). report_system_event(Event0) -> Event = {mnesia_system_event, Event0}, @@ -958,20 +1013,17 @@ report_system_event({'EXIT', Reason}, Event) -> unlink(Pid), %% We get an exit signal if server dies - receive - {'EXIT', Pid, _Reason} -> - {error, {node_not_running, node()}} - after 0 -> - gen_event:stop(mnesia_event), - ok + receive {'EXIT', Pid, _Reason} -> ok + after 0 -> gen_event:stop(mnesia_event) end; Error -> Msg = "Mnesia(~p): Cannot report event ~p: ~p (~p)~n", error_logger:format(Msg, [node(), Event, Reason, Error]) - end; + end, + ok; report_system_event(_Res, _Event) -> - ignore. + ok. %% important messages are reported regardless of debug level important(Format, Args) -> @@ -1007,7 +1059,7 @@ dbg_out(Format, Args) -> %% Keep the last 10 debug print outs save(DbgInfo) -> - catch save2(DbgInfo). + ?SAFE(save2(DbgInfo)). save2(DbgInfo) -> Key = {'$$$_report', current_pos}, @@ -1025,8 +1077,8 @@ copy_file(From, To) -> case file:open(To, [raw, binary, write]) of {ok, T} -> Res = copy_file_loop(F, T, 8000), - file:close(F), - file:close(T), + ok = file:close(F), + ok = file:close(T), Res; {error, Reason} -> {error, Reason} @@ -1038,7 +1090,7 @@ copy_file(From, To) -> copy_file_loop(F, T, ChunkSize) -> case file:read(F, ChunkSize) of {ok, Bin} -> - file:write(T, Bin), + ok = file:write(T, Bin), copy_file_loop(F, T, ChunkSize); eof -> ok; @@ -1055,18 +1107,24 @@ db_get(Tab, Key) -> db_get(val({Tab, storage_type}), Tab, Key). db_get(ram_copies, Tab, Key) -> ?ets_lookup(Tab, Key); db_get(disc_copies, Tab, Key) -> ?ets_lookup(Tab, Key); -db_get(disc_only_copies, Tab, Key) -> dets:lookup(Tab, Key). +db_get(disc_only_copies, Tab, Key) -> dets:lookup(Tab, Key); +db_get({ext, Alias, Mod}, Tab, Key) -> + Mod:lookup(Alias, Tab, Key). db_init_chunk(Tab) -> db_init_chunk(val({Tab, storage_type}), Tab, 1000). db_init_chunk(Tab, N) -> db_init_chunk(val({Tab, storage_type}), Tab, N). +db_init_chunk({ext, Alias, Mod}, Tab, N) -> + Mod:select(Alias, Tab, [{'_', [], ['$_']}], N); db_init_chunk(disc_only_copies, Tab, N) -> dets:select(Tab, [{'_', [], ['$_']}], N); db_init_chunk(_, Tab, N) -> ets:select(Tab, [{'_', [], ['$_']}], N). +db_chunk({ext, _Alias, Mod}, State) -> + Mod:select(State); db_chunk(disc_only_copies, State) -> dets:select(State); db_chunk(_, State) -> @@ -1077,46 +1135,82 @@ db_put(Tab, Val) -> db_put(ram_copies, Tab, Val) -> ?ets_insert(Tab, Val), ok; db_put(disc_copies, Tab, Val) -> ?ets_insert(Tab, Val), ok; -db_put(disc_only_copies, Tab, Val) -> dets:insert(Tab, Val). +db_put(disc_only_copies, Tab, Val) -> dets:insert(Tab, Val); +db_put({ext, Alias, Mod}, Tab, Val) -> + Mod:insert(Alias, Tab, Val). db_match_object(Tab, Pat) -> db_match_object(val({Tab, storage_type}), Tab, Pat). db_match_object(Storage, Tab, Pat) -> db_fixtable(Storage, Tab, true), - Res = catch_match_object(Storage, Tab, Pat), - db_fixtable(Storage, Tab, false), - case Res of - {'EXIT', Reason} -> exit(Reason); - _ -> Res + try + case Storage of + disc_only_copies -> dets:match_object(Tab, Pat); + {ext, Alias, Mod} -> Mod:select(Alias, Tab, [{Pat, [], ['$_']}]); + _ -> ets:match_object(Tab, Pat) + end + after + db_fixtable(Storage, Tab, false) end. -catch_match_object(disc_only_copies, Tab, Pat) -> - catch dets:match_object(Tab, Pat); -catch_match_object(_, Tab, Pat) -> - catch ets:match_object(Tab, Pat). +db_foldl(Fun, Acc, Tab) -> + db_foldl(val({Tab, storage_type}), Fun, Acc, Tab). + +db_foldl(Storage, Fun, Acc, Tab) -> + Limit = mnesia_monitor:get_env(fold_chunk_size), + db_foldl(Storage, Fun, Acc, Tab, [{'_', [], ['$_']}], Limit). + +db_foldl(ram_copies, Fun, Acc, Tab, Pat, Limit) -> + mnesia_lib:db_fixtable(ram_copies, Tab, true), + try select_foldl(db_select_init(ram_copies, Tab, Pat, Limit), + Fun, Acc, ram_copies) + after + mnesia_lib:db_fixtable(ram_copies, Tab, false) + end; +db_foldl(Storage, Fun, Acc, Tab, Pat, Limit) -> + select_foldl(mnesia_lib:db_select_init(Storage, Tab, Pat, Limit), Fun, Acc, Storage). + +select_foldl({Objs, Cont}, Fun, Acc, Storage) -> + select_foldl(mnesia_lib:db_select_cont(Storage, Cont, []), + Fun, lists:foldl(Fun, Acc, Objs), Storage); +select_foldl('$end_of_table', _, Acc, _) -> + Acc. db_select(Tab, Pat) -> db_select(val({Tab, storage_type}), Tab, Pat). db_select(Storage, Tab, Pat) -> db_fixtable(Storage, Tab, true), - Res = catch_select(Storage, Tab, Pat), - db_fixtable(Storage, Tab, false), - case Res of - {'EXIT', Reason} -> exit(Reason); - _ -> Res + try + case Storage of + disc_only_copies -> dets:select(Tab, Pat); + {ext, Alias, Mod} -> Mod:select(Alias, Tab, Pat); + _ -> ets:select(Tab, Pat) + end + after + db_fixtable(Storage, Tab, false) end. -catch_select(disc_only_copies, Tab, Pat) -> - catch dets:select(Tab, Pat); -catch_select(_, Tab, Pat) -> - catch ets:select(Tab, Pat). - +db_select_init({ext, Alias, Mod}, Tab, Pat, Limit) -> + case Mod:select(Alias, Tab, Pat, Limit) of + {Matches, Continuation} when is_list(Matches) -> + {Matches, {Alias, Continuation}}; + R -> + R + end; db_select_init(disc_only_copies, Tab, Pat, Limit) -> dets:select(Tab, Pat, Limit); db_select_init(_, Tab, Pat, Limit) -> ets:select(Tab, Pat, Limit). +db_select_cont({ext, Alias, Mod}, Cont0, Ms) -> + Cont = Mod:repair_continuation(Cont0, Ms), + case Mod:select(Cont) of + {Matches, Continuation} when is_list(Matches) -> + {Matches, {Alias, Continuation}}; + R -> + R + end; db_select_cont(disc_only_copies, Cont0, Ms) -> Cont = dets:repair_continuation(Cont0, Ms), dets:select(Cont); @@ -1133,13 +1227,18 @@ db_fixtable(disc_copies, Tab, Bool) -> db_fixtable(dets, Tab, Bool) -> dets:safe_fixtable(Tab, Bool); db_fixtable(disc_only_copies, Tab, Bool) -> - dets:safe_fixtable(Tab, Bool). + dets:safe_fixtable(Tab, Bool); +db_fixtable({ext, Alias, Mod}, Tab, Bool) -> + Mod:fixtable(Alias, Tab, Bool). db_erase(Tab, Key) -> db_erase(val({Tab, storage_type}), Tab, Key). db_erase(ram_copies, Tab, Key) -> ?ets_delete(Tab, Key), ok; db_erase(disc_copies, Tab, Key) -> ?ets_delete(Tab, Key), ok; -db_erase(disc_only_copies, Tab, Key) -> dets:delete(Tab, Key). +db_erase(disc_only_copies, Tab, Key) -> dets:delete(Tab, Key); +db_erase({ext, Alias, Mod}, Tab, Key) -> + Mod:delete(Alias, Tab, Key), + ok. db_match_erase(Tab, '_') -> db_delete_all(val({Tab, storage_type}),Tab); @@ -1147,7 +1246,10 @@ db_match_erase(Tab, Pat) -> db_match_erase(val({Tab, storage_type}), Tab, Pat). db_match_erase(ram_copies, Tab, Pat) -> ?ets_match_delete(Tab, Pat), ok; db_match_erase(disc_copies, Tab, Pat) -> ?ets_match_delete(Tab, Pat), ok; -db_match_erase(disc_only_copies, Tab, Pat) -> dets:match_delete(Tab, Pat). +db_match_erase(disc_only_copies, Tab, Pat) -> dets:match_delete(Tab, Pat); +db_match_erase({ext, Alias, Mod}, Tab, Pat) -> + Mod:match_delete(Alias, Tab, Pat), + ok. db_delete_all(ram_copies, Tab) -> ets:delete_all_objects(Tab); db_delete_all(disc_copies, Tab) -> ets:delete_all_objects(Tab); @@ -1157,31 +1259,41 @@ db_first(Tab) -> db_first(val({Tab, storage_type}), Tab). db_first(ram_copies, Tab) -> ?ets_first(Tab); db_first(disc_copies, Tab) -> ?ets_first(Tab); -db_first(disc_only_copies, Tab) -> dets:first(Tab). +db_first(disc_only_copies, Tab) -> dets:first(Tab); +db_first({ext, Alias, Mod}, Tab) -> + Mod:first(Alias, Tab). db_next_key(Tab, Key) -> db_next_key(val({Tab, storage_type}), Tab, Key). db_next_key(ram_copies, Tab, Key) -> ?ets_next(Tab, Key); db_next_key(disc_copies, Tab, Key) -> ?ets_next(Tab, Key); -db_next_key(disc_only_copies, Tab, Key) -> dets:next(Tab, Key). +db_next_key(disc_only_copies, Tab, Key) -> dets:next(Tab, Key); +db_next_key({ext, Alias, Mod}, Tab, Key) -> + Mod:next(Alias, Tab, Key). db_last(Tab) -> db_last(val({Tab, storage_type}), Tab). db_last(ram_copies, Tab) -> ?ets_last(Tab); db_last(disc_copies, Tab) -> ?ets_last(Tab); -db_last(disc_only_copies, Tab) -> dets:first(Tab). %% Dets don't have order +db_last(disc_only_copies, Tab) -> dets:first(Tab); %% Dets don't have order +db_last({ext, Alias, Mod}, Tab) -> + Mod:last(Alias, Tab). db_prev_key(Tab, Key) -> db_prev_key(val({Tab, storage_type}), Tab, Key). db_prev_key(ram_copies, Tab, Key) -> ?ets_prev(Tab, Key); db_prev_key(disc_copies, Tab, Key) -> ?ets_prev(Tab, Key); -db_prev_key(disc_only_copies, Tab, Key) -> dets:next(Tab, Key). %% Dets don't have order +db_prev_key(disc_only_copies, Tab, Key) -> dets:next(Tab, Key); %% Dets don't have order +db_prev_key({ext, Alias, Mod}, Tab, Key) -> + Mod:prev(Alias, Tab, Key). db_slot(Tab, Pos) -> db_slot(val({Tab, storage_type}), Tab, Pos). db_slot(ram_copies, Tab, Pos) -> ?ets_slot(Tab, Pos); db_slot(disc_copies, Tab, Pos) -> ?ets_slot(Tab, Pos); -db_slot(disc_only_copies, Tab, Pos) -> dets:slot(Tab, Pos). +db_slot(disc_only_copies, Tab, Pos) -> dets:slot(Tab, Pos); +db_slot({ext, Alias, Mod}, Tab, Pos) -> + Mod:slot(Alias, Tab, Pos). db_update_counter(Tab, C, Val) -> db_update_counter(val({Tab, storage_type}), Tab, C, Val). @@ -1190,13 +1302,16 @@ db_update_counter(ram_copies, Tab, C, Val) -> db_update_counter(disc_copies, Tab, C, Val) -> ?ets_update_counter(Tab, C, Val); db_update_counter(disc_only_copies, Tab, C, Val) -> - dets:update_counter(Tab, C, Val). + dets:update_counter(Tab, C, Val); +db_update_counter({ext, Alias, Mod}, Tab, C, Val) -> + Mod:update_counter(Alias, Tab, C, Val). db_erase_tab(Tab) -> db_erase_tab(val({Tab, storage_type}), Tab). db_erase_tab(ram_copies, Tab) -> ?ets_delete_table(Tab); db_erase_tab(disc_copies, Tab) -> ?ets_delete_table(Tab); -db_erase_tab(disc_only_copies, _Tab) -> ignore. +db_erase_tab(disc_only_copies, _Tab) -> ignore; +db_erase_tab({ext, _Alias, _Mod}, _Tab) -> ignore. %% assuming that Tab is a valid ets-table dets_to_ets(Tabname, Tab, File, Type, Rep, Lock) -> @@ -1205,7 +1320,7 @@ dets_to_ets(Tabname, Tab, File, Type, Rep, Lock) -> {keypos, 2}, {repair, Rep}]) of {ok, Tabname} -> Res = dets:to_ets(Tabname, Tab), - Close(Tabname), + ok = Close(Tabname), trav_ret(Res, Tab); Other -> Other @@ -1255,7 +1370,7 @@ dets_sync_open(Tab, Args) -> end. dets_sync_close(Tab) -> - catch dets:close(Tab), + ?SAFE(dets:close(Tab)), unlock_table(Tab), ok. @@ -1291,7 +1406,7 @@ readable_indecies(Tab) -> scratch_debug_fun() -> dbg_out("scratch_debug_fun(): ~p~n", [?DEBUG_TAB]), - (catch ?ets_delete_table(?DEBUG_TAB)), + ?SAFE(?ets_delete_table(?DEBUG_TAB)), ?ets_new_table(?DEBUG_TAB, [set, public, named_table, {keypos, 2}]). activate_debug_fun(FunId, Fun, InitialContext, File, Line) -> @@ -1304,43 +1419,45 @@ activate_debug_fun(FunId, Fun, InitialContext, File, Line) -> update_debug_info(Info). update_debug_info(Info) -> - case catch ?ets_insert(?DEBUG_TAB, Info) of - {'EXIT', _} -> + try ?ets_insert(?DEBUG_TAB, Info), + ok + catch error:_ -> scratch_debug_fun(), - ?ets_insert(?DEBUG_TAB, Info); - _ -> - ok + ?ets_insert(?DEBUG_TAB, Info) end, dbg_out("update_debug_info(~p)~n", [Info]), ok. deactivate_debug_fun(FunId, _File, _Line) -> - catch ?ets_delete(?DEBUG_TAB, FunId), + ?SAFE(?ets_delete(?DEBUG_TAB, FunId)), ok. eval_debug_fun(FunId, EvalContext, EvalFile, EvalLine) -> - case catch ?ets_lookup(?DEBUG_TAB, FunId) of - [] -> - ok; - [Info] -> - OldContext = Info#debug_info.context, - dbg_out("~s(~p): ~w " - "activated in ~s(~p)~n " - "eval_debug_fun(~w, ~w)~n", - [filename:basename(EvalFile), EvalLine, Info#debug_info.id, - filename:basename(Info#debug_info.file), Info#debug_info.line, - OldContext, EvalContext]), - Fun = Info#debug_info.function, - NewContext = Fun(OldContext, EvalContext), - - case catch ?ets_lookup(?DEBUG_TAB, FunId) of - [Info] when NewContext /= OldContext -> - NewInfo = Info#debug_info{context = NewContext}, - update_debug_info(NewInfo); - _ -> - ok - end; - {'EXIT', _} -> ok + try + case ?ets_lookup(?DEBUG_TAB, FunId) of + [] -> + ok; + [Info] -> + OldContext = Info#debug_info.context, + dbg_out("~s(~p): ~w " + "activated in ~s(~p)~n " + "eval_debug_fun(~w, ~w)~n", + [filename:basename(EvalFile), EvalLine, Info#debug_info.id, + filename:basename(Info#debug_info.file), Info#debug_info.line, + OldContext, EvalContext]), + Fun = Info#debug_info.function, + NewContext = Fun(OldContext, EvalContext), + + case ?ets_lookup(?DEBUG_TAB, FunId) of + [Info] when NewContext /= OldContext -> + NewInfo = Info#debug_info{context = NewContext}, + update_debug_info(NewInfo); + _ -> + ok + end + end + catch error -> + ok end. -ifdef(debug). diff --git a/lib/mnesia/src/mnesia_loader.erl b/lib/mnesia/src/mnesia_loader.erl index 4afbea1cc2..71e5829c87 100644 --- a/lib/mnesia/src/mnesia_loader.erl +++ b/lib/mnesia/src/mnesia_loader.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2013. All Rights Reserved. +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -35,7 +36,7 @@ val(Var) -> case ?catch_val(Var) of - {'EXIT', Reason} -> mnesia_lib:other_val(Var, Reason); + {'EXIT', _} -> mnesia_lib:other_val(Var); Value -> Value end. @@ -69,9 +70,10 @@ do_get_disc_copy2(Tab, Reason, Storage, Type) when Storage == disc_copies -> ignore; _ -> mnesia_monitor:mktab(Tab, Args), - Count = mnesia_log:dcd2ets(Tab, Repair), - case ets:info(Tab, size) of - X when X < Count * 4 -> + _Count = mnesia_log:dcd2ets(Tab, Repair), + case mnesia_monitor:get_env(dump_disc_copies_at_startup) + andalso mnesia_dumper:needs_dump_ets(Tab) of + true -> ok = mnesia_log:ets2dcd(Tab); _ -> ignore @@ -145,6 +147,19 @@ do_get_disc_copy2(Tab, Reason, Storage, Type) when Storage == disc_only_copies - {error, Error} -> {not_loaded, {"Failed to create dets table", Error}} end + end; + +do_get_disc_copy2(Tab, Reason, Storage = {ext, Alias, Mod}, _Type) -> + Cs = val({Tab, cstruct}), + case mnesia_monitor:unsafe_create_external(Tab, Alias, Mod, Cs) of + ok -> + ok = ext_load_table(Mod, Alias, Tab, Reason), + mnesia_index:init_index(Tab, Storage), + set({Tab, load_node}, node()), + set({Tab, load_reason}, Reason), + {loaded, ok}; + Other -> + {not_loaded, Other} end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -178,8 +193,7 @@ do_get_disc_copy2(Tab, Reason, Storage, Type) when Storage == disc_only_copies - -define(MAX_RAM_TRANSFERS, (?MAX_RAM_FILE_SIZE div ?MAX_TRANSFER_SIZE) + 1). -define(MAX_NOPACKETS, 20). -net_load_table(Tab, Reason, Ns, Cs) - when Reason == {dumper,add_table_copy} -> +net_load_table(Tab, {dumper,{add_table_copy, _}}=Reason, Ns, Cs) -> try_net_load_table(Tab, Reason, Ns, Cs); net_load_table(Tab, Reason, Ns, _Cs) -> try_net_load_table(Tab, Reason, Ns, val({Tab, cstruct})). @@ -231,7 +245,8 @@ do_snmpify(Tab, Us, Storage) -> set({Tab, {index, snmp}}, Snmp). %% Start the recieiver -init_receiver(Node, Tab, Storage, Cs, Reas={dumper,add_table_copy}) -> +init_receiver(Node, Tab, Storage, Cs, Reas={dumper,{add_table_copy, Tid}}) -> + rpc:call(Node, mnesia_lib, set, [{?MODULE, active_trans}, Tid]), case start_remote_sender(Node, Tab, Storage) of {SenderPid, TabSize, DetsData} -> start_receiver(Tab,Storage,Cs,SenderPid,TabSize,DetsData,Reas); @@ -257,7 +272,7 @@ init_receiver(Node, Tab,Storage,Cs,Reason) -> true = lists:member(Node, Active), {SenderPid, TabSize, DetsData} = start_remote_sender(Node,Tab,Storage), - Init = table_init_fun(SenderPid), + Init = table_init_fun(SenderPid, Storage), Args = [self(),Tab,Storage,Cs,SenderPid, TabSize,DetsData,Init], Pid = spawn_link(?MODULE, spawned_receiver, Args), @@ -287,7 +302,12 @@ start_remote_sender(Node,Tab,Storage) -> mnesia_controller:start_remote_sender(Node, Tab, self(), Storage), put(mnesia_table_sender_node, {Tab, Node}), receive - {SenderPid, {first, TabSize}} -> + {SenderPid, {first, _} = Msg} + when is_pid(SenderPid), element(1, Storage) == ext -> + {ext, Alias, Mod} = Storage, + {Sz, Data} = Mod:receiver_first_message(SenderPid, Msg, Alias, Tab), + {SenderPid, Sz, Data}; + {SenderPid, {first, TabSize}} =_M1 -> {SenderPid, TabSize, false}; {SenderPid, {first, TabSize, DetsData}} -> {SenderPid, TabSize, DetsData}; @@ -297,16 +317,18 @@ start_remote_sender(Node,Tab,Storage) -> down(Tab, Storage) end. -table_init_fun(SenderPid) -> +table_init_fun(SenderPid, Storage) -> fun(read) -> Receiver = self(), SenderPid ! {Receiver, more}, - get_data(SenderPid, Receiver) + get_data(SenderPid, Receiver, Storage); + (close) -> + ok end. %% Add_table_copy get's it's own locks. -start_receiver(Tab,Storage,Cs,SenderPid,TabSize,DetsData,{dumper,add_table_copy}) -> - Init = table_init_fun(SenderPid), +start_receiver(Tab,Storage,Cs,SenderPid,TabSize,DetsData,{dumper,{add_table_copy,_}}) -> + Init = table_init_fun(SenderPid, Storage), case do_init_table(Tab,Storage,Cs,SenderPid,TabSize,DetsData,self(), Init) of Err = {error, _} -> SenderPid ! {copier_done, node()}, @@ -330,7 +352,7 @@ wait_on_load_complete(Pid) -> {Pid, Res} -> Res; {'EXIT', Pid, Reason} -> - exit(Reason); + error(Reason); Else -> Pid ! Else, wait_on_load_complete(Pid) @@ -389,7 +411,15 @@ create_table(Tab, TabSize, Storage, Cs) -> {Storage, Tab}; Else -> Else - end + end; + element(1, Storage) == ext -> + {_, Alias, Mod} = Storage, + case mnesia_monitor:unsafe_create_external(Tab, Alias, Mod, Cs) of + ok -> + {Storage, Tab}; + Else -> + Else + end end. tab_receiver(Node, Tab, Storage, Cs, OrigTabRec) -> @@ -407,21 +437,25 @@ tab_receiver(Node, Tab, Storage, Cs, OrigTabRec) -> tab_receiver(Node, Tab, Storage, Cs, OrigTabRec) end. -make_table_fun(Pid, TabRec) -> +make_table_fun(Pid, TabRec, Storage) -> fun(close) -> ok; + ({read, Msg}) -> + Pid ! {TabRec, Msg}, + get_data(Pid, TabRec, Storage); (read) -> - get_data(Pid, TabRec) + get_data(Pid, TabRec, Storage) end. -get_data(Pid, TabRec) -> +get_data(Pid, TabRec, Storage) -> receive {Pid, {more_z, CompressedRecs}} when is_binary(CompressedRecs) -> - Pid ! {TabRec, more}, - {zlib_uncompress(CompressedRecs), make_table_fun(Pid,TabRec)}; + maybe_reply(Pid, {TabRec, more}, Storage), + {zlib_uncompress(CompressedRecs), + make_table_fun(Pid, TabRec, Storage)}; {Pid, {more, Recs}} -> - Pid ! {TabRec, more}, - {Recs, make_table_fun(Pid,TabRec)}; + maybe_reply(Pid, {TabRec, more}, Storage), + {Recs, make_table_fun(Pid, TabRec, Storage)}; {Pid, no_more} -> end_of_input; {copier_done, Node} -> @@ -429,29 +463,64 @@ get_data(Pid, TabRec) -> Node -> {copier_done, Node}; _ -> - get_data(Pid, TabRec) + get_data(Pid, TabRec, Storage) end; {'EXIT', Pid, Reason} -> handle_exit(Pid, Reason), - get_data(Pid, TabRec) + get_data(Pid, TabRec, Storage) end. +maybe_reply(_, _, {ext, _, _}) -> + ignore; +maybe_reply(Pid, Msg, _) -> + Pid ! Msg. + +ext_init_table(Alias, Mod, Tab, Fun, State, Sender) -> + ok = ext_load_table(Mod, Alias, Tab, {net_load, node(Sender)}), + ext_init_table(read, Alias, Mod, Tab, Fun, State, Sender). + +ext_load_table(Mod, Alias, Tab, Reason) -> + CS = val({Tab, cstruct}), + Mod:load_table(Alias, Tab, Reason, mnesia_schema:cs2list(CS)). + + +ext_init_table(Action, Alias, Mod, Tab, Fun, State, Sender) -> + case Fun(Action) of + {copier_done, Node} -> + verbose("Receiver of table ~p crashed on ~p (more)~n", [Tab, Node]), + down(Tab, {ext,Alias,Mod}); + {Data, NewFun} -> + case Mod:receive_data(Data, Alias, Tab, Sender, State) of + {more, NewState} -> + ext_init_table({read, more}, Alias, Mod, + Tab, NewFun, NewState, Sender); + {{more,Msg}, NewState} -> + ext_init_table({read, Msg}, Alias, Mod, + Tab, NewFun, NewState, Sender) + end; + end_of_input -> + Mod:receive_done(Alias, Tab, Sender, State), + ok = Fun(close) + end. + +init_table(Tab, {ext,Alias,Mod}, Fun, State, Sender) -> + ext_init_table(Alias, Mod, Tab, Fun, State, Sender); init_table(Tab, disc_only_copies, Fun, DetsInfo,Sender) -> ErtsVer = erlang:system_info(version), case DetsInfo of {ErtsVer, DetsData} -> - Res = (catch dets:is_compatible_bchunk_format(Tab, DetsData)), - case Res of - {'EXIT',{undef,[{dets,_,_,_}|_]}} -> - Sender ! {self(), {old_protocol, Tab}}, - dets:init_table(Tab, Fun); %% Old dets version - {'EXIT', What} -> - exit(What); + try dets:is_compatible_bchunk_format(Tab, DetsData) of false -> Sender ! {self(), {old_protocol, Tab}}, dets:init_table(Tab, Fun); %% Old dets version true -> dets:init_table(Tab, Fun, [{format, bchunk}]) + catch + error:{undef,[{dets,_,_,_}|_]} -> + Sender ! {self(), {old_protocol, Tab}}, + dets:init_table(Tab, Fun); %% Old dets version + error:What -> + What end; Old when Old /= false -> Sender ! {self(), {old_protocol, Tab}}, @@ -460,10 +529,10 @@ init_table(Tab, disc_only_copies, Fun, DetsInfo,Sender) -> dets:init_table(Tab, Fun) end; init_table(Tab, _, Fun, _DetsInfo,_) -> - case catch ets:init_table(Tab, Fun) of - true -> - ok; - {'EXIT', Else} -> Else + try + true = ets:init_table(Tab, Fun), + ok + catch _:Else -> {Else, erlang:get_stacktrace()} end. @@ -565,18 +634,24 @@ handle_last({ram_copies, Tab}, _Type, DatBin) -> ok; false -> ok - end. + end; + +handle_last(_Storage, _Type, nobin) -> + ok. down(Tab, Storage) -> case Storage of ram_copies -> - catch ?ets_delete_table(Tab); + ?SAFE(?ets_delete_table(Tab)); disc_copies -> - catch ?ets_delete_table(Tab); + ?SAFE(?ets_delete_table(Tab)); disc_only_copies -> TmpFile = mnesia_lib:tab2tmp(Tab), mnesia_lib:dets_sync_close(Tab), - file:delete(TmpFile) + file:delete(TmpFile); + {ext, Alias, Mod} -> + catch Mod:close_table(Alias, Tab), + catch Mod:delete_table(Alias, Tab) end, mnesia_checkpoint:tm_del_copy(Tab, node()), mnesia_controller:sync_del_table_copy_whereabouts(Tab, node()), @@ -601,21 +676,35 @@ db_erase({ram_copies, Tab}, Key) -> db_erase({disc_copies, Tab}, Key) -> true = ?ets_delete(Tab, Key); db_erase({disc_only_copies, Tab}, Key) -> - ok = dets:delete(Tab, Key). + ok = dets:delete(Tab, Key); +db_erase({{ext, Alias, Mod}, Tab}, Key) -> + ok = Mod:delete(Alias, Tab, Key). db_match_erase({ram_copies, Tab} , Pat) -> true = ?ets_match_delete(Tab, Pat); db_match_erase({disc_copies, Tab} , Pat) -> true = ?ets_match_delete(Tab, Pat); db_match_erase({disc_only_copies, Tab}, Pat) -> - ok = dets:match_delete(Tab, Pat). + ok = dets:match_delete(Tab, Pat); +db_match_erase({{ext, Alias, Mod}, Tab}, Pat) -> + % "ets style" is to return true + % "dets style" is to return N | { error, Reason } + % or sometimes ok (?) + % be nice and accept both + case Mod:match_delete(Alias, Tab, Pat) of + N when is_integer (N) -> ok; + true -> ok; + ok -> ok + end. db_put({ram_copies, Tab}, Val) -> true = ?ets_insert(Tab, Val); db_put({disc_copies, Tab}, Val) -> true = ?ets_insert(Tab, Val); db_put({disc_only_copies, Tab}, Val) -> - ok = dets:insert(Tab, Val). + ok = dets:insert(Tab, Val); +db_put({{ext, Alias, Mod}, Tab}, Val) -> + ok = Mod:insert(Alias, Tab, Val). %% This code executes at the remote site where the data is %% executes in a special copier process. @@ -634,65 +723,88 @@ send_table(Pid, Tab, RemoteS) -> unknown -> {error, {no_exists, Tab}}; Storage -> - %% Send first - TabSize = mnesia:table_info(Tab, size), - KeysPerTransfer = calc_nokeys(Storage, Tab), - ChunkData = dets:info(Tab, bchunk_format), - - UseDetsChunk = - Storage == RemoteS andalso - Storage == disc_only_copies andalso - ChunkData /= undefined, - if - UseDetsChunk == true -> - DetsInfo = erlang:system_info(version), - Pid ! {self(), {first, TabSize, {DetsInfo, ChunkData}}}; - true -> - Pid ! {self(), {first, TabSize}} - end, + do_send_table(Pid, Tab, Storage, RemoteS) + end. - %% Debug info - put(mnesia_table_sender, {Tab, node(Pid), Pid}), - {Init, Chunk} = reader_funcs(UseDetsChunk, Tab, Storage, KeysPerTransfer), - - SendIt = fun() -> - prepare_copy(Pid, Tab, Storage), - send_more(Pid, 1, Chunk, Init(), Tab), - finish_copy(Pid, Tab, Storage, RemoteS) - end, - - case catch SendIt() of - receiver_died -> - cleanup_tab_copier(Pid, Storage, Tab), - unlink(whereis(mnesia_tm)), - ok; - {_, receiver_died} -> - unlink(whereis(mnesia_tm)), - ok; - {atomic, no_more} -> - unlink(whereis(mnesia_tm)), - ok; - Reason -> - cleanup_tab_copier(Pid, Storage, Tab), - unlink(whereis(mnesia_tm)), - {error, Reason} - end +do_send_table(Pid, Tab, Storage, RemoteS) -> + {Init, Chunk} = + case Storage of + {ext, Alias, Mod} -> + case Mod:sender_init(Alias, Tab, RemoteS, Pid) of + {standard, I, C} -> + Pid ! {self(), {first, Mod:info(Alias, Tab, size)}}, + {I, C}; + {_, _} = Res -> + Res + end; + Storage -> + %% Send first + TabSize = mnesia:table_info(Tab, size), + KeysPerTransfer = calc_nokeys(Storage, Tab), + ChunkData = dets:info(Tab, bchunk_format), + + UseDetsChunk = + Storage == RemoteS andalso + Storage == disc_only_copies andalso + ChunkData /= undefined, + if + UseDetsChunk == true -> + DetsInfo = erlang:system_info(version), + Pid ! {self(), {first, TabSize, {DetsInfo, ChunkData}}}; + true -> + Pid ! {self(), {first, TabSize}} + end, + {_I, _C} = + reader_funcs(UseDetsChunk, Tab, Storage, KeysPerTransfer) + end, + %% Debug info + put(mnesia_table_sender, {Tab, node(Pid), Pid}), + + SendIt = fun() -> + NeedLock = need_lock(Tab), + {atomic, ok} = prepare_copy(Pid, Tab, Storage, NeedLock), + send_more(Pid, 1, Chunk, Init(), Tab, Storage), + finish_copy(Pid, Tab, Storage, RemoteS, NeedLock) + end, + + try SendIt() of + {_, receiver_died} -> ok; + {atomic, no_more} -> ok + catch + throw:receiver_died -> + cleanup_tab_copier(Pid, Storage, Tab), + ok; + error:Reason -> %% Prepare failed + cleanup_tab_copier(Pid, Storage, Tab), + {error, {tab_copier, Tab, {Reason, erlang:get_stacktrace()}}} + after + unlink(whereis(mnesia_tm)) end. -prepare_copy(Pid, Tab, Storage) -> +prepare_copy(Pid, Tab, Storage, NeedLock) -> Trans = fun() -> - mnesia:lock_table(Tab, load), + NeedLock andalso mnesia:lock_table(Tab, load), mnesia_subscr:subscribe(Pid, {table, Tab}), update_where_to_write(Tab, node(Pid)), mnesia_lib:db_fixtable(Storage, Tab, true), ok end, - case mnesia:transaction(Trans) of - {atomic, ok} -> - ok; - {aborted, Reason} -> - exit({tab_copier_prepare, Tab, Reason}) + mnesia:transaction(Trans). + + +need_lock(Tab) -> + case ?catch_val({?MODULE, active_trans}) of + #tid{} = Tid -> + %% move_table_copy grabs it's own table-lock + %% do not deadlock with it + mnesia_lib:unset({?MODULE, active_trans}), + case mnesia_locker:get_held_locks(Tab) of + [{write, Tid}|_] -> false; + _Locks -> true + end; + _ -> + true end. update_where_to_write(Tab, Node) -> @@ -716,20 +828,32 @@ update_where_to_write([H|T], Tab, AddNode) -> [{update_where_to_write, [add, Tab, AddNode], self()}]), update_where_to_write(T, Tab, AddNode). -send_more(Pid, N, Chunk, DataState, Tab) -> +send_more(Pid, N, Chunk, DataState, Tab, Storage) -> receive {NewPid, more} -> case send_packet(N - 1, NewPid, Chunk, DataState) of New when is_integer(New) -> New - 1; NewData -> - send_more(NewPid, ?MAX_NOPACKETS, Chunk, NewData, Tab) + send_more(NewPid, ?MAX_NOPACKETS, Chunk, NewData, + Tab, Storage) + end; + {NewPid, {more, Msg}} when element(1, Storage) == ext -> + {ext, Alias, Mod} = Storage, + {NewChunk, NewState} = + Mod:sender_handle_info(Msg, Alias, Tab, NewPid, DataState), + case send_packet(N - 1, NewPid, NewChunk, NewState) of + New when is_integer(New) -> + New -1; + NewData -> + send_more(NewPid, N, NewChunk, NewData, Tab, + Storage) end; {_NewPid, {old_protocol, Tab}} -> Storage = val({Tab, storage_type}), {Init, NewChunk} = reader_funcs(false, Tab, Storage, calc_nokeys(Storage, Tab)), - send_more(Pid, 1, NewChunk, Init(), Tab); + send_more(Pid, 1, NewChunk, Init(), Tab, Storage); {copier_done, Node} when Node == node(Pid)-> verbose("Receiver of table ~p crashed on ~p (more)~n", [Tab, Node]), @@ -789,12 +913,12 @@ send_packet(N, Pid, Chunk, {Recs, Cont}) when N < ?MAX_NOPACKETS -> send_packet(_N, _Pid, _Chunk, DataState) -> DataState. -finish_copy(Pid, Tab, Storage, RemoteS) -> +finish_copy(Pid, Tab, Storage, RemoteS, NeedLock) -> RecNode = node(Pid), DatBin = dat2bin(Tab, Storage, RemoteS), Trans = fun() -> - mnesia:read_lock_table(Tab), + NeedLock andalso mnesia:read_lock_table(Tab), A = val({Tab, access_mode}), mnesia_controller:sync_and_block_table_whereabouts(Tab, RecNode, RemoteS, A), cleanup_tab_copier(Pid, Storage, Tab), @@ -826,6 +950,6 @@ dat2bin(_Tab, _LocalS, _RemoteS) -> nobin. handle_exit(Pid, Reason) when node(Pid) == node() -> - exit(Reason); + error(Reason); handle_exit(_Pid, _Reason) -> %% Not from our node, this will be handled by ignore. %% mnesia_down soon. diff --git a/lib/mnesia/src/mnesia_locker.erl b/lib/mnesia/src/mnesia_locker.erl index 14011003d3..59fd89059f 100644 --- a/lib/mnesia/src/mnesia_locker.erl +++ b/lib/mnesia/src/mnesia_locker.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2012. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -21,13 +22,13 @@ -module(mnesia_locker). -export([ - get_held_locks/0, + get_held_locks/0, get_held_locks/1, get_lock_queue/0, global_lock/5, ixrlock/5, init/1, - mnesia_down/2, release_tid/1, + mnesia_down/2, async_release_tid/2, send_release_tid/2, receive_release_tid_acc/2, @@ -84,7 +85,7 @@ init(Parent) -> register(?MODULE, self()), process_flag(trap_exit, true), ?ets_new_table(mnesia_held_locks, [ordered_set, private, named_table]), - ?ets_new_table(mnesia_tid_locks, [bag, private, named_table]), + ?ets_new_table(mnesia_tid_locks, [ordered_set, private, named_table]), ?ets_new_table(mnesia_sticky_locks, [set, private, named_table]), ?ets_new_table(mnesia_lock_queue, [bag, private, named_table, {keypos, 2}]), @@ -98,12 +99,13 @@ init(Parent) -> val(Var) -> case ?catch_val(Var) of - {'EXIT', _ReASoN_} -> mnesia_lib:other_val(Var, _ReASoN_); + {'EXIT', _} -> mnesia_lib:other_val(Var); _VaLuE_ -> _VaLuE_ end. reply(From, R) -> - From ! {?MODULE, node(), R}. + From ! {?MODULE, node(), R}, + true. %% Quiets dialyzer l_request(Node, X, Store) -> {?MODULE, Node} ! {self(), X}, @@ -130,13 +132,29 @@ send_release_tid(Nodes, Tid) -> receive_release_tid_acc([Node | Nodes], Tid) -> receive {?MODULE, Node, {tid_released, Tid}} -> - receive_release_tid_acc(Nodes, Tid); - {mnesia_down, Node} -> receive_release_tid_acc(Nodes, Tid) + after 0 -> + receive + {?MODULE, Node, {tid_released, Tid}} -> + receive_release_tid_acc(Nodes, Tid); + {mnesia_down, Node} -> + receive_release_tid_acc(Nodes, Tid) + end end; receive_release_tid_acc([], _Tid) -> ok. +mnesia_down(Node, Pending) -> + case whereis(?MODULE) of + undefined -> {error, node_not_running}; + Pid -> + Ref = make_ref(), + Pid ! {{self(), Ref}, {release_remote_non_pending, Node, Pending}}, + receive %% No need to wait for anything else if process dies we die soon + {Ref,ok} -> ok + end + end. + loop(State) -> receive {From, {write, Tid, Oid}} -> @@ -213,9 +231,14 @@ loop(State) -> reply(From, {tid_released, Tid}), loop(State); - {release_remote_non_pending, Node, Pending} -> + {{From, Ref},{release_remote_non_pending, Node, Pending}} -> release_remote_non_pending(Node, Pending), - mnesia_monitor:mnesia_down(?MODULE, Node), + From ! {Ref, ok}, + loop(State); + + {From, {is_locked, Oid}} -> + Held = ?ets_lookup(mnesia_held_locks, Oid), + reply(From, Held), loop(State); {'EXIT', Pid, _} when Pid == State#state.supervisor -> @@ -236,13 +259,13 @@ loop(State) -> end. set_lock(Tid, Oid, Op, []) -> - ?ets_insert(mnesia_tid_locks, {Tid, Oid, Op}), + ?ets_insert(mnesia_tid_locks, {{Tid, Oid, Op}}), ?ets_insert(mnesia_held_locks, {Oid, Op, [{Op, Tid}]}); set_lock(Tid, Oid, read, [{Oid, Prev, Items}]) -> - ?ets_insert(mnesia_tid_locks, {Tid, Oid, read}), + ?ets_insert(mnesia_tid_locks, {{Tid, Oid, read}}), ?ets_insert(mnesia_held_locks, {Oid, Prev, [{read, Tid}|Items]}); set_lock(Tid, Oid, write, [{Oid, _Prev, Items}]) -> - ?ets_insert(mnesia_tid_locks, {Tid, Oid, write}), + ?ets_insert(mnesia_tid_locks, {{Tid, Oid, write}}), ?ets_insert(mnesia_held_locks, {Oid, write, [{write, Tid}|Items]}); set_lock(Tid, Oid, Op, undefined) -> set_lock(Tid, Oid, Op, ?ets_lookup(mnesia_held_locks, Oid)). @@ -258,7 +281,8 @@ try_sticky_lock(Tid, Op, Pid, {Tab, _} = Oid) -> try_lock(Tid, Op, Pid, Oid); [{_,N}] -> Req = {Pid, {Op, Tid, Oid}}, - Pid ! {?MODULE, node(), {switch, N, Req}} + Pid ! {?MODULE, node(), {switch, N, Req}}, + true end. try_lock(Tid, read_write, Pid, Oid) -> @@ -281,7 +305,7 @@ try_lock(Tid, Op, SimpleOp, Lock, Pid, Oid) -> ?ets_insert(mnesia_lock_queue, #queue{oid = Oid, tid = Tid, op = Op, pid = Pid, lucky = Lucky}), - ?ets_insert(mnesia_tid_locks, {Tid, Oid, {queued, Op}}) + ?ets_insert(mnesia_tid_locks, {{Tid, Oid, {queued, Op}}}) end. grant_lock(Tid, read, Lock, Oid = {Tab, Key}, Default) @@ -480,7 +504,7 @@ set_read_lock_on_all_keys(Tid, From, Tab, IxKey, Pos) -> ?ets_insert(mnesia_lock_queue, #queue{oid = Oid, tid = Tid, op = Op, pid = From, lucky = Lucky}), - ?ets_insert(mnesia_tid_locks, {Tid, Oid, {queued, Op}}) + ?ets_insert(mnesia_tid_locks, {{Tid, Oid, {queued, Op}}}) end. %%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -496,7 +520,8 @@ release_remote_non_pending(Node, Pending) -> %% running at the failed node and also simply remove all %% queue'd requests back to the failed node - AllTids = ?ets_match(mnesia_tid_locks, {'$1', '_', '_'}), + AllTids0 = ?ets_match(mnesia_tid_locks, {{'$1', '_', '_'}}), + AllTids = lists:usort(AllTids0), Tids = [T || [T] <- AllTids, Node == node(T#tid.pid), not lists:member(T, Pending)], do_release_tids(Tids). @@ -507,9 +532,10 @@ do_release_tids([]) -> ok. do_release_tid(Tid) -> - Locks = ?ets_lookup(mnesia_tid_locks, Tid), + Objects = ets:select(mnesia_tid_locks, [{{{Tid, '_', '_'}}, [], ['$_']}]), + Locks = lists:map(fun({L}) -> L end, Objects), ?dbg("Release ~p ~p ~n", [Tid, Locks]), - ?ets_delete(mnesia_tid_locks, Tid), + [?ets_delete(mnesia_tid_locks, L) || L <- Locks], release_locks(Locks), %% Removed queued locks which has had locks UniqueLocks = keyunique(lists:sort(Locks),[]), @@ -653,19 +679,6 @@ ix_read_res(Tab,IxKey,Pos) -> %% ********************* end server code ******************** %% The following code executes at the client side of a transactions -mnesia_down(N, Pending) -> - case whereis(?MODULE) of - undefined -> - %% Takes care of mnesia_down's in early startup - mnesia_monitor:mnesia_down(?MODULE, N); - Pid -> - %% Syncronously call needed in order to avoid - %% race with mnesia_tm's coordinator processes - %% that may restart and acquire new locks. - %% mnesia_monitor ensures the sync. - Pid ! {release_remote_non_pending, N, Pending} - end. - %% Aquire a write lock, but do a read, used by %% mnesia:wread/1 @@ -975,8 +988,14 @@ sticky_flush(Ns=[Node | Tail], Store) -> flush_remaining([], _SkipNode, Res) -> del_debug(), exit(Res); -flush_remaining([SkipNode | Tail ], SkipNode, Res) -> - flush_remaining(Tail, SkipNode, Res); +flush_remaining(Ns=[SkipNode | Tail ], SkipNode, Res) -> + add_debug(Ns), + receive + {?MODULE, SkipNode, _} -> + flush_remaining(Tail, SkipNode, Res) + after 0 -> + flush_remaining(Tail, SkipNode, Res) + end; flush_remaining(Ns=[Node | Tail], SkipNode, Res) -> add_debug(Ns), receive @@ -988,13 +1007,11 @@ flush_remaining(Ns=[Node | Tail], SkipNode, Res) -> opt_lookup_in_client(lookup_in_client, Oid, Lock) -> {Tab, Key} = Oid, - case catch mnesia_lib:db_get(Tab, Key) of - {'EXIT', _} -> + try mnesia_lib:db_get(Tab, Key) + catch error:_ -> %% Table has been deleted from this node, %% restart the transaction. - #cyclic{op = read, lock = Lock, oid = Oid, lucky = nowhere}; - Val -> - Val + #cyclic{op = read, lock = Lock, oid = Oid, lucky = nowhere} end; opt_lookup_in_client(Val, _Oid, _Lock) -> Val. @@ -1126,11 +1143,10 @@ send_requests([], _X) -> rec_requests([Node | Nodes], Oid, Store) -> Res = l_req_rec(Node, Store), - case catch rlock_get_reply(Node, Store, Oid, Res) of - {'EXIT', Reason} -> - flush_remaining(Nodes, Node, Reason); - _ -> - rec_requests(Nodes, Oid, Store) + try rlock_get_reply(Node, Store, Oid, Res) of + _ -> rec_requests(Nodes, Oid, Store) + catch _:Reason -> + flush_remaining(Nodes, Node, Reason) end; rec_requests([], _Oid, _Store) -> ok. @@ -1140,6 +1156,19 @@ get_held_locks() -> Locks = receive {mnesia_held_locks, Ls} -> Ls after 5000 -> [] end, rewrite_locks(Locks, []). +%% Mnesia internal usage only +get_held_locks(Tab) when is_atom(Tab) -> + Oid = {Tab, ?ALL}, + ?MODULE ! {self(), {is_locked, Oid}}, + receive + {?MODULE, _Node, Locks} -> + case Locks of + [] -> []; + [{Oid, _Prev, What}] -> What + end + end. + + rewrite_locks([{Oid, _, Ls}|Locks], Acc0) -> Acc = rewrite_locks(Ls, Oid, Acc0), rewrite_locks(Locks, Acc); diff --git a/lib/mnesia/src/mnesia_log.erl b/lib/mnesia/src/mnesia_log.erl index 18303869ed..9536effd42 100644 --- a/lib/mnesia/src/mnesia_log.erl +++ b/lib/mnesia/src/mnesia_log.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -200,7 +201,7 @@ log_header(Kind, Version) -> log_kind=Kind, mnesia_version=mnesia:system_info(version), node=node(), - now=now()}. + now=erlang:timestamp()}. version() -> "4.3". @@ -223,17 +224,12 @@ sappend(Log, Term) -> ok = disk_log:log(Log, Term). %% Write commit records to the latest_log -log(C) when C#commit.disc_copies == [], - C#commit.disc_only_copies == [], - C#commit.schema_ops == [] -> - ignore; log(C) -> - case mnesia_monitor:use_dir() of + case need_log(C) andalso mnesia_monitor:use_dir() of true -> if is_record(C, commit) -> - C2 = C#commit{ram_copies = [], snmp = []}, - append(latest_log, C2); + append(latest_log, strip_snmp(C)); true -> %% Either a commit record as binary %% or some decision related info @@ -246,17 +242,12 @@ log(C) -> %% Synced -slog(C) when C#commit.disc_copies == [], - C#commit.disc_only_copies == [], - C#commit.schema_ops == [] -> - ignore; slog(C) -> - case mnesia_monitor:use_dir() of + case need_log(C) andalso mnesia_monitor:use_dir() of true -> if is_record(C, commit) -> - C2 = C#commit{ram_copies = [], snmp = []}, - sappend(latest_log, C2); + sappend(latest_log, strip_snmp(C)); true -> %% Either a commit record as binary %% or some decision related info @@ -267,6 +258,13 @@ slog(C) -> ignore end. +need_log(#commit{disc_copies=[], disc_only_copies=[], schema_ops=[], ext=Ext}) -> + lists:keymember(ext_copies, 1, Ext); +need_log(_) -> true. + +strip_snmp(#commit{ext=[]}=CR) -> CR; +strip_snmp(#commit{ext=Ext}=CR) -> + CR#commit{ext=lists:keydelete(snmp, 1, Ext)}. %% Stuff related to the file LOG @@ -349,6 +347,8 @@ open_log(Name, Header, Fname, Exists, Repair, Mode) -> mnesia_lib:important("Data may be missing, log ~p repaired: Lost ~p bytes~n", [Fname, BadBytes]), Log; + {error, Reason = {file_error, _Fname, emfile}} -> + fatal("Cannot open log file ~p: ~p~n", [Fname, Reason]); {error, Reason} when Repair == true -> file:delete(Fname), mnesia_lib:important("Data may be missing, Corrupt logfile deleted: ~p, ~p ~n", @@ -393,13 +393,15 @@ unsafe_close_log(Log) -> purge_some_logs() -> mnesia_monitor:unsafe_close_log(latest_log), - file:delete(latest_log_file()), - file:delete(decision_tab_file()). + _ = file:delete(latest_log_file()), + _ = file:delete(decision_tab_file()), + ok. purge_all_logs() -> - file:delete(previous_log_file()), - file:delete(latest_log_file()), - file:delete(decision_tab_file()). + _ = file:delete(previous_log_file()), + _ = file:delete(latest_log_file()), + _ = file:delete(decision_tab_file()), + ok. %% Prepare dump by renaming the open logfile if possible %% Returns a tuple on the following format: {Res, OpenLog} @@ -460,7 +462,7 @@ chunk_log(Cont) -> chunk_log(_Log, eof) -> eof; chunk_log(Log, Cont) -> - case catch disk_log:chunk(Log, Cont) of + case disk_log:chunk(Log, Cont) of {error, Reason} -> fatal("Possibly truncated ~p file: ~p~n", [Log, Reason]); @@ -645,11 +647,11 @@ backup_checkpoint(Name, Opaque, Args) when is_list(Args) -> end. check_backup_args([Arg | Tail], B) -> - case catch check_backup_arg_type(Arg, B) of - {'EXIT', _Reason} -> - {error, {badarg, Arg}}; + try check_backup_arg_type(Arg, B) of B2 -> check_backup_args(Tail, B2) + catch error:_ -> + {error, {badarg, Arg}} end; check_backup_args([], B) -> @@ -672,11 +674,11 @@ check_backup_arg_type(Arg, B) -> backup_master(ClientPid, B) -> process_flag(trap_exit, true), - case catch do_backup_master(B) of - {'EXIT', Reason} -> - ClientPid ! {self(), ClientPid, {error, {'EXIT', Reason}}}; + try do_backup_master(B) of Res -> ClientPid ! {self(), ClientPid, Res} + catch _:Reason -> + ClientPid ! {self(), ClientPid, {error, {'EXIT', Reason}}} end, unlink(ClientPid), exit(normal). @@ -729,27 +731,30 @@ backup_schema(B, Tabs) -> safe_apply(B, write, [_, Items]) when Items == [] -> B; safe_apply(B, What, Args) -> - Abort = fun(R) -> abort_write(B, What, Args, R) end, + Abort = abort_write_fun(B, What, Args), receive {'EXIT', Pid, R} -> Abort({'EXIT', Pid, R}) after 0 -> Mod = B#backup_args.module, - case catch apply(Mod, What, Args) of + try apply(Mod, What, Args) of {ok, Opaque} -> B#backup_args{opaque=Opaque}; - {error, R} -> Abort(R); - R -> Abort(R) + {error, R} -> Abort(R) + catch _:R -> Abort(R) end end. +-spec abort_write_fun(_, _, _) -> fun((_) -> no_return()). +abort_write_fun(B, What, Args) -> + fun(R) -> abort_write(B, What, Args, R) end. + abort_write(B, What, Args, Reason) -> Mod = B#backup_args.module, Opaque = B#backup_args.opaque, dbg_out("Failed to perform backup. M=~p:F=~p:A=~p -> ~p~n", [Mod, What, Args, Reason]), - case catch apply(Mod, abort_write, [Opaque]) of - {ok, _Res} -> - throw({error, Reason}); - Other -> + try apply(Mod, abort_write, [Opaque]) of + {ok, _Res} -> throw({error, Reason}) + catch _:Other -> error("Failed to abort backup. ~p:~p~p -> ~p~n", [Mod, abort_write, [Opaque], Other]), throw({error, Reason}) @@ -890,10 +895,8 @@ tab_receiver(Pid, B, Tab, RecName, Slot) -> end. rec_filter(B, schema, _RecName, Recs) -> - case catch mnesia_bup:refresh_cookie(Recs, B#backup_args.cookie) of - Recs2 when is_list(Recs2) -> - Recs2; - {error, _Reason} -> + try mnesia_bup:refresh_cookie(Recs, B#backup_args.cookie) + catch throw:{error, _Reason} -> %% No schema table cookie Recs end; @@ -1004,13 +1007,14 @@ add_recs([{{Tab, _Key}, Val, delete_object} | Rest], N) -> add_recs(Rest, N+1); add_recs([{{Tab, Key}, Val, update_counter} | Rest], N) -> {RecName, Incr} = Val, - case catch ets:update_counter(Tab, Key, Incr) of - CounterVal when is_integer(CounterVal) -> - ok; - _ when Incr < 0 -> + try + CounterVal = ets:update_counter(Tab, Key, Incr), + true = (CounterVal >= 0) + catch + error:_ when Incr < 0 -> Zero = {RecName, Key, 0}, true = ets:insert(Tab, Zero); - _ -> + error:_ -> Zero = {RecName, Key, Incr}, true = ets:insert(Tab, Zero) end, diff --git a/lib/mnesia/src/mnesia_monitor.erl b/lib/mnesia/src/mnesia_monitor.erl index 7a788238fc..ab78c9b13e 100644 --- a/lib/mnesia/src/mnesia_monitor.erl +++ b/lib/mnesia/src/mnesia_monitor.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -31,6 +32,7 @@ init/0, mktab/2, unsafe_mktab/2, + unsafe_create_external/4, mnesia_down/2, needs_protocol_conversion/1, negotiate_protocol/1, @@ -44,6 +46,7 @@ set_env/2, start/0, start_proc/4, + sync_log/1, terminate_proc/3, unsafe_close_dets/1, unsafe_close_log/1, @@ -78,11 +81,11 @@ -record(state, {supervisor, pending_negotiators = [], going_down = [], tm_started = false, early_connects = [], - connecting, mq = []}). + connecting, mq = [], remote_node_status = []}). --define(current_protocol_version, {8,1}). +-define(current_protocol_version, {8,3}). --define(previous_protocol_version, {8,0}). +-define(previous_protocol_version, {8,2}). start() -> gen_server:start_link({local, ?MODULE}, ?MODULE, @@ -118,12 +121,17 @@ open_log(Args) -> reopen_log(Name, Fname, Head) -> unsafe_call({reopen_log, Name, Fname, Head}). +sync_log(Name) -> + unsafe_call({sync_log, Name}). + close_log(Name) -> unsafe_call({close_log, Name}). unsafe_close_log(Name) -> unsafe_call({unsafe_close_log, Name}). +unsafe_create_external(Tab, Alias, Mod, Cs) -> + unsafe_call({unsafe_create_external, Tab, Alias, Mod, Cs}). disconnect(Node) -> cast({disconnect, Node}). @@ -188,7 +196,7 @@ protocol_version() -> %% A sorted list of acceptable protocols the %% preferred protocols are first in the list acceptable_protocol_versions() -> - [protocol_version(), ?previous_protocol_version, {7,6}]. + [protocol_version(), ?previous_protocol_version, {8,1}]. needs_protocol_conversion(Node) -> case {?catch_val({protocol, Node}), protocol_version()} of @@ -202,7 +210,7 @@ needs_protocol_conversion(Node) -> cast(Msg) -> case whereis(?MODULE) of - undefined -> ignore; + undefined -> ok; Pid -> gen_server:cast(Pid, Msg) end. @@ -264,7 +272,7 @@ init([Parent]) -> set(version, Version), dbg_out("Version: ~p~n", [Version]), - case catch process_config_args(env()) of + try process_config_args(env()) of ok -> mnesia_lib:set({'$$$_report', current_pos}, 0), Level = mnesia_lib:val(debug), @@ -284,8 +292,8 @@ init([Parent]) -> set(pending_checkpoints, []), set(pending_checkpoint_pids, []), - {ok, #state{supervisor = Parent}}; - {'EXIT', Reason} -> + {ok, #state{supervisor = Parent}} + catch _:Reason -> mnesia_lib:report_fatal("Bad configuration: ~p~n", [Reason]), {stop, {bad_config, Reason}} end. @@ -319,25 +327,24 @@ non_empty_dir() -> %%---------------------------------------------------------------------- handle_call({mktab, Tab, Args}, _From, State) -> - case catch ?ets_new_table(Tab, Args) of - {'EXIT', ExitReason} -> + try ?ets_new_table(Tab, Args) of + Reply -> + {reply, Reply, State} + catch error:ExitReason -> Msg = "Cannot create ets table", Reason = {system_limit, Msg, Tab, Args, ExitReason}, fatal("~p~n", [Reason]), - {noreply, State}; - Reply -> - {reply, Reply, State} + {noreply, State} end; handle_call({unsafe_mktab, Tab, Args}, _From, State) -> - case catch ?ets_new_table(Tab, Args) of - {'EXIT', ExitReason} -> - {reply, {error, ExitReason}, State}; + try ?ets_new_table(Tab, Args) of Reply -> {reply, Reply, State} + catch error:ExitReason -> + {reply, {error, ExitReason}, State} end; - handle_call({open_dets, Tab, Args}, _From, State) -> case mnesia_lib:dets_sync_open(Tab, Args) of {ok, Tab} -> @@ -382,6 +389,9 @@ handle_call({reopen_log, Name, Fname, Head}, _From, State) -> {noreply, State} end; +handle_call({sync_log, Name}, _From, State) -> + {reply, disk_log:sync(Name), State}; + handle_call({close_log, Name}, _From, State) -> case disk_log:close(Name) of ok -> @@ -395,9 +405,17 @@ handle_call({close_log, Name}, _From, State) -> end; handle_call({unsafe_close_log, Name}, _From, State) -> - disk_log:close(Name), + _ = disk_log:close(Name), {reply, ok, State}; +handle_call({unsafe_create_external, Tab, Alias, Mod, Cs}, _From, State) -> + case catch Mod:create_table(Alias, Tab, mnesia_schema:cs2list(Cs)) of + {'EXIT', ExitReason} -> + {reply, {error, ExitReason}, State}; + Reply -> + {reply, Reply, State} + end; + handle_call({negotiate_protocol, Mon, _Version, _Protocols}, _From, State) when State#state.tm_started == false -> State2 = State#state{early_connects = [node(Mon) | State#state.early_connects]}, @@ -417,8 +435,6 @@ handle_call({negotiate_protocol, Mon, Version, Protocols}, From, State) case hd(Protocols) of ?previous_protocol_version -> accept_protocol(Mon, MyVersion, ?previous_protocol_version, From, State); - {7,6} -> - accept_protocol(Mon, MyVersion, {7,6}, From, State); _ -> verbose("Connection with ~p rejected. " "version = ~p, protocols = ~p, " @@ -439,7 +455,7 @@ handle_call({negotiate_protocol, Nodes}, From, State) -> end; handle_call(init, _From, State) -> - net_kernel:monitor_nodes(true), + _ = net_kernel:monitor_nodes(true), EarlyNodes = State#state.early_connects, State2 = State#state{tm_started = true}, {reply, EarlyNodes, State2}; @@ -482,27 +498,24 @@ handle_cast({mnesia_down, mnesia_controller, Node}, State) -> mnesia_tm:mnesia_down(Node), {noreply, State}; -handle_cast({mnesia_down, mnesia_tm, {Node, Pending}}, State) -> - mnesia_locker:mnesia_down(Node, Pending), - {noreply, State}; - -handle_cast({mnesia_down, mnesia_locker, Node}, State) -> +handle_cast({mnesia_down, mnesia_tm, Node}, State) -> Down = {mnesia_down, Node}, mnesia_lib:report_system_event(Down), GoingDown = lists:delete(Node, State#state.going_down), State2 = State#state{going_down = GoingDown}, Pending = State#state.pending_negotiators, + State3 = check_raise_conditon_nodeup(Node, State2), case lists:keysearch(Node, 1, Pending) of {value, {Node, Mon, ReplyTo, Reply}} -> %% Late reply to remote monitor link(Mon), %% link to remote Monitor gen_server:reply(ReplyTo, Reply), P2 = lists:keydelete(Node, 1,Pending), - State3 = State2#state{pending_negotiators = P2}, - process_q(State3); + State4 = State3#state{pending_negotiators = P2}, + process_q(State4); false -> %% No pending remote monitors - process_q(State2) + process_q(State3) end; handle_cast({disconnect, Node}, State) -> @@ -542,7 +555,7 @@ handle_info({'EXIT', Pid, fatal}, State) when node(Pid) == node() -> %% is in progress %% exit(State#state.supervisor, shutdown), %% It is better to kill an innocent process - catch exit(whereis(mnesia_locker), kill), + ?SAFE(exit(whereis(mnesia_locker), kill)), {noreply, State}; handle_info(Msg = {'EXIT',Pid,_}, State) -> @@ -568,27 +581,18 @@ handle_info({protocol_negotiated, From,Res}, State) -> gen_server:reply(From, Res), process_q(State#state{connecting = undefined}); -handle_info({nodeup, Node}, State) -> - %% Ok, we are connected to yet another Erlang node - %% Let's check if Mnesia is running there in order - %% to detect if the network has been partitioned - %% due to communication failure. +handle_info({check_nodeup, Node}, State) -> + State2 = check_mnesia_down(Node, State), + {noreply, State2}; - HasDown = mnesia_recover:has_mnesia_down(Node), - ImRunning = mnesia_lib:is_running(), - - if - %% If I'm not running the test will be made later. - HasDown == true, ImRunning == yes -> - spawn_link(?MODULE, detect_partitioned_network, [self(), Node]); - true -> - ignore - end, - {noreply, State}; +handle_info({nodeup, Node}, State) -> + State2 = remote_node_status(Node, up, State), + State3 = check_mnesia_down(Node, State2), + {noreply, State3}; -handle_info({nodedown, _Node}, State) -> - %% Ignore, we are only caring about nodeup's - {noreply, State}; +handle_info({nodedown, Node}, State) -> + State2 = remote_node_status(Node, down, State), + {noreply, State2}; handle_info({disk_log, _Node, Log, Info}, State) -> case Info of @@ -665,10 +669,12 @@ get_env(E) -> env() -> [ access_module, + allow_index_on_key, auto_repair, backup_module, debug, dir, + dump_disc_copies_at_startup, dump_log_load_regulation, dump_log_time_threshold, dump_log_update_in_place, @@ -677,19 +683,23 @@ env() -> extra_db_nodes, ignore_fallback_at_startup, fallback_error_function, + fold_chunk_size, max_wait_for_decision, schema_location, core_dir, pid_sort_order, no_table_loaders, dc_dump_limit, - send_compressed + send_compressed, + schema ]. default_env(access_module) -> mnesia; default_env(auto_repair) -> true; +default_env(allow_index_on_key) -> + false; default_env(backup_module) -> mnesia_backup; default_env(debug) -> @@ -697,6 +707,8 @@ default_env(debug) -> default_env(dir) -> Name = lists:concat(["Mnesia.", node()]), filename:absname(Name); +default_env(dump_disc_copies_at_startup) -> + true; default_env(dump_log_load_regulation) -> false; default_env(dump_log_time_threshold) -> @@ -713,6 +725,8 @@ default_env(ignore_fallback_at_startup) -> false; default_env(fallback_error_function) -> {mnesia, lkill}; +default_env(fold_chunk_size) -> + 100; default_env(max_wait_for_decision) -> infinity; default_env(schema_location) -> @@ -726,17 +740,17 @@ default_env(no_table_loaders) -> default_env(dc_dump_limit) -> 4; default_env(send_compressed) -> - 0. + 0; +default_env(schema) -> + []. check_type(Env, Val) -> - case catch do_check_type(Env, Val) of - {'EXIT', _Reason} -> - exit({bad_config, Env, Val}); - NewVal -> - NewVal + try do_check_type(Env, Val) + catch error:_ -> exit({bad_config, Env, Val}) end. do_check_type(access_module, A) when is_atom(A) -> A; +do_check_type(allow_index_on_key, B) -> bool(B); do_check_type(auto_repair, B) -> bool(B); do_check_type(backup_module, B) when is_atom(B) -> B; do_check_type(debug, debug) -> debug; @@ -746,6 +760,7 @@ do_check_type(debug, trace) -> trace; do_check_type(debug, true) -> debug; do_check_type(debug, verbose) -> verbose; do_check_type(dir, V) -> filename:absname(V); +do_check_type(dump_disc_copies_at_startup, B) -> bool(B); do_check_type(dump_log_load_regulation, B) -> bool(B); do_check_type(dump_log_time_threshold, I) when is_integer(I), I > 0 -> I; do_check_type(dump_log_update_in_place, B) -> bool(B); @@ -759,6 +774,8 @@ do_check_type(extra_db_nodes, L) when is_list(L) -> (A) when is_atom(A) -> true end, lists:filter(Fun, L); +do_check_type(fold_chunk_size, I) when is_integer(I), I > 0; + I =:= infinity -> I; do_check_type(max_wait_for_decision, infinity) -> infinity; do_check_type(max_wait_for_decision, I) when is_integer(I), I > 0 -> I; do_check_type(schema_location, M) -> media(M); @@ -772,7 +789,8 @@ do_check_type(pid_sort_order, "standard") -> standard; do_check_type(pid_sort_order, _) -> false; do_check_type(no_table_loaders, N) when is_integer(N), N > 0 -> N; do_check_type(dc_dump_limit,N) when is_number(N), N > 0 -> N; -do_check_type(send_compressed, L) when is_integer(L), L >= 0, L =< 9 -> L. +do_check_type(send_compressed, L) when is_integer(L), L >= 0, L =< 9 -> L; +do_check_type(schema, L) when is_list(L) -> L. bool(true) -> true; bool(false) -> false. @@ -782,12 +800,12 @@ media(opt_disc) -> opt_disc; media(ram) -> ram. patch_env(Env, Val) -> - case catch do_check_type(Env, Val) of - {'EXIT', _Reason} -> - {error, {bad_type, Env, Val}}; + try do_check_type(Env, Val) of NewVal -> application_controller:set_env(mnesia, Env, NewVal), NewVal + catch error:_ -> + {error, {bad_type, Env, Val}} end. detect_partitioned_network(Mon, Node) -> @@ -830,3 +848,48 @@ report_inconsistency([{badrpc, _Reason} | Replies], Context, Status) -> report_inconsistency(Replies, Context, Status); report_inconsistency([], _Context, Status) -> Status. + +remote_node_status(Node, Status, State) -> + {ok, Nodes} = mnesia_schema:read_nodes(), + case lists:member(Node, Nodes) of + true -> + update_node_status({Node, Status}, State); + _ -> + State + end. + +update_node_status({Node, down}, State = #state{remote_node_status = RNodeS}) -> + RNodeS2 = lists:ukeymerge(1, [{Node, down}], RNodeS), + State#state{remote_node_status = RNodeS2}; +update_node_status({Node, up}, State = #state{remote_node_status = RNodeS}) -> + case lists:keyfind(Node, 1, RNodeS) of + {Node, down} -> + RNodeS2 = lists:ukeymerge(1, [{Node, up}], RNodeS), + State#state{remote_node_status = RNodeS2}; + _ -> + State + end. + +check_raise_conditon_nodeup(Node, State = #state{remote_node_status = RNodeS}) -> + case lists:keyfind(Node, 1, RNodeS) of + {Node, up} -> + self() ! {check_nodeup, Node}; + _ -> + ignore + end, + State#state{remote_node_status = lists:keydelete(Node, 1, RNodeS)}. + +check_mnesia_down(Node, State = #state{remote_node_status = RNodeS}) -> + %% Check if the network has been partitioned + %% due to communication failure. + + HasDown = mnesia_recover:has_mnesia_down(Node), + ImRunning = mnesia_lib:is_running(), + if + %% If I'm not running the test will be made later. + HasDown == true, ImRunning == yes -> + spawn_link(?MODULE, detect_partitioned_network, [self(), Node]), + State#state{remote_node_status = lists:keydelete(Node, 1, RNodeS)}; + true -> + State + end. diff --git a/lib/mnesia/src/mnesia_recover.erl b/lib/mnesia/src/mnesia_recover.erl index 7aa03bda37..b204fb282f 100644 --- a/lib/mnesia/src/mnesia_recover.erl +++ b/lib/mnesia/src/mnesia_recover.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -178,7 +179,8 @@ log_decision(D) -> val(Var) -> case ?catch_val(Var) of - {'EXIT', Reason} -> mnesia_lib:other_val(Var, Reason); + {'EXIT', _Reason} -> + mnesia_lib:other_val(Var); Value -> Value end. @@ -369,11 +371,8 @@ log_master_nodes2([], _UseDir, IsRunning, WorstRes) -> get_master_node_info() -> Tab = mnesia_decision, Pat = {master_nodes, '_', '_'}, - case catch mnesia_lib:db_match_object(ram_copies,Tab, Pat) of - {'EXIT', _} -> - []; - Masters -> - Masters + try mnesia_lib:db_match_object(ram_copies,Tab, Pat) + catch error:_ -> [] end. get_master_node_tables() -> @@ -381,9 +380,8 @@ get_master_node_tables() -> [Tab || {master_nodes, Tab, _Nodes} <- Masters]. get_master_nodes(Tab) -> - case catch ?ets_lookup_element(mnesia_decision, Tab, 3) of - {'EXIT', _} -> []; - Nodes -> Nodes + try ?ets_lookup_element(mnesia_decision, Tab, 3) + catch error:_ -> [] end. %% Determine what has happened to the transaction @@ -481,8 +479,6 @@ load_decision_tab() -> load_decision_tab(Cont, load_decision_tab), mnesia_log:close_decision_tab(). -load_decision_tab(eof, _InitBy) -> - ok; load_decision_tab(Cont, InitBy) -> case mnesia_log:chunk_decision_tab(Cont) of {Cont2, Decisions} -> @@ -515,8 +511,6 @@ dump_decision_log(InitBy) -> Cont = mnesia_log:prepare_decision_log_dump(), perform_dump_decision_log(Cont, InitBy). -perform_dump_decision_log(eof, _InitBy) -> - confirm_decision_log_dump(); perform_dump_decision_log(Cont, InitBy) when InitBy == startup -> case mnesia_log:chunk_decision_log(Cont) of {Cont2, Decisions} -> @@ -685,12 +679,29 @@ handle_call({connect_nodes, Ns}, From, State) -> %% called from handle_info gen_server:reply(From, {[], AlreadyConnected}), {noreply, State}; - GoodNodes -> + ProbablyGoodNodes -> %% Now we have agreed upon a protocol with some new nodes - %% and we may use them when we recover transactions + %% and we may use them when we recover transactions. + %% + %% Just in case Mnesia was stopped on some of those nodes + %% between the protocol negotiation and now, we check one + %% more time the state of Mnesia. + %% + %% Of course, there is still a chance that mnesia_down + %% events occur during this check and we miss them. To + %% prevent it, handle_cast({mnesia_down, ...}, ...) removes + %% the down node again, in addition to mnesia_down/1. + %% + %% See a comment in handle_cast({mnesia_down, ...}, ...). + Verify = fun(N) -> + Run = mnesia_lib:is_running(N), + Run =:= yes orelse Run =:= starting + end, + GoodNodes = [N || N <- ProbablyGoodNodes, Verify(N)], + mnesia_lib:add_list(recover_nodes, GoodNodes), cast({announce_all, GoodNodes}), - case get_master_nodes(schema) of + case get_master_nodes(schema) of [] -> Context = starting_partitioned_network, mnesia_monitor:detect_inconcistency(GoodNodes, Context); @@ -838,6 +849,14 @@ handle_cast({what_decision, Node, OtherD}, State) -> {noreply, State}; handle_cast({mnesia_down, Node}, State) -> + %% The node was already removed from recover_nodes in mnesia_down/1, + %% but we do it again here in the mnesia_recover process, in case + %% another event incorrectly added it back. This can happen during + %% Mnesia startup which takes time betweenthe connection, the + %% protocol negotiation and the merge of the schema. + %% + %% See a comment in handle_call({connect_nodes, ...), ...). + mnesia_lib:del(recover_nodes, Node), case State#state.unclear_decision of undefined -> {noreply, State}; @@ -995,7 +1014,7 @@ decision(Tid) -> decision(Tid, tabs()). decision(Tid, [Tab | Tabs]) -> - case catch ?ets_lookup(Tab, Tid) of + try ?ets_lookup(Tab, Tid) of [D] when is_record(D, decision) -> D; [C] when is_record(C, transient_decision) -> @@ -1005,8 +1024,8 @@ decision(Tid, [Tab | Tabs]) -> ram_nodes = [] }; [] -> - decision(Tid, Tabs); - {'EXIT', _} -> + decision(Tid, Tabs) + catch error:_ -> %% Recently switched transient decision table decision(Tid, Tabs) end; @@ -1017,11 +1036,8 @@ outcome(Tid, Default) -> outcome(Tid, Default, tabs()). outcome(Tid, Default, [Tab | Tabs]) -> - case catch ?ets_lookup_element(Tab, Tid, 3) of - {'EXIT', _} -> - outcome(Tid, Default, Tabs); - Val -> - Val + try ?ets_lookup_element(Tab, Tid, 3) + catch error:_ -> outcome(Tid, Default, Tabs) end; outcome(_Tid, Default, []) -> Default. diff --git a/lib/mnesia/src/mnesia_registry.erl b/lib/mnesia/src/mnesia_registry.erl index 202689ae5e..55ddc3d2fd 100644 --- a/lib/mnesia/src/mnesia_registry.erl +++ b/lib/mnesia/src/mnesia_registry.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% Copyright Ericsson AB 1998-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/mnesia/src/mnesia_schema.erl b/lib/mnesia/src/mnesia_schema.erl index 6e43052fb0..0e4017e4c3 100644 --- a/lib/mnesia/src/mnesia_schema.erl +++ b/lib/mnesia/src/mnesia_schema.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -28,6 +29,16 @@ -module(mnesia_schema). -export([ + add_backend_type/2, + do_add_backend_type/2, + delete_backend_type/1, + do_delete_backend_type/1, + backend_types/0, + add_index_plugin/3, + do_add_index_plugin/3, + delete_index_plugin/1, + do_delete_index_plugin/1, + index_plugins/0, add_snmp/2, add_table_copy/3, add_table_index/2, @@ -54,14 +65,16 @@ dump_tables/1, ensure_no_schema/1, get_create_list/1, - get_initial_schema/2, + get_initial_schema/3, get_table_properties/1, info/0, info/1, init/1, + init_backends/0, insert_cstruct/3, is_remote_member/1, list2cs/1, + list2cs/2, lock_schema/0, merge_schema/0, merge_schema/1, @@ -109,7 +122,8 @@ do_delete_table/1, do_read_table_property/2, do_delete_table_property/2, - do_write_table_property/2]). + do_write_table_property/2, + do_change_table_copy_type/3]). -include("mnesia.hrl"). -include_lib("kernel/include/file.hrl"). @@ -137,12 +151,30 @@ init(IgnoreFallback) -> verbose("Schema initiated from: ~p~n", [Source]), set({schema, tables}, []), set({schema, local_tables}, []), + do_set_schema(schema), Tabs = set_schema(?ets_first(schema)), lists:foreach(fun(Tab) -> clear_whereabouts(Tab) end, Tabs), set({schema, where_to_read}, node()), set({schema, load_node}, node()), set({schema, load_reason}, initial), - mnesia_controller:add_active_replica(schema, node()). + mnesia_controller:add_active_replica(schema, node()), + init_backends(). + + +init_backends() -> + Backends = lists:foldl(fun({Alias, Mod}, Acc) -> + orddict:append(Mod, Alias, Acc) + end, orddict:new(), get_ext_types()), + [init_backend(Mod, Aliases) || {Mod, Aliases} <- Backends], + ok. + +init_backend(Mod, [_|_] = Aliases) -> + case Mod:init_backend() of + ok -> + Mod:add_aliases(Aliases); + Error -> + mnesia:abort({backend_init_error, Error}) + end. exit_on_error({error, Reason}) -> exit(Reason); @@ -151,7 +183,7 @@ exit_on_error(GoodRes) -> val(Var) -> case ?catch_val(Var) of - {'EXIT', Reason} -> mnesia_lib:other_val(Var, Reason); + {'EXIT', _} -> mnesia_lib:other_val(Var); Value -> Value end. @@ -179,6 +211,7 @@ do_set_schema(Tab, Cs) -> set({Tab, ram_copies}, Cs#cstruct.ram_copies), set({Tab, disc_copies}, Cs#cstruct.disc_copies), set({Tab, disc_only_copies}, Cs#cstruct.disc_only_copies), + set({Tab, external_copies}, Cs#cstruct.external_copies), set({Tab, load_order}, Cs#cstruct.load_order), set({Tab, access_mode}, Cs#cstruct.access_mode), set({Tab, majority}, Cs#cstruct.majority), @@ -194,15 +227,21 @@ do_set_schema(Tab, Cs) -> set({Tab, arity}, Arity), RecName = Cs#cstruct.record_name, set({Tab, record_name}, RecName), - set({Tab, record_validation}, {RecName, Arity, Type}), set({Tab, wild_pattern}, wild(RecName, Arity)), - set({Tab, index}, Cs#cstruct.index), + set({Tab, index}, [P || {P,_} <- Cs#cstruct.index]), + case Cs#cstruct.index of + [] -> + set({Tab, index_info}, mnesia_index:index_info(Type, [])); + _ -> + ignore + end, %% create actual index tabs later set({Tab, cookie}, Cs#cstruct.cookie), set({Tab, version}, Cs#cstruct.version), set({Tab, cstruct}, Cs), Storage = mnesia_lib:schema_cs_to_storage_type(node(), Cs), set({Tab, storage_type}, Storage), + set_record_validation(Tab, Storage, RecName, Arity, Type), mnesia_lib:add({schema, tables}, Tab), Ns = mnesia_lib:cs_to_nodes(Cs), case lists:member(node(), Ns) of @@ -212,7 +251,24 @@ do_set_schema(Tab, Cs) -> mnesia_lib:add({schema, local_tables}, Tab); false -> ignore - end. + end, + set_ext_types(Tab, get_ext_types(), Cs#cstruct.external_copies). + +set_record_validation(Tab, {ext,Alias,Mod}, RecName, Arity, Type) -> + set({Tab, record_validation}, {RecName, Arity, Type, Alias, Mod}); +set_record_validation(Tab, _, RecName, Arity, Type) -> + set({Tab, record_validation}, {RecName, Arity, Type}). + +set_ext_types(Tab, ExtTypes, ExtCopies) -> + lists:foreach( + fun({Type, _} = Key) -> + Nodes = case lists:keyfind(Key, 1, ExtCopies) of + {_, Ns} -> Ns; + false -> [] + end, + set({Tab, Type}, Nodes) + end, ExtTypes). + wild(RecName, Arity) -> Wp0 = list_to_tuple(lists:duplicate(Arity, '_')), @@ -262,7 +318,7 @@ incr_version(Cs) -> [] -> {Major + 1, 0}; % All replicas are active _ -> {Major, Minor + 1} % Some replicas are inactive end, - Cs#cstruct{version = {V, {node(), now()}}}. + Cs#cstruct{version = {V, {node(), erlang:timestamp()}}}. %% Returns table name insert_cstruct(Tid, Cs, KeepWhereabouts) -> @@ -524,9 +580,14 @@ do_read_disc_schema(Fname, Keep) -> Res. get_initial_schema(SchemaStorage, Nodes) -> + get_initial_schema(SchemaStorage, Nodes, []). + +get_initial_schema(SchemaStorage, Nodes, Properties) -> % + UserProps = initial_schema_properties(Properties), Cs = #cstruct{name = schema, record_name = schema, - attributes = [table, cstruct]}, + attributes = [table, cstruct], + user_properties = UserProps}, Cs2 = case SchemaStorage of ram_copies -> Cs#cstruct{ram_copies = Nodes}; @@ -534,6 +595,35 @@ get_initial_schema(SchemaStorage, Nodes) -> end, cs2list(Cs2). +initial_schema_properties(Props0) -> + DefaultProps = remove_duplicates(mnesia_monitor:get_env(schema)), + Props = lists:foldl( + fun({K,V}, Acc) -> + lists:keystore(K, 1, Acc, {K,V}) + end, DefaultProps, remove_duplicates(Props0)), + initial_schema_properties_(Props). + +initial_schema_properties_([{backend_types, Types}|Props]) -> + lists:foreach(fun({Name, Module}) -> + verify_backend_type(Name, Module) + end, Types), + [{mnesia_backend_types, Types}|initial_schema_properties_(Props)]; +initial_schema_properties_([{index_plugins, Plugins}|Props]) -> + lists:foreach(fun({Name, Module, Function}) -> + verify_index_plugin(Name, Module, Function) + end, Plugins), + [{mnesia_index_plugins, Plugins}|initial_schema_properties_(Props)]; +initial_schema_properties_([P|_Props]) -> + mnesia:abort({bad_schema_property, P}); +initial_schema_properties_([]) -> + []. + +remove_duplicates([{K,_} = H|T]) -> + [H | remove_duplicates([X || {K1,_} = X <- T, + K1 =/= K])]; +remove_duplicates([]) -> + []. + read_cstructs_from_disc() -> %% Assumptions: %% - local schema lock in global @@ -550,8 +640,9 @@ read_cstructs_from_disc() -> {type, set}], case dets:open_file(make_ref(), Args) of {ok, Tab} -> + ExtTypes = get_ext_types_disc(), Fun = fun({_, _, List}) -> - {continue, list2cs(List)} + {continue, list2cs(List, ExtTypes)} end, Cstructs = dets:traverse(Tab, Fun), dets:close(Tab), @@ -631,7 +722,7 @@ do_insert_schema_ops(_Store, []) -> api_list2cs(List) when is_list(List) -> Name = pick(unknown, name, List, must), - Keys = check_keys(Name, List, record_info(fields, cstruct)), + Keys = check_keys(Name, List), check_duplicates(Name, Keys), list2cs(List); api_list2cs(Other) -> @@ -645,55 +736,57 @@ cs2list(Cs) when is_record(Cs, cstruct) -> rec2list(Tags, Tags, 2, Cs); cs2list(CreateList) when is_list(CreateList) -> CreateList; -%% 4.6 -cs2list(Cs) when element(1, Cs) == cstruct, tuple_size(Cs) == 19 -> - Tags = [name,type,ram_copies,disc_copies,disc_only_copies, + +cs2list(Cs) when element(1, Cs) == cstruct, tuple_size(Cs) == 20 -> + Tags = [name,type, + ram_copies,disc_copies,disc_only_copies,external_copies, load_order,access_mode,majority,index,snmp,local_content, record_name,attributes, user_properties,frag_properties,storage_properties, cookie,version], rec2list(Tags, Tags, 2, Cs); -%% 4.4.19 -cs2list(Cs) when element(1, Cs) == cstruct, tuple_size(Cs) == 18 -> +%% since vsn-4.6 (protocol 8.2 or older) +cs2list(Cs) when element(1, Cs) == cstruct, tuple_size(Cs) == 19 -> Tags = [name,type,ram_copies,disc_copies,disc_only_copies, load_order,access_mode,majority,index,snmp,local_content, - record_name,attributes,user_properties,frag_properties, - cookie,version], - rec2list(Tags, Tags, 2, Cs); -%% 4.4.18 and earlier -cs2list(Cs) when element(1, Cs) == cstruct, tuple_size(Cs) == 17 -> - Tags = [name,type,ram_copies,disc_copies,disc_only_copies, - load_order,access_mode,index,snmp,local_content, - record_name,attributes,user_properties,frag_properties, + record_name,attributes, + user_properties,frag_properties,storage_properties, cookie,version], rec2list(Tags, Tags, 2, Cs). cs2list(false, Cs) -> cs2list(Cs); -cs2list(ver4_4_18, Cs) -> %% Or earlier - Orig = record_info(fields, cstruct), - Tags = [name,type,ram_copies,disc_copies,disc_only_copies, - load_order,access_mode,index,snmp,local_content, - record_name,attributes,user_properties,frag_properties, - cookie,version], - rec2list(Tags, Orig, 2, Cs); -cs2list(ver4_4_19, Cs) -> - Orig = record_info(fields, cstruct), - Tags = [name,type,ram_copies,disc_copies,disc_only_copies, - load_order,access_mode,majority,index,snmp,local_content, - record_name,attributes,user_properties,frag_properties, - cookie,version], - rec2list(Tags, Orig, 2, Cs); -cs2list(ver4_6, Cs) -> +cs2list({8,3}, Cs) -> + cs2list(Cs); +cs2list({8,Minor}, Cs) when Minor =:= 2; Minor =:= 1 -> Orig = record_info(fields, cstruct), Tags = [name,type,ram_copies,disc_copies,disc_only_copies, load_order,access_mode,majority,index,snmp,local_content, record_name,attributes, user_properties,frag_properties,storage_properties, cookie,version], - rec2list(Tags, Orig, 2, Cs). - + CsList = rec2list(Tags, Orig, 2, Cs), + case proplists:get_value(index, CsList, []) of + [] -> CsList; + NewFormat -> + OldFormat = [Pos || {Pos, _Pref} <- NewFormat], + lists:keyreplace(index, 1, CsList, {index, OldFormat}) + end. +rec2list([index | Tags], [index|Orig], Pos, Rec) -> + Val = element(Pos, Rec), + [{index, lists:map( + fun({_, _Type}=P) -> P; + (P) when is_integer(P); is_atom(P) -> {P, ordered} + end, Val)} | rec2list(Tags, Orig, Pos + 1, Rec)]; +rec2list([external_copies | Tags], Orig0, Pos, Rec) -> + Orig = case Orig0 of + [external_copies|Rest] -> Rest; + _ -> Orig0 + end, + Val = element(Pos, Rec), + [{Alias, Ns} || {{Alias,_}, Ns} <- Val] + ++ rec2list(Tags, Orig, Pos+1, Rec); rec2list([Tag | Tags], [Tag | Orig], Pos, Rec) -> Val = element(Pos, Rec), [{Tag, Val} | rec2list(Tags, Orig, Pos + 1, Rec)]; @@ -716,14 +809,19 @@ convert_cs(Version, Cs) -> Fields = [Value || {_, Value} <- cs2list(Version, Cs)], list_to_tuple([cstruct|Fields]). -list2cs(List) when is_list(List) -> +list2cs(List) -> + list2cs(List, get_ext_types()). + +list2cs(List, ExtTypes) when is_list(List) -> Name = pick(unknown, name, List, must), Type = pick(Name, type, List, set), Rc0 = pick(Name, ram_copies, List, []), Dc = pick(Name, disc_copies, List, []), Doc = pick(Name, disc_only_copies, List, []), - Rc = case {Rc0, Dc, Doc} of - {[], [], []} -> [node()]; + + Ext = pick_external_copies(List, ExtTypes), + Rc = case {Rc0, Dc, Doc, Ext} of + {[], [], [], []} -> [node()]; _ -> Rc0 end, LC = pick(Name, local_content, List, false), @@ -741,8 +839,6 @@ list2cs(List) when is_list(List) -> Ix = pick(Name, index, List, []), verify({alt, [nil, list]}, mnesia_lib:etype(Ix), {bad_type, Name, {index, [Ix]}}), - Ix2 = [attr_to_pos(I, Attrs) || I <- Ix], - Frag = pick(Name, frag_properties, List, []), verify({alt, [nil, list]}, mnesia_lib:etype(Frag), {badarg, Name, {frag_properties, Frag}}), @@ -769,24 +865,48 @@ list2cs(List) when is_list(List) -> DetsOpts = proplists:get_value(dets, BEProps, []), is_list(DetsOpts) orelse mnesia:abort({badarg, Name, {dets, DetsOpts}}), [CheckProp(Prop, BadDetsOpts) || Prop <- DetsOpts], - #cstruct{name = Name, - ram_copies = Rc, - disc_copies = Dc, - disc_only_copies = Doc, - type = Type, - index = Ix2, - snmp = Snmp, - load_order = LoadOrder, - access_mode = AccessMode, - majority = Majority, - local_content = LC, - record_name = RecName, - attributes = Attrs, - user_properties = lists:sort(UserProps), - frag_properties = lists:sort(Frag), - storage_properties = lists:sort(BEProps), - cookie = Cookie, - version = Version}. + + case lists:keymember(mnesia, 1, application:which_applications()) of + true -> + Keys = check_keys(Name, List), + check_duplicates(Name, Keys); + false -> + %% check_keys/2 cannot be executed when mnesia is not + %% running, due to it not being possible to read what ext + %% backends are loaded. + %%% this doesn't work - disabled for now: + %%%Keys = check_keys(Name, List, record_info(fields, cstruct)), + %%%check_duplicates(Name, Keys) + ignore + end, + + Cs0 = #cstruct{name = Name, + ram_copies = Rc, + disc_copies = Dc, + disc_only_copies = Doc, + external_copies = Ext, + type = Type, + index = Ix, + snmp = Snmp, + load_order = LoadOrder, + access_mode = AccessMode, + majority = Majority, + local_content = LC, + record_name = RecName, + attributes = Attrs, + user_properties = lists:sort(UserProps), + frag_properties = lists:sort(Frag), + storage_properties = lists:sort(BEProps), + cookie = Cookie, + version = Version}, + case Ix of + [] -> Cs0; + [_|_] -> + Ix2 = expand_index_attrs(Cs0), + Cs0#cstruct{index = Ix2} + end; +list2cs(Other, _ExtTypes) -> + mnesia:abort({badarg, Other}). pick(Tab, Key, List, Default) -> case lists:keysearch(Key, 1, List) of @@ -800,6 +920,79 @@ pick(Tab, Key, List, Default) -> mnesia:abort({bad_type, Tab, BadArg}) end. +pick_external_copies(_List, []) -> + []; +pick_external_copies(List, ExtTypes) -> + lists:foldr( + fun({K, Val}, Acc) -> + case lists:keyfind(K, 1, ExtTypes) of + false -> + Acc; + {_, Mod} -> + [{{K,Mod}, Val}|Acc] + end + end, [], List). + +expand_storage_type(S) when S==ram_copies; + S==disc_copies; + S==disc_only_copies -> + S; +expand_storage_type(S) -> + case lists:keyfind(S, 1, get_ext_types()) of + false -> + mnesia:abort({bad_type, {storage_type, S}}); + {Alias, Mod} -> + {ext, Alias, Mod} + end. + +get_ext_types() -> + get_schema_user_property(mnesia_backend_types). + +get_index_plugins() -> + get_schema_user_property(mnesia_index_plugins). + +get_schema_user_property(Key) -> + Tab = schema, + %% Must work reliably both within transactions and outside of transactions + Res = case get(mnesia_activity_state) of + undefined -> + dirty_read_table_property(Tab, Key); + _ -> + do_read_table_property(Tab, Key) + end, + case Res of + undefined -> + []; + {_, Types} -> + Types + end. + +get_ext_types_disc() -> + try get_ext_types_disc_() + catch + error:_ ->[] + end. + +get_ext_types_disc_() -> + case mnesia_schema:remote_read_schema() of + {ok, _, Prop} -> + K1 = user_properties, + case lists:keyfind(K1, 1, Prop) of + {K1, UserProp} -> + K2 = mnesia_backend_types, + case lists:keyfind(K2, 1, UserProp) of + {K2, Types} -> + Types; + _ -> + [] + end; + _ -> + [] + end; + _ -> + [] + end. + %% Convert attribute name to integer if neccessary attr_tab_to_pos(_Tab, Pos) when is_integer(Pos) -> Pos; @@ -807,6 +1000,7 @@ attr_tab_to_pos(Tab, Attr) -> attr_to_pos(Attr, val({Tab, attributes})). %% Convert attribute name to integer if neccessary +attr_to_pos({_} = P, _) -> P; attr_to_pos(Pos, _Attrs) when is_integer(Pos) -> Pos; attr_to_pos(Attr, Attrs) when is_atom(Attr) -> @@ -821,8 +1015,18 @@ attr_to_pos(Attr, [_ | Attrs], Pos) -> attr_to_pos(Attr, _, _) -> mnesia:abort({bad_type, Attr}). +check_keys(Tab, Attrs) -> + Types = [T || {T,_} <- get_ext_types()], + check_keys(Tab, Attrs, Types ++ record_info(fields, cstruct)). + check_keys(Tab, [{Key, _Val} | Tail], Items) -> - case lists:member(Key, Items) of + Key1 = if + is_tuple(Key) -> + element(1, Key); + true -> + Key + end, + case lists:member(Key1, Items) of true -> [Key | check_keys(Tab, Tail, Items)]; false -> mnesia:abort({badarg, Tab, Key}) end; @@ -846,7 +1050,92 @@ has_duplicates([]) -> false. %% This is the only place where we check the validity of data -verify_cstruct(Cs) when is_record(Cs, cstruct) -> + +verify_cstruct(#cstruct{} = Cs) -> + assert_correct_cstruct(Cs), + Cs1 = verify_external_copies( + Cs#cstruct{index = expand_index_attrs(Cs)}), + assert_correct_cstruct(Cs1), + Cs1. + +expand_index_attrs(#cstruct{index = Ix, attributes = Attrs, + name = Tab} = Cs) -> + Prefered = prefered_index_types(Cs), + expand_index_attrs(Ix, Tab, Attrs, Prefered). + +expand_index_attrs(Ix, Tab, Attrs, Prefered) -> + lists:map(fun(P) when is_integer(P); is_atom(P) -> + {attr_to_pos(P, Attrs), Prefered}; + ({A} = P) when is_atom(A) -> + {P, Prefered}; + ({P, Type}) -> + {attr_to_pos(P, Attrs), Type}; + (_Other) -> + mnesia:abort({bad_type, Tab, {index, Ix}}) + end, Ix). + +prefered_index_types(#cstruct{external_copies = Ext}) -> + ExtTypes = [mnesia_lib:semantics(S, index_types) || + {S,Ns} <- Ext, Ns =/= []], + case intersect_types(ExtTypes) of + [] -> ordered; + [Pref|_] -> Pref + end. + +intersect_types([]) -> + []; +intersect_types([S1, S2|Rest]) -> + intersect_types([S1 -- (S1 -- S2)|Rest]); +intersect_types([S]) -> + S. + +verify_external_copies(#cstruct{external_copies = []} = Cs) -> + Cs; +verify_external_copies(#cstruct{name = Tab, external_copies = EC} = Cs) -> + Bad = {bad_type, Tab, {external_copies, EC}}, + AllECNodes = lists:concat([Ns || {_, Ns} <- EC, + is_list(Ns)]), + verify(true, length(lists:usort(AllECNodes)) == length(AllECNodes), Bad), + CsL = cs2list(Cs), + CsL1 = lists:foldl( + fun({{Alias, Mod}, Ns} = _X, CsLx) -> + BadTab = fun(Why) -> + {Why, Tab, {{ext, Alias, Mod},Ns}} + end, + verify(atom, mnesia_lib:etype(Mod), BadTab), + verify(true, fun() -> + lists:all(fun is_atom/1, Ns) + end, BadTab), + check_semantics(Mod, Alias, BadTab, Cs), + try Mod:check_definition(Alias, Tab, Ns, CsLx) of + ok -> + CsLx; + {ok, CsLx1} -> + CsLx1; + {error, Reason} -> + mnesia:abort(BadTab(Reason)) + catch + error:E -> + mnesia:abort(BadTab(E)) + end; + (_, CsLx) -> + CsLx + end, CsL, EC), + list2cs(CsL1). + +check_semantics(Mod, Alias, BadTab, #cstruct{type = Type}) -> + Ext = {ext, Alias, Mod}, + case lists:member(mnesia_lib:semantics(Ext, storage), [ram_copies, disc_copies, + disc_only_copies]) of + false -> mnesia:abort(BadTab(invalid_storage)); + true -> ok + end, + case lists:member(Type, mnesia_lib:semantics(Ext, types)) of + false -> mnesia:abort(BadTab(bad_type)); + true -> ok + end. + +assert_correct_cstruct(Cs) when is_record(Cs, cstruct) -> verify_nodes(Cs), Tab = Cs#cstruct.name, @@ -887,22 +1176,30 @@ verify_cstruct(Cs) when is_record(Cs, cstruct) -> Attrs), Index = Cs#cstruct.index, + verify({alt, [nil, list]}, mnesia_lib:etype(Index), {bad_type, Tab, {index, Index}}), + IxPlugins = get_index_plugins(), + AllowIndexOnKey = check_if_allow_index_on_key(), IxFun = - fun(Pos) -> - verify(true, fun() -> - if - is_integer(Pos), - Pos > 2, - Pos =< Arity -> - true; - true -> false - end - end, - {bad_type, Tab, {index, [Pos]}}) - end, + fun(Pos) -> + verify( + true, fun() -> + I = index_pos(Pos), + case Pos of + {_, T} -> + (T==bag orelse T==ordered) + andalso good_ix_pos( + I, AllowIndexOnKey, + Arity, IxPlugins); + _ -> + good_ix_pos(Pos, AllowIndexOnKey, + Arity, IxPlugins) + end + end, + {bad_type, Tab, {index, [Pos]}}) + end, lists:foreach(IxFun, Index), LC = Cs#cstruct.local_content, @@ -926,7 +1223,9 @@ verify_cstruct(Cs) when is_record(Cs, cstruct) -> {badarg, Tab, {snmp, Snmp}}), CheckProp = fun(Prop) when is_tuple(Prop), size(Prop) >= 1 -> ok; - (Prop) -> mnesia:abort({bad_type, Tab, {user_properties, [Prop]}}) + (Prop) -> + mnesia:abort({bad_type, Tab, + {user_properties, [Prop]}}) end, lists:foreach(CheckProp, Cs#cstruct.user_properties), @@ -946,17 +1245,45 @@ verify_cstruct(Cs) when is_record(Cs, cstruct) -> mnesia:abort({bad_type, Tab, {version, Version}}) end. +good_ix_pos({_} = P, _, _, Plugins) -> + lists:keymember(P, 1, Plugins); +good_ix_pos(I, true, Arity, _) when is_integer(I) -> + I >= 0 andalso I =< Arity; +good_ix_pos(I, false, Arity, _) when is_integer(I) -> + I > 2 andalso I =< Arity; +good_ix_pos(_, _, _, _) -> + false. + + +check_if_allow_index_on_key() -> + case mnesia_monitor:get_env(allow_index_on_key) of + true -> + true; + _ -> + false + end. + verify_nodes(Cs) -> Tab = Cs#cstruct.name, Ram = Cs#cstruct.ram_copies, Disc = Cs#cstruct.disc_copies, DiscOnly = Cs#cstruct.disc_only_copies, + Ext = lists:append([Ns || {_,Ns} <- Cs#cstruct.external_copies]), LoadOrder = Cs#cstruct.load_order, verify({alt, [nil, list]}, mnesia_lib:etype(Ram), {bad_type, Tab, {ram_copies, Ram}}), verify({alt, [nil, list]}, mnesia_lib:etype(Disc), {bad_type, Tab, {disc_copies, Disc}}), + lists:foreach( + fun({BE, Ns}) -> + verify({alt, [nil, list]}, mnesia_lib:etype(Ns), + {bad_type, Tab, {BE, Ns}}), + lists:foreach(fun(N) -> + verify(atom, mnesia_lib:etype(N), + {bad_type, Tab, {BE, Ns}}) + end, Ns) + end, Cs#cstruct.external_copies), case Tab of schema -> verify([], DiscOnly, {bad_type, Tab, {disc_only_copies, DiscOnly}}); @@ -968,12 +1295,15 @@ verify_nodes(Cs) -> verify(integer, mnesia_lib:etype(LoadOrder), {bad_type, Tab, {load_order, LoadOrder}}), - Nodes = Ram ++ Disc ++ DiscOnly, + Nodes = Ram ++ Disc ++ DiscOnly ++ Ext, verify(list, mnesia_lib:etype(Nodes), {combine_error, Tab, - [{ram_copies, []}, {disc_copies, []}, {disc_only_copies, []}]}), + [{ram_copies, []}, {disc_copies, []}, + {disc_only_copies, []}, {external_copies, []}]}), verify(false, has_duplicates(Nodes), {combine_error, Tab, Nodes}), - AtomCheck = fun(N) -> verify(atom, mnesia_lib:etype(N), {bad_type, Tab, N}) end, + AtomCheck = fun(N) -> + verify(atom, mnesia_lib:etype(N), {bad_type, Tab, N}) + end, lists:foreach(AtomCheck, Nodes). verify(Expected, Fun, Error) when is_function(Fun) -> @@ -1056,28 +1386,194 @@ check_active([{badrpc, Reason} | _Replies], Expl, Tab) -> check_active([], _Expl, _Tab) -> ok. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Function for definining an external backend type + +add_backend_type(Name, Module) -> + case schema_transaction(fun() -> do_add_backend_type(Name, Module) end) of + {atomic, NeedsInit} -> + case NeedsInit of + true -> + Module:init_backend(); + false -> + ignore + end, + Module:add_aliases([Name]), + {atomic, ok}; + Other -> + Other + end. + +do_add_backend_type(Name, Module) -> + verify_backend_type(Name, Module), + Types = case do_read_table_property(schema, mnesia_backend_types) of + undefined -> + []; + {_, Ts} -> + case lists:keymember(Name, 1, Ts) of + true -> + mnesia:abort({backend_type_already_exists, Name}); + false -> + Ts + end + end, + ModuleRegistered = lists:keymember(Module, 2, Types), + do_write_table_property(schema, {mnesia_backend_types, + [{Name, Module}|Types]}), + ModuleRegistered. + +delete_backend_type(Name) -> + schema_transaction(fun() -> do_delete_backend_type(Name) end). + +do_delete_backend_type(Name) -> + case do_read_table_property(schema, mnesia_backend_types) of + undefined -> + []; + {_, Ts} -> + case lists:keyfind(Name, 1, Ts) of + {_, Mod} -> + case using_backend_type(Name, Mod) of + [_|_] = Tabs -> + mnesia:abort({backend_in_use, {Name, Tabs}}); + [] -> + do_write_table_property( + schema, {mnesia_backend_types, + lists:keydelete(Name, 1, Ts)}) + end; + false -> + mnesia:abort({no_such_backend, Name}) + end + end. + +using_backend_type(Name, Mod) -> + Ext = ets:select(mnesia_gvar, + [{ {{'$1',external_copies},'$2'}, [], [{{'$1','$2'}}] }]), + Entry = {Name, Mod}, + [T || {T,C} <- Ext, + lists:keymember(Entry, 1, C)]. + +verify_backend_type(Name, Module) -> + case legal_backend_name(Name) of + false -> + mnesia:abort({bad_type, {backend_type,Name,Module}}); + true -> + ok + end, + ExpectedExports = mnesia_backend_type:behaviour_info(callbacks), + Exports = try Module:module_info(exports) + catch + error:_ -> + mnesia:abort({undef_backend, Module}) + end, + case ExpectedExports -- Exports of + [] -> + ok; + _Other -> + io:fwrite(user, "Missing backend_type exports: ~p~n", [_Other]), + mnesia:abort({bad_type, {backend_type,Name,Module}}) + end. + +legal_backend_name(Name) -> + is_atom(Name) andalso + (not lists:member(Name, record_info(fields, cstruct))). + +%% Used e.g. by mnesia:system_info(backend_types). +backend_types() -> + [ram_copies, disc_copies, disc_only_copies | + [T || {T,_} <- get_ext_types()]]. + +add_index_plugin(Name, Module, Function) -> + schema_transaction( + fun() -> do_add_index_plugin(Name, Module, Function) end). + +do_add_index_plugin(Name, Module, Function) -> + verify_index_plugin(Name, Module, Function), + Plugins = case do_read_table_property(schema, mnesia_index_plugins) of + undefined -> + []; + {_, Ps} -> + case lists:keymember(Name, 1, Ps) of + true -> + mnesia:abort({index_plugin_already_exists, Name}); + false -> + Ps + end + end, + do_write_table_property(schema, {mnesia_index_plugins, + [{Name, Module, Function}|Plugins]}). + +delete_index_plugin(P) -> + schema_transaction( + fun() -> do_delete_index_plugin(P) end). + +do_delete_index_plugin({A} = P) when is_atom(A) -> + Plugins = get_index_plugins(), + case lists:keyfind(P, 1, Plugins) of + false -> + mnesia:abort({no_exists, {index_plugin, P}}); + _Found -> + case ets:select(mnesia_gvar, + [{ {{'$1',{index,{P,'_'}}},'_'},[],['$1']}, + { {{'$1',{index,P}},'_'},[],['$1']}], 1) of + {[_], _} -> + mnesia:abort({plugin_in_use, P}); + '$end_of_table' -> + do_write_table_property( + schema, {mnesia_index_plugins, + lists:keydelete(P, 1, Plugins)}) + end + end. + +verify_index_plugin({A} = Name, Module, Function) + when is_atom(A), is_atom(Module), is_atom(Function) -> + case code:ensure_loaded(Module) of + {error, nofile} -> + mnesia:abort({bad_type, {index_plugin,Name,Module,Function}}); + {module,_} -> + %% Index plugins are called as Module:Function(Tab, Pos, Obj) + case erlang:function_exported(Module, Function, 3) of + true -> + ok; + false -> + mnesia:abort( + {bad_type, {index_plugin,Name,Module,Function}}) + end + end; +verify_index_plugin(Name, Module, Function) -> + mnesia:abort({bad_type, {index_plugin,Name,Module,Function}}). + + +%% Used e.g. by mnesia:system_info(backend_types). +index_plugins() -> + get_index_plugins(). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% Here's the real interface function to create a table -create_table(TabDef) -> - schema_transaction(fun() -> do_multi_create_table(TabDef) end). +create_table([_|_] = TabDef) -> + schema_transaction(fun() -> do_multi_create_table(TabDef) end); +create_table(Arg) -> {aborted, {badarg, Arg}}. %% And the corresponding do routines .... do_multi_create_table(TabDef) -> get_tid_ts_and_lock(schema, write), ensure_writable(schema), + do_create_table(TabDef), + ok. + +do_create_table(TabDef) when is_list(TabDef) -> Cs = api_list2cs(TabDef), case Cs#cstruct.frag_properties of [] -> - do_create_table(Cs); + do_create_table_1(Cs); _Props -> CsList = mnesia_frag:expand_cstruct(Cs), - lists:foreach(fun do_create_table/1, CsList) - end, - ok. + lists:foreach(fun do_create_table_1/1, CsList) + end. -do_create_table(Cs) -> +do_create_table_1(Cs) -> {_Mod, _Tid, Ts} = get_tid_ts_and_lock(schema, none), Store = Ts#tidstore.store, do_insert_schema_ops(Store, make_create_table(Cs)). @@ -1087,14 +1583,9 @@ make_create_table(Cs) -> verify(false, check_if_exists(Tab), {already_exists, Tab}), unsafe_make_create_table(Cs). -% unsafe_do_create_table(Cs) -> -% {_Mod, Tid, Ts} = get_tid_ts_and_lock(schema, none), -% Store = Ts#tidstore.store, -% do_insert_schema_ops(Store, unsafe_make_create_table(Cs)). - -unsafe_make_create_table(Cs) -> +unsafe_make_create_table(Cs0) -> {_Mod, Tid, Ts} = get_tid_ts_and_lock(schema, none), - verify_cstruct(Cs), + Cs = verify_cstruct(Cs0), Tab = Cs#cstruct.name, %% Check that we have all disc replica nodes running @@ -1141,6 +1632,7 @@ do_delete_table(Tab) -> ensure_writable(schema), insert_schema_ops(TidTs, make_delete_table(Tab, whole_table)). +-dialyzer({no_improper_lists, make_delete_table/2}). make_delete_table(Tab, Mode) -> case existed_before(Tab) of false -> @@ -1241,8 +1733,7 @@ make_add_table_copy(Tab, Node, Storage) -> Cs = incr_version(val({Tab, cstruct})), Ns = mnesia_lib:cs_to_nodes(Cs), verify(false, lists:member(Node, Ns), {already_exists, Tab, Node}), - Cs2 = new_cs(Cs, Node, Storage, add), - verify_cstruct(Cs2), + Cs2 = verify_cstruct(new_cs(Cs, Node, Storage, add)), %% Check storage and if node is running IsRunning = lists:member(Node, val({current, db_nodes})), @@ -1290,13 +1781,14 @@ make_del_table_copy(Tab, Node) -> _ when Tab == schema -> %% ensure_active(Cs2), ensure_not_active(Tab, Node), - verify_cstruct(Cs2), + Cs3 = verify_cstruct(Cs2), Ops = remove_node_from_tabs(val({schema, tables}), Node), - [{op, del_table_copy, ram_copies, Node, vsn_cs2list(Cs2)} | Ops]; + [{op, del_table_copy, ram_copies, Node, vsn_cs2list(Cs3)} | Ops]; _ -> ensure_active(Cs), - verify_cstruct(Cs2), - [{op, del_table_copy, Storage, Node, vsn_cs2list(Cs2)}] + Cs3 = verify_cstruct(Cs2), + get_tid_ts_and_lock(Tab, write), + [{op, del_table_copy, Storage, Node, vsn_cs2list(Cs3)}] end. remove_node_from_tabs([], _Node) -> @@ -1322,8 +1814,9 @@ remove_node_from_tabs([Tab|Rest], Node) -> [{op, delete_table, vsn_cs2list(Cs)} | remove_node_from_tabs(Rest, Node)]; _Ns -> - verify_cstruct(Cs2), - [{op, del_table_copy, ram_copies, Node, vsn_cs2list(Cs2)}| + Cs3 = verify_cstruct(Cs2), + get_tid_ts_and_lock(Tab, write), + [{op, del_table_copy, ram_copies, Node, vsn_cs2list(Cs3)}| remove_node_from_tabs(Rest, Node)] end end. @@ -1341,8 +1834,34 @@ new_cs(Cs, Node, disc_copies, del) -> new_cs(Cs, Node, disc_only_copies, del) -> Cs#cstruct{disc_only_copies = lists:delete(Node , Cs#cstruct.disc_only_copies)}; -new_cs(Cs, _Node, Storage, _Op) -> - mnesia:abort({badarg, Cs#cstruct.name, Storage}). +new_cs(#cstruct{external_copies = ExtCps} = Cs, Node, Storage0, Op) -> + Storage = case Storage0 of + {ext, Alias, _} -> Alias; + Alias -> Alias + end, + ExtTypes = get_ext_types(), + case lists:keyfind(Storage, 1, ExtTypes) of + false -> + mnesia:abort({badarg, Cs#cstruct.name, Storage}); + {_, Mod} -> + Key = {Storage, Mod}, + case {lists:keymember(Key, 1, ExtCps), Op} of + {false, del} -> + Cs; + {false, add} -> + Cs#cstruct{external_copies = [{Key, [Node]}|ExtCps]}; + {true, _} -> + F = fun({K, Ns}) when K == Key -> + case Op of + del -> {K, lists:delete(Node, Ns)}; + add -> {K, opt_add(Node, Ns)} + end; + (X) -> + X + end, + Cs#cstruct{external_copies = lists:map(F, ExtCps)} + end + end. opt_add(N, L) -> [N | lists:delete(N, L)]. @@ -1354,6 +1873,11 @@ do_move_table(schema, _FromNode, _ToNode) -> mnesia:abort({bad_type, schema}); do_move_table(Tab, FromNode, ToNode) when is_atom(FromNode), is_atom(ToNode) -> TidTs = get_tid_ts_and_lock(schema, write), + AnyOld = lists:any(fun(Node) -> mnesia_monitor:needs_protocol_conversion(Node) end, + [ToNode|val({Tab, where_to_write})]), + if AnyOld -> ignore; %% Leads to deadlock on old nodes + true -> get_tid_ts_and_lock(Tab, write) + end, insert_schema_ops(TidTs, make_move_table(Tab, FromNode, ToNode)); do_move_table(Tab, FromNode, ToNode) -> mnesia:abort({badarg, Tab, FromNode, ToNode}). @@ -1373,8 +1897,7 @@ make_move_table(Tab, FromNode, ToNode) -> verify(true, lists:member(ToNode, Running), {not_active, schema, ToNode}), Cs2 = new_cs(Cs, ToNode, Storage, add), - Cs3 = new_cs(Cs2, FromNode, Storage, del), - verify_cstruct(Cs3), + Cs3 = verify_cstruct(new_cs(Cs2, FromNode, Storage, del)), [{op, add_table_copy, Storage, ToNode, vsn_cs2list(Cs2)}, {op, sync_trans}, {op, del_table_copy, Storage, FromNode, vsn_cs2list(Cs3)}]. @@ -1401,9 +1924,11 @@ make_change_table_copy_type(Tab, Node, ToS) -> Cs = incr_version(val({Tab, cstruct})), FromS = mnesia_lib:storage_type_at_node(Node, Tab), - case compare_storage_type(false, FromS, ToS) of + ToSExp = expand_storage_type(ToS), + + case compare_storage_type(false, FromS, ToSExp) of {same, _} -> - mnesia:abort({already_exists, Tab, Node, ToS}); + mnesia:abort({already_exists, Tab, Node, ToSExp}); {diff, _} -> ignore; incompatible -> @@ -1411,10 +1936,8 @@ make_change_table_copy_type(Tab, Node, ToS) -> end, Cs2 = new_cs(Cs, Node, FromS, del), - Cs3 = new_cs(Cs2, Node, ToS, add), - verify_cstruct(Cs3), - - [{op, change_table_copy_type, Node, FromS, ToS, vsn_cs2list(Cs3)}]. + Cs3 = verify_cstruct(new_cs(Cs2, Node, ToS, add)), + [{op, change_table_copy_type, Node, FromS, ToSExp, vsn_cs2list(Cs3)}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% change index functions .... @@ -1436,11 +1959,12 @@ make_add_table_index(Tab, Pos) -> Cs = incr_version(val({Tab, cstruct})), ensure_active(Cs), Ix = Cs#cstruct.index, - verify(false, lists:member(Pos, Ix), {already_exists, Tab, Pos}), + verify(false, lists:keymember(index_pos(Pos), 1, Ix), + {already_exists, Tab, Pos}), Ix2 = lists:sort([Pos | Ix]), - Cs2 = Cs#cstruct{index = Ix2}, - verify_cstruct(Cs2), - [{op, add_index, Pos, vsn_cs2list(Cs2)}]. + Cs2 = verify_cstruct(Cs#cstruct{index = Ix2}), + NewPosInfo = lists:keyfind(Pos, 1, Cs2#cstruct.index), + [{op, add_index, NewPosInfo, vsn_cs2list(Cs2)}]. del_table_index(Tab, Pos) -> schema_transaction(fun() -> do_del_table_index(Tab, Pos) end). @@ -1458,9 +1982,8 @@ make_del_table_index(Tab, Pos) -> Cs = incr_version(val({Tab, cstruct})), ensure_active(Cs), Ix = Cs#cstruct.index, - verify(true, lists:member(Pos, Ix), {no_exists, Tab, Pos}), - Cs2 = Cs#cstruct{index = lists:delete(Pos, Ix)}, - verify_cstruct(Cs2), + verify(true, lists:keymember(Pos, 1, Ix), {no_exists, Tab, Pos}), + Cs2 = verify_cstruct(Cs#cstruct{index = lists:keydelete(Pos, 1, Ix)}), [{op, del_index, Pos, vsn_cs2list(Cs2)}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1482,8 +2005,7 @@ make_add_snmp(Tab, Ustruct) -> verify([], Cs#cstruct.snmp, {already_exists, Tab, snmp}), Error = {badarg, Tab, snmp, Ustruct}, verify(true, mnesia_snmp_hook:check_ustruct(Ustruct), Error), - Cs2 = Cs#cstruct{snmp = Ustruct}, - verify_cstruct(Cs2), + Cs2 = verify_cstruct(Cs#cstruct{snmp = Ustruct}), [{op, add_snmp, Ustruct, vsn_cs2list(Cs2)}]. del_snmp(Tab) -> @@ -1500,8 +2022,7 @@ make_del_snmp(Tab) -> ensure_writable(schema), Cs = incr_version(val({Tab, cstruct})), ensure_active(Cs), - Cs2 = Cs#cstruct{snmp = []}, - verify_cstruct(Cs2), + Cs2 = verify_cstruct(Cs#cstruct{snmp = []}), [{op, del_snmp, vsn_cs2list(Cs2)}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1530,19 +2051,21 @@ make_transform(Tab, Fun, NewAttrs, NewRecName) -> Cs = incr_version(val({Tab, cstruct})), ensure_active(Cs), ensure_writable(Tab), - case mnesia_lib:val({Tab, index}) of + case Cs#cstruct.index of [] -> - Cs2 = Cs#cstruct{attributes = NewAttrs, record_name = NewRecName}, - verify_cstruct(Cs2), + Cs2 = verify_cstruct( + Cs#cstruct{attributes = NewAttrs, + record_name = NewRecName}), [{op, transform, Fun, vsn_cs2list(Cs2)}]; PosList -> - DelIdx = fun(Pos, Ncs) -> + DelIdx = fun({Pos,_}, Ncs) -> Ix = Ncs#cstruct.index, - Ncs1 = Ncs#cstruct{index = lists:delete(Pos, Ix)}, + Ix2 = lists:keydelete(Pos, 1, Ix), + Ncs1 = Ncs#cstruct{index = Ix2}, Op = {op, del_index, Pos, vsn_cs2list(Ncs1)}, {Op, Ncs1} end, - AddIdx = fun(Pos, Ncs) -> + AddIdx = fun({_,_} = Pos, Ncs) -> Ix = Ncs#cstruct.index, Ix2 = lists:sort([Pos | Ix]), Ncs1 = Ncs#cstruct{index = Ix2}, @@ -1552,10 +2075,16 @@ make_transform(Tab, Fun, NewAttrs, NewRecName) -> {DelOps, Cs1} = lists:mapfoldl(DelIdx, Cs, PosList), Cs2 = Cs1#cstruct{attributes = NewAttrs, record_name = NewRecName}, {AddOps, Cs3} = lists:mapfoldl(AddIdx, Cs2, PosList), - verify_cstruct(Cs3), - lists:flatten([DelOps, {op, transform, Fun, vsn_cs2list(Cs2)}, AddOps]) + _ = verify_cstruct(Cs3), % just a sanity check + lists:flatten([DelOps, {op, transform, Fun, vsn_cs2list(Cs2)}, + AddOps]) end. +index_pos({Pos,_}) -> Pos; +index_pos(Pos) when is_integer(Pos) -> Pos; +index_pos({P} = Pos) when is_atom(P) -> Pos. + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% @@ -1575,8 +2104,7 @@ make_change_table_access_mode(Tab, Mode) -> ensure_active(Cs), OldMode = Cs#cstruct.access_mode, verify(false, OldMode == Mode, {already_exists, Tab, Mode}), - Cs2 = Cs#cstruct{access_mode = Mode}, - verify_cstruct(Cs2), + Cs2 = verify_cstruct(Cs#cstruct{access_mode = Mode}), [{op, change_table_access_mode, vsn_cs2list(Cs2), OldMode, Mode}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1596,8 +2124,7 @@ make_change_table_load_order(Tab, LoadOrder) -> Cs = incr_version(val({Tab, cstruct})), ensure_active(Cs), OldLoadOrder = Cs#cstruct.load_order, - Cs2 = Cs#cstruct{load_order = LoadOrder}, - verify_cstruct(Cs2), + Cs2 = verify_cstruct(Cs#cstruct{load_order = LoadOrder}), [{op, change_table_load_order, vsn_cs2list(Cs2), OldLoadOrder, LoadOrder}]. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1643,6 +2170,7 @@ write_table_property(Tab, Prop) when is_tuple(Prop), size(Prop) >= 1 -> schema_transaction(fun() -> do_write_table_property(Tab, Prop) end); write_table_property(Tab, Prop) -> {aborted, {bad_type, Tab, Prop}}. + do_write_table_property(Tab, Prop) -> TidTs = get_tid_ts_and_lock(schema, write), {_, _, Ts} = TidTs, @@ -1674,8 +2202,7 @@ make_write_table_properties(Tab, [Prop | Props], Cs) -> PropKey = element(1, Prop), DelProps = lists:keydelete(PropKey, 1, OldProps), MergedProps = lists:merge(DelProps, [Prop]), - Cs2 = Cs#cstruct{user_properties = MergedProps}, - verify_cstruct(Cs2), + Cs2 = verify_cstruct(Cs#cstruct{user_properties = MergedProps}), [{op, write_property, vsn_cs2list(Cs2), Prop} | make_write_table_properties(Tab, Props, Cs2)]; make_write_table_properties(_Tab, [], _Cs) -> @@ -1720,22 +2247,38 @@ do_read_table_property(Tab, Key) -> {_, _, Ts} = TidTs, Store = Ts#tidstore.store, Props = ets:foldl( - fun({op, create_table, [{name, T}|Opts]}, _Acc) - when T==Tab -> + fun({op, announce_im_running,_,Opts,_,_}, _Acc) when Tab==schema -> + find_props(Opts); + ({op, create_table, [{name, T}|Opts]}, _Acc) + when T==Tab -> find_props(Opts); ({op, Op, [{name,T}|Opts], _Prop}, _Acc) - when T==Tab, Op==write_property; Op==delete_property -> + when T==Tab, Op==write_property; + T==Tab, Op==delete_property -> find_props(Opts); ({op, delete_table, [{name,T}|_]}, _Acc) when T==Tab -> []; (_Other, Acc) -> Acc - end, [], Store), - case lists:keysearch(Key, 1, Props) of - {value, Property} -> - Property; - false -> + end, undefined, Store), + case Props of + undefined -> + get_tid_ts_and_lock(Tab, read), + dirty_read_table_property(Tab, Key); + _ when is_list(Props) -> + case lists:keyfind(Key, 1, Props) of + false -> + undefined; + Other -> + Other + end + end. + +dirty_read_table_property(Tab, Key) -> + try ets:lookup_element(mnesia_gvar, {Tab,user_property,Key}, 2) + catch + error:_ -> undefined end. @@ -1795,8 +2338,7 @@ make_delete_table_properties(Tab, PropKeys) -> make_delete_table_properties(Tab, [PropKey | PropKeys], Cs) -> OldProps = Cs#cstruct.user_properties, Props = lists:keydelete(PropKey, 1, OldProps), - Cs2 = Cs#cstruct{user_properties = Props}, - verify_cstruct(Cs2), + Cs2 = verify_cstruct(Cs#cstruct{user_properties = Props}), [{op, delete_property, vsn_cs2list(Cs2), PropKey} | make_delete_table_properties(Tab, PropKeys, Cs2)]; make_delete_table_properties(_Tab, [], _Cs) -> @@ -1937,6 +2479,11 @@ prepare_op(Tid, {op, create_table, TabDef}, _WaitFor) -> create_disc_only_table(Tab,Cs), insert_cstruct(Tid, Cs, false), {true, optional}; + {ext, Alias, Mod} -> + mnesia_lib:set({Tab, create_table},true), + create_external_table(Alias, Tab, Mod, Cs), + insert_cstruct(Tid, Cs, false), + {true, optional}; unknown -> %% No replica on this node mnesia_lib:set({Tab, create_table},true), insert_cstruct(Tid, Cs, false), @@ -1961,7 +2508,7 @@ prepare_op(Tid, {op, add_table_copy, Storage, Node, TabDef}, _WaitFor) -> end, %% Tables are created by mnesia_loader get_network code insert_cstruct(Tid, Cs, true), - case mnesia_controller:get_network_copy(Tab, Cs) of + case mnesia_controller:get_network_copy(Tid, Tab, Cs) of {loaded, ok} -> {true, optional}; {not_loaded, ErrReason} -> @@ -1992,28 +2539,11 @@ prepare_op(Tid, {op, add_table_copy, Storage, Node, TabDef}, _WaitFor) -> {true, optional} end; -prepare_op(Tid, {op, del_table_copy, _Storage, Node, TabDef}, _WaitFor) -> +prepare_op(_Tid, {op, del_table_copy, _Storage, Node, TabDef}, _WaitFor) -> Cs = list2cs(TabDef), Tab = Cs#cstruct.name, - - if - %% Schema table lock is always required to run a schema op. - %% No need to look it. - node(Tid#tid.pid) == node(), Tab /= schema -> - Self = self(), - Pid = spawn_link(fun() -> lock_del_table(Tab, Node, Cs, Self) end), - put(mnesia_lock, Pid), - receive - {Pid, updated} -> - {true, optional}; - {Pid, FailReason} -> - mnesia:abort(FailReason); - {'EXIT', Pid, Reason} -> - mnesia:abort(Reason) - end; - true -> - {true, optional} - end; + set_where_to_read(Tab, Node, Cs), + {true, optional}; prepare_op(_Tid, {op, change_table_copy_type, N, FromS, ToS, TabDef}, _WaitFor) when N == node() -> @@ -2022,6 +2552,12 @@ prepare_op(_Tid, {op, change_table_copy_type, N, FromS, ToS, TabDef}, _WaitFor) NotActive = mnesia_lib:not_active_here(Tab), + if Tab =/= schema -> + check_if_disc_required(FromS, ToS); + true -> + ok + end, + if NotActive == true -> mnesia:abort({not_active, Tab, node()}); @@ -2061,6 +2597,15 @@ prepare_op(_Tid, {op, change_table_copy_type, N, FromS, ToS, TabDef}, _WaitFor) mnesia:abort({combine_error, Tab, ToS}) end; + element(1,FromS) == ext; element(1,ToS) == ext -> + if ToS == ram_copies -> + create_ram_table(Tab, Cs); + true -> + ok + end, + mnesia_dumper:dump_to_logfile(FromS, Tab), + mnesia_checkpoint:tm_change_table_copy_type(Tab, FromS, ToS); + FromS == ram_copies -> case mnesia_monitor:use_dir() of true -> @@ -2076,7 +2621,9 @@ prepare_op(_Tid, {op, change_table_copy_type, N, FromS, ToS, TabDef}, _WaitFor) disc_only_copies -> mnesia_dumper:raw_named_dump_table(Tab, dmp) end, - mnesia_checkpoint:tm_change_table_copy_type(Tab, FromS, ToS) + mnesia_checkpoint:tm_change_table_copy_type(Tab, + FromS, + ToS) end; false -> mnesia:abort({has_no_disc, node()}) @@ -2084,6 +2631,7 @@ prepare_op(_Tid, {op, change_table_copy_type, N, FromS, ToS, TabDef}, _WaitFor) FromS == disc_copies, ToS == disc_only_copies -> mnesia_dumper:raw_named_dump_table(Tab, dmp); + FromS == disc_only_copies -> Type = Cs#cstruct.type, create_ram_table(Tab, Cs), @@ -2095,6 +2643,7 @@ prepare_op(_Tid, {op, change_table_copy_type, N, FromS, ToS, TabDef}, _WaitFor) Err = "Failed to copy disc data to ram", mnesia:abort({system_limit, Tab, {Err,Reason}}) end; + true -> ignore end, @@ -2151,14 +2700,16 @@ prepare_op(_Tid, {op, transform, Fun, TabDef}, _WaitFor) -> mnesia_lib:db_fixtable(Storage, Tab, true), Key = mnesia_lib:db_first(Tab), Op = {op, transform, Fun, TabDef}, - case catch transform_objs(Fun, Tab, RecName, - Key, NewArity, Storage, Type, [Op]) of - {'EXIT', Reason} -> - mnesia_lib:db_fixtable(Storage, Tab, false), - exit({"Bad transform function", Tab, Fun, node(), Reason}); + try transform_objs(Fun, Tab, RecName, Key, + NewArity, Storage, Type, [Op]) of Objs -> mnesia_lib:db_fixtable(Storage, Tab, false), {true, Objs, mandatory} + catch _:Reason -> + mnesia_lib:db_fixtable(Storage, Tab, false), + mnesia_lib:important("Transform function failed: '~p' in '~p'", + [Reason, erlang:get_stacktrace()]), + exit({"Bad transform function", Tab, Fun, node(), Reason}) end end; @@ -2174,6 +2725,22 @@ prepare_op(_Tid, {op, merge_schema, TabDef}, _WaitFor) -> prepare_op(_Tid, _Op, _WaitFor) -> {true, optional}. +check_if_disc_required(FromS, ToS) -> + FromSem = mnesia_lib:semantics(FromS, storage), + ToSem = mnesia_lib:semantics(ToS, storage), + case {FromSem, ToSem} of + {ram_copies, _} when ToSem == disc_copies; + ToSem == disc_only_copies -> + case mnesia_monitor:use_dir() of + true -> + ok; + false -> + mnesia:abort({has_no_disc, node()}) + end; + _ -> + ok + end. + create_ram_table(Tab, #cstruct{type=Type, storage_properties=Props}) -> EtsOpts = proplists:get_value(ets, Props, []), Args = [{keypos, 2}, public, named_table, Type | EtsOpts], @@ -2215,6 +2782,14 @@ create_disc_only_table(Tab, #cstruct{type=Type, storage_properties=Props}) -> mnesia:abort({system_limit, Tab, {Err,Reason}}) end. +create_external_table(Alias, Tab, Mod, Cs) -> + case mnesia_monitor:unsafe_create_external(Tab, Alias, Mod, Cs) of + ok -> + ok; + {error,Reason} -> + Err = "Failed to create external table", + mnesia:abort({system_limit, Tab, {Err,Reason}}) + end. receive_sync([], Pids) -> Pids; @@ -2227,46 +2802,6 @@ receive_sync(Nodes, Pids) -> {abort, Else} end. -lock_del_table(Tab, NewNode, Cs0, Father) -> - Ns = val({schema, active_replicas}), - process_flag(trap_exit,true), - Lock = fun() -> - mnesia:write_lock_table(Tab), - %% Sigh using cs record - Set = fun(Node) -> - [Cs] = normalize_cs([Cs0], Node), - rpc:call(Node, ?MODULE, set_where_to_read, [Tab, NewNode, Cs]) - end, - Res = [Set(Node) || Node <- Ns], - Filter = fun(ok) -> - false; - ({badrpc, {'EXIT', {undef, _}}}) -> - %% This will be the case we talks with elder nodes - %% than 3.8.2, they will set where_to_read without - %% getting a lock. - false; - (_) -> - true - end, - case lists:filter(Filter, Res) of - [] -> - Father ! {self(), updated}, - %% When transaction is commited the process dies - %% and the lock is released. - receive _ -> ok end; - Err -> - Father ! {self(), {bad_commit, Err}} - end, - ok - end, - case mnesia:transaction(Lock) of - {atomic, ok} -> ok; - {aborted, R} -> Father ! {self(), R} - end, - unlink(Father), - unlink(whereis(mnesia_tm)), - exit(normal). - set_where_to_read(Tab, Node, Cs) -> case mnesia_lib:val({Tab, where_to_read}) of Node -> @@ -2342,7 +2877,7 @@ undo_prepare_commit(Tid, Commit) -> ignore; Ops -> %% Catch to allow failure mnesia_controller may not be started - catch mnesia_controller:release_schema_commit_lock(), + ?SAFE(mnesia_controller:release_schema_commit_lock()), undo_prepare_ops(Tid, Ops) end, Commit. @@ -2389,7 +2924,10 @@ undo_prepare_op(Tid, {op, create_table, TabDef}) -> mnesia_monitor:unsafe_close_dets(Tab), Dat = mnesia_lib:tab2dat(Tab), %% disc_delete_table(Tab, Storage), - file:delete(Dat) + file:delete(Dat); + {ext, Alias, Mod} -> + Mod:close_table(Alias, Tab), + Mod:delete_table(Alias, Tab) end; undo_prepare_op(Tid, {op, add_table_copy, Storage, Node, TabDef}) -> @@ -2417,13 +2955,19 @@ undo_prepare_op(Tid, {op, add_table_copy, Storage, Node, TabDef}) -> insert_cstruct(Tid, Cs2, true) % Don't care about the version end; -undo_prepare_op(_Tid, {op, del_table_copy, _, Node, TabDef}) - when Node == node() -> - WriteLocker = get(mnesia_lock), - WriteLocker =/= undefined andalso (WriteLocker ! die), +undo_prepare_op(_Tid, {op, del_table_copy, _, Node, TabDef}) -> Cs = list2cs(TabDef), Tab = Cs#cstruct.name, - mnesia_lib:set({Tab, where_to_read}, Node); + if node() =:= Node -> + mnesia_lib:set({Tab, where_to_read}, Node); + true -> + case mnesia_lib:val({Tab, where_to_read}) of + nowhere -> + mnesia_lib:set_remote_where_to_read(Tab); + _ -> + ignore + end + end; undo_prepare_op(_Tid, {op, change_table_copy_type, N, FromS, ToS, TabDef}) when N == node() -> @@ -2483,20 +3027,22 @@ ram_delete_table(Tab, Storage) -> case Storage of unknown -> ignore; + {ext, _, _} -> + ignore; disc_only_copies -> ignore; _Else -> %% delete possible index files and data ..... %% Got to catch this since if no info has been set in the %% mnesia_gvar it will crash - catch mnesia_index:del_transient(Tab, Storage), + ?CATCH(mnesia_index:del_transient(Tab, Storage)), case ?catch_val({Tab, {index, snmp}}) of {'EXIT', _} -> ignore; Etab -> - catch mnesia_snmp_hook:delete_table(Tab, Etab) + ?SAFE(mnesia_snmp_hook:delete_table(Tab, Etab)) end, - catch ?ets_delete_table(Tab) + ?SAFE(?ets_delete_table(Tab)) end. purge_dir(Dir, KeepFiles) -> @@ -2544,13 +3090,27 @@ purge_known_files([File | Tail], KeepFiles, Dir, Suffixes) -> ignore; true -> AbsFile = filename:join([Dir, File]), - file:delete(AbsFile) - end + delete_recursive(AbsFile) + end end, purge_known_files(Tail, KeepFiles, Dir, Suffixes); purge_known_files([], _KeepFiles, _Dir, _Suffixes) -> ok. +%% Removes a directory or file recursively +delete_recursive(Path) -> + case filelib:is_dir(Path) of + true -> + {ok, Names} = file:list_dir(Path), + lists:foreach(fun(Name) -> + delete_recursive(filename:join(Path, Name)) + end, + Names), + file:del_dir(Path); + false -> + file:delete(Path) + end. + has_known_suffix(_File, _Suffixes, true) -> true; has_known_suffix(File, [Suffix | Tail], false) -> @@ -2558,11 +3118,33 @@ has_known_suffix(File, [Suffix | Tail], false) -> has_known_suffix(_File, [], Bool) -> Bool. -known_suffixes() -> real_suffixes() ++ tmp_suffixes(). +known_suffixes() -> known_suffixes(get_ext_types_disc()). -real_suffixes() -> [".DAT", ".LOG", ".BUP", ".DCL", ".DCD"]. +known_suffixes(Ext) -> real_suffixes(Ext) ++ tmp_suffixes(Ext). -tmp_suffixes() -> [".TMP", ".BUPTMP", ".RET", ".DMP"]. +real_suffixes(Ext) -> [".DAT", ".LOG", ".BUP", ".DCL", ".DCD"] ++ ext_real_suffixes(Ext). + +tmp_suffixes() -> tmp_suffixes(get_ext_types_disc()). + +tmp_suffixes(Ext) -> [".TMP", ".BUPTMP", ".RET", ".DMP", "."] ++ ext_tmp_suffixes(Ext). + +ext_real_suffixes(Ext) -> + try lists:foldl(fun(Mod, Acc) -> Acc++Mod:real_suffixes() end, [], + [M || {_,M} <- Ext]) + catch + error:E -> + verbose("Cant find real ext suffixes (~p)~n", [E]), + [] + end. + +ext_tmp_suffixes(Ext) -> + try lists:foldl(fun(Mod, Acc) -> Acc++Mod:tmp_suffixes() end, [], + [M || {_,M} <- Ext]) + catch + error:E -> + verbose("Cant find tmp ext suffixes (~p)~n", [E]), + [] + end. info() -> Tabs = lists:sort(val({schema, tables})), @@ -2584,10 +3166,7 @@ info2(_, []) -> io:format("~n", []). get_table_properties(Tab) -> - case catch mnesia_lib:db_match_object(ram_copies, - mnesia_gvar, {{Tab, '_'}, '_'}) of - {'EXIT', _} -> - mnesia:abort({no_exists, Tab, all}); + try mnesia_lib:db_match_object(ram_copies, mnesia_gvar, {{Tab, '_'}, '_'}) of RawGvar -> case [{Item, Val} || {{_Tab, Item}, Val} <- RawGvar] of [] -> @@ -2598,6 +3177,8 @@ get_table_properties(Tab) -> Master = {master_nodes, mnesia:table_info(Tab, master_nodes)}, lists:sort([Size, Memory, Master | Gvar]) end + catch error:_ -> + mnesia:abort({no_exists, Tab, all}) end. %%%%%%%%%%% RESTORE %%%%%%%%%%% @@ -2620,15 +3201,15 @@ restore(_Opaque, BadArg) -> {aborted, {badarg, BadArg}}. restore(Opaque, Args, Module) when is_list(Args), is_atom(Module) -> InitR = #r{opaque = Opaque, module = Module}, - case catch lists:foldl(fun check_restore_arg/2, InitR, Args) of + try lists:foldl(fun check_restore_arg/2, InitR, Args) of R when is_record(R, r) -> case mnesia_bup:read_schema(R#r.module, Opaque) of {error, Reason} -> {aborted, Reason}; BupSchema -> schema_transaction(fun() -> do_restore(R, BupSchema) end) - end; - {'EXIT', Reason} -> + end + catch exit:Reason -> {aborted, Reason} end; restore(_Opaque, Args, Module) -> @@ -2681,12 +3262,12 @@ do_restore(R, BupSchema) -> arrange_restore(R, Fun, Recs) -> R2 = R#r{insert_op = Fun, recs = Recs}, - case mnesia_bup:iterate(R#r.module, fun restore_items/4, R#r.opaque, R2) of + case mnesia_bup:iterate(R#r.module, fun restore_items/5, R#r.opaque, R2) of {ok, R3} -> R3#r.recs; {error, Reason} -> mnesia:abort(Reason) end. -restore_items([Rec | Recs], Header, Schema, R) -> +restore_items([Rec | Recs], Header, Schema, Ext, R) -> Tab = element(1, Rec), case lists:keysearch(Tab, 1, R#r.tables) of {value, {Tab, Where0, Snmp, RecName}} -> @@ -2699,13 +3280,13 @@ restore_items([Rec | Recs], Header, Schema, R) -> {Rest, NRecs} = restore_tab_items([Rec | Recs], Tab, RecName, Where, Snmp, R#r.recs, R#r.insert_op), - restore_items(Rest, Header, Schema, R#r{recs = NRecs}); + restore_items(Rest, Header, Schema, Ext, R#r{recs = NRecs}); false -> Rest = skip_tab_items(Recs, Tab), - restore_items(Rest, Header, Schema, R) + restore_items(Rest, Header, Schema, Ext, R) end; -restore_items([], _Header, _Schema, R) -> +restore_items([], _Header, _Schema, _Ext, R) -> R. restore_func(Tab, R) -> @@ -2719,8 +3300,14 @@ restore_func(Tab, R) -> where_to_commit(Tab, CsList) -> Ram = [{N, ram_copies} || N <- pick(Tab, ram_copies, CsList, [])], Disc = [{N, disc_copies} || N <- pick(Tab, disc_copies, CsList, [])], - DiscO = [{N, disc_only_copies} || N <- pick(Tab, disc_only_copies, CsList, [])], - Ram ++ Disc ++ DiscO. + DiscO = [{N, disc_only_copies} || + N <- pick(Tab, disc_only_copies, CsList, [])], + ExtNodes = [{Alias, Mod, pick(Tab, Alias, CsList, [])} || + {Alias, Mod} <- get_ext_types()], + Ext = lists:foldl(fun({Alias, Mod, Ns}, Acc) -> + [{N, {ext, Alias, Mod}} || N <- Ns] ++ Acc + end, [], ExtNodes), + Ram ++ Disc ++ DiscO ++ Ext. %% Changes of the Meta info of schema itself is not allowed restore_schema([{schema, schema, _List} | Schema], R) -> @@ -2799,7 +3386,13 @@ make_dump_tables([schema | _Tabs]) -> make_dump_tables([Tab | Tabs]) -> get_tid_ts_and_lock(Tab, read), TabDef = get_create_list(Tab), - DiscResident = val({Tab, disc_copies}) ++ val({Tab, disc_only_copies}), + DiscResident = + val({Tab, disc_copies}) ++ + val({Tab, disc_only_copies}) ++ + lists:concat([Ns || {{A,M},Ns} <- val({Tab, external_copies}), + lists:member( + mnesia_lib:semantics({ext,A,M},storage), + [disc_copies, disc_only_copies])]), verify([], DiscResident, {"Only allowed on ram_copies", Tab, DiscResident}), [{op, dump_table, unknown, TabDef} | make_dump_tables(Tabs)]; @@ -2811,7 +3404,9 @@ merge_schema() -> schema_transaction(fun() -> do_merge_schema([]) end). merge_schema(UserFun) -> - schema_transaction(fun() -> UserFun(fun(Arg) -> do_merge_schema(Arg) end) end). + schema_transaction(fun() -> + UserFun(fun(Arg) -> do_merge_schema(Arg) end) + end). do_merge_schema(LockTabs0) -> {_Mod, Tid, Ts} = get_tid_ts_and_lock(schema, write), @@ -2885,39 +3480,22 @@ do_merge_schema(LockTabs0) -> end. fetch_cstructs(Node) -> - case need_old_cstructs([Node]) of - false -> - rpc:call(Node, mnesia_controller, get_remote_cstructs, []); - _Ver -> - case rpc:call(Node, mnesia_controller, get_cstructs, []) of - {cstructs, Cs0, RR} -> - {cstructs, [list2cs(cs2list(Cs)) || Cs <- Cs0], RR}; - Err -> Err - end + Convert = mnesia_monitor:needs_protocol_conversion(Node), + case rpc:call(Node, mnesia_controller, get_remote_cstructs, []) of + {cstructs, Cs0, RemoteRunning1} when Convert -> + {cstructs, [list2cs(cs2list(Cs)) || Cs <- Cs0], RemoteRunning1}; + Result -> + Result end. need_old_cstructs() -> need_old_cstructs(val({schema, where_to_write})). need_old_cstructs(Nodes) -> - Filter = fun(Node) -> not mnesia_monitor:needs_protocol_conversion(Node) end, - case lists:dropwhile(Filter, Nodes) of + Filter = fun(Node) -> mnesia_monitor:needs_protocol_conversion(Node) end, + case lists:filter(Filter, Nodes) of [] -> false; - [Node|_] -> - case rpc:call(Node, mnesia_lib, val, [{schema,cstruct}]) of - #cstruct{} -> - %% mnesia_lib:warning("Mnesia on ~p do not need to convert cstruct (~p)~n", - %% [node(), Node]), - false; - {badrpc, _} -> - need_old_cstructs(lists:delete(Node,Nodes)); - Cs when element(1, Cs) == cstruct, tuple_size(Cs) == 17 -> - ver4_4_18; % Without majority - Cs when element(1, Cs) == cstruct, tuple_size(Cs) == 18 -> - ver4_4_19; % With majority - Cs when element(1, Cs) == cstruct, tuple_size(Cs) == 19 -> - ver4_6 % With storage_properties - end + Ns -> lists:min([element(1, ?catch_val({protocol, Node})) || Node <- Ns]) end. tab_to_nodes(Tab) when is_atom(Tab) -> @@ -3071,28 +3649,26 @@ do_make_merge_schema(Node, NeedsConv, RemoteCs = #cstruct{}) -> %% invariants must be enforced in order to allow merge of cstructs. %% %% Returns a new cstruct or issues a fatal error -merge_cstructs(Cs, RemoteCs, Force) -> - verify_cstruct(Cs), - case catch do_merge_cstructs(Cs, RemoteCs, Force) of - {'EXIT', {aborted, _Reason}} when Force == true -> - Cs; - {'EXIT', Reason} -> - exit(Reason); +merge_cstructs(Cs0, RemoteCs, Force) -> + Cs = verify_cstruct(Cs0), + try do_merge_cstructs(Cs, RemoteCs, Force) of MergedCs when is_record(MergedCs, cstruct) -> - MergedCs; - Other -> - throw(Other) + MergedCs + catch exit:{aborted, _Reason} when Force == true -> + Cs; + exit:Reason -> exit(Reason); + error:Reason -> exit(Reason) end. -do_merge_cstructs(Cs, RemoteCs, Force) -> - verify_cstruct(RemoteCs), +do_merge_cstructs(Cs, RemoteCs0, Force) -> + RemoteCs = verify_cstruct(RemoteCs0), Ns = mnesia_lib:uniq(mnesia_lib:cs_to_nodes(Cs) ++ mnesia_lib:cs_to_nodes(RemoteCs)), {AnythingNew, MergedCs} = merge_storage_type(Ns, false, Cs, RemoteCs, Force), - MergedCs2 = merge_versions(AnythingNew, MergedCs, RemoteCs, Force), - verify_cstruct(MergedCs2), - MergedCs2. + verify_cstruct( + merge_versions(AnythingNew, MergedCs, RemoteCs, Force)). + merge_storage_type([N | Ns], AnythingNew, Cs, RemoteCs, Force) -> Local = mnesia_lib:cs_to_storage_type(N, Cs), @@ -3236,4 +3812,3 @@ unannounce_im_running([N | Ns]) -> unannounce_im_running(Ns); unannounce_im_running([]) -> ok. - diff --git a/lib/mnesia/src/mnesia_snmp_hook.erl b/lib/mnesia/src/mnesia_snmp_hook.erl index 893b39f3c0..58a20c6051 100644 --- a/lib/mnesia/src/mnesia_snmp_hook.erl +++ b/lib/mnesia/src/mnesia_snmp_hook.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -30,11 +31,6 @@ -include("mnesia.hrl"). -val(Var) -> - case ?catch_val(Var) of - {'EXIT', _ReASoN_} -> mnesia_lib:other_val(Var, _ReASoN_); - _VaLuE_ -> _VaLuE_ - end. check_ustruct([]) -> true; %% default value, not SNMP'ified @@ -81,12 +77,12 @@ delete_table(_MnesiaTab, Tree) -> %%----------------------------------------------------------------- update({clear_table, MnesiaTab}) -> - Tree = val({MnesiaTab, {index, snmp}}), + Tree = mnesia_lib:val({MnesiaTab, {index, snmp}}), b_clear(Tree), ok; update({Op, MnesiaTab, MnesiaKey, SnmpKey}) -> - Tree = val({MnesiaTab, {index, snmp}}), + Tree = mnesia_lib:val({MnesiaTab, {index, snmp}}), update(Op, Tree, MnesiaKey, SnmpKey). update(Op, Tree, MnesiaKey, SnmpKey) -> @@ -116,7 +112,7 @@ update(Op, Tree, MnesiaKey, SnmpKey) -> %%----------------------------------------------------------------- key_to_oid(Tab,Key) -> - Types = val({Tab,snmp}), + Types = mnesia_lib:val({Tab,snmp}), key_to_oid(Tab, Key, Types). key_to_oid(Tab, Key, [{key, Types}]) -> @@ -140,7 +136,7 @@ keys_to_oid(N, Key, Oid, Types) -> %% This can be lookup up in tree but that might be on a remote node. %% It's probably faster to look it up, but use when it migth be remote oid_to_key(Oid, Tab) -> - [{key, Types}] = val({Tab,snmp}), + [{key, Types}] = mnesia_lib:val({Tab,snmp}), oid_to_key_1(Types, Oid). oid_to_key_1(integer, [Key]) -> Key; diff --git a/lib/mnesia/src/mnesia_snmp_sup.erl b/lib/mnesia/src/mnesia_snmp_sup.erl index 7e86281428..ed579d01c5 100644 --- a/lib/mnesia/src/mnesia_snmp_sup.erl +++ b/lib/mnesia/src/mnesia_snmp_sup.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/mnesia/src/mnesia_sp.erl b/lib/mnesia/src/mnesia_sp.erl index 58a177513f..d65c659690 100644 --- a/lib/mnesia/src/mnesia_sp.erl +++ b/lib/mnesia/src/mnesia_sp.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/mnesia/src/mnesia_subscr.erl b/lib/mnesia/src/mnesia_subscr.erl index 8f78dc55e8..c2748f5bae 100644 --- a/lib/mnesia/src/mnesia_subscr.erl +++ b/lib/mnesia/src/mnesia_subscr.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -186,11 +187,11 @@ patch_record(Tab, Obj) -> end. what(Tab, Tid, {RecName, Key}, delete, undefined) -> - case catch mnesia_lib:db_get(Tab, Key) of - Old when is_list(Old) -> %% Op only allowed for set table. - {mnesia_table_event, {delete, Tab, {RecName, Key}, Old, Tid}}; - _ -> - %% Record just deleted by a dirty_op or + try mnesia_lib:db_get(Tab, Key) of + Old -> %% Op only allowed for set table. + {mnesia_table_event, {delete, Tab, {RecName, Key}, Old, Tid}} + catch error:_ -> + %% Record just deleted by a dirty_op or %% the whole table has been deleted ignore end; @@ -199,12 +200,14 @@ what(Tab, Tid, Obj, delete, Old) -> what(Tab, Tid, Obj, delete_object, _Old) -> {mnesia_table_event, {delete, Tab, Obj, [Obj], Tid}}; what(Tab, Tid, Obj, write, undefined) -> - case catch mnesia_lib:db_get(Tab, element(2, Obj)) of - Old when is_list(Old) -> - {mnesia_table_event, {write, Tab, Obj, Old, Tid}}; - {'EXIT', _} -> + try mnesia_lib:db_get(Tab, element(2, Obj)) of + Old -> + {mnesia_table_event, {write, Tab, Obj, Old, Tid}} + catch error:_ -> ignore - end. + end; +what(Tab, Tid, Obj, write, Old) -> + {mnesia_table_event, {write, Tab, Obj, Old, Tid}}. deliver(_, ignore) -> ok; @@ -223,7 +226,7 @@ call(Msg) -> Res = gen_server:call(Pid, Msg, infinity), %% We get an exit signal if server dies receive - {'EXIT', _Pid, _Reason} -> + {'EXIT', Pid, _Reason} -> {error, {node_not_running, node()}} after 0 -> Res @@ -384,12 +387,12 @@ activate(ClientPid, What, Var, OldSubscribers, SubscrTab) -> case lists:member(ClientPid, Old) of false -> %% Don't care about checking old links - case catch link(ClientPid) of + try link(ClientPid) of true -> ?ets_insert(SubscrTab, {ClientPid, What}), add_subscr(Var, What, ClientPid), - {ok, node()}; - {'EXIT', _Reason} -> + {ok, node()} + catch error:_ -> {error, {no_exists, ClientPid}} end; true -> @@ -441,11 +444,10 @@ add_subscr({Tab, commit_work}, What, Pid) -> deactivate(ClientPid, What, Var, SubscrTab) -> ?ets_match_delete(SubscrTab, {ClientPid, What}), - case catch ?ets_lookup_element(SubscrTab, ClientPid, 1) of - List when is_list(List) -> - ignore; - {'EXIT', _} -> - unlink(ClientPid) + try + ?ets_lookup_element(SubscrTab, ClientPid, 1), + ignore + catch error:_ -> unlink(ClientPid) end, try del_subscr(Var, What, ClientPid), diff --git a/lib/mnesia/src/mnesia_sup.erl b/lib/mnesia/src/mnesia_sup.erl index 8443fefe7f..4aece81308 100644 --- a/lib/mnesia/src/mnesia_sup.erl +++ b/lib/mnesia/src/mnesia_sup.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -59,9 +60,10 @@ init() -> Flags = {one_for_all, 0, 3600}, % Should be rest_for_one policy Event = event_procs(), + Ext = ext_procs(), Kernel = kernel_procs(), - {ok, {Flags, Event ++ Kernel}}. + {ok, {Flags, Event ++ Ext ++ Kernel}}. event_procs() -> KillAfter = timer:seconds(30), @@ -74,6 +76,11 @@ kernel_procs() -> KA = infinity, [{K, {K, start, []}, permanent, KA, supervisor, [K, supervisor]}]. +ext_procs() -> + K = mnesia_ext_sup, + KA = infinity, + [{K, {K, start, []}, permanent, KA, supervisor, [K, supervisor]}]. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% event handler diff --git a/lib/mnesia/src/mnesia_text.erl b/lib/mnesia/src/mnesia_text.erl index 0906d18da9..21adca813a 100644 --- a/lib/mnesia/src/mnesia_text.erl +++ b/lib/mnesia/src/mnesia_text.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2011. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -84,8 +85,12 @@ validate_tab({Tabname, RecName, List}) -> validate_tab(_) -> error(badtab). make_tabs([{Tab, Def} | Tail]) -> - case catch mnesia:table_info(Tab, where_to_read) of - {'EXIT', _} -> %% non-existing table + try mnesia:table_info(Tab, where_to_read) of + Node -> + io:format("** Table ~w already exists on ~p, just entering data~n", + [Tab, Node]), + make_tabs(Tail) + catch exit:_ -> %% non-existing table case mnesia:create_table(Tab, Def) of {aborted, Reason} -> io:format("** Failed to create table ~w ~n" @@ -95,11 +100,7 @@ make_tabs([{Tab, Def} | Tail]) -> _ -> io:format("New table ~w~n", [Tab]), make_tabs(Tail) - end; - Node -> - io:format("** Table ~w already exists on ~p, just entering data~n", - [Tab, Node]), - make_tabs(Tail) + end end; make_tabs([]) -> @@ -118,11 +119,9 @@ load_data(L) -> parse(File) -> case file(File) of {ok, Terms} -> - case catch collect(Terms) of - {error, X} -> - {error, X}; - Other -> - {ok, Other} + try collect(Terms) of + Other -> {ok, Other} + catch throw:Error -> Error end; Other -> Other diff --git a/lib/mnesia/src/mnesia_tm.erl b/lib/mnesia/src/mnesia_tm.erl index e54e5c4e88..b116b48312 100644 --- a/lib/mnesia/src/mnesia_tm.erl +++ b/lib/mnesia/src/mnesia_tm.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2013. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -41,7 +42,8 @@ put_activity_id/2, block_tab/1, unblock_tab/1, - fixtable/3 + fixtable/3, + new_cr_format/1 ]). %% sys callback functions @@ -51,6 +53,7 @@ ]). -include("mnesia.hrl"). + -import(mnesia_lib, [set/2]). -import(mnesia_lib, [fatal/2, verbose/2, dbg_out/2]). @@ -119,7 +122,7 @@ init(Parent) -> val(Var) -> case ?catch_val(Var) of - {'EXIT', _ReASoN_} -> mnesia_lib:other_val(Var, _ReASoN_); + {'EXIT', _} -> mnesia_lib:other_val(Var); _VaLuE_ -> _VaLuE_ end. @@ -181,9 +184,10 @@ mnesia_down(Node) -> %% mnesia_monitor takes care of the sync case whereis(?MODULE) of undefined -> - mnesia_monitor:mnesia_down(?MODULE, {Node, []}); + mnesia_monitor:mnesia_down(?MODULE, Node); Pid -> - Pid ! {mnesia_down, Node} + Pid ! {mnesia_down, Node}, + ok end. prepare_checkpoint(Nodes, Cp) -> @@ -203,10 +207,10 @@ doit_loop(#state{coordinators=Coordinators,participants=Participants,supervisor= {_From, {async_dirty, Tid, Commit, Tab}} -> case lists:member(Tab, State#state.blocked_tabs) of false -> - do_async_dirty(Tid, Commit, Tab), + do_async_dirty(Tid, new_cr_format(Commit), Tab), doit_loop(State); true -> - Item = {async_dirty, Tid, Commit, Tab}, + Item = {async_dirty, Tid, new_cr_format(Commit), Tab}, State2 = State#state{dirty_queue = [Item | State#state.dirty_queue]}, doit_loop(State2) end; @@ -214,20 +218,16 @@ doit_loop(#state{coordinators=Coordinators,participants=Participants,supervisor= {From, {sync_dirty, Tid, Commit, Tab}} -> case lists:member(Tab, State#state.blocked_tabs) of false -> - do_sync_dirty(From, Tid, Commit, Tab), + do_sync_dirty(From, Tid, new_cr_format(Commit), Tab), doit_loop(State); true -> - Item = {sync_dirty, From, Tid, Commit, Tab}, + Item = {sync_dirty, From, Tid, new_cr_format(Commit), Tab}, State2 = State#state{dirty_queue = [Item | State#state.dirty_queue]}, doit_loop(State2) end; {From, start_outer} -> %% Create and associate ets_tab with Tid - case catch ?ets_new_table(mnesia_trans_store, [bag, public]) of - {'EXIT', Reason} -> %% system limit - Msg = "Cannot create an ets table for the " - "local transaction store", - reply(From, {error, {system_limit, Msg, Reason}}, State); + try ?ets_new_table(mnesia_trans_store, [bag, public]) of Etab -> tmlink(From), C = mnesia_recover:incr_trans_tid_serial(), @@ -236,12 +236,17 @@ doit_loop(#state{coordinators=Coordinators,participants=Participants,supervisor= A2 = gb_trees:insert(Tid,[Etab],Coordinators), S2 = State#state{coordinators = A2}, reply(From, {new_tid, Tid, Etab}, S2) + catch error:Reason -> %% system limit + Msg = "Cannot create an ets table for the " + "local transaction store", + reply(From, {error, {system_limit, Msg, Reason}}, State) end; - {From, {ask_commit, Protocol, Tid, Commit, DiscNs, RamNs}} -> + {From, {ask_commit, Protocol, Tid, Commit0, DiscNs, RamNs}} -> ?eval_debug_fun({?MODULE, doit_ask_commit}, [{tid, Tid}, {prot, Protocol}]), mnesia_checkpoint:tm_enter_pending(Tid, DiscNs, RamNs), + Commit = new_cr_format(Commit0), Pid = case Protocol of asym_trans when node(Tid#tid.pid) /= node() -> @@ -338,15 +343,15 @@ doit_loop(#state{coordinators=Coordinators,participants=Participants,supervisor= end; {From, {add_store, Tid}} -> %% new store for nested transaction - case catch ?ets_new_table(mnesia_trans_store, [bag, public]) of - {'EXIT', Reason} -> %% system limit - Msg = "Cannot create an ets table for a nested " - "local transaction store", - reply(From, {error, {system_limit, Msg, Reason}}, State); + try ?ets_new_table(mnesia_trans_store, [bag, public]) of Etab -> A2 = add_coord_store(Coordinators, Tid, Etab), reply(From, {new_store, Etab}, State#state{coordinators = A2}) + catch error:Reason -> %% system limit + Msg = "Cannot create an ets table for a nested " + "local transaction store", + reply(From, {error, {system_limit, Msg, Reason}}, State) end; {From, {del_store, Tid, Current, Obsolete, PropagateStore}} -> @@ -403,7 +408,9 @@ doit_loop(#state{coordinators=Coordinators,participants=Participants,supervisor= Tids = gb_trees:keys(Participants), reconfigure_participants(N, gb_trees:values(Participants)), NewState = clear_fixtable(N, State), - mnesia_monitor:mnesia_down(?MODULE, {N, Tids}), + + mnesia_locker:mnesia_down(N, Tids), + mnesia_monitor:mnesia_down(?MODULE, N), doit_loop(NewState); {From, {unblock_me, Tab}} -> @@ -468,13 +475,13 @@ doit_loop(#state{coordinators=Coordinators,participants=Participants,supervisor= do_sync_dirty(From, Tid, Commit, _Tab) -> ?eval_debug_fun({?MODULE, sync_dirty, pre}, [{tid, Tid}]), - Res = (catch do_dirty(Tid, Commit)), + Res = do_dirty(Tid, Commit), ?eval_debug_fun({?MODULE, sync_dirty, post}, [{tid, Tid}]), From ! {?MODULE, node(), {dirty_res, Res}}. do_async_dirty(Tid, Commit, _Tab) -> ?eval_debug_fun({?MODULE, async_dirty, pre}, [{tid, Tid}]), - catch do_dirty(Tid, Commit), + do_dirty(Tid, Commit), ?eval_debug_fun({?MODULE, async_dirty, post}, [{tid, Tid}]). @@ -498,7 +505,7 @@ process_dirty_queue(_Tab, []) -> []. prepare_pending_coordinators([{Tid, [Store | _Etabs]} | Coords], IgnoreNew) -> - case catch ?ets_lookup(Store, pending) of + try ?ets_lookup(Store, pending) of [] -> prepare_pending_coordinators(Coords, IgnoreNew); [Pending] -> @@ -508,8 +515,8 @@ prepare_pending_coordinators([{Tid, [Store | _Etabs]} | Coords], IgnoreNew) -> true -> ignore end, - prepare_pending_coordinators(Coords, IgnoreNew); - {'EXIT', _} -> + prepare_pending_coordinators(Coords, IgnoreNew) + catch error:_ -> prepare_pending_coordinators(Coords, IgnoreNew) end; prepare_pending_coordinators([], _IgnoreNew) -> @@ -570,11 +577,7 @@ recover_coordinator(Tid, Etabs) -> Store = hd(Etabs), CheckNodes = get_elements(nodes,Store), TellNodes = CheckNodes -- [node()], - case catch arrange(Tid, Store, async) of - {'EXIT', Reason} -> - dbg_out("Recovery of coordinator ~p failed:~n", [Tid, Reason]), - Protocol = asym_trans, - tell_outcome(Tid, Protocol, node(), CheckNodes, TellNodes); + try arrange(Tid, Store, async) of {_N, Prep} -> %% Tell the participants about the outcome Protocol = Prep#prep.protocol, @@ -593,6 +596,11 @@ recover_coordinator(Tid, Etabs) -> false -> %% When killed before store havn't been copied to ok %% to the new nested trans store. end + catch _:Reason -> + dbg_out("Recovery of coordinator ~p failed:~n", + [Tid, {Reason, erlang:get_stacktrace()}]), + Protocol = asym_trans, + tell_outcome(Tid, Protocol, node(), CheckNodes, TellNodes) end, erase_ets_tabs(Etabs), transaction_terminated(Tid), @@ -721,33 +729,25 @@ non_transaction(OldState={_,_,Trans}, Fun, Args, ActivityKind, Mod) _ -> async end, case transaction(OldState, Fun, Args, infinity, Mod, Kind) of - {atomic, Res} -> - Res; - {aborted,Res} -> - exit(Res) + {atomic, Res} -> Res; + {aborted,Res} -> exit(Res) end; non_transaction(OldState, Fun, Args, ActivityKind, Mod) -> Id = {ActivityKind, self()}, NewState = {Mod, Id, non_transaction}, put(mnesia_activity_state, NewState), - %% I Want something uniqe here, references are expensive - Ref = mNeSia_nOn_TrAnSacTioN, - RefRes = (catch {Ref, apply(Fun, Args)}), - case OldState of - undefined -> erase(mnesia_activity_state); - _ -> put(mnesia_activity_state, OldState) - end, - case RefRes of - {Ref, Res} -> - case Res of - {'EXIT', Reason} -> exit(Reason); - {aborted, Reason} -> mnesia:abort(Reason); - _ -> Res - end; - {'EXIT', Reason} -> - exit(Reason); - Throw -> - throw(Throw) + try apply(Fun, Args) of + {'EXIT', Reason} -> exit(Reason); + {aborted, Reason} -> mnesia:abort(Reason); + Res -> Res + catch + throw:Throw -> throw(Throw); + _:Reason -> exit(Reason) + after + case OldState of + undefined -> erase(mnesia_activity_state); + _ -> put(mnesia_activity_state, OldState) + end end. transaction(OldTidTs, Fun, Args, Retries, Mod, Type) -> @@ -807,23 +807,28 @@ insert_objs([], _Tab) -> ok. execute_transaction(Fun, Args, Factor, Retries, Type) -> - case catch apply_fun(Fun, Args, Type) of - {'EXIT', Reason} -> - check_exit(Fun, Args, Factor, Retries, Reason, Type); + try apply_fun(Fun, Args, Type) of {atomic, Value} -> mnesia_lib:incr_counter(trans_commits), erase(mnesia_activity_state), %% no need to clear locks, already done by commit ... %% Flush any un processed mnesia_down messages we might have flush_downs(), - catch unlink(whereis(?MODULE)), + ?SAFE(unlink(whereis(?MODULE))), {atomic, Value}; + {do_abort, Reason} -> + check_exit(Fun, Args, Factor, Retries, {aborted, Reason}, Type); {nested_atomic, Value} -> mnesia_lib:incr_counter(trans_commits), - {atomic, Value}; - Value -> %% User called throw + {atomic, Value} + catch throw:Value -> %% User called throw Reason = {aborted, {throw, Value}}, - return_abort(Fun, Args, Reason) + return_abort(Fun, Args, Reason); + error:Reason -> + ST = erlang:get_stacktrace(), + check_exit(Fun, Args, Factor, Retries, {Reason,ST}, Type); + _:Reason -> + check_exit(Fun, Args, Factor, Retries, Reason, Type) end. apply_fun(Fun, Args, Type) -> @@ -833,10 +838,10 @@ apply_fun(Fun, Args, Type) -> {atomic, Result}; do_commit_nested -> {nested_atomic, Result}; - {do_abort, {aborted, Reason}} -> - {'EXIT', {aborted, Reason}}; - {do_abort, Reason} -> - {'EXIT', {aborted, Reason}} + {do_abort, {aborted, Reason}} -> + {do_abort, Reason}; + {do_abort, _} = Abort -> + Abort end. check_exit(Fun, Args, Factor, Retries, Reason, Type) -> @@ -940,7 +945,7 @@ return_abort(Fun, Args, Reason) -> OldStore = Ts#tidstore.store, Nodes = get_elements(nodes, OldStore), intercept_friends(Tid, Ts), - catch mnesia_lib:incr_counter(trans_failures), + ?SAFE(mnesia_lib:incr_counter(trans_failures)), Level = Ts#tidstore.level, if Level == 1 -> @@ -948,7 +953,7 @@ return_abort(Fun, Args, Reason) -> ?MODULE ! {delete_transaction, Tid}, erase(mnesia_activity_state), flush_downs(), - catch unlink(whereis(?MODULE)), + ?SAFE(unlink(whereis(?MODULE))), {aborted, mnesia_lib:fix_error(Reason)}; true -> %% Nested transaction @@ -1002,11 +1007,11 @@ erase_activity_id() -> erase(mnesia_activity_state). get_elements(Type,Store) -> - case catch ?ets_lookup(Store, Type) of + try ?ets_lookup(Store, Type) of [] -> []; [{_,Val}] -> [Val]; - {'EXIT', _} -> []; Vals -> [Val|| {_,Val} <- Vals] + catch error:_ -> [] end. opt_propagate_store(_Current, _Obsolete, false) -> @@ -1029,7 +1034,7 @@ intercept_friends(_Tid, Ts) -> intercept_best_friend([],_) -> ok; intercept_best_friend([{stop,Fun} | R],Ignore) -> - catch Fun(), + ?CATCH(Fun()), intercept_best_friend(R,Ignore); intercept_best_friend([Pid | R],false) -> Pid ! {activity_ended, undefined, self()}, @@ -1043,25 +1048,12 @@ wait_for_best_friend(Pid, Timeout) -> {'EXIT', Pid, _} -> ok; {activity_ended, _, Pid} -> ok after Timeout -> - case my_process_is_alive(Pid) of + case erlang:is_process_alive(Pid) of true -> wait_for_best_friend(Pid, 1000); false -> ok end end. -my_process_is_alive(Pid) -> - case catch erlang:is_process_alive(Pid) of % New BIF in R5 - true -> - true; - false -> - false; - {'EXIT', _} -> % Pre R5 backward compatibility - case process_info(Pid, message_queue_len) of - undefined -> false; - _ -> true - end - end. - dirty(Protocol, Item) -> {{Tab, Key}, _Val, _Op} = Item, Tid = {dirty, self()}, @@ -1141,30 +1133,20 @@ arrange(Tid, Store, Type) -> async -> #prep{protocol = sym_trans, records = Recs}; sync -> #prep{protocol = sync_sym_trans, records = Recs} end, - case catch do_arrange(Tid, Store, Key, Prep, N) of - {'EXIT', Reason} -> - dbg_out("do_arrange failed ~p ~p~n", [Reason, Tid]), - case Reason of - {aborted, R} -> - mnesia:abort(R); - _ -> - mnesia:abort(Reason) - end; - {New, Prepared} -> - {New, Prepared#prep{records = reverse(Prepared#prep.records)}} - end. + {New, Prepared} = do_arrange(Tid, Store, Key, Prep, N), + {New, Prepared#prep{records = reverse(Prepared#prep.records)}}. reverse([]) -> []; reverse([H=#commit{ram_copies=Ram, disc_copies=DC, - disc_only_copies=DOC,snmp = Snmp} + disc_only_copies=DOC, ext=Ext} |R]) -> [ H#commit{ - ram_copies = lists:reverse(Ram), - disc_copies = lists:reverse(DC), - disc_only_copies = lists:reverse(DOC), - snmp = lists:reverse(Snmp) + ram_copies = lists:reverse(Ram), + disc_copies = lists:reverse(DC), + disc_only_copies = lists:reverse(DOC), + ext = [{Type, lists:reverse(E)} || {Type,E} <- Ext] } | reverse(R)]. @@ -1331,8 +1313,13 @@ pick_node({dirty,_}, Node, [], Done) -> pick_node(_Tid, Node, [], _Done) -> mnesia:abort({bad_commit, {missing_lock, Node}}). -prepare_node(Node, Storage, [Item | Items], Rec, Kind) when Kind == snmp -> - Rec2 = Rec#commit{snmp = [Item | Rec#commit.snmp]}, +prepare_node(Node, Storage, [Item | Items], #commit{ext=Ext0}=Rec, Kind) when Kind == snmp -> + Rec2 = case lists:keytake(snmp, 1, Ext0) of + false -> + Rec#commit{ext = [{snmp,[Item]}|Ext0]}; + {_, {snmp,Snmp},Ext} -> + Rec#commit{ext = [{snmp,[Item|Snmp]}|Ext]} + end, prepare_node(Node, Storage, Items, Rec2, Kind); prepare_node(Node, Storage, [Item | Items], Rec, Kind) when Kind /= schema -> Rec2 = @@ -1343,7 +1330,15 @@ prepare_node(Node, Storage, [Item | Items], Rec, Kind) when Kind /= schema -> Rec#commit{disc_copies = [Item | Rec#commit.disc_copies]}; disc_only_copies -> Rec#commit{disc_only_copies = - [Item | Rec#commit.disc_only_copies]} + [Item | Rec#commit.disc_only_copies]}; + {ext, Alias, Mod} -> + Ext0 = Rec#commit.ext, + case lists:keytake(ext_copies, 1, Ext0) of + false -> + Rec#commit{ext = [{ext_copies, [{{ext,Alias,Mod}, Item}]}|Ext0]}; + {_,{_,EC},Ext} -> + Rec#commit{ext = [{ext_copies, [{{ext,Alias,Mod}, Item}|EC]}|Ext]} + end end, prepare_node(Node, Storage, Items, Rec2, Kind); prepare_node(_Node, _Storage, Items, Rec, Kind) @@ -1519,7 +1514,7 @@ multi_commit(asym_trans, Majority, Tid, CR, Store) -> Pending = mnesia_checkpoint:tm_enter_pending(Tid, DiscNs, RamNs), ?ets_insert(Store, Pending), {WaitFor, Local} = ask_commit(asym_trans, Tid, CR2, DiscNs, RamNs), - SchemaPrep = (catch mnesia_schema:prepare_commit(Tid, Local, {coord, WaitFor})), + SchemaPrep = ?CATCH(mnesia_schema:prepare_commit(Tid, Local, {coord, WaitFor})), {Votes, Pids} = rec_all(WaitFor, Tid, do_commit, []), ?eval_debug_fun({?MODULE, multi_commit_asym_got_votes}, @@ -1586,7 +1581,7 @@ rec_acc_pre_commit([Pid | Tail], Tid, Store, Commit, Res, DumperMode, GoodPids, SchemaAckPids); {mnesia_down, Node} when Node == node(Pid) -> AbortRes = {do_abort, {bad_commit, Node}}, - catch Pid ! {Tid, AbortRes}, %% Tell him that he has died + ?SAFE(Pid ! {Tid, AbortRes}), %% Tell him that he has died rec_acc_pre_commit(Tail, Tid, Store, Commit, AbortRes, DumperMode, GoodPids, SchemaAckPids) end; @@ -1663,7 +1658,7 @@ commit_participant(Coord, Tid, C = #commit{}, DiscNs, RamNs) -> commit_participant(Coord, Tid, Bin, C0, DiscNs, _RamNs) -> ?eval_debug_fun({?MODULE, commit_participant, pre}, [{tid, Tid}]), - case catch mnesia_schema:prepare_commit(Tid, C0, {part, Coord}) of + try mnesia_schema:prepare_commit(Tid, C0, {part, Coord}) of {Modified, C = #commit{}, DumperMode} -> %% If we can not find any local unclear decision %% we should presume abort at startup recovery @@ -1712,13 +1707,10 @@ commit_participant(Coord, Tid, Bin, C0, DiscNs, _RamNs) -> ?eval_debug_fun({?MODULE, commit_participant, undo_prepare}, [{tid, Tid}]); - {'EXIT', _, _} -> + {'EXIT', _MnesiaTM, Reason} -> + reply(Coord, {do_abort, Tid, self(), {bad_commit,Reason}}), mnesia_recover:log_decision(D#decision{outcome = aborted}), - ?eval_debug_fun({?MODULE, commit_participant, exit_log_abort}, - [{tid, Tid}]), - mnesia_schema:undo_prepare_commit(Tid, C0), - ?eval_debug_fun({?MODULE, commit_participant, exit_undo_prepare}, - [{tid, Tid}]); + mnesia_schema:undo_prepare_commit(Tid, C0); Msg -> verbose("** ERROR ** commit_participant ~p, got unexpected msg: ~p~n", @@ -1739,9 +1731,8 @@ commit_participant(Coord, Tid, Bin, C0, DiscNs, _RamNs) -> reply(Coord, {do_abort, Tid, self(), {bad_commit,internal}}), verbose("** ERROR ** commit_participant ~p, got unexpected msg: ~p~n", [Tid, Msg]) - end; - - {'EXIT', Reason} -> + end + catch _:Reason -> ?eval_debug_fun({?MODULE, commit_participant, vote_no}, [{tid, Tid}]), reply(Coord, {vote_no, Tid, Reason}), @@ -1774,180 +1765,192 @@ do_commit(Tid, Bin) when is_binary(Bin) -> do_commit(Tid, binary_to_term(Bin)); do_commit(Tid, C) -> do_commit(Tid, C, optional). + do_commit(Tid, Bin, DumperMode) when is_binary(Bin) -> do_commit(Tid, binary_to_term(Bin), DumperMode); do_commit(Tid, C, DumperMode) -> mnesia_dumper:update(Tid, C#commit.schema_ops, DumperMode), - R = do_snmp(Tid, C#commit.snmp), + R = do_snmp(Tid, proplists:get_value(snmp, C#commit.ext, [])), R2 = do_update(Tid, ram_copies, C#commit.ram_copies, R), R3 = do_update(Tid, disc_copies, C#commit.disc_copies, R2), R4 = do_update(Tid, disc_only_copies, C#commit.disc_only_copies, R3), + R5 = do_update_ext(Tid, C#commit.ext, R4), mnesia_subscr:report_activity(Tid), - R4. + R5. + +%% This could/should be optimized +do_update_ext(_Tid, [], OldRes) -> OldRes; +do_update_ext(Tid, Ext, OldRes) -> + case lists:keyfind(ext_copies, 1, Ext) of + false -> OldRes; + {_, Ops} -> + Do = fun({{ext, _,_} = Storage, Op}, R) -> + do_update(Tid, Storage, [Op], R) + end, + lists:foldl(Do, OldRes, Ops) + end. %% Update the items do_update(Tid, Storage, [Op | Ops], OldRes) -> - case catch do_update_op(Tid, Storage, Op) of - ok -> - do_update(Tid, Storage, Ops, OldRes); - {'EXIT', Reason} -> + try do_update_op(Tid, Storage, Op) of + ok -> do_update(Tid, Storage, Ops, OldRes); + NewRes -> do_update(Tid, Storage, Ops, NewRes) + catch _:Reason -> %% This may only happen when we recently have %% deleted our local replica, changed storage_type %% or transformed table %% BUGBUG: Updates may be lost if storage_type is changed. %% Determine actual storage type and try again. %% BUGBUG: Updates may be lost if table is transformed. - + ST = erlang:get_stacktrace(), verbose("do_update in ~w failed: ~p -> {'EXIT', ~p}~n", - [Tid, Op, Reason]), - do_update(Tid, Storage, Ops, OldRes); - NewRes -> - do_update(Tid, Storage, Ops, NewRes) + [Tid, Op, {Reason, ST}]), + do_update(Tid, Storage, Ops, OldRes) end; do_update(_Tid, _Storage, [], Res) -> Res. do_update_op(Tid, Storage, {{Tab, K}, Obj, write}) -> - commit_write(?catch_val({Tab, commit_work}), Tid, + commit_write(?catch_val({Tab, commit_work}), Tid, Storage, Tab, K, Obj, undefined), mnesia_lib:db_put(Storage, Tab, Obj); do_update_op(Tid, Storage, {{Tab, K}, Val, delete}) -> - commit_delete(?catch_val({Tab, commit_work}), Tid, Tab, K, Val, undefined), + commit_delete(?catch_val({Tab, commit_work}), Tid, Storage, Tab, K, Val, undefined), mnesia_lib:db_erase(Storage, Tab, K); do_update_op(Tid, Storage, {{Tab, K}, {RecName, Incr}, update_counter}) -> {NewObj, OldObjs} = - case catch mnesia_lib:db_update_counter(Storage, Tab, K, Incr) of - NewVal when is_integer(NewVal), NewVal >= 0 -> - {{RecName, K, NewVal}, [{RecName, K, NewVal - Incr}]}; - _ when Incr > 0 -> + try + NewVal = mnesia_lib:db_update_counter(Storage, Tab, K, Incr), + true = is_integer(NewVal) andalso (NewVal >= 0), + {{RecName, K, NewVal}, [{RecName, K, NewVal - Incr}]} + catch error:_ when Incr > 0 -> New = {RecName, K, Incr}, mnesia_lib:db_put(Storage, Tab, New), {New, []}; - _ -> + error:_ -> Zero = {RecName, K, 0}, mnesia_lib:db_put(Storage, Tab, Zero), {Zero, []} end, - commit_update(?catch_val({Tab, commit_work}), Tid, Tab, + commit_update(?catch_val({Tab, commit_work}), Tid, Storage, Tab, K, NewObj, OldObjs), element(3, NewObj); do_update_op(Tid, Storage, {{Tab, Key}, Obj, delete_object}) -> commit_del_object(?catch_val({Tab, commit_work}), - Tid, Tab, Key, Obj, undefined), + Tid, Storage, Tab, Key, Obj), mnesia_lib:db_match_erase(Storage, Tab, Obj); do_update_op(Tid, Storage, {{Tab, Key}, Obj, clear_table}) -> - commit_clear(?catch_val({Tab, commit_work}), Tid, Tab, Key, Obj), + commit_clear(?catch_val({Tab, commit_work}), Tid, Storage, Tab, Key, Obj), mnesia_lib:db_match_erase(Storage, Tab, Obj). -commit_write([], _, _, _, _, _) -> ok; -commit_write([{checkpoints, CpList}|R], Tid, Tab, K, Obj, Old) -> +commit_write([], _, _, _, _, _, _) -> ok; +commit_write([{checkpoints, CpList}|R], Tid, Storage, Tab, K, Obj, Old) -> mnesia_checkpoint:tm_retain(Tid, Tab, K, write, CpList), - commit_write(R, Tid, Tab, K, Obj, Old); -commit_write([H|R], Tid, Tab, K, Obj, Old) + commit_write(R, Tid, Storage, Tab, K, Obj, Old); +commit_write([H|R], Tid, Storage, Tab, K, Obj, Old) when element(1, H) == subscribers -> mnesia_subscr:report_table_event(H, Tab, Tid, Obj, write, Old), - commit_write(R, Tid, Tab, K, Obj, Old); -commit_write([H|R], Tid, Tab, K, Obj, Old) + commit_write(R, Tid, Storage, Tab, K, Obj, Old); +commit_write([H|R], Tid, Storage, Tab, K, Obj, Old) when element(1, H) == index -> - mnesia_index:add_index(H, Tab, K, Obj, Old), - commit_write(R, Tid, Tab, K, Obj, Old). + mnesia_index:add_index(H, Storage, Tab, K, Obj, Old), + commit_write(R, Tid, Storage, Tab, K, Obj, Old). -commit_update([], _, _, _, _, _) -> ok; -commit_update([{checkpoints, CpList}|R], Tid, Tab, K, Obj, _) -> +commit_update([], _, _, _, _, _, _) -> ok; +commit_update([{checkpoints, CpList}|R], Tid, Storage, Tab, K, Obj, _) -> Old = mnesia_checkpoint:tm_retain(Tid, Tab, K, write, CpList), - commit_update(R, Tid, Tab, K, Obj, Old); -commit_update([H|R], Tid, Tab, K, Obj, Old) + commit_update(R, Tid, Storage, Tab, K, Obj, Old); +commit_update([H|R], Tid, Storage, Tab, K, Obj, Old) when element(1, H) == subscribers -> mnesia_subscr:report_table_event(H, Tab, Tid, Obj, write, Old), - commit_update(R, Tid, Tab, K, Obj, Old); -commit_update([H|R], Tid, Tab, K, Obj, Old) + commit_update(R, Tid, Storage, Tab, K, Obj, Old); +commit_update([H|R], Tid,Storage, Tab, K, Obj, Old) when element(1, H) == index -> - mnesia_index:add_index(H, Tab, K, Obj, Old), - commit_update(R, Tid, Tab, K, Obj, Old). + mnesia_index:add_index(H, Storage, Tab, K, Obj, Old), + commit_update(R, Tid, Storage, Tab, K, Obj, Old). -commit_delete([], _, _, _, _, _) -> ok; -commit_delete([{checkpoints, CpList}|R], Tid, Tab, K, Obj, _) -> +commit_delete([], _, _, _, _, _, _) -> ok; +commit_delete([{checkpoints, CpList}|R], Tid, Storage, Tab, K, Obj, _) -> Old = mnesia_checkpoint:tm_retain(Tid, Tab, K, delete, CpList), - commit_delete(R, Tid, Tab, K, Obj, Old); -commit_delete([H|R], Tid, Tab, K, Obj, Old) + commit_delete(R, Tid, Storage, Tab, K, Obj, Old); +commit_delete([H|R], Tid, Storage, Tab, K, Obj, Old) when element(1, H) == subscribers -> mnesia_subscr:report_table_event(H, Tab, Tid, Obj, delete, Old), - commit_delete(R, Tid, Tab, K, Obj, Old); -commit_delete([H|R], Tid, Tab, K, Obj, Old) + commit_delete(R, Tid, Storage, Tab, K, Obj, Old); +commit_delete([H|R], Tid, Storage, Tab, K, Obj, Old) when element(1, H) == index -> - mnesia_index:delete_index(H, Tab, K), - commit_delete(R, Tid, Tab, K, Obj, Old). + mnesia_index:delete_index(H, Storage, Tab, K), + commit_delete(R, Tid, Storage, Tab, K, Obj, Old). commit_del_object([], _, _, _, _, _) -> ok; -commit_del_object([{checkpoints, CpList}|R], Tid, Tab, K, Obj, _) -> - Old = mnesia_checkpoint:tm_retain(Tid, Tab, K, delete_object, CpList), - commit_del_object(R, Tid, Tab, K, Obj, Old); -commit_del_object([H|R], Tid, Tab, K, Obj, Old) - when element(1, H) == subscribers -> - mnesia_subscr:report_table_event(H, Tab, Tid, Obj, delete_object, Old), - commit_del_object(R, Tid, Tab, K, Obj, Old); -commit_del_object([H|R], Tid, Tab, K, Obj, Old) - when element(1, H) == index -> - mnesia_index:del_object_index(H, Tab, K, Obj, Old), - commit_del_object(R, Tid, Tab, K, Obj, Old). - -commit_clear([], _, _, _, _) -> ok; -commit_clear([{checkpoints, CpList}|R], Tid, Tab, K, Obj) -> +commit_del_object([{checkpoints, CpList}|R], Tid, Storage, Tab, K, Obj) -> + mnesia_checkpoint:tm_retain(Tid, Tab, K, delete_object, CpList), + commit_del_object(R, Tid, Storage, Tab, K, Obj); +commit_del_object([H|R], Tid, Storage, Tab, K, Obj) when element(1, H) == subscribers -> + mnesia_subscr:report_table_event(H, Tab, Tid, Obj, delete_object), + commit_del_object(R, Tid, Storage, Tab, K, Obj); +commit_del_object([H|R], Tid, Storage, Tab, K, Obj) when element(1, H) == index -> + mnesia_index:del_object_index(H, Storage, Tab, K, Obj), + commit_del_object(R, Tid, Storage, Tab, K, Obj). + +commit_clear([], _, _, _, _, _) -> ok; +commit_clear([{checkpoints, CpList}|R], Tid, Storage, Tab, K, Obj) -> mnesia_checkpoint:tm_retain(Tid, Tab, K, clear_table, CpList), - commit_clear(R, Tid, Tab, K, Obj); -commit_clear([H|R], Tid, Tab, K, Obj) + commit_clear(R, Tid, Storage, Tab, K, Obj); +commit_clear([H|R], Tid, Storage, Tab, K, Obj) when element(1, H) == subscribers -> mnesia_subscr:report_table_event(H, Tab, Tid, Obj, clear_table, undefined), - commit_clear(R, Tid, Tab, K, Obj); -commit_clear([H|R], Tid, Tab, K, Obj) + commit_clear(R, Tid, Storage, Tab, K, Obj); +commit_clear([H|R], Tid, Storage, Tab, K, Obj) when element(1, H) == index -> mnesia_index:clear_index(H, Tab, K, Obj), - commit_clear(R, Tid, Tab, K, Obj). + commit_clear(R, Tid, Storage, Tab, K, Obj). do_snmp(_, []) -> ok; -do_snmp(Tid, [Head | Tail]) -> - case catch mnesia_snmp_hook:update(Head) of - {'EXIT', Reason} -> +do_snmp(Tid, [Head|Tail]) -> + try mnesia_snmp_hook:update(Head) + catch _:Reason -> %% This should only happen when we recently have %% deleted our local replica or recently deattached %% the snmp table - + ST = erlang:get_stacktrace(), verbose("do_snmp in ~w failed: ~p -> {'EXIT', ~p}~n", - [Tid, Head, Reason]); - ok -> - ignore + [Tid, Head, {Reason, ST}]) end, do_snmp(Tid, Tail). -commit_nodes([C | Tail], AccD, AccR) - when C#commit.disc_copies == [], - C#commit.disc_only_copies == [], - C#commit.schema_ops == [] -> - commit_nodes(Tail, AccD, [C#commit.node | AccR]); commit_nodes([C | Tail], AccD, AccR) -> - commit_nodes(Tail, [C#commit.node | AccD], AccR); + case C of + #commit{disc_copies=[], disc_only_copies=[], schema_ops=[], ext=Ext} -> + case lists:keyfind(ext_copies, 1, Ext) of + false -> commit_nodes(Tail, AccD, [C#commit.node | AccR]); + _ -> commit_nodes(Tail, [C#commit.node | AccD], AccR) + end; + _ -> + commit_nodes(Tail, [C#commit.node | AccD], AccR) + end; commit_nodes([], AccD, AccR) -> {AccD, AccR}. commit_decision(D, [C | Tail], AccD, AccR) -> N = C#commit.node, {D2, Tail2} = - case C#commit.schema_ops of - [] when C#commit.disc_copies == [], - C#commit.disc_only_copies == [] -> - commit_decision(D, Tail, AccD, [N | AccR]); - [] -> + case C of + #commit{disc_copies=[], disc_only_copies=[], schema_ops=[], ext=Ext} -> + case lists:keyfind(ext_copies, 1, Ext) of + false -> commit_decision(D, Tail, AccD, [N | AccR]); + _ -> commit_decision(D, Tail, [N | AccD], AccR) + end; + #commit{schema_ops=[]} -> commit_decision(D, Tail, [N | AccD], AccR); - Ops -> + #commit{schema_ops=Ops} -> case ram_only_ops(N, Ops) of - true -> - commit_decision(D, Tail, AccD, [N | AccR]); - false -> - commit_decision(D, Tail, [N | AccD], AccR) + true -> commit_decision(D, Tail, AccD, [N | AccR]); + false -> commit_decision(D, Tail, [N | AccD], AccR) end end, {D2, [C#commit{decision = D2} | Tail2]}; @@ -1975,7 +1978,7 @@ sync_send_dirty(Tid, [Head | Tail], Tab, WaitFor) -> Res = do_dirty(Tid, Head), {WF, Res}; true -> - {?MODULE, Node} ! {self(), {sync_dirty, Tid, Head, Tab}}, + {?MODULE, Node} ! {self(), {sync_dirty, Tid, ext_format(Head), Tab}}, sync_send_dirty(Tid, Tail, Tab, [Node | WaitFor]) end; sync_send_dirty(_Tid, [], _Tab, WaitFor) -> @@ -1994,11 +1997,11 @@ async_send_dirty(Tid, [Head | Tail], Tab, ReadNode, WaitFor, Res) -> NewRes = do_dirty(Tid, Head), async_send_dirty(Tid, Tail, Tab, ReadNode, WaitFor, NewRes); ReadNode == Node -> - {?MODULE, Node} ! {self(), {sync_dirty, Tid, Head, Tab}}, + {?MODULE, Node} ! {self(), {sync_dirty, Tid, ext_format(Head), Tab}}, NewRes = {'EXIT', {aborted, {node_not_running, Node}}}, async_send_dirty(Tid, Tail, Tab, ReadNode, [Node | WaitFor], NewRes); true -> - {?MODULE, Node} ! {self(), {async_dirty, Tid, Head, Tab}}, + {?MODULE, Node} ! {self(), {async_dirty, Tid, ext_format(Head), Tab}}, async_send_dirty(Tid, Tail, Tab, ReadNode, WaitFor, Res) end; async_send_dirty(_Tid, [], _Tab, _ReadNode, WaitFor, Res) -> @@ -2055,23 +2058,29 @@ ask_commit(Protocol, Tid, [Head | Tail], DiscNs, RamNs, WaitFor, Local) -> Node == node() -> ask_commit(Protocol, Tid, Tail, DiscNs, RamNs, WaitFor, Head); true -> - Bin = opt_term_to_binary(Protocol, Head, DiscNs++RamNs), - Msg = {ask_commit, Protocol, Tid, Bin, DiscNs, RamNs}, + CR = ext_format(Head), + Msg = {ask_commit, Protocol, Tid, CR, DiscNs, RamNs}, {?MODULE, Node} ! {self(), Msg}, ask_commit(Protocol, Tid, Tail, DiscNs, RamNs, [Node | WaitFor], Local) end; ask_commit(_Protocol, _Tid, [], _DiscNs, _RamNs, WaitFor, Local) -> {WaitFor, Local}. -%% This used to test protocol conversion between mnesia-nodes -%% but it is really dependent on the emulator version on the -%% two nodes (if funs are sent which they are in transform table op). -%% to be safe we let erts do the translation (many times maybe and thus -%% slower but it works. -% opt_term_to_binary(asym_trans, Head, Nodes) -> -% opt_term_to_binary(Nodes, Head); -opt_term_to_binary(_Protocol, Head, _Nodes) -> - Head. +ext_format(#commit{ext=[]}=CR) -> CR; +ext_format(#commit{node=Node, ext=Ext}=CR) -> + case mnesia_monitor:needs_protocol_conversion(Node) of + true -> + case lists:keyfind(snmp, 1, Ext) of + false -> CR#commit{ext=[]}; + {snmp, List} -> CR#commit{ext=List} + end; + false -> CR + end. + +new_cr_format(#commit{ext=[]}=Cr) -> Cr; +new_cr_format(#commit{ext=[{_,_}|_]}=Cr) -> Cr; +new_cr_format(#commit{ext=Snmp}=Cr) -> + Cr#commit{ext=[{snmp,Snmp}]}. rec_all([Node | Tail], Tid, Res, Pids) -> receive @@ -2090,7 +2099,7 @@ rec_all([Node | Tail], Tid, Res, Pids) -> %% Make sure that mnesia_tm knows it has died %% it may have been restarted Abort = {do_abort, {bad_commit, Node}}, - catch {?MODULE, Node} ! {Tid, Abort}, + ?SAFE({?MODULE, Node} ! {Tid, Abort}), rec_all(Tail, Tid, Abort, Pids) end; rec_all([], _Tid, Res, Pids) -> @@ -2234,8 +2243,6 @@ reconfigure_coordinators(N, [{Tid, [Store | _]} | Coordinators]) -> true -> send_mnesia_down(Tid, Store, N) end; - aborted -> - ignore; % avoid spurious mnesia_down messages _ -> %% Tell the coordinator about the mnesia_down send_mnesia_down(Tid, Store, N) |