aboutsummaryrefslogtreecommitdiffstats
path: root/lib/mnesia/src
diff options
context:
space:
mode:
authorUlf Wiger <[email protected]>2010-05-09 12:15:56 +0200
committerUlf Wiger <[email protected]>2010-05-09 12:15:56 +0200
commit82c090488eb76d16e8b828bb8f050afd0949e4c4 (patch)
tree6332210528c3bc10ad60fb526f415c1b7055e8a5 /lib/mnesia/src
parenta21a9dac550bcb976fa074d7005499a4d6b55791 (diff)
downloadotp-82c090488eb76d16e8b828bb8f050afd0949e4c4.tar.gz
otp-82c090488eb76d16e8b828bb8f050afd0949e4c4.tar.bz2
otp-82c090488eb76d16e8b828bb8f050afd0949e4c4.zip
Enable continuous monitoring of mnesia overload status
Mnesia currently issues an event whenever it detects an overload condition. It recognizes two different types of overload: - whenever the message queue of mnesia_tm process grows large - when a log dump interval triggers before the previous dump is done These events could be used to trigger a load regulation mechanism to reduce the load until the condition seizes. The missing piece is that there is no facility to ask mnesia whether the overload condition still exists. This patch implements a couple of functions in mnesia_lib that can be used to sample the overload status. It has been tested in a load regulator component being developed by Erlang Solutions. No mnesia test suites have been run, since they are not available. No documentation has been updated. The functions in mnesia_lib are at any rate undocumented (as are all other functions in that module). A decision would have to be made about whether to provide a documented API on top of these functions. The internal state record of mnesia_recover has been modified. For this reason, a code change hook has been provided.
Diffstat (limited to 'lib/mnesia/src')
-rw-r--r--lib/mnesia/src/mnesia_controller.erl25
-rw-r--r--lib/mnesia/src/mnesia_lib.erl30
-rw-r--r--lib/mnesia/src/mnesia_recover.erl31
3 files changed, 74 insertions, 12 deletions
diff --git a/lib/mnesia/src/mnesia_controller.erl b/lib/mnesia/src/mnesia_controller.erl
index 9bc480e619..14e28bdc14 100644
--- a/lib/mnesia/src/mnesia_controller.erl
+++ b/lib/mnesia/src/mnesia_controller.erl
@@ -1842,17 +1842,20 @@ reply(ReplyTo, Reply) ->
add_worker(Worker = #dump_log{}, State) ->
InitBy = Worker#dump_log.initiated_by,
Queue = State#state.dumper_queue,
- case lists:keymember(InitBy, #dump_log.initiated_by, Queue) of
- true when Worker#dump_log.opt_reply_to == undefined ->
- %% The same threshold has been exceeded again,
- %% before we have had the possibility to
- %% process the older one.
- DetectedBy = {dump_log, InitBy},
- Event = {mnesia_overload, DetectedBy},
- mnesia_lib:report_system_event(Event);
- _ ->
- ignore
- end,
+ Status =
+ case lists:keymember(InitBy, #dump_log.initiated_by, Queue) of
+ true when Worker#dump_log.opt_reply_to == undefined ->
+ %% The same threshold has been exceeded again,
+ %% before we have had the possibility to
+ %% process the older one.
+ DetectedBy = {dump_log, InitBy},
+ Event = {mnesia_overload, DetectedBy},
+ mnesia_lib:report_system_event(Event),
+ true;
+ _ ->
+ false
+ end,
+ mnesia_recover:log_dump_overload(Status),
Queue2 = Queue ++ [Worker],
State2 = State#state{dumper_queue = Queue2},
opt_start_worker(State2);
diff --git a/lib/mnesia/src/mnesia_lib.erl b/lib/mnesia/src/mnesia_lib.erl
index dba808e66e..168e225289 100644
--- a/lib/mnesia/src/mnesia_lib.erl
+++ b/lib/mnesia/src/mnesia_lib.erl
@@ -113,6 +113,9 @@
mkcore/1,
not_active_here/1,
other_val/2,
+ overload_read/0,
+ overload_read/1,
+ overload_set/2,
pad_name/3,
random_time/2,
read_counter/1,
@@ -551,6 +554,33 @@ cs_to_nodes(Cs) ->
Cs#cstruct.disc_only_copies ++
Cs#cstruct.disc_copies ++
Cs#cstruct.ram_copies.
+
+overload_types() ->
+ [mnesia_tm, mnesia_dump_log].
+
+valid_overload_type(T) ->
+ case lists:member(T, overload_types()) of
+ false ->
+ erlang:error(bad_type);
+ true ->
+ true
+ end.
+
+overload_set(Type, Bool) when is_boolean(Bool) ->
+ valid_overload_type(Type),
+ set({overload, Type}, Bool).
+
+overload_read() ->
+ [{T, overload_read(T)} || T <- overload_types()].
+
+overload_read(T) ->
+ case ?catch_val({overload, T}) of
+ {'EXIT',_} ->
+ valid_overload_type(T),
+ false;
+ Flag when is_boolean(Flag) ->
+ Flag
+ end.
dist_coredump() ->
dist_coredump(all_nodes()).
diff --git a/lib/mnesia/src/mnesia_recover.erl b/lib/mnesia/src/mnesia_recover.erl
index 6c53c2e752..051b193da7 100644
--- a/lib/mnesia/src/mnesia_recover.erl
+++ b/lib/mnesia/src/mnesia_recover.erl
@@ -36,6 +36,7 @@
incr_trans_tid_serial/0,
init/0,
log_decision/1,
+ log_dump_overload/1,
log_master_nodes/3,
log_mnesia_down/1,
log_mnesia_up/1,
@@ -70,6 +71,7 @@
unclear_decision,
unclear_waitfor,
tm_queue_len = 0,
+ log_dump_overload = false,
initiated = false,
early_msgs = []
}).
@@ -277,6 +279,9 @@ mnesia_down(Node) ->
cast({mnesia_down, Node})
end.
+log_dump_overload(Flag) when is_boolean(Flag) ->
+ cast({log_dump_overload, Flag}).
+
log_master_nodes(Args, UseDir, IsRunning) ->
if
IsRunning == yes ->
@@ -818,6 +823,12 @@ handle_cast({announce_all, Nodes}, State) ->
announce_all(Nodes),
{noreply, State};
+handle_cast({log_dump_overload, Flag}, State) when is_boolean(Flag) ->
+ Prev = State#state.log_dump_overload,
+ Overload = Prev orelse Flag,
+ mnesia_lib:overload_set(mnesia_dump_log, Overload),
+ {noreply, State#state{log_dump_overload = Flag}};
+
handle_cast(Msg, State) ->
error("~p got unexpected cast: ~p~n", [?MODULE, Msg]),
{noreply, State}.
@@ -851,12 +862,14 @@ handle_info(check_overload, S) ->
Len > Threshold, Prev > Threshold ->
What = {mnesia_tm, message_queue_len, [Prev, Len]},
mnesia_lib:report_system_event({mnesia_overload, What}),
+ mnesia_lib:overload_set(mnesia_tm, true),
{noreply, S#state{tm_queue_len = 0}};
Len > Threshold ->
{noreply, S#state{tm_queue_len = Len}};
true ->
+ mnesia_lib:overload_set(mnesia_tm, false),
{noreply, S#state{tm_queue_len = 0}}
end;
undefined ->
@@ -905,7 +918,23 @@ terminate(Reason, State) ->
%% Purpose: Upgrade process when its code is to be changed
%% Returns: {ok, NewState}
%%----------------------------------------------------------------------
-code_change(_OldVsn, State, _Extra) ->
+code_change(_OldVsn, {state,
+ Supervisor,
+ Unclear_pid,
+ Unclear_decision,
+ Unclear_waitfor,
+ Tm_queue_len,
+ Initiated,
+ Early_msgs
+ }, _Extra) ->
+ {ok, #state{supervisor = Supervisor,
+ unclear_pid = Unclear_pid,
+ unclear_decision = Unclear_decision,
+ unclear_waitfor = Unclear_waitfor,
+ tm_queue_len = Tm_queue_len,
+ initiated = Initiated,
+ early_msgs = Early_msgs}};
+code_change(_OldVsn, #state{} = State, _Extra) ->
{ok, State}.
%%%----------------------------------------------------------------------