Enable continuous monitoring of mnesia overload status

Mnesia currently issues an event whenever it detects an overload
condition. It recognizes two different types of overload:
- whenever the message queue of mnesia_tm process grows large
- when a log dump interval triggers before the previous dump is done

These events could be used to trigger a load regulation mechanism
to reduce the load until the condition seizes. The missing piece
is that there is no facility to ask mnesia whether the overload
condition still exists.

This patch implements a couple of functions in mnesia_lib that
can be used to sample the overload status. It has been tested
in a load regulator component being developed by Erlang Solutions.

No mnesia test suites have been run, since they are not available.
No documentation has been updated. The functions in mnesia_lib
are at any rate undocumented (as are all other functions in that
module). A decision would have to be made about whether to provide
a documented API on top of these functions.

The internal state record of mnesia_recover has been modified.
For this reason, a code change hook has been provided.

3 files changed, 74 insertions, 12 deletions

diff --git a/lib/mnesia/src/mnesia_controller.erl b/lib/mnesia/src/mnesia_controller.erl
index 9bc480e619..14e28bdc14 100644
--- a/lib/mnesia/src/mnesia_controller.erl
+++ b/lib/mnesia/src/mnesia_controller.erl
@@ -1842,17 +1842,20 @@ reply(ReplyTo, Reply) ->
 add_worker(Worker = #dump_log{}, State) ->
     InitBy = Worker#dump_log.initiated_by,
     Queue = State#state.dumper_queue,
-    case lists:keymember(InitBy, #dump_log.initiated_by, Queue) of
-	true when Worker#dump_log.opt_reply_to == undefined ->
-	    %% The same threshold has been exceeded again,
-	    %% before we have had the possibility to
-	    %% process the older one.
-	    DetectedBy = {dump_log, InitBy},
-	    Event = {mnesia_overload, DetectedBy},
-	    mnesia_lib:report_system_event(Event);
-	_ ->
-	    ignore
-    end,
+    Status =
+        case lists:keymember(InitBy, #dump_log.initiated_by, Queue) of
+            true when Worker#dump_log.opt_reply_to == undefined ->
+                %% The same threshold has been exceeded again,
+                %% before we have had the possibility to
+                %% process the older one.
+                DetectedBy = {dump_log, InitBy},
+                Event = {mnesia_overload, DetectedBy},
+                mnesia_lib:report_system_event(Event),
+                true;
+            _ ->
+                false
+        end,
+    mnesia_recover:log_dump_overload(Status),
     Queue2 = Queue ++ [Worker],
     State2 = State#state{dumper_queue = Queue2},
     opt_start_worker(State2);
diff --git a/lib/mnesia/src/mnesia_lib.erl b/lib/mnesia/src/mnesia_lib.erl
index dba808e66e..168e225289 100644
--- a/lib/mnesia/src/mnesia_lib.erl
+++ b/lib/mnesia/src/mnesia_lib.erl
@@ -113,6 +113,9 @@
 	 mkcore/1,
 	 not_active_here/1,
 	 other_val/2,
+         overload_read/0,
+         overload_read/1,
+         overload_set/2,
 	 pad_name/3,
 	 random_time/2,
 	 read_counter/1,
@@ -551,6 +554,33 @@ cs_to_nodes(Cs) ->
     Cs#cstruct.disc_only_copies ++
     Cs#cstruct.disc_copies ++
     Cs#cstruct.ram_copies.
+
+overload_types() ->
+    [mnesia_tm, mnesia_dump_log].
+
+valid_overload_type(T) ->
+    case lists:member(T, overload_types()) of
+        false ->
+            erlang:error(bad_type);
+        true ->
+            true
+    end.
+
+overload_set(Type, Bool) when is_boolean(Bool) ->
+    valid_overload_type(Type),
+    set({overload, Type}, Bool).
+
+overload_read() ->
+    [{T, overload_read(T)} || T <- overload_types()].
+
+overload_read(T) ->
+    case ?catch_val({overload, T}) of
+        {'EXIT',_} ->
+            valid_overload_type(T),
+            false;
+        Flag when is_boolean(Flag) ->
+            Flag
+    end.
  
 dist_coredump() ->
     dist_coredump(all_nodes()).
diff --git a/lib/mnesia/src/mnesia_recover.erl b/lib/mnesia/src/mnesia_recover.erl
index 6c53c2e752..051b193da7 100644
--- a/lib/mnesia/src/mnesia_recover.erl
+++ b/lib/mnesia/src/mnesia_recover.erl
@@ -36,6 +36,7 @@
 	 incr_trans_tid_serial/0,
 	 init/0,
 	 log_decision/1,
+         log_dump_overload/1,
 	 log_master_nodes/3,
 	 log_mnesia_down/1,
 	 log_mnesia_up/1,
@@ -70,6 +71,7 @@
 		unclear_decision,
 		unclear_waitfor,
 		tm_queue_len = 0,
+                log_dump_overload = false,
 		initiated = false,
 		early_msgs = []
 	       }).
@@ -277,6 +279,9 @@ mnesia_down(Node) ->
 	    cast({mnesia_down, Node})
     end.
 
+log_dump_overload(Flag) when is_boolean(Flag) ->
+    cast({log_dump_overload, Flag}).
+
 log_master_nodes(Args, UseDir, IsRunning) ->
     if
 	IsRunning == yes ->
@@ -818,6 +823,12 @@ handle_cast({announce_all, Nodes}, State) ->
     announce_all(Nodes),
     {noreply, State};
 
+handle_cast({log_dump_overload, Flag}, State) when is_boolean(Flag) ->
+    Prev = State#state.log_dump_overload,
+    Overload = Prev orelse Flag,
+    mnesia_lib:overload_set(mnesia_dump_log, Overload),
+    {noreply, State#state{log_dump_overload = Flag}};
+
 handle_cast(Msg, State) ->
     error("~p got unexpected cast: ~p~n", [?MODULE, Msg]),
     {noreply, State}.
@@ -851,12 +862,14 @@ handle_info(check_overload, S) ->
 		Len > Threshold, Prev > Threshold ->
 		    What = {mnesia_tm, message_queue_len, [Prev, Len]},
 		    mnesia_lib:report_system_event({mnesia_overload, What}),
+                    mnesia_lib:overload_set(mnesia_tm, true),
 		    {noreply, S#state{tm_queue_len = 0}};
 		
 		Len > Threshold ->
 		    {noreply, S#state{tm_queue_len = Len}};
 		
 		true ->
+                    mnesia_lib:overload_set(mnesia_tm, false),
 		    {noreply, S#state{tm_queue_len = 0}}
 	    end;
 	undefined ->
@@ -905,7 +918,23 @@ terminate(Reason, State) ->
 %% Purpose: Upgrade process when its code is to be changed
 %% Returns: {ok, NewState}
 %%----------------------------------------------------------------------
-code_change(_OldVsn, State, _Extra) ->
+code_change(_OldVsn, {state,
+                      Supervisor,
+                      Unclear_pid,
+                      Unclear_decision,
+                      Unclear_waitfor,
+                      Tm_queue_len,
+                      Initiated,
+                      Early_msgs
+                     }, _Extra) ->
+    {ok, #state{supervisor = Supervisor,
+                unclear_pid = Unclear_pid,
+                unclear_decision = Unclear_decision,
+                unclear_waitfor = Unclear_waitfor,
+                tm_queue_len = Tm_queue_len,
+                initiated = Initiated,
+                early_msgs = Early_msgs}};
+code_change(_OldVsn, #state{} = State, _Extra) ->
     {ok, State}.
 
 %%%----------------------------------------------------------------------