Merge branch 'rickard/node-refc-tests-21' into rickard/node-refc-tests-22

* rickard/node-refc-tests-21:
  Node container refc test for persistent terms
  Include persistent term storage in node/dist refc check
  Fix node refc test for system message queue

7 files changed, 276 insertions, 34 deletions

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index bb1b2e5b27..4c8ee5178a 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1799,6 +1799,78 @@ erts_queue_release_literals(Process* c_p, ErtsLiteralArea* literals)
     }
 }
 
+struct debug_la_oh {
+    void (*func)(ErlOffHeap *, void *);
+    void *arg;
+};
+
+static void debug_later_cleanup_literal_area_off_heap(void *vfap,
+                                                      ErtsThrPrgrVal val,
+                                                      void *vlrlap)
+{
+    struct debug_la_oh *fap = vfap;
+    ErtsLaterReleasLiteralArea *lrlap = vlrlap;
+    ErtsLiteralArea *lap = lrlap->la;
+    if (!erts_debug_have_accessed_literal_area(lap)) {
+        ErlOffHeap oh;
+        ERTS_INIT_OFF_HEAP(&oh);
+        oh.first = lap->off_heap;
+        (*fap->func)(&oh, fap->arg);
+        erts_debug_save_accessed_literal_area(lap);
+    }
+}
+
+static void debug_later_complete_literal_area_switch_off_heap(void *vfap,
+                                                              ErtsThrPrgrVal val,
+                                                              void *vlap)
+{
+    struct debug_la_oh *fap = vfap;
+    ErtsLiteralArea *lap = vlap;
+    if (lap && !erts_debug_have_accessed_literal_area(lap)) {
+        ErlOffHeap oh;
+        ERTS_INIT_OFF_HEAP(&oh);
+        oh.first = lap->off_heap;
+        (*fap->func)(&oh, fap->arg);
+        erts_debug_save_accessed_literal_area(lap);
+    }
+}
+
+
+void
+erts_debug_foreach_release_literal_area_off_heap(void (*func)(ErlOffHeap *, void *), void *arg)
+{
+    ErtsLiteralArea *lap;
+    ErlOffHeap oh;
+    ErtsLiteralAreaRef *ref;
+    struct debug_la_oh fa;
+    erts_mtx_lock(&release_literal_areas.mtx);
+    for (ref = release_literal_areas.first; ref; ref = ref->next) {
+        lap = ref->literal_area;
+        if (!erts_debug_have_accessed_literal_area(lap)) {
+            ERTS_INIT_OFF_HEAP(&oh);
+            oh.first = lap->off_heap;
+            (*func)(&oh, arg);
+            erts_debug_save_accessed_literal_area(lap);
+        }
+    }
+    erts_mtx_unlock(&release_literal_areas.mtx);
+    lap = ERTS_COPY_LITERAL_AREA();
+    if (lap && !erts_debug_have_accessed_literal_area(lap)) {
+        ERTS_INIT_OFF_HEAP(&oh);
+        oh.first = lap->off_heap;
+        (*func)(&oh, arg);
+        erts_debug_save_accessed_literal_area(lap);
+    }
+    fa.func = func;
+    fa.arg = arg;
+    erts_debug_later_op_foreach(later_release_literal_area,
+                                debug_later_cleanup_literal_area_off_heap,
+                                (void *) &fa);
+    erts_debug_later_op_foreach(complete_literal_area_switch,
+                                debug_later_complete_literal_area_switch_off_heap,
+                                (void *) &fa);
+}
+
 /*
  * Move code from current to old and null all export entries for the module
  */
diff --git a/erts/emulator/beam/erl_bif_persistent.c b/erts/emulator/beam/erl_bif_persistent.c
index f38e0cc5cb..8662a9e33a 100644
--- a/erts/emulator/beam/erl_bif_persistent.c
+++ b/erts/emulator/beam/erl_bif_persistent.c
@@ -1234,3 +1234,102 @@ next_to_delete(void)
     erts_mtx_unlock(&delete_queue_mtx);
     return table;
 }
+
+/*
+ * test/debug functionality follow...
+ */
+
+static Uint accessed_literal_areas_size;
+static Uint accessed_no_literal_areas;
+static ErtsLiteralArea **accessed_literal_areas;
+
+int
+erts_debug_have_accessed_literal_area(ErtsLiteralArea *lap)
+{
+    Uint i;
+    for (i = 0; i < accessed_no_literal_areas; i++) {
+        if (accessed_literal_areas[i] == lap)
+            return !0;
+    }
+    return 0;
+}
+
+void
+erts_debug_save_accessed_literal_area(ErtsLiteralArea *lap)
+{
+    if (accessed_no_literal_areas == accessed_literal_areas_size) {
+        accessed_literal_areas_size += 10;
+        accessed_literal_areas = erts_realloc(ERTS_ALC_T_TMP,
+                                              accessed_literal_areas,
+                                              (sizeof(ErtsLiteralArea *)
+                                               *accessed_literal_areas_size));
+    }
+    accessed_literal_areas[accessed_no_literal_areas++] = lap;
+}
+
+static void debug_foreach_off_heap(HashTable *tbl, void (*func)(ErlOffHeap *, void *), void *arg)
+{
+    int i;
+    
+    for (i = 0; i < tbl->allocated; i++) {
+        Eterm term = tbl->term[i];
+        if (is_tuple_arity(term, 2)) {
+            ErtsLiteralArea *lap = term_to_area(term);
+            ErlOffHeap oh;
+            if (!erts_debug_have_accessed_literal_area(lap)) {
+                ERTS_INIT_OFF_HEAP(&oh);
+                oh.first = lap->off_heap;
+                (*func)(&oh, arg);
+                erts_debug_save_accessed_literal_area(lap);
+            }
+        }
+    }
+}
+
+struct debug_la_oh {
+    void (*func)(ErlOffHeap *, void *);
+    void *arg;
+};
+
+static void debug_handle_table(void *vfap,
+                               ErtsThrPrgrVal val,
+                               void *vtbl)
+{
+    struct debug_la_oh *fap = vfap;
+    HashTable *tbl = vtbl;
+    debug_foreach_off_heap(tbl, fap->func, fap->arg);
+}
+
+void
+erts_debug_foreach_persistent_term_off_heap(void (*func)(ErlOffHeap *, void *), void *arg)
+{
+    HashTable *tbl;
+    struct debug_la_oh fa;
+    accessed_no_literal_areas = 0;
+    accessed_literal_areas_size = 10;
+    accessed_literal_areas = erts_alloc(ERTS_ALC_T_TMP,
+                                        (sizeof(ErtsLiteralArea *)
+                                         * accessed_literal_areas_size));
+    
+    tbl = (HashTable *) erts_atomic_read_nob(&the_hash_table);
+    debug_foreach_off_heap(tbl, func, arg);
+    erts_mtx_lock(&delete_queue_mtx);
+    for (tbl = delete_queue_head; tbl; tbl = tbl->delete_next)
+        debug_foreach_off_heap(tbl, func, arg);
+    erts_mtx_unlock(&delete_queue_mtx);
+    fa.func = func;
+    fa.arg = arg;
+    erts_debug_later_op_foreach(table_updater,
+                                debug_handle_table,
+                                (void *) &fa);
+    erts_debug_later_op_foreach(table_deleter,
+                                debug_handle_table,
+                                (void *) &fa);
+    erts_debug_foreach_release_literal_area_off_heap(func, arg);
+    
+    erts_free(ERTS_ALC_T_TMP, accessed_literal_areas);
+    accessed_no_literal_areas = 0;
+    accessed_literal_areas_size = 0;
+    accessed_literal_areas = NULL;
+}
+
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index 4eb6c3e214..9cba707440 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -1188,6 +1188,7 @@ static Eterm AM_delayed_delete_timer;
 static Eterm AM_thread_progress_delete_timer;
 static Eterm AM_sequence;
 static Eterm AM_signal;
+static Eterm AM_persistent_term;
 
 static void setup_reference_table(void);
 static Eterm reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp);
@@ -1284,6 +1285,7 @@ erts_get_node_and_dist_references(struct process *proc)
         INIT_AM(thread_progress_delete_timer);
 	INIT_AM(signal);
 	INIT_AM(sequence);
+	INIT_AM(persistent_term);
 	references_atoms_need_init = 0;
     }
 
@@ -1916,6 +1918,14 @@ insert_dist_suspended_procs(DistEntry *dep)
     }
 }
 
+static void
+insert_persistent_term(ErlOffHeap *ohp, void *arg)
+{
+    Eterm heap[3];
+    insert_offheap(ohp, SYSTEM_REF,
+                   TUPLE2(&heap[0], AM_system, AM_persistent_term));
+}
+
 #ifdef ERL_NODE_BOOKKEEP
 void
 erts_node_bookkeep(ErlNode *np, Eterm term, int what)
@@ -1990,8 +2000,8 @@ setup_reference_table(void)
 	if (proc)
             insert_process(proc);
     }
-
-    erts_foreach_sys_msg_in_q(insert_sys_msg);
+    
+    erts_debug_foreach_sys_msg_in_q(insert_sys_msg);
 
     /* Insert all ports */
     max = erts_ptab_max(&erts_port);
@@ -2096,6 +2106,10 @@ setup_reference_table(void)
     /* Insert all bif timers */
     erts_debug_bif_timer_foreach(insert_bif_timer, NULL);
 
+    /* Insert persistent term storage */
+    erts_debug_foreach_persistent_term_off_heap(insert_persistent_term,
+                                                NULL);
+
     /* Insert node table (references to dist) */
     hash_foreach(&erts_node_table, insert_erl_node, NULL);
 }
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index c85a7df5ec..d96a32d406 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -2190,11 +2190,12 @@ sys_msg_dispatcher_wait(void *vwait_p)
     erts_mtx_unlock(&smq_mtx);
 }
 
+static ErtsSysMsgQ *local_sys_message_queue = NULL;
+
 static void *
 sys_msg_dispatcher_func(void *unused)
 {
     ErtsThrPrgrCallbacks callbacks;
-    ErtsSysMsgQ *local_sys_message_queue = NULL;
     ErtsThrPrgrData *tpd;
     int wait = 0;
 
@@ -2202,6 +2203,8 @@ sys_msg_dispatcher_func(void *unused)
     erts_lc_set_thread_name("system message dispatcher");
 #endif
 
+    local_sys_message_queue = NULL;
+
     callbacks.arg = (void *) &wait;
     callbacks.wakeup = sys_msg_dispatcher_wakeup;
     callbacks.prepare_wait = sys_msg_dispatcher_prep_wait;
@@ -2258,6 +2261,8 @@ sys_msg_dispatcher_func(void *unused)
 	    Process *proc = NULL;
 	    Port *port = NULL;
 
+            ASSERT(is_value(smqp->msg));
+
 	    if (erts_thr_progress_update(tpd))
 		erts_thr_progress_leader_update(tpd);
 
@@ -2370,6 +2375,7 @@ sys_msg_dispatcher_func(void *unused)
 		erts_fprintf(stderr, "dropped\n");
 #endif
 	    }
+            smqp->msg = THE_NON_VALUE;
 	}
     }
 
@@ -2377,32 +2383,38 @@ sys_msg_dispatcher_func(void *unused)
 }
 
 void
-erts_foreach_sys_msg_in_q(void (*func)(Eterm,
-				       Eterm,
-				       Eterm,
-				       ErlHeapFragment *))
+erts_debug_foreach_sys_msg_in_q(void (*func)(Eterm,
+                                             Eterm,
+                                             Eterm,
+                                             ErlHeapFragment *))
 {
-    ErtsSysMsgQ *sm;
-    erts_mtx_lock(&smq_mtx);
-    for (sm = sys_message_queue; sm; sm = sm->next) {
-	Eterm to;
-	switch (sm->type) {
-	case SYS_MSG_TYPE_SYSMON:
-	    to = erts_get_system_monitor();
-	    break;
-	case SYS_MSG_TYPE_SYSPROF:
-	    to = erts_get_system_profile();
-	    break;
-	case SYS_MSG_TYPE_ERRLGR:
-	    to = erts_get_system_logger();
-	    break;
-	default:
-	    to = NIL;
-	    break;
-	}
-	(*func)(sm->from, to, sm->msg, sm->bp);
+    ErtsSysMsgQ *smq[] = {sys_message_queue, local_sys_message_queue};
+    int i;
+
+    ERTS_LC_ASSERT(erts_thr_progress_is_blocking());
+
+    for (i = 0; i < sizeof(smq)/sizeof(smq[0]); i++) {
+        ErtsSysMsgQ *sm;
+        for (sm = smq[i]; sm; sm = sm->next) {
+            Eterm to;
+            switch (sm->type) {
+            case SYS_MSG_TYPE_SYSMON:
+                to = erts_get_system_monitor();
+                break;
+            case SYS_MSG_TYPE_SYSPROF:
+                to = erts_get_system_profile();
+                break;
+            case SYS_MSG_TYPE_ERRLGR:
+                to = erts_get_system_logger();
+                break;
+            default:
+                to = NIL;
+                break;
+            }
+            if (is_value(sm->msg))
+                (*func)(sm->from, to, sm->msg, sm->bp);
+        }
     }
-    erts_mtx_unlock(&smq_mtx);
 }
 
 
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index b7844d1cb0..af38ef52db 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -90,10 +90,10 @@ int erts_is_tracer_valid(Process* p);
 void erts_check_my_tracer_proc(Process *);
 void erts_block_sys_msg_dispatcher(void);
 void erts_release_sys_msg_dispatcher(void);
-void erts_foreach_sys_msg_in_q(void (*func)(Eterm,
-					    Eterm,
-					    Eterm,
-					    ErlHeapFragment *));
+void erts_debug_foreach_sys_msg_in_q(void (*func)(Eterm,
+                                                  Eterm,
+                                                  Eterm,
+                                                  ErlHeapFragment *));
 Eterm erts_set_system_logger(Eterm);
 Eterm erts_get_system_logger(void);
 void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 4c8d3d3dbe..64fb080250 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -907,7 +907,8 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2);
 /* beam_bif_load.c */
 Eterm erts_check_process_code(Process *c_p, Eterm module, int *redsp, int fcalls);
 Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed);
-
+void erts_debug_foreach_release_literal_area_off_heap(void (*func)(ErlOffHeap *, void *),
+                                                      void *arg);
 typedef struct ErtsLiteralArea_ {
     struct erl_off_heap_header *off_heap;
     Eterm *end;
@@ -1257,6 +1258,10 @@ Uint erts_persistent_term_count(void);
 void erts_init_persistent_dumping(void);
 extern ErtsLiteralArea** erts_persistent_areas;
 extern Uint erts_num_persistent_areas;
+void erts_debug_foreach_persistent_term_off_heap(void (*func)(ErlOffHeap *, void *),
+                                                 void *arg);
+int erts_debug_have_accessed_literal_area(ErtsLiteralArea *lap);
+void erts_debug_save_accessed_literal_area(ErtsLiteralArea *lap);
 
 /* external.c */
 void erts_init_external(void);
diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl
index 7d2db5257c..c44693f8d9 100644
--- a/erts/emulator/test/node_container_SUITE.erl
+++ b/erts/emulator/test/node_container_SUITE.erl
@@ -51,7 +51,8 @@
          unique_pid/1,
          iter_max_procs/1,
          magic_ref/1,
-         dist_entry_gc/1]).
+         dist_entry_gc/1,
+         persistent_term/1]).
 
 suite() ->
     [{ct_hooks,[ts_install_cth]},
@@ -63,7 +64,8 @@ all() ->
      node_table_gc, dist_link_refc, dist_monitor_refc,
      node_controller_refc, ets_refc, match_spec_refc,
      timer_refc, pid_wrap, port_wrap, bad_nc,
-     unique_pid, iter_max_procs, magic_ref].
+     unique_pid, iter_max_procs,
+     magic_ref, persistent_term].
 
 init_per_suite(Config) ->
     Config.
@@ -906,6 +908,44 @@ magic_ref(Config) when is_list(Config) ->
     true = erts_debug:get_internal_state({magic_ref,MRef2}),
     ok.
 
+persistent_term(Config) when is_list(Config) ->
+    {ok, Node} = start_node(get_nodefirstname()),
+    Self = self(),
+    NcData = make_ref(),
+    RPid = spawn_link(Node,
+                      fun () ->
+                              Self ! {NcData, self(), hd(erlang:ports()), erlang:make_ref()}
+                      end),
+    Data = receive
+               {NcData, RPid, RPort, RRef} ->
+                   {RPid, RPort, RRef}
+           end,
+    unlink(RPid),
+    stop_node(Node),
+    Stuff = lists:foldl(fun (N, Acc) ->
+                                persistent_term:put({?MODULE, N}, Data),
+                                persistent_term:erase({?MODULE, N-1}),
+                                node_container_refc_check(node()),
+                                Data = persistent_term:get({?MODULE, N}),
+                                try
+                                    persistent_term:get({?MODULE, N-1})
+                                catch
+                                    error:badarg ->
+                                        ok
+                                end,
+                                case N rem 4 of
+                                    0 -> [persistent_term:get({?MODULE, N})|Acc];
+                                    _ -> Acc
+                                end
+                        end,
+                        [],
+                        lists:seq(1, 100)),
+    persistent_term:erase({?MODULE, 100}),
+    receive after 2000 -> ok end, %% give literal gc some time to run...
+    node_container_refc_check(node()),
+    id(Stuff),
+    ok.
+
 
 lost_pending_connection(Node) ->
     _ = (catch erts_internal:new_connection(Node)),