Honor the max heap size when copying literals after purging

When a module has been purged from memory, any literals belonging
to that module will be copied to all processes that hold references
to them.

The max heap size limit would be ignored in the garbage collection
initiated when copying literals to a process. If the max heap size
was exceeded, the process would typically be terminated in the
following garbage collection.

Since the process would be killed anyway later, kill the process
before copying a literal that would make it exceed its max heap
size.

While at it, also fix a potential bug in `erlang:garbage_collect/0`.
If it was found that the max heap sized had been exceeded while
executing `erlang:garbage_collect/0`, the process would enter a
kind of zombie state instead of being properly terminated.

3 files changed, 58 insertions, 2 deletions

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index f18af8bcd7..015c051cc1 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3622,6 +3622,10 @@ erts_internal_garbage_collect_1(BIF_ALIST_1)
     default:        BIF_ERROR(BIF_P, BADARG);
     }
     erts_garbage_collect(BIF_P, 0, NULL, 0);
+    if (ERTS_PROC_IS_EXITING(BIF_P)) {
+        /* The max heap size limit was reached. */
+        return THE_NON_VALUE;
+    }
     return am_true;
 }
 
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index a65dbbf42b..47dd115c82 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1133,9 +1133,28 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
     reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, 0,
 				    p->arg_reg, p->arity, fcalls,
 				    ygen_usage);
+    if (ERTS_PROC_IS_EXITING(p)) {
+        return 0;
+    }
 
     ASSERT(!(p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)));
 
+    if (MAX_HEAP_SIZE_GET(p)) {
+        Uint new_heap_size;
+        Uint old_heap_size;
+        Uint total_heap_size;
+
+        new_heap_size = HEAP_END(p) - HEAP_START(p);
+        old_heap_size = erts_next_heap_size(lit_size, 0);
+        total_heap_size = new_heap_size + old_heap_size;
+        if (MAX_HEAP_SIZE_GET(p) < total_heap_size &&
+            reached_max_heap_size(p, total_heap_size,
+                                  new_heap_size, old_heap_size)) {
+            erts_set_self_exiting(p, am_killed);
+            return 0;
+        }
+    }
+
     /*
      * Set GC state.
      */
diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 9c6dc3ff83..0444ba4f89 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -28,7 +28,7 @@
          fake_literals/1,
          false_dependency/1,coverage/1,fun_confusion/1,
          t_copy_literals/1, t_copy_literals_frags/1,
-         erl_544/1]).
+         erl_544/1, max_heap_size/1]).
 
 -define(line_trace, 1).
 -include_lib("common_test/include/ct.hrl").
@@ -43,7 +43,7 @@ all() ->
      constant_pools, constant_refc_binaries, fake_literals,
      false_dependency,
      coverage, fun_confusion, t_copy_literals, t_copy_literals_frags,
-     erl_544].
+     erl_544, max_heap_size].
 
 init_per_suite(Config) ->
     erts_debug:set_internal_state(available_internal_state, true),
@@ -968,6 +968,39 @@ erl_544(Config) when is_list(Config) ->
             {skipped, "Only run when native file name encoding is utf8"}
     end.
 
+%% Test that the copying of literals to a process during purging of
+%% literals will cause the process to be killed if the max heap size
+%% is exceeded.
+max_heap_size(_Config) ->
+    Mod = ?FUNCTION_NAME,
+    Value = [I || I <- lists:seq(1, 5000)],
+    Code = gen_lit(Mod, [{term,Value}]),
+    {module,Mod} = erlang:load_module(Mod, Code),
+    SpawnOpts = [monitor,
+                 {max_heap_size,
+                  #{size=>1024,
+                    kill=>true,
+                    error_logger=>true}}],
+    {Pid,Ref} = spawn_opt(fun() ->
+                                  max_heap_size_proc(Mod)
+                          end, SpawnOpts),
+    receive
+        {'DOWN',Ref,process,Pid,Reason} ->
+            killed = Reason;
+        Other ->
+            ct:fail({unexpected_message,Other})
+    after 10000 ->
+            ct:fail({process_did_not_die, Pid, erlang:process_info(Pid)})
+    end.
+
+max_heap_size_proc(Mod) ->
+    Value = Mod:term(),
+    code:delete(Mod),
+    code:purge(Mod),
+    receive
+        _ -> Value
+    end.
+
 %% Utilities.
 
 make_sub_binary(Bin) when is_binary(Bin) ->