Check that messages outside of the heap are not corrupted

Waiting messages for a process may be stored in a queue
outside of any heap or heap fragment belonging to the process.
This is an optimization added in a recent major release to
avoid garbage collection messages again and again if there
is a long message queue.

Until such message has been matched and accepted by
the remove_message/0 instruction, the message must not be
included in the root set for a garbage collection, as that
would corrupt the message. The loop_rec/2 instruction explicitly
turns off garbage collection of the process as long messages
are being matched.

However, if the compiler were to put references to a message
outside of the heap in an Y register (on the stack) and there
happened to be a GC when the process had been scheduled out,
the message would be corrupted and the runtime system would
crash sooner or later.

To ensure that doesn't happen, teach beam_validator to check
for references on the stack to messages outside of the heap.

1 files changed, 92 insertions, 0 deletions

diff --git a/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.erl b/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.erl
new file mode 100644
index 0000000000..b95fa9ca62
--- /dev/null
+++ b/lib/compiler/test/beam_validator_SUITE_data/receive_stacked.erl
@@ -0,0 +1,92 @@
+-module(receive_stacked).
+-compile([export_all,nowarn_export_all]).
+
+%% Messages may be stored outside any process heap until they
+%% have been accepted by the 'remove_message' instruction.
+%% When matching of a message fails, it is not allowed to
+%% leave references to the message or any part of it in
+%% the Y registers. An experimental code generator could
+%% do that, causing an emulator crash if there happenened to
+%% be a garbage collection.
+%%
+%% The 'S' file corresponding to this file was compiled with
+%% that experimental code generator.
+
+f1() ->
+    receive
+        X when is_integer(X) ->
+            id(42),
+            X
+    end.
+
+f2() ->
+    receive
+        [X] ->
+            Res = {ok,X},
+            id(42),
+            {Res,X}
+    end.
+
+f3() ->
+    receive
+        [H|_] when is_integer(H) ->
+            Res = {ok,H},
+            id(42),
+            {Res,H}
+    end.
+
+f4() ->
+    receive
+        [_|T] when is_list(T) ->
+            Res = {ok,T},
+            id(42),
+            {Res,T}
+    end.
+
+f5() ->
+    receive
+        {X} when is_integer(X) ->
+            Res = #{key=>X},
+            id(42),
+            {Res,X}
+    end.
+
+f6() ->
+    receive
+        <<_:8,T/binary>> when byte_size(T) > 8 ->
+            id(42),
+            T
+    end.
+
+f7() ->
+    receive
+        <<_:8,T/binary>> when is_binary(T) ->
+            id(42),
+            T
+    end.
+
+f8() ->
+    receive
+        <<_:8,T/binary>> = Bin when is_binary(Bin) ->
+            id(42),
+            T
+    end.
+
+m1() ->
+    receive
+        #{key:=V} when is_integer(V) ->
+            id(42),
+            [V]
+    end.
+
+m2() ->
+    K1 = id(key1),
+    K2 = id(key2),
+    receive
+        #{K1:=V1,K2:=V2} when is_integer(V1), is_integer(V2) ->
+            id(42),
+            {V1,V2}
+    end.
+
+id(I) ->
+    I.