check_process_code: Sweep HiPE stack for literals

Because check_process_code neglected checking the HiPE stack for
references to the literal area, such references would survive the purge
and subsequent deletion of a module and its literal area. These dangling
references would then cause incorrect behaviour or even hard crashes of
the VM.

By simply adding a scan of the HiPE stack to check_process_code and
erts_garbage_collect_literals, this problem is fixed.

In order to support full stack walks without deleting the graylimit
trap, a new stack walking interface function,
nstack_walk_init_sdesc_ignore_trap() was introduced.

2 files changed, 90 insertions, 0 deletions

diff --git a/erts/emulator/test/hipe_SUITE_data/literals.erl b/erts/emulator/test/hipe_SUITE_data/literals.erl
new file mode 100644
index 0000000000..31e443970f
--- /dev/null
+++ b/erts/emulator/test/hipe_SUITE_data/literals.erl
@@ -0,0 +1,26 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(literals).
+
+-export([a/0, b/0]).
+
+a() -> [a,b,c].
+b() -> {a,b,c}.
diff --git a/erts/emulator/test/hipe_SUITE_data/ref_cell.erl b/erts/emulator/test/hipe_SUITE_data/ref_cell.erl
new file mode 100644
index 0000000000..2654e4077b
--- /dev/null
+++ b/erts/emulator/test/hipe_SUITE_data/ref_cell.erl
@@ -0,0 +1,64 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ref_cell).
+
+-export([start_link/0, start_link_deep/0, call/2]).
+
+-compile(native).
+
+-define(DEPTH, 100).
+-define(ALLOCS, 500).
+
+start_link() ->
+    spawn_link(fun() -> loop(undefined) end).
+
+start_link_deep() ->
+    spawn_link(fun() -> go_deep(?DEPTH) end).
+
+%% Create a stack large enough to get a graylimit trap placed next time there's
+%% a minor gc.
+go_deep(0) ->
+    alloc_some(?ALLOCS),
+    loop(undefined),
+    0;
+go_deep(Depth) ->
+    go_deep(Depth-1)+1.
+
+%% Do some allocation to trigger a minor gc
+alloc_some(Amount) ->
+    Check = (Amount * (Amount + 1)) div 2,
+    Check = lists:sum(lists:seq(1, Amount)).
+
+call(Pid, Call) ->
+    Pid ! {Call, self()},
+    receive {Pid, Res} -> Res end.
+
+loop(Thing) ->
+    receive
+	{done, Pid} -> Pid ! {self(), done};
+	{{put_res_of, Fun}, Pid} ->
+	    NewThing = Fun(),
+	    Pid ! {self(), put},
+	    loop(NewThing);
+	{get, Pid} ->
+	    Pid ! {self(), Thing},
+	    loop(Thing)
+    end.