Restart emulator before running upgrade script when erts is upgraded

If the version of erts differs between two releases, the release
handler automatically adds a 'restart_new_emulator' instruction to the
upgrade script (relup). Earlier, this instruction was always added at
the end of the upgrade script, causing the following sequence of
operations during an upgrade (a bit simplified):

1. suspend processes
2. load new code
3. execute code_change functions
4. resume processes
5. restart emulator with new erts version

Obviously, this caused the new code to be loaded into the old emulator
and this would fail if the beam format had been changed in the new
version of the emulator.

To overcome this problem, this commit changes the order of the
instructions, so for upgrade with changed erts version we now have:

1. restart emulator with new erts, kernel, stdlib and sasl versions,
   but old versions of all other applications.
2. suspend processes
3. load new code
4. execute code_change functions
5. resume processes

This is implemented by creating a temporary release, including new
erts, kernel, stdlib and sasl from the new release and all other
applications from the old release. A new boot file for this temporary
release is created, which includes a new 'apply' instruction to run
release_handler:new_emulator_upgrade/2. Then the emulator is restarted
using this boot file - and release_handler:new_emulator_upgrade/2
executes the rest of the upgrade script.

For downgrade, the order will be as before:

1. suspend processes
2. execute code_change functions with 'down'-indication
3. load old code
4. resume processes
5. restart emulator with old erts version

1 files changed, 425 insertions, 137 deletions

diff --git a/lib/sasl/test/installer.erl b/lib/sasl/test/installer.erl
index f5ceab0dc4..d77b0e8538 100644
--- a/lib/sasl/test/installer.erl
+++ b/lib/sasl/test/installer.erl
@@ -19,20 +19,32 @@
 
 -module(installer).
 
+-include("test_lib.hrl").
+
 %%-compile(export_all).
 -export([install_1/2]).
 -export([install_2/1]).
 -export([install_3/2]).
--export([install_3a/1]).
+-export([install_6a/1]).
 -export([install_4/1]).
 -export([install_5/1]).
 -export([install_5a/1]).
 -export([install_6/1]).
 -export([install_7/1]).
+-export([install_7a/1]).
 -export([install_8/1]).
+-export([install_8a/1]).
 -export([install_9/1]).
 -export([install_10/1]).
 -export([install_11/1]).
+-export([install_12/1]).
+-export([install_13/1]).
+-export([install_14/1]).
+-export([upgrade_restart_1/2]).
+-export([upgrade_restart_1a/1]).
+-export([upgrade_restart_2/1]).
+-export([upgrade_restart_2a/1]).
+-export([upgrade_restart_3/1]).
 -export([client1_1/4]).
 -export([client2/3]).
 -export([stop/1]).
@@ -46,28 +58,35 @@
 -define(fail(Term), exit({?MODULE, ?LINE, Term})).
 -define(fail_line(Line,Term), exit({?MODULE, Line, Term})).
 
--define(check_release(Vsn,Status,Apps),
-	check_release(TestNode,node(),Vsn,Status,Apps,?LINE)).
--define(check_release_client(Node,Vsn,Status,Apps),
-	check_release(TestNode,Node,Vsn,Status,Apps,?LINE)).
+-define(check_release_states(States),
+	check_release_states(TestNode,node(),States,?LINE)).
+-define(check_release_states_client(Node,States),
+	check_release_states(TestNode,Node,States,?LINE)).
+
+-define(check_release_lib(Vsn,Apps),
+	check_release_lib(TestNode,node(),Vsn,Apps,?LINE)).
+-define(check_release_lib_client(Node,Vsn,Apps),
+	check_release_lib(TestNode,Node,Vsn,Apps,?LINE)).
 
 -define(check_running_app(App,Vsn),
 	check_running_app(TestNode,node(),App,Vsn,?LINE)).
 -define(check_running_app_client(Node,App,Vsn),
 	check_running_app(TestNode,Node,App,Vsn,?LINE)).
 
+-define(check_disallowed_calls,check_disallowed_calls(TestNode,?LINE)).
+
 
 install_1(TestNode,PrivDir) ->
     ?print([TestNode]),
     ?print(["install_1 start"]),
+    ?check_release_states([permanent]),
 
     % Unpack and install P1H
     {ok, "P1H"} = unpack_release(PrivDir,"rel1"),
-    ?print(["unpack_release P1H ok"]),
-    ?check_release("P1H",unpacked,["a-1.0"]),
+    ?check_release_states([permanent,unpacked]),
+    ?check_release_lib("P1H",["a-1.0"]),
     {ok,"P1G",[new_appl]} = release_handler:install_release("P1H"),
-    ?print(["install_release P1H ok"]),
-    ?check_release("P1H",current,["a-1.0"]),
+    ?check_release_states([permanent,current]),
     ?check_running_app(a,"1.0"),
     X = a:a(),
     ?print(["X", X]),
@@ -81,172 +100,337 @@ install_2(TestNode) ->
     ?print(["install_2 start"]),
 
     % Check that P1H is still unpacked, install it and make_permanent
-    ?check_release("P1H",unpacked,["a-1.0"]),
-    ?print(["install_2 P1H unpacked"]),
+    ?check_release_states([permanent,unpacked]),
     {ok,"P1G",[new_appl]} = release_handler:install_release("P1H"),
     ?print(["install_2 install_release ok"]),
-    ?check_release("P1H",current,["a-1.0"]),
+    ?check_release_states([permanent,current]),
+    ?check_running_app(a,"1.0"),
+    ok = release_handler:make_permanent("P1H"),
+    ?print(["install_2 make permanent P1H ok"]),
+    ?check_release_states([old,permanent]),
     ?check_running_app(a,"1.0"),
-    ok = release_handler:make_permanent("P1H").
+    ok.
     % release_handler_SUITE will reboot this node now!
 
 install_3(TestNode,PrivDir) ->
     ?print(["install_3 start"]),
 
     % Check that P1H is permanent
-    ?check_release("P1H",permanent,["a-1.0"]),
+    ?check_release_states([old,permanent]),
+    ?check_running_app(a,"1.0"),
     X = a:a(),
     {key2, val2} = lists:keyfind(key2, 1, X),
     {key1, val1} = lists:keyfind(key1, 1, X),
 
     % Unpack and install P1I
     {ok, "P1I"} = unpack_release(PrivDir,"rel2"),
-    ?print(["install_3 unpack_release P1I ok"]),
-    ?check_release("P1I",unpacked,["a-1.1"]),
+    ?check_release_states([old,permanent,unpacked]),
+    ?check_release_lib("P1I",["a-1.1"]),
     {ok,"P1H",[{extra, gott}]} = release_handler:check_install_release("P1I"),
+    ?print(["install_3 check_install_release P1I ok"]),
     {error,_} = release_handler:check_install_release("P1J"),
+    ?print(["install_3 check_install_release P1J fails - ok"]),
     {ok,"P1H",[{extra, gott}]} = release_handler:install_release("P1I"),
-    ?print(["install_3 install_release P1I ok"]),
-    ?check_release("P1I",current,["a-1.1"]),
+    ?check_release_states([old,permanent,current]),
     ?check_running_app(a,"1.1"),
     X2 = a:a(),
     {key2, newval2} = lists:keyfind(key2, 1, X2),
     {key1, val1} = lists:keyfind(key1, 1, X2),
     {ok, bval} = a:b(),
+    ?print(["install_3 env ok"]),
 
-    % Unpack and install P2A
+    % Unpack P2A
     {ok, "P2A"} = unpack_release(PrivDir,"rel3"),
-    ?print(["install_3 unpack_release P2A ok"]),
-    ?check_release("P2A",unpacked,["a-1.1"]),
+    ?check_release_states([old,permanent,current,unpacked]),
+    ?check_release_lib("P2A",["a-1.1"]),
     {ok, "P1I", [new_emu]} = release_handler:check_install_release("P2A"),
     ?print(["install_3 check_install_release P2A ok"]),
-    ok = release_handler:make_permanent("P1I"),
-    ?print(["install_3 make_permanent P1I ok"]),
-    ?check_release("P1I",permanent,["a-1.1"]),
     ok.
+    % release_handler_SUITE will reboot this node now!
+
+install_4(TestNode) ->
+    ?print(["install_4 start"]),
 
-install_3a(TestNode) ->
-    {ok, "P1I", [new_emu]} = release_handler:install_release("P2A"),
+    %% Check that P1H is the one that is used
+    ?check_release_states([old,permanent,unpacked,unpacked]),
+    ?check_running_app(a,"1.0"),
+
+    %% Install P2A
+    {continue_after_restart, "P1H", [new_emu,new_appl]} =
+	release_handler:install_release("P2A"),
     %% Node is rebooted by the release_handler:install_release
     %% (init:reboot) because P2A includes a new erts vsn and the relup
     %% file contains a 'restart_new_emulator' instruction.
-    ?print(["install_3 P2A installed"]),
+    ?print(["install_4 P2A installed"]),
     ok.
 
 
+install_5(TestNode) ->
+    ?print(["install_5 start"]),
 
-install_4(TestNode) ->
-    ?print(["install_4 start"]),
+    %% Check that the upgrade was done via a temporary release due to
+    %% new emulator version.
+    {"SASL-test","__new_emulator__P1H"} = init:script_id(),
+
+    %% Check that P2A is in use.
+    ?check_release_states([old,permanent,unpacked,current]),
+    ?check_running_app(a,"1.1"),
+    X = a:a(),
+    {key2, newval2} = lists:keyfind(key2, 1, X),
+    {key1, val1} = lists:keyfind(key1, 1, X),
+    {ok, bval} = a:b(),
+    ?print(["install_5 check env ok"]),
+    ok.
+
+install_5a(TestNode) ->
+    ?print(["install_5a start"]),
+
+    %% Install P1I (this will be a downgrade)
+    {ok, "P1I", [old_emu]} = release_handler:install_release("P1I"),
+    %% Node is rebooted by the release_handler:install_release
+    %% (init:reboot) because P2A includes a new erts vsn and the relup
+    %% file contains a 'restart_new_emulator' instruction.
+    ?print(["install_5a P1I installed"]),
+    ok.
+
+install_6(TestNode) ->
+    ?print(["install_6 start"]),
+
+    %% Check that P1I is used
+    ?check_release_states([old,permanent,current,old]),
+    ?check_running_app(a,"1.1"),
+
+    %% Make P1I permanent
+    ok = release_handler:make_permanent("P1I"),
+    ?check_release_states([old,old,permanent,old]),
+    ?check_running_app(a,"1.1"),
+    ok.
+
+install_6a(TestNode) ->
+    %% Install P2A
+    {continue_after_restart, "P1I", [new_emu]} =
+	release_handler:install_release("P2A"),
+    %% Node is rebooted by the release_handler:install_release
+    %% (init:reboot) because P2A includes a new erts vsn and the relup
+    %% file contains a 'restart_new_emulator' instruction.
+    ?print(["install_6a P2A installed"]),
+    ok.
+
+install_7(TestNode) ->
+    ?print(["install_7 start"]),
+
+    %% Check that the upgrade was done via a temporary release due to
+    %% new emulator version.
+    {"SASL-test","__new_emulator__P1I"} = init:script_id(),
 
     % Check that P2A is in use.
-    ?check_release("P2A",current,["a-1.1"]),
+    ?check_release_states([old,old,permanent,current]),
     ?check_running_app(a,"1.1"),
     X = a:a(),
     {key2, newval2} = lists:keyfind(key2, 1, X),
     {key1, val1} = lists:keyfind(key1, 1, X),
     {ok, bval} = a:b(),
+    ?print(["install_7 check env ok"]),
     ok.
-    % release_handler_SUITE will reboot this node now!
 
-install_5(TestNode) ->
-    ?print(["install_5 start"]),
+install_7a(TestNode) ->
+    %% Install P1H (this will be a downgrade)
+    {ok, "P1H", [old_emu,old_appl]} = release_handler:install_release("P1H"),
+    %% Node is rebooted by the release_handler:install_release
+    %% (init:reboot) because P2A includes a new erts vsn and the relup
+    %% file contains a 'restart_new_emulator' instruction.
+    ?print(["install_7a P1H installed"]),
+    ok.
 
-    % Check that P1I is used
-    {ok, "P1I", [new_emu]} = release_handler:check_install_release("P2A"),
+install_8(TestNode) ->
+    ?print(["install_8 start"]),
+
+    %% Check that P1H is used
+    ?check_release_states([old,current,permanent,old]),
+    ?check_running_app(a,"1.0"),
+    {ok,"P1H",[new_emu,new_appl]} = release_handler:check_install_release("P2A"),
+    ?print(["install_8 check_install_release P2A ok"]),
+
+    %% Install P1I and check that it is permanent
+    {ok,"P1H",[{extra, gott}]} = release_handler:install_release("P1I"),
+    ?check_release_states([old,old,permanent,old]),
+    ?check_running_app(a,"1.1"),
     ok.
 
-install_5a(TestNode) ->
+install_8a(TestNode) ->
     % Install P2A again
-    {ok, "P1I", [new_emu]} = release_handler:install_release("P2A"),
+    {continue_after_restart, "P1I", [new_emu]} =
+	release_handler:install_release("P2A"),
     %% Node is rebooted by the release_handler:install_release
     %% (init:reboot) because P2A includes a new erts vsn and the relup
     %% file contains a 'restart_new_emulator' instruction.
-    ?print(["install_5 P2A installed"]),
+    ?print(["install_8a P2A installed"]),
     ok.
 
-install_6(TestNode) ->
-    ?print(["install_6 start"]),
+install_9(TestNode) ->
+    ?print(["install_9 start"]),
+
+    %% Check that the upgrade was done via a temporary release due to
+    %% new emulator version.
+    {"SASL-test","__new_emulator__P1I"} = init:script_id(),
 
     % Check that P2A is used
-    ?check_release("P2A",current,["a-1.1"]),
+    ?check_release_states([old,old,permanent,current]),
     ?check_running_app(a,"1.1"),
     X = a:a(),
     {key2, newval2} = lists:keyfind(key2, 1, X),
     {key1, val1} = lists:keyfind(key1, 1, X),
     {ok, bval} = a:b(),
-    ok = release_handler:make_permanent("P2A").
+    ?print(["install_9 check env ok"]),
+    ok = release_handler:make_permanent("P2A"),
+    ?check_release_states([old,old,old,permanent]),
+    ?check_running_app(a,"1.1"),
+    ok.
     % release_handler_SUITE will reboot this node now!
 
 
-install_7(TestNode) ->
-    ?print(["install_7 start"]),
+install_10(TestNode) ->
+    ?print(["install_10 start"]),
 
     % Check that P2A is used
-    ?check_release("P2A",permanent,["a-1.1"]),
+    ?check_release_states([old,old,old,permanent]),
+    ?check_running_app(a,"1.1"),
 
     % Install old P1H
     ok = release_handler:reboot_old_release("P1H"),
+    ?print(["install_10 reboot_old ok"]),
     ok.
 
-install_8(TestNode) ->
-    ?print(["install_8 start"]),
+
+install_11(TestNode) ->
+    ?print(["install_11 start"]),
 
     % Check that P1H is permanent
-    ?check_release("P1H",permanent,["a-1.0"]),
+    ?check_release_states([old,permanent,old,old]),
+    ?check_running_app(a,"1.0"),
     X = a:a(),
     {key2, val2} = lists:keyfind(key2, 1, X),
     {key1, val1} = lists:keyfind(key1, 1, X),
+    ?print(["install_11 check env ok"]),
 
     %% Remove P1I and P2A and check that a-1.1 and erts-<latest> are removed
     ok = release_handler:remove_release("P2A"),
+    ?check_release_states([old,permanent,old]),
     ok = release_handler:remove_release("P1I"),
+    ?check_release_states([old,permanent]),
     {ok, Libs} = file:list_dir(code:lib_dir()),
     {_,_,StdlibVsn} = lists:keyfind(stdlib,1,application:which_applications()),
     true = lists:member("stdlib-"++StdlibVsn, Libs),
     true = lists:member("a-1.0", Libs),
     false = lists:member("a-1.1", Libs),
     {ok, Dirs} = file:list_dir(code:root_dir()),
-    ["erts-4.4"] = lists:filter(fun(Dir) -> lists:prefix("erts-",Dir) end, Dirs),
+    ErtsDir = "erts-"++?ertsvsn,
+    [ErtsDir] = lists:filter(fun(Dir) -> lists:prefix("erts-",Dir) end, Dirs),
+    ?print(["install_11 file checks ok"]),
     ok.
     % release_handler_SUITE will reboot this node now!
 
-install_9(TestNode) ->
-    ?print(["install_9 start"]),
+install_12(TestNode) ->
+    ?print(["install_12 start"]),
 
     % Check that P1H is permanent
-    ?check_release("P1H",permanent,["a-1.0"]),
+    ?check_release_states([old,permanent]),
+    ?check_running_app(a,"1.0"),
     X = a:a(),
     {key2, val2} = lists:keyfind(key2, 1, X),
     {key1, val1} = lists:keyfind(key1, 1, X),
+    ?print(["install_12 check env ok"]),
 
     % Install old P1G
     ok = release_handler:reboot_old_release("P1G"),
+    ?print(["install_12 reboot_old ok"]),
     ok.
 
-install_10(TestNode) ->
-    ?print(["install_10 start"]),
+install_13(TestNode) ->
+    ?print(["install_13 start"]),
 
     % Check that P1G is permanent
-    ?check_release("P1G",permanent,[]),
-    ?check_release("P1H",old,["a-1.0"]),
+    ?check_release_states([permanent,old]),
+    false = lists:keysearch(a,1,application:loaded_applications()),
+    ?print(["install_13 no a application found - ok"]),
 
     %% Remove P1H and check that both versions of application a is removed
     ok = release_handler:remove_release("P1H"),
+    ?check_release_states([permanent]),
     {ok, Libs} = file:list_dir(code:lib_dir()),
     {_,_,StdlibVsn} = lists:keyfind(stdlib,1,application:which_applications()),
     true = lists:member("stdlib-"++StdlibVsn, Libs),
     false = lists:member("a-1.0", Libs),
     false = lists:member("a-1.1", Libs),
+    ?print(["install_13 file checks ok"]),
     ok.
     % release_handler_SUITE will reboot this node now!
 
-install_11(TestNode) ->
-    ?print(["install_11 start"]),
+install_14(TestNode) ->
+    ?print(["install_14 start"]),
 
     % Check that P1G is permanent
-    ?check_release("P1G",permanent,[]),
+    ?check_release_states([permanent]),
+    false = lists:keysearch(a,1,application:loaded_applications()),
+    ?print(["install_13 no a application found - ok"]),
+    ok.
+
+
+%%%-----------------------------------------------------------------
+%%% Ths test checks that an upgrade which both upgrades to a new
+%%% emulator version, and had a restart_emulator option to
+%%% systools:make_relup will be restarted twice on upgrade.
+%%% (On downgrade it will happen only once.)
+upgrade_restart_1(TestNode,PrivDir) ->
+    ?print([TestNode]),
+    ?print(["upgrade_restart_1 start"]),
+    ?check_release_states([permanent]),
+
+    {ok, "P2B"} = unpack_release(PrivDir,"rel4"),
+    ?check_release_states([permanent,unpacked]),
+    ?check_release_lib("P2B",["a-1.1"]),
+    ok.
+
+upgrade_restart_1a(TestNode) ->
+    ?print(["upgrade_restart_1a start"]),
+
+    {continue_after_restart,"P1G",[new_emu,add_appl]} =
+	release_handler:install_release("P2B"),
+    ?print(["upgrade_restart_1a P2B installed"]),
+    ok.
+
+upgrade_restart_2(TestNode) ->
+    ?print(["upgrade_restart_2 start"]),
+
+    %% Check that the node has been restarted once more after the tmp release
+    {"SASL-test","P2B"} = init:script_id(),
+    ?check_release_states([permanent,current]),
+    ?check_running_app(a,"1.1"),
+
+    ok = release_handler:make_permanent("P2B"),
+    ?check_release_states([old,permanent]),
+
     ok.
 
+upgrade_restart_2a(TestNode) ->
+    ?print(["upgrade_restart_2a start"]),
+
+    {ok,"P1G",[old_emu,rm_appl]} = release_handler:install_release("P1G"),
+    ?print(["upgrade_restart_2a P1G installed"]),
+    ok.
+
+upgrade_restart_3(TestNode) ->
+    ?print(["upgrade_restart_3 start"]),
+
+    %% Ideally we should test that the node has only been restarted
+    %% once... but that's not so easy. Let's just check that P1G is running.
+    ?check_release_states([current,permanent]),
+    false = lists:keysearch(a,1,application:loaded_applications()),
+    ?print(["upgrade_restart_3 no a application found - ok"]),
+
+    ok.
+
+
 
 
 %%-----------------------------------------------------------------
@@ -272,6 +456,8 @@ client1_1(TestNode,PrivDir,MasterDir,ClientSname) ->
     Node = start_client(TestNode,client1,ClientSname),
     trace_disallowed_calls(Node),
 
+    ?check_release_states_client(Node,[permanent]),
+
     %% Check env var for SASL on client node
     SaslEnv = rpc:call(Node, application, get_all_env, [sasl]),
     ?print([{client1_1,sasl_env},SaslEnv]),
@@ -300,13 +486,14 @@ client1_1(TestNode,PrivDir,MasterDir,ClientSname) ->
     %% as default. But it is given here in order to force hitting the
     %% release_handler:check_path function so it can be checked that
     %% it does not use file:read_file_info on the client node, see
-    %% trace_disallowed_calls/1 and check_disallowed_calls/0 below.
+    %% trace_disallowed_calls/1 and check_disallowed_calls/2 below.
     %% (OTP-9142)
     {ok, "P1H"} = rpc:call(Node, release_handler, set_unpacked,
 			   [filename:join(P1HDir, "rel1.rel"),
 			    [{a,"1.0",filename:join(MasterDir,lib)}]]),
     
-    ?check_release_client(Node,"P1H",unpacked,["a-1.0"]),
+    ?check_release_states_client(Node,[permanent,unpacked]),
+    ?check_release_lib_client(Node,"P1H",["a-1.0"]),
     
     ok = rpc:call(Node, release_handler, install_file,
 		  ["P1H", filename:join(P1HDir, "start.boot")]),
@@ -323,13 +510,16 @@ client1_1(TestNode,PrivDir,MasterDir,ClientSname) ->
     {ok,"P1G",[new_appl]} = 
 	rpc:call(Node, release_handler, install_release, ["P1H"]),
 
+    ?check_release_states_client(Node,[permanent,current]),
+    ?check_running_app_client(Node,a,"1.0"),
+
     Apps = rpc:call(Node, application, which_applications, []),
     {a,"A  CXC 138 11","1.0"} = lists:keyfind(a, 1, Apps),
     X = rpc:call(Node, a, a, []),
     {key2, val2} = lists:keyfind(key2, 1, X),
     {key1, val1} = lists:keyfind(key1, 1, X),
 
-    check_disallowed_calls(),
+    ?check_disallowed_calls,
     reboot(TestNode,Node),
     trace_disallowed_calls(Node),
 
@@ -339,17 +529,17 @@ client1_2(TestNode,PrivDir,Node) ->
     ?print(["client1_2 start"]),
 
     %% Check that P1H is still unpacked, install it and make_permanent
-    ?check_release_client(Node,"P1H",unpacked,["a-1.0"]),
+    ?check_release_states_client(Node,[permanent,unpacked]),
 
     {ok,"P1G",[new_appl]} = 
 	rpc:call(Node, release_handler, install_release, ["P1H"]),
-    ?check_release_client(Node,"P1H",current,["a-1.0"]),
+    ?check_release_states_client(Node,[permanent,current]),
     ?check_running_app_client(Node,a,"1.0"),
 
     ok = rpc:call(Node, release_handler, make_permanent, ["P1H"]),
-    ?check_release_client(Node,"P1H",permanent,["a-1.0"]),
+    ?check_release_states_client(Node,[old,permanent]),
 
-    check_disallowed_calls(),
+    ?check_disallowed_calls,
     reboot(TestNode,Node),
     trace_disallowed_calls(Node),
 
@@ -359,10 +549,8 @@ client1_3(TestNode,PrivDir,Node) ->
     ?print(["client1_3 start"]),
 
     %% Check that P1H is permanent
-    ?check_release_client(Node,"P1H",permanent,["a-1.0"]),
-    X = rpc:call(Node, a, a, []),
-    {key2, val2} = lists:keyfind(key2, 1, X),
-    {key1, val1} = lists:keyfind(key1, 1, X),
+    ?check_release_states_client(Node,[old,permanent]),
+    ?check_running_app_client(Node,a,"1.0"),
 
     %% Unpack P1I on master
     {ok, "P1I"} = unpack_release(PrivDir,"rel2"),
@@ -374,7 +562,8 @@ client1_3(TestNode,PrivDir,Node) ->
     {ok, "P1I"} = rpc:call(Node, release_handler, set_unpacked,
 			   [filename:join(P1IDir, "rel2.rel"),[]]),
 
-    ?check_release_client(Node,"P1I",unpacked,["a-1.1"]),
+    ?check_release_states_client(Node,[old,permanent,unpacked]),
+    ?check_release_lib_client(Node,"P1I",["a-1.1"]),
 
     ok = rpc:call(Node, release_handler, install_file,
                   ["P1I", filename:join(P1IDir, "start.boot")]),
@@ -389,6 +578,7 @@ client1_3(TestNode,PrivDir,Node) ->
     {ok,"P1H",[{extra, gott}]} =
         rpc:call(Node, release_handler, install_release, ["P1I"]),
 
+    ?check_release_states_client(Node,[old,permanent,current]),
     ?check_running_app_client(Node,a,"1.1"),
     X2 = rpc:call(Node, a, a, []),
     {key2, newval2} = lists:keyfind(key2, 1, X2),
@@ -404,6 +594,9 @@ client1_3(TestNode,PrivDir,Node) ->
         rpc:call(Node, release_handler, set_unpacked,
                  [filename:join(P2ADir, "rel3.rel"),[]]),
 
+    ?check_release_states_client(Node,[old,permanent,current,unpacked]),
+    ?check_release_lib_client(Node,"P2A",["a-1.1"]),
+
     ok = rpc:call(Node, release_handler, install_file,
                   ["P2A", filename:join(P2ADir, "start.boot")]),
     ok = rpc:call(Node, release_handler, install_file,
@@ -413,66 +606,136 @@ client1_3(TestNode,PrivDir,Node) ->
 
     {ok, "P1I", [new_emu]} = 
 	rpc:call(Node, release_handler, check_install_release, ["P2A"]),
+
+    %% Reboot from P1H
+    ?check_disallowed_calls,
+    reboot(TestNode,Node),
+    trace_disallowed_calls(Node),
+
+    client1_4(TestNode,Node).
+
+client1_4(TestNode,Node) ->
+    ?print(["client1_4 start"]),
+
+    %% check that P1H is used
+    ?check_release_states_client(Node,[old,permanent,unpacked,unpacked]),
+
+    %% since the install_release below reboot the node...
+    ?check_disallowed_calls,
+    cover_client(TestNode,Node,stop_cover),
+
+    {continue_after_restart, "P1H", [new_emu,new_appl]} =
+	rpc:call(Node, release_handler, install_release, ["P2A"]),
+    %% Reboots the client !
+
+    check_reboot(TestNode,Node),
+    trace_disallowed_calls(Node),
+
+    client1_5(TestNode,Node).
+
+client1_5(TestNode,Node) ->
+    ?print(["client1_5 start"]),
+
+    %% Check that P2A is in use.
+    ?check_release_states_client(Node,[old,permanent,unpacked,current]),
+    ?check_running_app_client(Node,a,"1.1"),
+    X = rpc:call(Node, a, a, []),
+    {key2, newval2} = lists:keyfind(key2, 1, X),
+    {key1, val1} = lists:keyfind(key1, 1, X),
+    {ok, bval} = rpc:call(Node, a, b, []),
+
+    %% since the install_release below reboot the node...
+    ?check_disallowed_calls,
+    cover_client(TestNode,Node,stop_cover),
+
+    {ok,"P1I",[old_emu]} =
+        rpc:call(Node, release_handler, install_release, ["P1I"]),
+
+    check_reboot(TestNode,Node),
+    trace_disallowed_calls(Node),
+
+    client1_6(TestNode,Node).
+
+client1_6(TestNode,Node) ->
+    ?print(["client1_6 start"]),
+
+    ?check_release_states_client(Node,[old,permanent,current,old]),
+    ?check_running_app_client(Node,a,"1.1"),
+
     ok = rpc:call(Node, release_handler, make_permanent, ["P1I"]),
-    ?check_release_client(Node,"P1I",permanent,["a-1.1"]),
+    ?check_release_states_client(Node,[old,old,permanent,old]),
 
     %% since the install_release below reboot the node...
-    check_disallowed_calls(),
+    ?check_disallowed_calls,
     cover_client(TestNode,Node,stop_cover), 
 
-    {ok, "P1I", [new_emu]} = 
+    {continue_after_restart, "P1I", [new_emu]} =
 	rpc:call(Node, release_handler, install_release, ["P2A"]),
     %% Reboots the client !
 
     check_reboot(TestNode,Node),
     trace_disallowed_calls(Node),
 
-    client1_4(TestNode,Node).
+    client1_7(TestNode,Node).
 
-client1_4(TestNode,Node) ->
-    ?print(["client1_4 start"]),
+client1_7(TestNode,Node) ->
+    ?print(["client1_7 start"]),
 
     %% Check that P2A is in use.
-    ?check_release_client(Node,"P2A",current,["a-1.1"]),
+    ?check_release_states_client(Node,[old,old,permanent,current]),
     ?check_running_app_client(Node,a,"1.1"),
     X = rpc:call(Node, a, a, []),
     {key2, newval2} = lists:keyfind(key2, 1, X),
     {key1, val1} = lists:keyfind(key1, 1, X),
     {ok, bval} = rpc:call(Node, a, b, []),
 
-    %% Reboot from P1I
-    check_disallowed_calls(),
-    reboot(TestNode,Node),
+    %% since the install_release below reboot the node...
+    ?check_disallowed_calls,
+    cover_client(TestNode,Node,stop_cover),
+
+    {ok,"P1H",[old_emu,old_appl]} =
+        rpc:call(Node, release_handler, install_release, ["P1H"]),
+
+    check_reboot(TestNode,Node),
     trace_disallowed_calls(Node),
 
-    client1_5(TestNode,Node).
+    client1_8(TestNode,Node).
 
-client1_5(TestNode,Node) ->
-    ?print(["client1_5 start"]),
+client1_8(TestNode,Node) ->
+    ?print(["client1_8 start"]),
 
-    %% Check that P1I is used
-    {ok, "P1I", [new_emu]} = 
+    %% Check that P1H is used
+    ?check_release_states_client(Node,[old,current,permanent,old]),
+    ?check_running_app_client(Node,a,"1.0"),
+    {ok, "P1H", [new_emu,new_appl]} =
 	rpc:call(Node, release_handler, check_install_release, ["P2A"]),
 
+
+    {ok,"P1H",[{extra, gott}]} =
+        rpc:call(Node, release_handler, install_release, ["P1I"]),
+    ?check_release_states_client(Node,[old,old,permanent,old]),
+    ?check_running_app_client(Node,a,"1.1"),
+
+
     %% since the install_release below will reboot the node...
-    check_disallowed_calls(),
+    ?check_disallowed_calls,
     cover_client(TestNode,Node,stop_cover),
 
     %% Install P2A again
-    {ok, "P1I", [new_emu]} = 
+    {continue_after_restart, "P1I", [new_emu]} =
 	rpc:call(Node, release_handler, install_release, ["P2A"]),
 
     %% We are rebooted again.
     check_reboot(TestNode,Node),
     trace_disallowed_calls(Node),
 
-    client1_6(TestNode,Node).
+    client1_9(TestNode,Node).
 
-client1_6(TestNode,Node) ->
-    ?print(["client1_6 start"]),
+client1_9(TestNode,Node) ->
+    ?print(["client1_9 start"]),
 
     %% Check that P2A is used
-    ?check_release_client(Node,"P2A",current,["a-1.1"]),
+    ?check_release_states_client(Node,[old,old,permanent,current]),
     ?check_running_app_client(Node,a,"1.1"),
     X = rpc:call(Node, a, a, []),
     {key2, newval2} = lists:keyfind(key2, 1, X),
@@ -481,22 +744,23 @@ client1_6(TestNode,Node) ->
 
     %% Make P2A permanent
     ok = rpc:call(Node, release_handler, make_permanent, ["P2A"]),
+    ?check_release_states_client(Node,[old,old,old,permanent]),
 
     %% Reboot from P2A
-    check_disallowed_calls(),
+    ?check_disallowed_calls,
     reboot(TestNode,Node),
     trace_disallowed_calls(Node),
 
-    client1_7(TestNode,Node).
+    client1_10(TestNode,Node).
 
-client1_7(TestNode,Node) ->
-    ?print(["client1_7 start"]),
+client1_10(TestNode,Node) ->
+    ?print(["client1_10 start"]),
 
     %% Check that P2A is used
-    ?check_release_client(Node,"P2A",permanent,["a-1.1"]),
+    ?check_release_states_client(Node,[old,old,old,permanent]),
 
     %% since the reboot_old_release below will reboot the node
-    check_disallowed_calls(),
+    ?check_disallowed_calls,
     cover_client(TestNode,Node,stop_cover),
 
     %% Install old P1H
@@ -505,41 +769,45 @@ client1_7(TestNode,Node) ->
     check_reboot(TestNode,Node),
     trace_disallowed_calls(Node),
 
-    client1_8(TestNode,Node).
+    client1_11(TestNode,Node).
 
-client1_8(TestNode,Node) ->
-    ?print(["client1_8 start"]),
+client1_11(TestNode,Node) ->
+    ?print(["client1_11 start"]),
 
     %% Check that P1H is permanent
-    ?check_release_client(Node,"P1H",permanent,["a-1.0"]),
+    ?check_release_states_client(Node,[old,permanent,old,old]),
+    ?check_running_app_client(Node,a,"1.0"),
     X = rpc:call(Node, a, a, []),
     {key2, val2} = lists:keyfind(key2, 1, X),
     {key1, val1} = lists:keyfind(key1, 1, X),
 
-    %% Remove P1I and P2I from client
+    %% Remove P1I and P2A from client
     ok = rpc:call(Node, release_handler, set_removed, ["P2A"]),
+    ?check_release_states_client(Node,[old,permanent,old]),
     ok = rpc:call(Node, release_handler, set_removed, ["P1I"]),
+    ?check_release_states_client(Node,[old,permanent]),
 
-    check_disallowed_calls(),
-    reboot(TestNode,Node),
-    trace_disallowed_calls(Node),
-
-    client1_9(TestNode,Node).
-
-client1_9(TestNode,Node) ->
-    ?print(["client1_9 start"]),
-
-    %% Check that P2A and P1I does not exists and that PiH is permanent.
+    %% Check that P2A and P1I does not exists
     Rels = rpc:call(Node, release_handler, which_releases, []),
     false = lists:keysearch("P2A", 2, Rels),
     false = lists:keysearch("P1I", 2, Rels),
-    ?check_release_client(Node,"P1H",permanent,["a-1.0"]),
     X = rpc:call(Node, a, a, []),
     {key2, val2} = lists:keyfind(key2, 1, X),
     {key1, val1} = lists:keyfind(key1, 1, X),
 
+    ?check_disallowed_calls,
+    reboot(TestNode,Node),
+    trace_disallowed_calls(Node),
+
+    client1_12(TestNode,Node).
+
+client1_12(TestNode,Node) ->
+    ?print(["client1_12 start"]),
+
+    ?check_release_states_client(Node,[old,permanent]),
+
     %% since the reboot_old_release below will reboot the node
-    check_disallowed_calls(),
+    ?check_disallowed_calls,
     cover_client(TestNode,Node,stop_cover),
 
     %% Install old P1G
@@ -548,33 +816,34 @@ client1_9(TestNode,Node) ->
     check_reboot(TestNode,Node),
     trace_disallowed_calls(Node),
 
-    client1_10(TestNode,Node).
+    client1_13(TestNode,Node).
 
-client1_10(TestNode,Node) ->
-    ?print(["client1_10 start"]),
+client1_13(TestNode,Node) ->
+    ?print(["client1_13 start"]),
 
     %% Check that P1G is permanent
-    ?check_release_client(Node,"P1G",permanent,[]),
-    ?check_release_client(Node,"P1H",old,["a-1.0"]),
+    ?check_release_states_client(Node,[permanent,old]),
     {error,client_node} = rpc:call(Node,release_handler,remove_release,["P1H"]),
     ok = rpc:call(Node, release_handler, set_removed, ["P1H"]),
+    ?check_release_states_client(Node,[permanent]),
 
-    check_disallowed_calls(),
+    ?check_disallowed_calls,
     reboot(TestNode,Node),
     trace_disallowed_calls(Node),
 
-    client1_11(TestNode,Node).
+    client1_14(TestNode,Node).
 
-client1_11(TestNode,Node) ->
-    ?print(["client1_11 start"]),
+client1_14(TestNode,Node) ->
+    ?print(["client1_14 start"]),
 
     %% Check that P1G is permanent
-    ?check_release_client(Node,"P1G",permanent,[]),
+    ?check_release_states_client(Node,[permanent]),
 
-    check_disallowed_calls(),
+    ?check_disallowed_calls,
     stop_client(TestNode,Node),  %% TEST IS OK !!
     net_kernel:monitor_nodes(false),
 
+    %% Remove releases from master
     ok = release_handler:remove_release("P2A"),
     ok = release_handler:remove_release("P1I"),
     ok = release_handler:remove_release("P1H"),
@@ -595,9 +864,10 @@ trace_disallowed_calls(Node) ->
     rpc:call(Node,dbg,p,[all,call]),
     rpc:call(Node,dbg,tp,[file,[{'_',[],[{message,{caller}}]}]]).
 
-check_disallowed_calls() ->
+check_disallowed_calls(TestNode,Line) ->
     receive
 	Trace when element(1,Trace)==trace ->
+	    ?print_line(Line,["Disallowed function called",Trace]),
 	    exit({disallowed_function_call,Trace})
     after 0 ->
 	    ok
@@ -628,7 +898,7 @@ start_client_unix(TestNode,Sname,Node) ->
 start_client_win32(TestNode,Client,ClientSname) ->
     Name = atom_to_list(ClientSname) ++ "_P1G",
     RootDir = code:root_dir(),
-    ErtsBinDir = filename:join(RootDir,"erts-4.4/bin"),
+    ErtsBinDir = filename:join([RootDir,"erts-"++?ertsvsn,"bin"]),
 
     {ClientArgs,RelClientDir} = rh_test_lib:get_client_args(Client,ClientSname,
 							    RootDir),
@@ -729,8 +999,10 @@ client2(TestNode,PrivDir,ClientSname) ->
 	    ok
     end,
 
+    %% Unpack P1H on master
     {ok, "P1H"} = unpack_release(PrivDir,"rel1"),
 
+    %% Try to set P1H unpacked on client
     Root = code:root_dir(),
     {error,{bad_masters,[Master2]}} =
 	rpc:call(Node, release_handler, set_unpacked,
@@ -755,15 +1027,17 @@ stop(Now) ->
 
 unpack_p1h(TestNode,PrivDir) ->
     {ok, "P1H"} = unpack_release(PrivDir,"rel1"), 
-    ?check_release("P1H",unpacked,["a-1.0"]),
+    ?check_release_states([permanent,unpacked]),
+    ?check_release_lib("P1H",["a-1.0"]),
     ok.
 
 permanent_p1h(TestNode) ->
-    ?check_release("P1H",unpacked,["a-1.0"]),
+    ?check_release_states([permanent,unpacked]),
+    ?check_release_lib("P1H",["a-1.0"]),
     {ok,"P1G",[new_appl]} = release_handler:install_release("P1H"),
-    ?check_release("P1H",current,["a-1.0"]),
+    ?check_release_states([permanent,current]),
     ok = release_handler:make_permanent("P1H"),
-    ?check_release("P1H",permanent,["a-1.0"]),
+    ?check_release_states([old,permanent]),
     ok.
 
 
@@ -779,24 +1053,38 @@ registered_loop(_Name) ->
             exit(killed)
     end.
 
-check_release(TestNode,Node,Vsn,Status,Apps,Line) ->
+%% Checks that the list of states for all releases (sorted on vsn)
+%% equals the input States
+check_release_states(TestNode,Node,States,Line) ->
+    case rpc:call(Node,release_handler,which_releases,[]) of
+	{badrpc,_}=Error ->
+	    ?fail_line(Line,{check_release_states,Node,States,Error});
+	Rels ->
+	    ?print_line(Line,["check_release_states:", Rels]),
+	    States = [Status || {_,_,_,Status} <- lists:keysort(2,Rels)],
+	    ok
+    end.
+
+%% Check that the given release (Vsn) sees the correct vsn of App.
+check_release_lib(TestNode,Node,Vsn,Apps,Line) ->
     case rpc:call(Node,release_handler,which_releases,[]) of
 	{badrpc,_}=Error ->
-	    ?fail_line(Line,{check_release,Node,Vsn,Status,Error});
+	    ?fail_line(Line,{check_release_lib,Node,Vsn,Apps,Error});
 	Rels ->
-	    ?print_line(Line,["check_release:", Rels]),
-	    {"SASL-test", Vsn, Libs, Status} = lists:keyfind(Vsn, 2, Rels),
+	    ?print_line(Line,["check_release_lib:", Rels]),
+	    {"SASL-test", Vsn, Libs, _Status} = lists:keyfind(Vsn, 2, Rels),
 	    true = lists:all(fun(App) -> lists:member(App,Libs) end,Apps),
 	    ok
     end.
 
+%% Check that the given Vsn of App is executed
 check_running_app(TestNode,Node,App,Vsn,Line) ->
     case rpc:call(Node,application,which_applications,[]) of
 	{badrpc,_}=Error ->
 	    ?fail_line(Line,{check_running_app,Node,App,Vsn,Error});
 	Apps ->
 	    ?print_line(Line,["check_running_app:", Apps]),
-	    {App, _, Vsn} = lists:keyfind(a, 1, Apps),
+	    {App, _, Vsn} = lists:keyfind(App, 1, Apps),
 	    ok
     end.