Merge branch 'siri/sasl/upgrade-erts/OTP-9438'

* siri/sasl/upgrade-erts/OTP-9438:
  Fix bug in erts upgrade on windows
  Add release vsn info to erts_vsn_changed warning
  Check for sasl application in systools:make_script and make_relup
  Add syntax check of relup to check_install_release and install_release
  Add documentation for upgrade from pre R15 to post R15 sasl
  Handle upgrade from pre R15 to post R15 sasl
  Step version of sasl to 2.2 for R15
  Document upgrade instructions restart_new_emulator and restart_emulator
  Wait for two restarts in upgrade_restart test
  Add restart_new_emulator instruction to kernel, stdlib and sasl appups
  Distinguish restart_new_emulator from restart_emulator in upgrade instructions
  Upgrade erts: merge sys.config for tmp release instead of using old
  Allow regexp for version in .appup
  Restart emulator before running upgrade script when erts is upgraded

Conflicts:
	lib/sasl/src/release_handler.erl
	lib/sasl/test/release_handler_SUITE.erl

1 files changed, 252 insertions, 94 deletions

diff --git a/lib/sasl/test/release_handler_SUITE.erl b/lib/sasl/test/release_handler_SUITE.erl
index 286aff9bb0..11f8bbe4fe 100644
--- a/lib/sasl/test/release_handler_SUITE.erl
+++ b/lib/sasl/test/release_handler_SUITE.erl
@@ -19,6 +19,7 @@
 -module(release_handler_SUITE).
 
 -include_lib("common_test/include/ct.hrl").
+-include("test_lib.hrl").
 
 -compile(export_all).
 
@@ -30,6 +31,7 @@ suite() ->
     [{ct_hooks, [ts_install_cth]}].
 
 init_per_suite(Config) ->
+    init_priv_dir(Config),
     application:start(sasl),
     Config.
 
@@ -59,7 +61,7 @@ cases() ->
      otp_9395_check_old_code, otp_9395_check_and_purge,
      otp_9395_update_many_mods, otp_9395_rm_many_mods,
      instructions, eval_appup, supervisor_which_children_timeout,
-     release_handler_which_releases].
+     release_handler_which_releases, install_release_syntax_check].
 
 groups() ->
     [{release,[],
@@ -70,6 +72,7 @@ groups() ->
      {release_single,[],
       [
        upgrade,
+       upgrade_restart,
        client1,
        client2
       ]},
@@ -83,7 +86,7 @@ groups() ->
 init_per_group(release, Config) ->
     Dog = ?t:timetrap(?default_timeout),
     P1gInstall = filename:join(priv_dir(Config),p1g_install),
-    ok = do_create_p1g(Config,P1gInstall),
+    ok = create_p1g(Config,P1gInstall),
     ok = create_p1h(Config),
     ?t:timetrap_cancel(Dog);
 
@@ -96,6 +99,7 @@ init_per_group(release_single, Config) ->
     %% Create some more releases to upgrade to
     ok = create_p1i(Config),
     ok = create_p2a(Config),
+    ok = create_p2b(Config),
 
     ?t:timetrap_cancel(Dog);
 
@@ -156,7 +160,7 @@ end_per_group(release, Config) ->
 	{win32,_} -> delete_all_services();
 	_ -> ok
     end,
-    delete_release(Config),
+    clean_priv_dir(Config,true),
     ?t:timetrap_cancel(Dog),
     Config;
 end_per_group(_GroupName, Config) ->
@@ -192,7 +196,10 @@ end_per_testcase(Case, Config) ->
 	    FailDir = filename:join(SaveDir,lists:concat(["failed-",Case])),
 	    ok = filelib:ensure_dir(filename:join(FailDir,"*")),
 
-	    LogDirs = filelib:wildcard(filename:join([PrivDir,"*",log])),
+	    LogDirs =
+		filelib:wildcard(filename:join([PrivDir,"*",log])) ++
+		filelib:wildcard(filename:join([PrivDir,"*",clients,
+						type1,"*",log])),
 
 	    lists:foreach(
 	      fun(LogDir) ->
@@ -237,7 +244,7 @@ break(Config) ->
 	?t:break(priv_dir(Config)),
 	ok.
 
-%% Test upgrade and downgrade of erts
+%% Test upgrade and downgrade of erts and other apps on embedded node
 upgrade(Conf) when is_list(Conf) ->
     reg_print_proc(), %% starts a printer process on test_server node
     ?t:format("upgrade ~p~n",[reg_print_proc]),
@@ -260,54 +267,75 @@ upgrade(Conf) when is_list(Conf) ->
     stop_cover(TestNode),
     reboot_and_wait(TestNode,"install_2",[a]),
 
-    %% check that P1H is permanent, unpack and install P1I, unpack and install P2A
-    TestNodeInit1 = rpc:call(TestNode,erlang,whereis,[init]),
+    %% check that P1H is permanent, unpack and install P1I, unpack P2A
     ok = rpc_inst(TestNode, install_3, [PrivDir]),
     stop_cover(TestNode),
-    ok = rpc_inst(TestNode, install_3a, []),
-    wait_nodes_up([{TestNode,TestNodeInit1}],"install_3",[a]),
+    reboot_and_wait(TestNode,"install_3",[a]),
 
-    %% check that P2A is used, reboot from P1I
-    ok = rpc_inst(TestNode, install_4, []),
+    %% check that P1H is used, install P2A
+    TestNodeInit1 = rpc:call(TestNode,erlang,whereis,[init]),
     stop_cover(TestNode),
-    reboot_and_wait(TestNode,"install_4",[a]),
+    ok = rpc_inst(TestNode, install_4, []),
+    wait_nodes_up([{TestNode,TestNodeInit1}],"install_4",[a]),
 
-    %% check that P1I, reinstall P2A
+    %% check that P2A is used, then downgrade to P1I
     TestNodeInit2 = rpc:call(TestNode,erlang,whereis,[init]),
     ok = rpc_inst(TestNode, install_5, []),
     stop_cover(TestNode),
     ok = rpc_inst(TestNode, install_5a, []),
     wait_nodes_up([{TestNode,TestNodeInit2}],"install_5",[a]),
 
-    %% check that P2A is used, make P2A permanent
+    %% Check that P1I is used, then make P1I permanent and install P2A
+    TestNodeInit3 = rpc:call(TestNode,erlang,whereis,[init]),
     ok = rpc_inst(TestNode, install_6, []),
     stop_cover(TestNode),
-    reboot_and_wait(TestNode,"install_6",[a]),
+    ok = rpc_inst(TestNode, install_6a, []),
+    wait_nodes_up([{TestNode,TestNodeInit3}],"install_6",[a]),
 
-    %% check that P2A is permanent, install old P1H
-    TestNodeInit3 = rpc:call(TestNode,erlang,whereis,[init]),
-    stop_cover(TestNode),
+    %% check that P2A is used, then downgrade to P1H
+    TestNodeInit4 = rpc:call(TestNode,erlang,whereis,[init]),
     ok = rpc_inst(TestNode, install_7, []),
-    wait_nodes_up([{TestNode,TestNodeInit3}],"install_7",[a]),
+    stop_cover(TestNode),
+    ok = rpc_inst(TestNode, install_7a, []),
+    wait_nodes_up([{TestNode,TestNodeInit4}],"install_7",[a]),
 
-    %% check that P1H is permanent, remove P1I and P2A
+    %% check that P1H is used, then install P1I and check that it is permanent
+    %% then reinstall P2A
+    TestNodeInit5 = rpc:call(TestNode,erlang,whereis,[init]),
     ok = rpc_inst(TestNode, install_8, []),
     stop_cover(TestNode),
-    reboot_and_wait(TestNode,"install_8",[a]),
+    ok = rpc_inst(TestNode, install_8a, []),
+    wait_nodes_up([{TestNode,TestNodeInit5}],"install_8",[a]),
+
+    %% check that P2A is used, make P2A permanent
+    ok = rpc_inst(TestNode, install_9, []),
+    stop_cover(TestNode),
+    reboot_and_wait(TestNode,"install_9",[a]),
+
+    %% check that P2A is permanent, reboot to old P1H
+    TestNodeInit6 = rpc:call(TestNode,erlang,whereis,[init]),
+    stop_cover(TestNode),
+    ok = rpc_inst(TestNode, install_10, []),
+    wait_nodes_up([{TestNode,TestNodeInit6}],"install_10",[a]),
+
+    %% check that P1H is permanent, remove P1I and P2A
+    ok = rpc_inst(TestNode, install_11, []),
+    stop_cover(TestNode),
+    reboot_and_wait(TestNode,"install_11",[a]),
 
     %% check that P1H is permanent, reboot old P1G
-    TestNodeInit4 = rpc:call(TestNode,erlang,whereis,[init]),
+    TestNodeInit7 = rpc:call(TestNode,erlang,whereis,[init]),
     stop_cover(TestNode),
-    ok = rpc_inst(TestNode, install_9, []),
-    wait_nodes_up([{TestNode,TestNodeInit4}],"install_9"),
+    ok = rpc_inst(TestNode, install_12, []),
+    wait_nodes_up([{TestNode,TestNodeInit7}],"install_12"),
 
     %% check that P1G is permanent, remove P1H
-    ok = rpc_inst(TestNode, install_10, []),
+    ok = rpc_inst(TestNode, install_13, []),
     stop_cover(TestNode),
-    reboot_and_wait(TestNode,"install_10"),
+    reboot_and_wait(TestNode,"install_13"),
 
     %% check that P1G is permanent
-    ok = rpc_inst(TestNode, install_11, []),
+    ok = rpc_inst(TestNode, install_14, []),
 
     ok.
 
@@ -324,6 +352,54 @@ reboot_and_wait(Node,Tag,Apps) ->
     wait_nodes_up([{Node,InitPid}],Tag,Apps).
 
 
+%% Test upgrade and downgrade of erts in combination with the
+%% restart_emulator option to systools:make_relup. For upgrade, this
+%% should cause one restart before the upgrade code, and one
+%% after. For downgrade, there will be one restart only - at the end.
+upgrade_restart(Conf) when is_list(Conf) ->
+    reg_print_proc(), %% starts a printer process on test_server node
+    ?t:format("upgrade_restart ~p~n",[reg_print_proc]),
+    PrivDir = priv_dir(Conf),
+    Sname = tc_sname(Conf), % nodename for use in this testcase
+
+    %% Copy the P1G release to a directory for use in this testcase
+    ok = copy_installed(Conf,p1g_install,[Sname]),
+
+    %% start the test node
+    [TestNode] = start_nodes(Conf,[Sname],"upgrade_restart start"),
+
+    %% unpack and install P2B
+    TestNodeInit1 = rpc:call(TestNode,erlang,whereis,[init]),
+    ok = rpc_inst(TestNode, upgrade_restart_1, [PrivDir]),
+    stop_cover(TestNode),
+    ok = rpc_inst(TestNode, upgrade_restart_1a, []),
+    wait_nodes_up([{TestNode,TestNodeInit1}],"upgrade_restart_1",[a]),
+
+    %% install P1G
+    case rpc_inst(TestNode, upgrade_restart_2, []) of
+	ok ->
+	    ok;
+	{wait,TestNodeInit2a} ->
+	    %% We catched the node too early - it was supposed to
+	    %% restart twice, so let's wait for one more restart.
+	    wait_nodes_up([{TestNode,TestNodeInit2a}],"upgrade_restart_2a",[]),
+	    ok = rpc_inst(TestNode, upgrade_restart_2a, [])
+    end,
+    TestNodeInit2 = rpc:call(TestNode,erlang,whereis,[init]),
+    stop_cover(TestNode),
+    ok = rpc_inst(TestNode, upgrade_restart_2b, []),
+    wait_nodes_up([{TestNode,TestNodeInit2}],"upgrade_restart_2b",[]),
+
+    %% Check that P1G is going again
+    ok = rpc_inst(TestNode, upgrade_restart_3, []),
+
+    ok.
+
+upgrade_restart(cleanup,Config) ->
+    TestNode = tc_full_node_name(Config),
+    ok = stop_nodes([TestNode]).
+
+
 %% Test upgrade and downgrade of erts, diskless
 client1(Conf) when is_list(Conf) ->
     reg_print_proc(), %% starts a printer process on test_server node
@@ -543,9 +619,25 @@ supervisor_which_children_timeout(Conf) ->
 
     ok.
 
-supervisor_which_children_timeout(cleanup, Conf) ->
+supervisor_which_children_timeout(cleanup, _Conf) ->
     stop_node(node_name(supervisor_which_children_timeout)).
 
+
+%% Test that check_install_release will fail for illegal relup
+%% instructions, even after point of no return.
+install_release_syntax_check(Conf) when is_list(Conf) ->
+
+    S1 = [point_of_no_return, illegal_instruction],
+    {error,{illegal_instruction_after_point_of_no_return,illegal_instruction}} =
+	release_handler_1:check_script(S1,[]),
+
+    S2 = [point_of_no_return,restart_new_emulator],
+    {error,{illegal_instruction_after_point_of_no_return,restart_new_emulator}} =
+	release_handler_1:check_script(S2,[]),
+
+    ok.
+
+
 %%-----------------------------------------------------------------
 %% release_handler:which_releases/0 and 1 test
 %%-----------------------------------------------------------------
@@ -1174,11 +1266,10 @@ target_system(Conf) when is_list(Conf) ->
 
 
     %% Create the .rel file
-    ErtsVsn = erlang:system_info(version),
     RelName = filename:join(TargetCreateDir,"ts-1.0"),
     RelFile = RelName++".rel",
     RelVsn = "R1A",
-    create_rel_file(RelFile,RelName,RelVsn,ErtsVsn,[{a, "1.0"}]),
+    create_rel_file(RelFile,RelName,RelVsn,current,[{a, "1.0"}]),
 
     %% Build the target_system module
     ExamplesEbin = filename:join([code:lib_dir(sasl),examples,ebin]),
@@ -1206,11 +1297,12 @@ target_system(Conf) when is_list(Conf) ->
     code:del_path(TSPath),
 
     %% Check that all files exist in installation
-    true = filelib:is_dir(filename:join(TargetInstallDir,"erts-"++ErtsVsn)),
+    ErtsDir = app_dir(erts,current),
+    true = filelib:is_dir(filename:join(TargetInstallDir,ErtsDir)),
     LibDir = filename:join(TargetInstallDir,lib),
-    {ok,KernelVsn} = application:get_key(kernel,vsn),
-    {ok,StdlibVsn} = application:get_key(stdlib,vsn),
-    {ok,SaslVsn} = application:get_key(sasl,vsn),
+    KernelVsn = vsn(kernel,current),
+    StdlibVsn = vsn(stdlib,current),
+    SaslVsn = vsn(sasl,current),
     true = filelib:is_dir(filename:join(LibDir,"kernel-"++KernelVsn)),
     true = filelib:is_dir(filename:join(LibDir,"stdlib-"++StdlibVsn)),
     true = filelib:is_dir(filename:join(LibDir,"sasl-"++SaslVsn)),
@@ -1232,6 +1324,7 @@ target_system(Conf) when is_list(Conf) ->
     true = filelib:is_regular(filename:join(BinDir,to_erl)),
 
     %% Check content of files
+    ErtsVsn = vsn(erts,current),
     {ok,SED} = file:read_file(filename:join(RelDir,"start_erl.data")),
     [ErtsVsn,RelVsn] = string:tokens(binary_to_list(SED),"\s\n"),
     ok.
@@ -1483,7 +1576,7 @@ copy_client(Conf,Master,Sname,Client) ->
     ok.
 
 
-delete_release(Conf) ->
+clean_priv_dir(Conf,Save) ->
     PrivDir = priv_dir(Conf),
 
     {ok, OrigWd} = file:get_cwd(),
@@ -1493,7 +1586,7 @@ delete_release(Conf) ->
     {ok, Dirs} = file:list_dir(PrivDir),
     ?t:format("========  deleting  ~p~n",[Dirs]),
 
-    ok = delete_release_os(Dirs--["save"]),
+    ok = clean_dirs_os(Dirs,Save),
     {ok,Remaining} = file:list_dir(PrivDir),
     ?t:format("========  remaining  ~p~n",[Remaining]),
 
@@ -1501,7 +1594,7 @@ delete_release(Conf) ->
 	[] ->
 	    ok;
 	_ ->
-	    delete_release_os(Remaining),
+	    clean_dirs_os(Remaining,Save),
 	    Remaining2 = file:list_dir(PrivDir),
 	    ?t:format("========  remaining after second try ~p~n",[Remaining2])
     end,
@@ -1510,22 +1603,22 @@ delete_release(Conf) ->
     ok.
 
 
-delete_release_os(Dirs) ->
+clean_dirs_os(Dirs,Save) ->
     case os:type() of
 	{unix, _} ->
-	    delete_release_unix(Dirs);
+	    clean_dirs_unix(Dirs,Save);
 	{win32, _} ->
-	    delete_release_win32(Dirs);
+	    clean_dirs_win32(Dirs,Save);
 	Os ->
 	    test_server:fail({error, {not_yet_implemented_os, Os}})
     end.
 
 
-delete_release_unix([]) ->
+clean_dirs_unix([],_) ->
     ok;
-delete_release_unix(["save"|Dirs]) ->
-    delete_release_unix(Dirs);
-delete_release_unix([Dir|Dirs]) ->
+clean_dirs_unix(["save"|Dirs],Save) when Save ->
+    clean_dirs_unix(Dirs,Save);
+clean_dirs_unix([Dir|Dirs],Save) ->
     Rm = string:concat("rm -rf ", Dir),
     ?t:format("============== COMMAND ~p~n",[Rm]),
     case file:list_dir(Dir) of
@@ -1542,13 +1635,13 @@ delete_release_unix([Dir|Dirs]) ->
 		  ?t:format("------- ls -al  ~p~n",[os:cmd("ls -al " ++ Dir)])
 	  end,
 
-    delete_release_unix(Dirs).
+    clean_dirs_unix(Dirs,Save).
 
-delete_release_win32([]) ->
+clean_dirs_win32([],_) ->
     ok;
-delete_release_win32(["save"|Dirs]) ->
-    delete_release_win32(Dirs);
-delete_release_win32([Dir|Dirs]) ->
+clean_dirs_win32(["save"|Dirs],Save) when Save ->
+    clean_dirs_win32(Dirs,Save);
+clean_dirs_win32([Dir|Dirs],Save) ->
     Rm =
        case filelib:is_dir(Dir) of
           true ->
@@ -1558,7 +1651,7 @@ delete_release_win32([Dir|Dirs]) ->
        end,
     ?t:format("============== COMMAND ~p~n",[Rm]),
     [] = os:cmd(Rm),
-    delete_release_win32(Dirs).
+    clean_dirs_win32(Dirs,Save).
 
 
 node_name(Sname) when is_atom(Sname) ->
@@ -1684,9 +1777,17 @@ priv_dir(Conf) ->
 %%    filename:absname(?config(priv_dir, Conf)). % Get rid of trailing slash
     %% Due to problem with long paths on windows => creating a new
     %% priv_dir under data_dir
+    filename:absname(filename:join(?config(data_dir, Conf),priv_dir)).
+
+init_priv_dir(Conf) ->
     Dir = filename:absname(filename:join(?config(data_dir, Conf),priv_dir)),
-    filelib:ensure_dir(filename:join(Dir,"*")),
-    Dir.
+    case filelib:is_dir(Dir) of
+	true ->
+	    clean_priv_dir(Conf,false);
+	false ->
+	    ok
+    end,
+    filelib:ensure_dir(filename:join(Dir,"*")).
 
 latest_version(Dir) ->
     List = filelib:wildcard(Dir ++ "*"),
@@ -1721,14 +1822,22 @@ stop_print_proc() ->
 
 %% Create the first target release, vsn P1G. This release is used for
 %% all test cases in {group,release}
-create_p1g(Conf,Sname) ->
-    do_create_p1g(Conf,filename:join(priv_dir(Conf),Sname)).
-
-do_create_p1g(Conf,TargetDir) ->
-    PrivDir = priv_dir(Conf),
+create_p1g(Conf,TargetDir) ->
     DataDir = ?config(data_dir,Conf),
-    ErtsVsn = "4.4",
-    ErtsDir = "erts-"++ErtsVsn,
+    PrivDir = priv_dir(Conf),
+    ErtsDir = app_dir(erts,old),
+    KernelDir = app_dir(kernel,old),
+    StdlibDir = app_dir(stdlib,old),
+
+    %% Fake earlier version of kernel and stdlib
+    SystemLib = system_lib(PrivDir),
+    ok = filelib:ensure_dir(filename:join(SystemLib,"*")),
+    KernelLib = code:lib_dir(kernel),
+    StdlibLib = code:lib_dir(stdlib),
+    ok = copy_tree(Conf,KernelLib,KernelDir,SystemLib),
+    ok = copy_tree(Conf,StdlibLib,StdlibDir,SystemLib),
+    fix_version(SystemLib,kernel),
+    fix_version(SystemLib,stdlib),
 
     %% Create dirs
     BinDir = filename:join(TargetDir,bin),
@@ -1772,17 +1881,15 @@ do_create_p1g(Conf,TargetDir) ->
     RelFileName = filename:join(RelDir,RelName),
     RelFile = RelFileName ++ ".rel",
     ok = filelib:ensure_dir(RelFile),
-    LibPath = filename:join([DataDir,lib,"*",ebin]),
 
-    TarFile = create_basic_release(Conf, RelFile, RelVsn, {ErtsVsn,false},
-				   LibPath, [], [], [], []),
+    TarFile = create_basic_release(Conf,RelFile,RelVsn,{old,false}),
 
     %% Extract tar file in target directory (i.e. same directory as erts etc.)
     ok = erl_tar:extract(TarFile, [{cwd, TargetDir}, compressed]),
 
     %% Create start_erl.data
     StartErlDataFile = filename:join([ReleasesDir, "start_erl.data"]),
-    StartErlData = io_lib:fwrite("~s ~s~n", [ErtsVsn, RelVsn]),
+    StartErlData = io_lib:fwrite("~s ~s~n", [vsn(erts,old), RelVsn]),
     ok = file:write_file(StartErlDataFile, StartErlData),
 
     %% Create RELEASES
@@ -1790,60 +1897,98 @@ do_create_p1g(Conf,TargetDir) ->
 
     ok.
 
+fix_version(SystemLib,App) ->
+    FromVsn = vsn(App,current),
+    ToVsn = vsn(App,old),
+    Rootname = filename:join([SystemLib,app_dir(App,old),ebin,atom_to_list(App)]),
+
+    AppFile = Rootname ++ ".app",
+    {ok,OrigApp} = file:read_file(AppFile),
+    ok = file:write_file(AppFile,re:replace(OrigApp,FromVsn,ToVsn,
+					    [{return,binary}])),
+    AppupFile = Rootname ++ ".appup",
+    {ok,OrigAppup} = file:read_file(AppupFile),
+    ok = file:write_file(AppupFile,re:replace(OrigAppup,FromVsn,ToVsn,
+					      [{return,binary}])).
+
+
 %% Create version P1H - which is P1G + a-1.0
 %% Must have run create_p1g first!!
 create_p1h(Conf) ->
-    create_upgrade_release(Conf,"rel1","P1H",{"4.4",false},[{a,"1.0"}],
-			   [{a,[{key2,val2}]}],{"rel0",[new_appl]}).
+    create_upgrade_release(Conf,"rel1","P1H",{old,false},[{a,"1.0"}],
+			   [{a,[{key2,val2}]}],[{"rel0",[new_appl]}]).
 
 %% Create version P1I - which is P1H, but with application a upgraded to a-1.1
 %% Must have run create_p1h first!!
 create_p1i(Conf) ->
-    create_upgrade_release(Conf,"rel2","P1I",{"4.4",false},[{a,"1.1"}],
+    create_upgrade_release(Conf,"rel2","P1I",{old,false},[{a,"1.1"}],
 			   [{a,[{key2,newval2}]}],
-			   {"rel1",[{extra,gott}]}).
+			   [{"rel1",[{extra,gott}]}]).
 
 %% Create version P2A - which is P1I, but with erts-<latest>
 %% Must have run create_p1i first!!
 create_p2a(Conf) ->
-    ErtsVsn = erlang:system_info(version),
-    create_upgrade_release(Conf,"rel3","P2A",{ErtsVsn,code:root_dir()},
+    create_upgrade_release(Conf,"rel3","P2A",{current,code:root_dir()},
+			   [{a,"1.1"}],[{a,[{key2,newval2}]}],
+			   [{"rel1",[new_emu,new_appl]},{"rel2",[new_emu]}],
+			   [{"rel1",[old_emu,old_appl]},{"rel2",[old_emu]}]).
+
+%% Create version P2B - which is P2A, but with relup containing an
+%% extra reboot.
+%% Can be upgraded to from P1G - so must have run create_p1g first!!
+create_p2b(Conf) ->
+    create_upgrade_release(Conf,"rel4","P2B",{current,code:root_dir()},
 			   [{a,"1.1"}],[{a,[{key2,newval2}]}],
-			   {"rel2",[new_emu]}).
+			   [{"rel0",[new_emu,add_appl]}],
+			   [{"rel0",[old_emu,rm_appl]}],
+			   [restart_emulator]).
 
 %% Create a release tar package which can be installed on top of P1G
-create_upgrade_release(Conf,RelName,RelVsn,Erts,Apps,Config,{UpFromName,Descr}) ->
+create_upgrade_release(Conf,RelName,RelVsn,Erts,Apps,Config,UpFrom) ->
+    create_upgrade_release(Conf,RelName,RelVsn,Erts,Apps,Config,UpFrom,[]).
+create_upgrade_release(Conf,RelName,RelVsn,Erts,Apps,Config,UpFrom,DownTo) ->
+    create_upgrade_release(Conf,RelName,RelVsn,Erts,Apps,Config,UpFrom,DownTo,[]).
+create_upgrade_release(Conf,RelName,RelVsn,Erts,Apps,Config,UpFrom0,DownTo0,RelupOpts) ->
     PrivDir = priv_dir(Conf),
-    DataDir = ?config(data_dir,Conf),
-
     RelDir = filename:join(PrivDir,RelName),
     RelFileName = filename:join(RelDir,RelName),
     RelFile = RelFileName ++ ".rel",
     ok = filelib:ensure_dir(RelFile),
-    LibPath = filename:join([DataDir,lib,"*",ebin]),
 
-    UpFrom = [{filename:join([PrivDir,UpFromName,UpFromName]),Descr}],
+    UpFrom = [{filename:join([PrivDir,UpFromName,UpFromName]),Descr} ||
+		 {UpFromName,Descr} <- UpFrom0],
+    DownTo = [{filename:join([PrivDir,DownToName,DownToName]),Descr} ||
+		 {DownToName,Descr} <- DownTo0],
 
-    create_basic_release(Conf, RelFile, RelVsn, Erts, LibPath,
-			 Apps, Config, UpFrom, []),
+    create_basic_release(Conf,RelFile,RelVsn,Erts,Apps,Config,
+			 UpFrom,DownTo,RelupOpts),
     ok.
 
 %% Create .rel, .script, .boot, sys.config and tar
-create_basic_release(Conf, RelFile,RelVsn,{ErtsVsn,ErtsDir},LibPath,ExtraApps,Config,UpFrom,DownTo) ->
+create_basic_release(Conf,RelFile,RelVsn,{Erts,ErtsDir}) ->
+    create_basic_release(Conf, RelFile,RelVsn,{Erts,ErtsDir},[],[],[],[],[]).
+create_basic_release(Conf,RelFile,RelVsn,{Erts,ErtsDir},ExtraApps,Config,UpFrom,DownTo,RelupOpts) ->
+    DataDir = ?config(data_dir,Conf),
+    PrivDir = priv_dir(Conf),
+    SystemLib = system_lib(PrivDir),
+    LibPath = [filename:join([SystemLib,"*",ebin]),
+	       filename:join([DataDir,lib,"*",ebin])],
+
     RelDir = filename:dirname(RelFile),
     RelFileName = filename:rootname(RelFile),
 
     %% Create .rel file
-    create_installer_rel_file(RelFile,RelVsn,ErtsVsn,ExtraApps),
+    create_installer_rel_file(RelFile,RelVsn,Erts,ExtraApps),
 
     %% Generate .script and .boot
     ok = systools:make_script(RelFileName,
-			      [{path,[LibPath]},
+			      [{path,LibPath},
 			       {outdir,RelDir}]),
 
     %% Generate relup
-    ok = systools:make_relup(RelFileName,UpFrom,DownTo,[{path,[LibPath]},
-							      {outdir,RelDir}]),
+    ok = systools:make_relup(RelFileName,UpFrom,DownTo,[{path,LibPath},
+							{outdir,RelDir} |
+							RelupOpts]),
 
     %% Create sys.config
     ok = write_term_file(filename:join(RelDir,"sys.config"),Config),
@@ -1851,7 +1996,7 @@ create_basic_release(Conf, RelFile,RelVsn,{ErtsVsn,ErtsDir},LibPath,ExtraApps,Co
 
     %% Create tar file (i.e. collect all lib/app-*/* and system files)
     ok = systools:make_tar(RelFileName,
-			   [{path,[LibPath]},
+			   [{path,LibPath},
 			    {outdir,RelDir} |
 			    case ErtsDir of
 				false -> [];
@@ -1868,18 +2013,19 @@ create_basic_release(Conf, RelFile,RelVsn,{ErtsVsn,ErtsDir},LibPath,ExtraApps,Co
     TarFileName.
 
 %% Create a .rel file
-create_installer_rel_file(RelFile,RelVsn,ErtsVsn,ExtraApps) ->
-    create_rel_file(RelFile,"SASL-test",RelVsn,ErtsVsn,
+create_installer_rel_file(RelFile,RelVsn,Erts,ExtraApps) ->
+    create_rel_file(RelFile,"SASL-test",RelVsn,Erts,
 		    [{installer,"1.0"}|ExtraApps]).
 
-create_rel_file(RelFile,RelName,RelVsn,ErtsVsn,ExtraApps) ->
-    {ok,KernelVsn} = application:get_key(kernel,vsn),
-    {ok,StdlibVsn} = application:get_key(stdlib,vsn),
-    {ok,SaslVsn} = application:get_key(sasl,vsn),
+create_rel_file(RelFile,RelName,RelVsn,Erts,ExtraApps) ->
+    ErtsVsn = vsn(erts,Erts),
+    KernelVsn = vsn(kernel,Erts),
+    StdlibVsn = vsn(stdlib,Erts),
+    SaslVsn = vsn(sasl,current),
     application:load(tools),
-    {ok,ToolsVsn} = application:get_key(tools,vsn),
+    ToolsVsn = vsn(tools,current),
     application:load(runtime_tools),
-    {ok,RuntimeToolsVsn} = application:get_key(runtime_tools,vsn),
+    RuntimeToolsVsn = vsn(runtime_tools,current),
     
     RelFileContent = {release,
 		      {RelName, RelVsn},
@@ -2060,7 +2206,7 @@ start_node_unix(Sname,NodeDir) ->
 
 start_node_win32(Sname,NodeDir) ->
     Name = atom_to_list(Sname) ++ "_P1G",
-    ErtsBinDir = filename:join(NodeDir,"erts-4.4/bin"),
+    ErtsBinDir = filename:join([NodeDir,app_dir(erts,old),"bin"]),
 
     StartErlArgs = rh_test_lib:get_start_erl_args(NodeDir),
     ServiceArgs = rh_test_lib:get_service_args(NodeDir, Sname, StartErlArgs),
@@ -2185,7 +2331,6 @@ create_fake_release(Dir,RelName,RelVsn,AppDirs) ->
     RelDir = filename:join(Dir,"rel_" ++ RelVsn),
     Rel = filename:join([RelDir,"rel_" ++ RelVsn]),
     ok = filelib:ensure_dir(Rel),
-    ErtsVsn = erlang:system_info(version),
 
     {Apps,Paths} = 
 	lists:foldl(fun({App,Vsn,Lib},{As,Ps}) ->
@@ -2195,7 +2340,7 @@ create_fake_release(Dir,RelName,RelVsn,AppDirs) ->
 		    {[],[]},
 		    AppDirs),
 
-    create_rel_file(Rel++".rel",RelName,RelVsn,ErtsVsn,Apps),
+    create_rel_file(Rel++".rel",RelName,RelVsn,current,Apps),
     
     %% Generate boot scripts
     ok = systools:make_script(Rel,[local,
@@ -2244,3 +2389,16 @@ modify_tar_win32(Conf, TarFileName) ->
     [ok = erl_tar:add(T,filename:join(TmpDir,F),F,[]) || F <- Fs],
     ok = erl_tar:close(T),
     ok.
+
+app_dir(App,Vsn) ->
+    atom_to_list(App) ++ "-" ++ vsn(App,Vsn).
+vsn(erts,old) -> ?ertsvsn;
+vsn(kernel,old) -> ?kernelvsn;
+vsn(stdlib,old) -> ?stdlibvsn;
+vsn(erts,current) -> erlang:system_info(version);
+vsn(App,current) ->
+    {ok,Vsn} = application:get_key(App,vsn),
+    Vsn.
+
+system_lib(PrivDir) ->
+    filename:join(PrivDir,"system_lib").