aboutsummaryrefslogblamecommitdiffstats
path: root/lib/sasl/test/release_handler_SUITE.erl
blob: b44da72d35fd9bbc593094197e2c96671979649f (plain) (tree)
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397




















































                                                                         
                                


                                         
                                                      


                                                       



























































































                                                                                



                                           




















                                                               



                                            




































                                                                              













                                                                    



                                   

































































































                                                                                  
                                                 























                                                                     
                                                 




































                                                          
                                                








                                                 
                                                                


                                     
                                                                  




                                              
                                                 































                                                         
                                                             












































































































                                                                                        
                                                                                








                                                                           
                                                                        

                                                    

       



                                   
























                                                                         
                                                                  






















































                                                                                   

       


                                   















                                                                      
                                                                 








                                                                                   
                                             











                                                                   









                                                                    
                                             










                                                                    
                                             


       


                                   
 













                                                                     
                                                                 






































                                                                                   



                                   



                                                                     
                                                   

                  

                                                                                 
 

                                                          
 


                                                      



                                                               




                                                                          
 
              




                                                    
                         
 
                                                          





                                                
                                                                     











                                                                           























































































































































































































                                                                                  

 











































































































































































































































































































































                                                                                 
                                        




                                                     

                                                                                
 
                                                






                                                                 









                                                                     




















                                                                            
                                           
                                            

                                                       


                     
            



                                                                               
 





                              






                                                                   





























                                                                                







                                                  



























































































































                                                                         





                                                                            






















































                                                                          
















                                                                                  


                                                                 

                                                                 









                                                    
                                                                          












                                                                              



































                                                                                  
                                                              



                                                   
                                                                                                      



























                                                                                







                                                                              


































































































                                                                                 















                                                                           

                  
                                                    




































                                                                                  
          






                                                                                  







                                                         






                                                         





                             












                                                                  
                                                                             








                                                                     




































































































                                                                                

                                                                  






































                                                                              




























                                                                          
%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 2011. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
%% compliance with the License. You should have received a copy of the
%% Erlang Public License along with this software. If not, it can be
%% retrieved online at http://www.erlang.org/.
%%
%% Software distributed under the License is distributed on an "AS IS"
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
%% the License for the specific language governing rights and limitations
%% under the License.
%%
%% %CopyrightEnd%
%%
-module(release_handler_SUITE).

-include_lib("common_test/include/ct.hrl").

-compile(export_all).

% Default timetrap timeout (set in init_per_testcase).
%-define(default_timeout, ?t:minutes(40)).
-define(default_timeout, ?t:minutes(10)).

suite() ->
    [{ct_hooks, [ts_install_cth]}].

init_per_suite(Config) ->
    application:start(sasl),
    Config.

end_per_suite(_Config) ->
    ok.

all() -> 
    case os:type() of
	{unix, _} -> unix_cases();
	{win32, _} -> win32_cases()
    end.

unix_cases() -> 
    RunErl = filename:join([code:root_dir(),"bin","run_erl"]),
    RunErlCases = case filelib:is_file(RunErl) of
		true ->  [{group, release}];
		false -> [no_run_erl]
	    end,
    [target_system] ++ RunErlCases ++ cases().

win32_cases() -> 
    [{group,release} | cases()].

%% Cases that can be run on all platforms
cases() ->
    [otp_2740, otp_2760, otp_5761, otp_9402, otp_9417,
     otp_9395_check_old_code, otp_9395_check_and_purge,
     otp_9395_update_many_mods, otp_9395_rm_many_mods,
     instructions, eval_appup].

groups() ->
    [{release,[],
      [
       {group,release_single},
       {group,release_gg}
      ]},
     {release_single,[],
      [
       upgrade,
       client1,
       client2
      ]},
     {release_gg,[],
      [
       upgrade_gg
      ]}].

%% {group,release}
%% Top group for all cases using run_erl
init_per_group(release, Config) ->
    Dog = ?t:timetrap(?default_timeout),
    P1gInstall = filename:join(priv_dir(Config),p1g_install),
    ok = do_create_p1g(Config,P1gInstall),
    ok = create_p1h(Config),
    ?t:timetrap_cancel(Dog);

%% {group,release_single}
%% Subgroup of {group,release}, contains all cases that are not
%% related to global_group
init_per_group(release_single, Config) ->
    Dog = ?t:timetrap(?default_timeout),

    %% Create some more releases to upgrade to
    ok = create_p1i(Config),
    ok = create_p2a(Config),

    ?t:timetrap_cancel(Dog);

%% {group,release_gg}
%% Subgroup of {group,release}. global_group tests.
init_per_group(release_gg, Config0) ->
    Config = [{sname_prefix,release_gg}|Config0],

    PrivDir = priv_dir(Config),
    Dog = ?t:timetrap(?default_timeout),

    reg_print_proc(), %% starts a printer process on this node

    Snames = [Gg1Sname,Gg2Sname,Gg3Sname,Gg4Sname,Gg5Sname,Gg6Sname] =
	gg_node_snames(Config),

    %% kill all possible nodes which are to be used
    ok = stop_nodes([node_name(Sname) || Sname <- Snames]),

    %% For gg1, gg3, gg4 and gg5: create a target system running
    %% P1G, and with P1H unpacked.
    %% For gg2 and gg6: create a target system running P1H.
    %% Use gg2 for unpacking and permanenting P1H.
    ok = copy_installed(Config,p1g_install,[Gg2Sname]),
    InstallNode = unpack_p1h(Config,Gg2Sname),
    ok = copy_installed(Config,Gg2Sname,[Gg1Sname,Gg3Sname,Gg4Sname,Gg5Sname]),
    ok = permanent_p1h(InstallNode),
    ok = stop_nodes([InstallNode]),
    ok = copy_installed(Config,Gg2Sname,[Gg6Sname]),

    %% Replace the sys.config files
    %% The reason for not creating the releases with these configs in
    %% the first place (create_p1g, create_p1h) is that then the
    %% InstallNode (gg2) will be very slow started since it will try
    %% to synch with the other nodes in the global group.
    %% Also, the rpc call for installing the P1H release (in
    %% permanent_p1h/1) would return {rpc,nodedown} due to change of
    %% global groups.
    lists:foreach(
      fun(Sname) ->
	      ReleasesDir = filename:join([PrivDir,Sname,"releases"]),
	      write_term_file(filename:join([ReleasesDir,"P1G","sys.config"]),
			      gg_config([Gg1Sname,Gg3Sname,Gg4Sname,Gg5Sname])),
	      write_term_file(filename:join([ReleasesDir,"P1H","sys.config"]),
			      gg_config([Gg1Sname,Gg2Sname,Gg4Sname,
					 Gg5Sname,Gg6Sname]))
      end,
      Snames),

    ?t:timetrap_cancel(Dog),
    [{snames,Snames}|Config].


end_per_group(release, Config) ->
    Dog = ?t:timetrap(?default_timeout),
    stop_print_proc(),
    case os:type() of
	{win32,_} -> delete_all_services();
	_ -> ok
    end,
    delete_release(Config),
    ?t:timetrap_cancel(Dog),
    Config;
end_per_group(_GroupName, Config) ->
    Config.


init_per_testcase(Case, Config0) ->
    Dog = test_server:timetrap(?default_timeout),
    Config = [{sname_prefix,Case},{watchdog, Dog}|Config0],
    try apply(?MODULE,Case,[cleanup,Config])
    catch error:undef -> ok
    end,
    ?t:format("~n======= init_per_testcase done =======~n",[]),
    Config.

end_per_testcase(Case, Config) ->
    ?t:format("~n======= start end_per_testcase =======~n",[]),
    Dog=?config(watchdog, Config),
    test_server:timetrap_cancel(Dog),

    try apply(?MODULE,Case,[cleanup,Config])
    catch error:undef -> ok
    end,

    %% DEBUG
    case ?config(tc_status,Config) of
	ok ->
	    ok;
	_Fail ->
	    %% save logs from master and client nodes
	    PrivDir = priv_dir(Config),
	    SaveDir = filename:join(PrivDir,save),
	    FailDir = filename:join(SaveDir,lists:concat(["failed-",Case])),
	    ok = filelib:ensure_dir(filename:join(FailDir,"*")),

	    LogDirs = filelib:wildcard(filename:join([PrivDir,"*",log])),

	    lists:foreach(
	      fun(LogDir) ->
		      ["log",Sname|_] = lists:reverse(filename:split(LogDir)),
		      copy_tree(Config,LogDir,Sname,FailDir)
	      end,
	      LogDirs),

	    case filelib:is_file("sasl_erl_crash.dump") of
		true ->
		    copy_file("sasl_erl_crash.dump",FailDir);
		_ ->
		    ok
	    end

    end,
    %% End DEBUG

    %% Remove any remaining sasl_erl_crash.dump
    %% These can occur when a new master@<host> is started, before
    %% the old usage of the name is unregistered, causing the node to
    %% terminate. (This has no effect on the test case, as the node is
    %% immediately restarted by heart and the test cases wait until
    %% the node is actually up and running -- see wait_nodes_up/2)
    file:delete("sasl_erl_crash.dump"),
    ok.

gg_node_snames(Config) ->
    [tc_sname(Config,X) || X <- [gg1,gg2,gg3,gg4,gg5,gg6]].


%%%-----------------------------------------------------------------
%%% TEST CASES


%% Executed instead of release group when no run_erl program exists
no_run_erl(Config) when is_list(Config) ->
    {comment, "No run_erl program"}.

break(Config) ->
	erlang:display(test_break),
	?t:break(priv_dir(Config)),
	ok.

%% Test upgrade and downgrade of erts
upgrade(Conf) when is_list(Conf) ->
    reg_print_proc(), %% starts a printer process on test_server node
    ?t:format("upgrade ~p~n",[reg_print_proc]),
    PrivDir = priv_dir(Conf),
    Sname = tc_sname(Conf), % nodename for use in this testcase

    %% Copy the P1G release to a directory for use in this testcase
    ok = copy_installed(Conf,p1g_install,[Sname]),

    %% start the test node
    [TestNode] = start_nodes(Conf,[Sname],"upgrade start"),

    %% unpack and install P1H
    ok = rpc_inst(TestNode, install_1, [PrivDir]),
    stop_cover(TestNode),
    reboot_and_wait(TestNode,"install_1"),

    %% reinstall P1H and make it permanent
    ok = rpc_inst(TestNode, install_2, []),
    stop_cover(TestNode),
    reboot_and_wait(TestNode,"install_2",[a]),

    %% check that P1H is permanent, unpack and install P1I, unpack and install P2A
    TestNodeInit1 = rpc:call(TestNode,erlang,whereis,[init]),
    ok = rpc_inst(TestNode, install_3, [PrivDir]),
    stop_cover(TestNode),
    ok = rpc_inst(TestNode, install_3a, []),
    wait_nodes_up([{TestNode,TestNodeInit1}],"install_3",[a]),

    %% check that P2A is used, reboot from P1I
    ok = rpc_inst(TestNode, install_4, []),
    stop_cover(TestNode),
    reboot_and_wait(TestNode,"install_4",[a]),

    %% check that P1I, reinstall P2A
    TestNodeInit2 = rpc:call(TestNode,erlang,whereis,[init]),
    ok = rpc_inst(TestNode, install_5, []),
    stop_cover(TestNode),
    ok = rpc_inst(TestNode, install_5a, []),
    wait_nodes_up([{TestNode,TestNodeInit2}],"install_5",[a]),

    %% check that P2A is used, make P2A permanent
    ok = rpc_inst(TestNode, install_6, []),
    stop_cover(TestNode),
    reboot_and_wait(TestNode,"install_6",[a]),

    %% check that P2A is permanent, install old P1H
    TestNodeInit3 = rpc:call(TestNode,erlang,whereis,[init]),
    stop_cover(TestNode),
    ok = rpc_inst(TestNode, install_7, []),
    wait_nodes_up([{TestNode,TestNodeInit3}],"install_7",[a]),

    %% check that P1H is permanent, remove P1I and P2A
    ok = rpc_inst(TestNode, install_8, []),
    stop_cover(TestNode),
    reboot_and_wait(TestNode,"install_8",[a]),

    %% check that P1H is permanent, reboot old P1G
    TestNodeInit4 = rpc:call(TestNode,erlang,whereis,[init]),
    stop_cover(TestNode),
    ok = rpc_inst(TestNode, install_9, []),
    wait_nodes_up([{TestNode,TestNodeInit4}],"install_9"),

    %% check that P1G is permanent, remove P1H
    ok = rpc_inst(TestNode, install_10, []),
    stop_cover(TestNode),
    reboot_and_wait(TestNode,"install_10"),

    %% check that P1G is permanent
    ok = rpc_inst(TestNode, install_11, []),

    ok.

upgrade(cleanup,Config) ->
    TestNode = tc_full_node_name(Config),
    ok = stop_nodes([TestNode]).

reboot_and_wait(Node,Tag) ->
    reboot_and_wait(Node,Tag,[]).

reboot_and_wait(Node,Tag,Apps) ->
    InitPid = rpc:call(Node,erlang,whereis,[init]),
    ok = rpc:call(Node,init,reboot,[]),
    wait_nodes_up([{Node,InitPid}],Tag,Apps).


%% Test upgrade and downgrade of erts, diskless
client1(Conf) when is_list(Conf) ->
    reg_print_proc(), %% starts a printer process on test_server node
    PrivDir = priv_dir(Conf),
    Master = tc_sname(Conf,master),
    Client = tc_sname(Conf,client),
    MasterDir = filename:join(PrivDir,Master),

    %% Copy the P1G release to a directory for use in this testcase
    ok = copy_installed(Conf,p1g_install,[Master]),
    ok = copy_client(Conf,Master,Client,client1),

    %% start the master node
    [TestNode] = start_nodes(Conf,[Master],"client1"),

    ok = rpc_inst(TestNode, client1_1, [PrivDir,MasterDir,Client]),

    ok.

client1(cleanup,Config) ->
    MasterNode = tc_full_node_name(Config,master),
    ClientNode = tc_full_node_name(Config,client),
    ok = stop_nodes([MasterNode,ClientNode]).



%% Test diskless release handling when illegal master node
client2(Conf) when is_list(Conf) ->
    reg_print_proc(), %% starts a printer process on test_server node
    PrivDir = priv_dir(Conf),
    Master = tc_sname(Conf,master),
    Client = tc_sname(Conf,client),

    %% Copy the P1G release to a directory for use in this testcase
    ok = copy_installed(Conf,p1g_install,[Master]),
    ok = copy_client(Conf,Master,Client,client2),

    %% start the master node
    [TestNode] = start_nodes(Conf,[Master],"client2"),

    ok = rpc_inst(TestNode, client2, [PrivDir,Client]),

    ok.

client2(cleanup,Config) ->
    MasterNode = tc_full_node_name(Config,master),
    ClientNode = tc_full_node_name(Config,client),
    ok = stop_nodes([MasterNode,ClientNode]).



%% Test instructions _not_ tested by the installer module.
instructions(Conf) when is_list(Conf) ->
    DataDir = ?config(data_dir, Conf),

    Dir = filename:join(DataDir, "c"),
    true = code:add_path(Dir),
    check_bstate("no", []),
    ok = application:start(c),
    ok = wait_for(bb),
    check_bstate("first", []),
    FirstBB = whereis(bb),

    case whereis(cc) of
        Pid when is_pid(Pid) -> ok;
        _ -> ?t:fail("cc not started")
    end,

    %% Stop and start cc process
    S1 = [point_of_no_return,
          {stop, [aa]},
          {apply, {?MODULE, no_cc, []}},
          {start, [aa]}],
    {ok, _} = release_handler_1:eval_script(S1),

    case whereis(cc) of
        Pid2 when is_pid(Pid2) -> ok;
        _ -> ?t:fail("cc not started")
    end,

    %% Make bb run old version of b.
    S2 = [point_of_no_return,
          {remove, {b, soft_purge, soft_purge}}],
    {ok, [{b, soft_purge}]} = release_handler_1:eval_script(S2),
    check_bstate("first", [FirstBB]),

    false = code:is_loaded(b),
    {error,{old_processes,b}} = release_handler_1:eval_script(S2),
    check_bstate("first", [FirstBB]),

    %% Let supervisor restart bb with new code
    S3 = [point_of_no_return,
          {purge, [b]}],
    {ok, []} = release_handler_1:eval_script(S3),
    ok = wait_for(bb),
    check_bstate("second", []),
    SecondBB = whereis(bb),

    if
        SecondBB =:= FirstBB -> ?t:fail("bb not killed");
        true -> ok
    end,

    %% Restart bb yet another time
    ok = application:stop(c),
    ok = application:start(c),
    ok = wait_for(bb),
    check_bstate("third", []),
    ThirdBB = whereis(bb),

    case ThirdBB of
        _ when is_pid(ThirdBB) -> ok;
        undefined -> ?t:fail("bb not started")
    end,

    %% Make bb run old version of b.
    %%c:l(b),
    check_bstate("third", []),
    false = code:purge(b),
    check_bstate("third", []),
    {module,b} = code:load_file(b),
    check_bstate("third", [ThirdBB]),

    %% Let supervisor restart bb yet another time
    S4 = [point_of_no_return,
          {remove, {b, brutal_purge, soft_purge}}],
    {ok, HopefullyEmpty} = release_handler_1:eval_script(S4),
    ok = wait_for(bb),
    FourthBB = whereis(bb),

    case HopefullyEmpty of
	[{b, soft_purge}] ->
	    %% The process managed to start between purge and delete
	    check_bstate("fourth", [FourthBB]);
	[] ->
	    %% The process started after delete
	    check_bstate("fourth", [])
    end,

    application:stop(c),
    check_bstate("no", []),
    ok.

instructions(cleanup,Conf) ->
    application:stop(c),
    really_del_code([aa,b,c_sup]),
    code:del_path(filename:join(?config(data_dir,Conf), "c")),
    ok.

really_del_code(Mods) ->
    lists:foreach(fun(Mod) ->
			  code:purge(Mod), % remove old code
			  code:delete(Mod),% make current code old
			  code:purge(Mod)  % remove old code
		  end,
		  Mods).

check_bstate(Slogan,ExpectedProcs) ->
    BB = whereis(bb),
    ActualProcs = lists:sort([P || P <- processes(),
				   erlang:check_process_code(P, b)]),
    ExpectedProcs2 = lists:sort(ExpectedProcs),
    ?t:format("check_bstate:~n~p~n~p~n",
	      [{"bb process", Slogan, BB},
	       {"Processes running old b code", ActualProcs}]),
    if
	Slogan =:= "no", BB =/= undefined ->
	    ?t:fail("instructions failed; process bb is running");
	Slogan =/= "no", BB =:= undefined ->
	    ?t:fail("instructions failed; process bb is not running");
	ExpectedProcs2 =:= [], ActualProcs =/= ExpectedProcs2 ->
	    ?t:fail("instructions failed; old b processes are running");
	ActualProcs =/= ExpectedProcs2 ->
	    ?t:fail("instructions failed; wrong number of old b processes are running");
	true ->
	    ok
    end.

wait_for(Name) ->
    case whereis(Name) of
        undefined ->
            timer:sleep(100),
            wait_for(Name);
        Pid when is_pid(Pid) ->
            ok
    end.

no_cc() ->
    case whereis(cc) of
        Pid when is_pid(Pid) -> ?t:fail("cc not stopped");
        _ -> ok
    end.



%%%-----------------------------------------------------------------
%%% Testing of reported bugs and other tickets.
%%%-----------------------------------------------------------------

%%-----------------------------------------------------------------
%% Ticket: OTP-2740
%% Slogan: vsn not numeric doesn't work so good in release_handling
%%-----------------------------------------------------------------
%% Test vsn.
otp_2740(Conf) ->
    DataDir = ?config(data_dir, Conf),
    Dir = filename:join(DataDir, "otp_2740"),
    true = code:add_path(Dir),

    {module, vsn_numeric} = c:l(vsn_numeric),
    {module, vsn_tuple} = c:l(vsn_tuple),
    {module, vsn_list} = c:l(vsn_list),
    {module, vsn_atom} = c:l(vsn_atom),
    {module, vsn_string} = c:l(vsn_string),

    231894 = release_handler_1:get_current_vsn(vsn_numeric),
    {tuple,["of",terms]} = release_handler_1:get_current_vsn(vsn_tuple),
    [list,"of",{some,terms}] = release_handler_1:get_current_vsn(vsn_list),
    atom = release_handler_1:get_current_vsn(vsn_atom),
    "a string" = release_handler_1:get_current_vsn(vsn_string),

    true = code:del_path(Dir),
    ok.

%%-----------------------------------------------------------------
%% Ticket: OTP-2760
%% Slogan: when an application is removed from a node it is not unloaded
%%-----------------------------------------------------------------
%% Test that when an application is removed from a node it is also unloaded.
otp_2760(Conf) ->
    PrivDir = priv_dir(Conf),
    Dir = filename:join(PrivDir,"otp_2760"),
    DataDir = ?config(data_dir,Conf),
    LibDir = filename:join([DataDir,app1_app2,lib1]),

    Rel1 = create_and_install_fake_first_release(Dir,[{app1,"1.0",LibDir}]),
    Rel2 = create_fake_upgrade_release(Dir,"after",[],{[Rel1],[Rel1],[LibDir]}),
    Rel2Dir = filename:dirname(Rel2),

    %% Start a node with Rel1.boot and check that the app1 module is loaded
    {ok, Node} = t_start_node(otp_2760, Rel1, []),
    {file, _} = rpc:call(Node, code, is_loaded, [app1]),

    %% Execute the relup script and check that app1 is unloaded
    {ok, [{"after", [{_Rel1Vsn, _Descr, Script}], _}]} =
	file:consult(filename:join(Rel2Dir, "relup")),
    {ok, []} = rpc:call(Node, release_handler_1, eval_script, [Script]),
    false = rpc:call(Node, code, is_loaded, [app1]),

    ok.

otp_2760(cleanup,_Conf) ->
    stop_node(node_name(otp_2760)).


%% Test upgrade using other filesystem than the defined in OTP and
%% option {update_paths, true}
otp_5761(Conf) when is_list(Conf) ->

    %% In the following test case, the release upgrade is somewhat
    %% simplified (since it is not this procedure in itself we want to
    %% test, but that application code directories are set correctly.)
    %% Existing Erlang release is used as base, instead of creating
    %% a new one.

    %% Set some paths
    PrivDir = priv_dir(Conf),
    Dir = filename:join(PrivDir,"otp_5761"),
    RelDir = filename:join(?config(data_dir, Conf), "app1_app2"),
    LibDir1 = filename:join(RelDir, "lib1"),
    LibDir2 = filename:join(RelDir, "lib2"),

    %% Create the releases
    Rel1 = create_and_install_fake_first_release(Dir,
						 [{app1,"1.0",LibDir1},
						  {app2,"1.0",LibDir1}]),
    Rel2 = create_fake_upgrade_release(Dir,
				       "2",
				       [{app1,"2.0",LibDir2},
					{app2,"1.0",LibDir2}],
				       {[Rel1],[Rel1],[LibDir1]}),
    Rel1Dir = filename:dirname(Rel1),
    Rel2Dir = filename:dirname(Rel2),
    
    %% Start a slave node
    {ok, Node} = t_start_node(otp_5761, Rel1, filename:join(Rel1Dir,"sys.config")),

    %% Bind some variable names that will be used in patternmatching below 
    App11Dir = filename:join([LibDir1, "app1-1.0"]),
    App12Dir = filename:join([LibDir2, "app1-2.0"]),
    App2aDir = filename:join([LibDir1, "app2-1.0"]),
    App2bDir = filename:join([LibDir2, "app2-1.0"]),

    %% Make sure correct code paths are used
    App11Dir = rpc:call(Node, code, lib_dir, [app1]),
    App2aDir = rpc:call(Node, code, lib_dir, [app2]),

    %% Unpack rel2 (make sure it does not work if an AppDir is bad)
    LibDir3 = filename:join(RelDir, "lib3"),
    {error, {no_such_directory, _}} =
	rpc:call(Node, release_handler, set_unpacked,
		 [Rel2++".rel", [{app1,"2.0",LibDir2}, {app2,"1.0",LibDir3}]]),
    {ok, RelVsn2} =
	rpc:call(Node, release_handler, set_unpacked,
		 [Rel2++".rel", [{app1,"2.0",LibDir2}, {app2,"1.0",LibDir2}]]),
    ok = rpc:call(Node, release_handler, install_file,
			[RelVsn2, filename:join(Rel2Dir, "relup")]),
    ok = rpc:call(Node, release_handler, install_file,
			[RelVsn2, filename:join(Rel2Dir, "start.boot")]),
    ok = rpc:call(Node, release_handler, install_file,
			[RelVsn2, filename:join(Rel2Dir, "sys.config")]),

    %% Install RelVsn2 without {update_paths, true} option
    {ok, RelVsn1, []} =
	rpc:call(Node, release_handler, install_release, [RelVsn2]),
    App12Dir = rpc:call(Node, code, lib_dir, [app1]),
    App2aDir = rpc:call(Node, code, lib_dir, [app2]),

    %% Install RelVsn1 again
    {ok, RelVsn1, []} =
	rpc:call(Node, release_handler, install_release, [RelVsn1]),

    %% Install RelVsn2 with {update_paths, true} option
    {ok, RelVsn1, []} =
	rpc:call(Node, release_handler, install_release,
		 [RelVsn2, [{update_paths, true}]]),
    App12Dir = rpc:call(Node, code, lib_dir, [app1]),
    App2bDir = rpc:call(Node, code, lib_dir, [app2]),

    %% Install RelVsn1 again
    {ok, RelVsn1, []} =
	rpc:call(Node, release_handler, install_release,
		 [RelVsn1, [{update_paths, true}]]),
    App11Dir = rpc:call(Node, code, lib_dir, [app1]),
    App2aDir = rpc:call(Node, code, lib_dir, [app2]),

    ok.

otp_5761(cleanup,_Conf) ->
    stop_node(node_name(otp_5761)).


%% When a new version of an application is added, but no module is
%% changed - the path was not updated - i.e. code:priv_dir would point
%% to the old location.
otp_9402(Conf) when is_list(Conf) ->
    %% Set some paths
    PrivDir = priv_dir(Conf),
    Dir = filename:join(PrivDir,"otp_9402"),
    LibDir = filename:join(?config(data_dir, Conf), "lib"),

    %% Create the releases
    Rel1 = create_and_install_fake_first_release(Dir,
						 [{a,"1.1",LibDir}]),
    Rel2 = create_fake_upgrade_release(Dir,
				       "2",
				       [{a,"1.2",LibDir}],
				       {[Rel1],[Rel1],[LibDir]}),
    Rel1Dir = filename:dirname(Rel1),
    Rel2Dir = filename:dirname(Rel2),

    %% Start a slave node
    {ok, Node} = t_start_node(otp_9402, Rel1, filename:join(Rel1Dir,"sys.config")),

    %% Check path
    Dir1 = filename:join([LibDir, "a-1.1"]),
    Dir1 = rpc:call(Node, code, lib_dir, [a]),
    ABeam = rpc:call(Node, code, which, [a]),

    %% Install second release, with no changed modules
    {ok, RelVsn2} =
	rpc:call(Node, release_handler, set_unpacked,
		 [Rel2++".rel", [{a,"1.2",LibDir}]]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "relup")]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "start.boot")]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "sys.config")]),

    {ok, RelVsn1, []} =
	rpc:call(Node, release_handler, install_release, [RelVsn2]),

    %% Check path
    Dir2 = filename:join([LibDir, "a-1.2"]),
    Dir2 = rpc:call(Node, code, lib_dir, [a]),
    APrivDir2 = rpc:call(Node, code, priv_dir, [a]),
    true = filelib:is_regular(filename:join(APrivDir2,"file")),

    %% Just to make sure no modules have been re-loaded
    ABeam = rpc:call(Node, code, which, [a]),

    %% Install RelVsn1 again
    {ok, _OtherVsn, []} =
	rpc:call(Node, release_handler, install_release, [RelVsn1]),

    %% Check path
    Dir1 = rpc:call(Node, code, lib_dir, [a]),
    APrivDir1 = rpc:call(Node, code, priv_dir, [a]),
    false = filelib:is_regular(filename:join(APrivDir1,"file")),

    %% Just to make sure no modules have been re-loaded
    ABeam = rpc:call(Node, code, which, [a]),

    ok.

otp_9402(cleanup,_Conf) ->
    stop_node(node_name(otp_9402)).


%% When a module is deleted in an appup instruction, the upgrade
%% failed if the module was not loaded.
otp_9417(Conf) when is_list(Conf) ->
    %% Set some paths
    PrivDir = priv_dir(Conf),
    Dir = filename:join(PrivDir,"otp_9417"),
    LibDir = filename:join(?config(data_dir, Conf), "lib"),

    %% Create the releases
    Rel1 = create_and_install_fake_first_release(Dir,
						 [{b,"1.0",LibDir}]),
    Rel2 = create_fake_upgrade_release(Dir,
				       "2",
				       [{b,"2.0",LibDir}],
				       {[Rel1],[Rel1],[LibDir]}),
    Rel1Dir = filename:dirname(Rel1),
    Rel2Dir = filename:dirname(Rel2),

    %% Start a slave node
    {ok, Node} = t_start_node(otp_9417, Rel1, filename:join(Rel1Dir,"sys.config")),

    %% Check paths
    Dir1 = filename:join([LibDir, "b-1.0"]),
    Dir1 = rpc:call(Node, code, lib_dir, [b]),
    BLibBeam = filename:join([Dir1,"ebin","b_lib.beam"]),
    BLibBeam = rpc:call(Node,code,which,[b_lib]),
    false = rpc:call(Node,code,is_loaded,[b_lib]),
    false = rpc:call(Node,code,is_loaded,[b_server]),

    %% Install second release, which removes b_lib module
    {ok, RelVsn2} =
	rpc:call(Node, release_handler, set_unpacked,
		 [Rel2++".rel", [{b,"2.0",LibDir}]]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "relup")]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "start.boot")]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "sys.config")]),

    {ok, _RelVsn1, []} =
	rpc:call(Node, release_handler, install_release, [RelVsn2]),

    %% Check that the module does no longer exist
    false = rpc:call(Node, code, is_loaded, [b_lib]),
    non_existing = rpc:call(Node, code, which, [b_lib]),

    %% And check some paths
    Dir2 = filename:join([LibDir, "b-2.0"]),
    Dir2 = rpc:call(Node, code, lib_dir, [b]),
    BServerBeam = filename:join([Dir2,"ebin","b_server.beam"]),
    {file,BServerBeam} = rpc:call(Node,code,is_loaded,[b_server]),
    ok.

otp_9417(cleanup,_Conf) ->
    stop_node(node_name(otp_9417)).


%% OTP-9395 - performance problems when there are MANY processes
%% Test that the procedure of checking for old code before an upgrade
%% can be started is "very much faster" when there is no old code in
%% the system.
otp_9395_check_old_code(Conf) when is_list(Conf) ->

    NProcs = 1000,
    MPath = filename:join([?config(data_dir,Conf),"lib","many_mods-1.0","ebin"]),
    code:add_path(MPath),

    %% Start NProc processes, each referencing each module
    {Modules,Pids} = m:start(NProcs),

    %% Load each module again in order to get old code
    [code:load_file(Mod) || Mod <- Modules],
    true = erlang:check_old_code(m10),

    S = [point_of_no_return |
	 [{remove,{M,soft_purge,soft_purge}} || M <- Modules]],

    %% Do the old code check, then purge, and redo
    {T1,{ok,PurgeMods}} = timer:tc(release_handler_1,check_script,[S,[]]),
    true = (lists:sort(PurgeMods) == lists:sort(Modules)),
    [code:purge(M) || M <- PurgeMods],
    {T2,{ok,[]}} = timer:tc(release_handler_1,check_script,[S,[]]),

    %% Cleanup
    lists:foreach(fun(Pid) -> Pid ! stop end, Pids),
    lists:foreach(fun(Mod) -> code:purge(Mod),
			      code:delete(Mod),
			      code:purge(Mod)
		  end, Modules),
    code:del_path(MPath),

    %% Test that second run was much faster than the first
    if T2 > 0 ->
	    X = T1/T2,
	    ct:log("~p procs, ~p mods -> ~n"
		   "\tWith old code: ~.2f sec~n"
		   "\tAfter purge: ~.2f sec~n"
		   "\tT1/T2: ~.2f",
		   [NProcs,length(Modules),T1/1000000,T2/1000000,X]),
	    if X < 1000 ->
		    ct:fail({not_enough_improvement_after_purge,round(X)});
	       true ->
		    ok
	    end;
       T1 > 0 -> %% Means T1/T2 = infinite
	    ok;
       true ->
	    ct:fail({unexpected_values,T1,T2})
    end,
    ok.


%% OTP-9395 - performance problems when there are MANY processes
%% Added option 'purge' to check_install_release
otp_9395_check_and_purge(Conf) when is_list(Conf) ->
    %% Set some paths
    PrivDir = priv_dir(Conf),
    Dir = filename:join(PrivDir,"otp_9395_check_and_purge"),
    LibDir = filename:join(?config(data_dir, Conf), "lib"),

    %% Create the releases
    Rel1 = create_and_install_fake_first_release(Dir,
						 [{b,"1.0",LibDir}]),
    Rel2 = create_fake_upgrade_release(Dir,
				       "2",
				       [{b,"2.0",LibDir}],
				       {[Rel1],[Rel1],[LibDir]}),
    Rel1Dir = filename:dirname(Rel1),
    Rel2Dir = filename:dirname(Rel2),

    %% Start a slave node
    {ok, Node} = t_start_node(otp_9395_check_and_purge, Rel1,
			      filename:join(Rel1Dir,"sys.config")),

    %% Make sure there is old code for b_lib and b_server
    rpc:call(Node,code,load_file,[b_lib]),
    rpc:call(Node,code,load_file,[b_lib]),
    rpc:call(Node,code,load_file,[b_server]),
    rpc:call(Node,code,load_file,[b_server]),
    true = rpc:call(Node,erlang,check_old_code,[b_lib]),
    true = rpc:call(Node,erlang,check_old_code,[b_server]),

    %% Unpack second release, which removes b_lib module and loads b_server
    {ok, RelVsn2} =
	rpc:call(Node, release_handler, set_unpacked,
		 [Rel2++".rel", [{b,"2.0",LibDir}]]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "relup")]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "start.boot")]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "sys.config")]),

    %% Do check_install_release, and check that old code still exists
    {ok, _RelVsn1, []} =
	rpc:call(Node, release_handler, check_install_release, [RelVsn2]),
    true = rpc:call(Node,erlang,check_old_code,[b_lib]),
    true = rpc:call(Node,erlang,check_old_code,[b_server]),

    %% Do check_install_release with option 'purge' and check that old
    %% code is gone
    {ok, _RelVsn1, []} =
	rpc:call(Node, release_handler, check_install_release, [RelVsn2,[purge]]),
    false = rpc:call(Node,erlang,check_old_code,[b_lib]),
    false = rpc:call(Node,erlang,check_old_code,[b_server]),

    ok.

otp_9395_check_and_purge(cleanup,_Conf) ->
    stop_node(node_name(otp_9395_check_and_purge)).


%% OTP-9395 - performance problems when there are MANY processes
%% Upgrade which updates many modules (brutal_purge)
otp_9395_update_many_mods(Conf) when is_list(Conf) ->
    %% Set some paths
    PrivDir = priv_dir(Conf),
    Dir = filename:join(PrivDir,"otp_9395_update_many_mods"),
    LibDir = filename:join(?config(data_dir, Conf), "lib"),

    %% Create the releases
    Rel1 = create_and_install_fake_first_release(Dir,
						 [{many_mods,"1.0",LibDir}]),
    Rel2 = create_fake_upgrade_release(Dir,
				       "2",
				       [{many_mods,"1.1",LibDir}],
				       {[Rel1],[Rel1],[LibDir]}),
    Rel1Dir = filename:dirname(Rel1),
    Rel2Dir = filename:dirname(Rel2),

    %% Start a slave node
    {ok, Node} = t_start_node(otp_9395_update_many_mods, Rel1,
			      filename:join(Rel1Dir,"sys.config")),

    %% Start a lot of processes on the new node, all with refs to each
    %% module that will be updated
    NProcs = 1000,
    {Modules,Pids1} = rpc:call(Node,m,start,[NProcs]),

    %% Then load modules in order to get old code
    [rpc:call(Node,code,load_file,[Mod]) || Mod <- Modules],
    true = rpc:call(Node,erlang,check_old_code,[m10]),

    %% Unpack second release, which updates all mX modules
    {ok, RelVsn2} =
	rpc:call(Node, release_handler, set_unpacked,
		 [Rel2++".rel", [{many_mods,"1.1",LibDir}]]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "relup")]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "start.boot")]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "sys.config")]),

    %% First, install release directly and check how much time it takes
    {TInst0,{ok, _, []}} =
	timer:tc(rpc,call,[Node, release_handler, install_release, [RelVsn2]]),
    ct:log("install_release: ~.2f",[TInst0/1000000]),

    %% Restore to old release, spawn processes again and load to get old code
    {_,RelVsn1} = init:script_id(),
    {_TInst1,{ok, _, []}} =
	timer:tc(rpc,call,[Node, release_handler, install_release, [RelVsn1]]),
%    ct:log("install_release: ~.2f",[_TInst1/1000000]),

    [exit(Pid,kill) || Pid <- Pids1],
    {Modules,_Pids2} = rpc:call(Node,m,start,[NProcs]),
    [rpc:call(Node,code,load_file,[Mod]) || Mod <- Modules],
    true = rpc:call(Node,erlang,check_old_code,[m10]),

    %% Run check_install_release with purge before install this time
    {TCheck,{ok, _RelVsn1, []}} =
	timer:tc(rpc,call,[Node, release_handler, check_install_release,
			   [RelVsn2,[purge]]]),
    ct:log("check_install_release with purge: ~.2f",[TCheck/1000000]),

    %% Finally install release after check and purge, and check that
    %% this install was faster than the first.
    {TInst2,{ok, _RelVsn1, []}} =
	timer:tc(rpc,call,[Node, release_handler, install_release, [RelVsn2]]),
    ct:log("install_release: ~.2f",[TInst2/1000000]),

    true = (TInst2 < TInst0),

    ok.

otp_9395_update_many_mods(cleanup,_Conf) ->
    stop_node(node_name(otp_9395_update_many_mods)).


%% OTP-9395 - performance problems when there are MANY processes
%% Upgrade which removes many modules (brutal_purge)
otp_9395_rm_many_mods(Conf) when is_list(Conf) ->
    %% Set some paths
    PrivDir = priv_dir(Conf),
    Dir = filename:join(PrivDir,"otp_9395_rm_many_mods"),
    LibDir = filename:join(?config(data_dir, Conf), "lib"),

    %% Create the releases
    Rel1 = create_and_install_fake_first_release(Dir,
						 [{many_mods,"1.0",LibDir}]),
    Rel2 = create_fake_upgrade_release(Dir,
				       "2",
				       [{many_mods,"2.0",LibDir}],
				       {[Rel1],[Rel1],[LibDir]}),
    Rel1Dir = filename:dirname(Rel1),
    Rel2Dir = filename:dirname(Rel2),

    %% Start a slave node
    {ok, Node} = t_start_node(otp_9395_rm_many_mods, Rel1,
			      filename:join(Rel1Dir,"sys.config")),

    %% Start a lot of processes on the new node, all with refs to each
    %% module that will be updated
    NProcs = 1000,
    {Modules,Pids1} = rpc:call(Node,m,start,[NProcs]),

    %% Then load modules in order to get old code
    [rpc:call(Node,code,load_file,[Mod]) || Mod <- Modules],
    true = rpc:call(Node,erlang,check_old_code,[m10]),

    %% Unpack second release, which removes all mX modules
    {ok, RelVsn2} =
	rpc:call(Node, release_handler, set_unpacked,
		 [Rel2++".rel", [{many_mods,"2.0",LibDir}]]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "relup")]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "start.boot")]),
    ok = rpc:call(Node, release_handler, install_file,
		  [RelVsn2, filename:join(Rel2Dir, "sys.config")]),

    %% First, install release directly and check how much time it takes
    {TInst0,{ok, _, []}} =
	timer:tc(rpc,call,[Node, release_handler, install_release, [RelVsn2]]),
    ct:log("install_release: ~.2f",[TInst0/1000000]),

    %% Restore to old release, spawn processes again and load to get old code
    {_,RelVsn1} = init:script_id(),
    {_TInst1,{ok, _, []}} =
	timer:tc(rpc,call,[Node, release_handler, install_release, [RelVsn1]]),
%    ct:log("install_release: ~.2f",[_TInst1/1000000]),

    [exit(Pid,kill) || Pid <- Pids1],
    {Modules,_Pids2} = rpc:call(Node,m,start,[NProcs]),
    [rpc:call(Node,code,load_file,[Mod]) || Mod <- Modules],
    true = rpc:call(Node,erlang,check_old_code,[m10]),

    %% Run check_install_release with purge before install this time
    {TCheck,{ok, _RelVsn1, []}} =
	timer:tc(rpc,call,[Node, release_handler, check_install_release,
			   [RelVsn2,[purge]]]),
    ct:log("check_install_release with purge: ~.2f",[TCheck/1000000]),

    %% Finally install release after check and purge, and check that
    %% this install was faster than the first.
    {TInst2,{ok, _RelVsn1, []}} =
	timer:tc(rpc,call,[Node, release_handler, install_release, [RelVsn2]]),
    ct:log("install_release: ~.2f",[TInst2/1000000]),

    true = (TInst2 =< TInst0),

    ok.

otp_9395_rm_many_mods(cleanup,_Conf) ->
    stop_node(node_name(otp_9395_rm_many_mods)).



%% Test upgrade and downgrade of applications
eval_appup(Conf) when is_list(Conf) ->

    %% OTP-6162
    %% Create an ETS table which is updated by app1 if there is any
    %% change made to the application configuration parameter 'var'
    %% (see config_change/3 in myrel/lib1|2/app1-1|2.0/src/app1.erl)
    ets:new(otp_6162, [set, public, named_table]),

    %% Set some paths
    RelDir = filename:join(?config(data_dir, Conf), "app1_app2"),
    App11Dir = filename:join([RelDir, "lib1", "app1-1.0"]),
    App12Dir = filename:join([RelDir, "lib2", "app1-2.0"]),
    EbinDir = filename:join(App11Dir, "ebin"),

    %% Start app1-1.0
    code:add_patha(EbinDir),
    ok = application:start(app1),
    App11Dir = code:lib_dir(app1),
    ok = gen_server:call(harry, error),

    %% Upgrade to app1-2.0
    {ok, []} = release_handler:upgrade_app(app1, App12Dir),
    App12Dir = code:lib_dir(app1),
    error = gen_server:call(harry, error),

    %% OTP-6162
    %% Value of config parameter 'var' should now be 'val2'
    %% (see myrel/lib2/app1-2.0/ebin/app1.app)
    [{var,val2}] = ets:lookup(otp_6162, var),

    %% Downgrade to app1-1.0
    {ok, []} = release_handler:downgrade_app(app1,"1.0",App11Dir),
    App11Dir = code:lib_dir(app1),
    ok = gen_server:call(harry, error),

    %% OTP-6162
    %% Value of config parameter 'var' should now be 'val1'
    %% (see myrel/lib1/app1-1.0/ebin/app1.app)
    [{var,val1}] = ets:lookup(otp_6162, var),

    ok = application:stop(app1),
    ok = application:unload(app1),

    true = code:del_path(EbinDir),
    ok.


%% Test the example/target_system.erl module
target_system(Conf) when is_list(Conf) ->
    PrivDir = priv_dir(Conf),
    DataDir = ?config(data_dir,Conf),

    TargetCreateDir = filename:join([PrivDir,"target_system","create"]),
    TargetInstallDir = filename:join([PrivDir,"target_system","install"]),

    ok = filelib:ensure_dir(filename:join(TargetCreateDir,"xx")),
    ok = filelib:ensure_dir(filename:join(TargetInstallDir,"xx")),


    %% Create the .rel file
    ErtsVsn = erlang:system_info(version),
    RelName = filename:join(TargetCreateDir,"ts-1.0"),
    RelFile = RelName++".rel",
    RelVsn = "R1A",
    create_rel_file(RelFile,RelName,RelVsn,ErtsVsn,[{a, "1.0"}]),

    %% Build the target_system module
    ExamplesEbin = filename:join([code:lib_dir(sasl),examples,ebin]),
    TSPath =
	case filelib:is_file(filename:join(ExamplesEbin,"target_system.beam")) of
	    true ->
		ExamplesEbin;
	    false ->
		{ok,_} =
		    compile:file(filename:join(DataDir,"target_system.erl"),
				 [{outdir,TargetCreateDir}]),
		TargetCreateDir
    end,
    code:add_path(TSPath),

    %% Create the release
    target_system:create(RelName,[{path,[filename:join([DataDir,
							lib,
							"a-1.0",
							ebin])]}]),

    %% Install the release
    target_system:install(RelName,TargetInstallDir),

    code:del_path(TSPath),

    %% Check that all files exist in installation
    true = filelib:is_dir(filename:join(TargetInstallDir,"erts-"++ErtsVsn)),
    LibDir = filename:join(TargetInstallDir,lib),
    {ok,KernelVsn} = application:get_key(kernel,vsn),
    {ok,StdlibVsn} = application:get_key(stdlib,vsn),
    {ok,SaslVsn} = application:get_key(sasl,vsn),
    true = filelib:is_dir(filename:join(LibDir,"kernel-"++KernelVsn)),
    true = filelib:is_dir(filename:join(LibDir,"stdlib-"++StdlibVsn)),
    true = filelib:is_dir(filename:join(LibDir,"sasl-"++SaslVsn)),
    true = filelib:is_dir(filename:join(LibDir,"a-1.0")),
    RelDir = filename:join(TargetInstallDir,releases),
    true = filelib:is_regular(filename:join(RelDir,"RELEASES")),
    true = filelib:is_regular(filename:join(RelDir,"start_erl.data")),
    true = filelib:is_regular(filename:join(RelDir,
						  filename:basename(RelFile))),
    true = filelib:is_dir(filename:join(RelDir,RelVsn)),
    true = filelib:is_regular(filename:join([RelDir,RelVsn,"start.boot"])),
    BinDir = filename:join(TargetInstallDir,bin),
    true = filelib:is_regular(filename:join(BinDir,"start.boot")),
    true = filelib:is_regular(filename:join(BinDir,erl)),
    true = filelib:is_regular(filename:join(BinDir,start_erl)),
    true = filelib:is_regular(filename:join(BinDir,start)),
    true = filelib:is_regular(filename:join(BinDir,epmd)),
    true = filelib:is_regular(filename:join(BinDir,run_erl)),
    true = filelib:is_regular(filename:join(BinDir,to_erl)),

    %% Check content of files
    {ok,SED} = file:read_file(filename:join(RelDir,"start_erl.data")),
    [ErtsVsn,RelVsn] = string:tokens(binary_to_list(SED),"\s\n"),
    ok.



%%%=================================================================
%%% Testing global groups.
%%%=================================================================

%% This test case involves P1G and P1H with the sys.config as
%% specified in gg_config/1. The test case checks that the global
%% group information is correct before and after the upgrade and also
%% after terminating one of the nodes. The flow is as follows:
%% 1. Start all four nodes of global group gg1 with P1G
%% 2. Terminate one of the nodes, and upgrade the others to P1H. P1H
%%    config adds to more nodes to the global group.
%% 3. Start the two remaining nodes with P1H
upgrade_gg(Conf) ->
    [Gg1Sname,Gg2Sname,Gg3Sname,Gg4Sname,Gg5Sname,Gg6Sname] =
	?config(snames,Conf),

    %% start gg1, gg3, gg4, gg5 and check that global group info is ok
    Nodes1 = [Gg1,Gg3,Gg4,Gg5] =
	start_nodes(Conf,[Gg1Sname,Gg3Sname,Gg4Sname,Gg5Sname],"upgrade_gg"),

    %% Give some time to synch nodes, then check global group info.
    timer:sleep(1000),
    [check_gg_info(Node,Nodes1,[],Nodes1--[Node]) || Node <- Nodes1],

    %% register a process on each of the nodes
    ok = rpc:call(Gg1, installer, reg_proc, [reg1]),
    ok = rpc:call(Gg3, installer, reg_proc, [reg3]),
    ok = rpc:call(Gg4, installer, reg_proc, [reg4]),
    ok = rpc:call(Gg5, installer, reg_proc, [reg5]),
    are_names_reg_gg(Gg1, [reg1, reg3, reg4, reg5]),

    %% Stop gg3, then upgrade gg1, gg4 and gg5 to P1H
    ok = stop_nodes([Gg3]),

    ok = install_release_changed_gg(Gg1,"P1H"),
    ok = install_release_changed_gg(Gg4,"P1H"),
    ok = install_release_changed_gg(Gg5,"P1H"),

    %% Check global group info
    Gg2 = node_name(Gg2Sname),
    Gg6 = node_name(Gg6Sname),
    Nodes2 = [Gg1,Gg4,Gg5],
    [check_gg_info(Node,Nodes2,[Gg2,Gg6],Nodes2--[Node]) || Node <- Nodes2],

    %% start gg2 and gg6
    [Gg2,Gg6] = start_nodes(Conf,[Gg2Sname,Gg6Sname],"upgrade_gg start gg2/gg6"),

    %% reg proc on each of the nodes
    ok = rpc:call(Gg2, installer, reg_proc, [reg2]),
    ok = rpc:call(Gg6, installer, reg_proc, [reg6]),
    are_names_reg_gg(Gg1, [reg1, reg2, reg4, reg5, reg6]),

    %% Check global group info
    Nodes3 = [Gg1,Gg2,Gg4,Gg5,Gg6],
    [check_gg_info(Node,Nodes3,[],Nodes3--[Node]) || Node <- Nodes3],

    ok.

upgrade_gg(cleanup,Config) ->
    Snames = ?config(snames,Config),
    NodeNames = [node_name(Sname) || Sname <- Snames],
    ok = stop_nodes(NodeNames).




%%%=================================================================
%%% Misceleaneous functions
%%%=================================================================
stop_nodes(Nodes) ->
    ?t:format("Stopping nodes: ~p~n",[Nodes]),
    Running = 
	lists:foldl(fun(Node,Acc) -> 
			    Now = now(),
			    stop_cover(Node),
			    case rpc:call(Node,installer,stop,[Now]) of
				{badrpc,nodedown} ->
				    Acc;
				Other ->
				    ?t:format("Stop ~p(~p): ~p~n",
					      [Node,Now,Other]),
				    [Node|Acc]
			    end
		    end, [], Nodes),
    wait_nodes_down(Running).


wait_nodes_down(Nodes) ->
    ?t:format( "wait_nodes_down ~p:",[Nodes]),
    wait_nodes_down(Nodes, 30).

wait_nodes_down(Nodes, 0) ->
    test_server:fail({error, {"could not kill nodes", Nodes}});
wait_nodes_down(Nodes, N) ->
    Fun = fun(Node, A) ->
		  case net_adm:ping(Node) of
		      pong ->
			  ?t:format( "  net_adm:ping(~p) = pong", [Node]),
			  [Node|A];
		      pang ->
			  ?t:format( "  net_adm:ping(~p) = pang", [Node]),
			  A
		  end
	  end,
    Pang = lists:foldl(Fun, [], Nodes),
    case Pang of
	[] -> 
	    ?t:format("",[]),
	    ok;
	_ ->
	    timer:sleep(1000),
	    wait_nodes_down(Pang, N-1)
    end.



wait_nodes_up(Nodes, Tag) ->
    wait_nodes_up(Nodes, Tag, []).

wait_nodes_up(Nodes0, Tag, Apps) ->
    ?t:format("wait_nodes_up(~p, ~p, ~p):",[Nodes0, Tag, Apps]),
    Nodes = fix_nodes(Nodes0),
    wait_nodes_up(Nodes, Tag, lists:umerge(Apps,[kernel,stdlib,sasl]), 30).

fix_nodes([{Node,InitPid}|Nodes]) ->
    [{Node,InitPid} | fix_nodes(Nodes)];
fix_nodes([Node|Nodes]) ->
    [{Node,fake_init_pid} | fix_nodes(Nodes)];
fix_nodes([]) ->
    [].

wait_nodes_up(Nodes, Tag, Apps, 0) ->
    test_server:fail({error, {"nodes not started", Nodes, Tag, Apps}});
wait_nodes_up(Nodes, Tag, Apps, N) ->
    Fun = 
	fun(NodeInfo={Node,OldInitPid}, A) ->
		case rpc:call(Node, application, which_applications, []) of
		    {badrpc, nodedown} ->
			?t:format( "  ~p = {badarg, nodedown}",[Node]),
			[NodeInfo | A];
		    List when is_list(List)->
			?t:format( "  ~p = [~p]",[Node, List]),
			case lists:all(fun(App) -> 
					       lists:keymember(App,1,List) 
				       end, Apps) of
			    true ->
				case rpc:call(Node,erlang,whereis,[init]) of
				    OldInitPid ->
					[NodeInfo | A];
				    _ ->
					start_cover(Node),
					A
				end;
			    false ->
				[NodeInfo | A]
			end
		end
	end,
    Pang = lists:foldl(Fun,[],Nodes),
    case Pang of
	[] ->
	    ?t:format("",[]),
	    ok;
	_ ->
	    timer:sleep(1000),
	    wait_nodes_up(Pang, Tag, Apps, N-1)
    end.




are_names_reg_gg(Node, Names) ->
    ?t:format( "are_names_reg_gg ~p~n",[Names]),
    are_names_reg_gg(Node, Names, 30).

are_names_reg_gg(Node, Names, N) ->
    case lists:sort(rpc:call(Node, global, registered_names, [])) of
	Names ->
	    ok;
	Regs when N > 0 ->
	    timer:sleep(1000),
	    ?t:format( "are_names_reg_gg Regs ~p~n",[Regs]),
	    are_names_reg_gg(Node, Names, N-1);
	Regs ->
	    ?t:fail({error, {"Names not registered",
			     {{"should :", Names},
			      {"was :", Regs}}}})
    end.



t_start_node(Name, Boot, SysConfig) ->
    Args = 
	case Boot of
	    [] -> [];
	    _ -> " -boot " ++ Boot
	end ++
	case SysConfig of
	    [] -> [];
	    _ -> " -config " ++ SysConfig
	end,
    test_server:start_node(Name, slave, [{args, Args}]).

stop_node(Node) ->
    ?t:stop_node(Node).


copy_client(Conf,Master,Sname,Client) ->
    io:format("copy_client(Conf)"),

    DataDir = ?config(data_dir, Conf),
    MasterDir = filename:join(priv_dir(Conf),Master),

    {ClientArgs,RelCliDir} = rh_test_lib:get_client_args(Client,Sname,MasterDir,
							 node_name(Master)),

    Cli = filename:join([MasterDir, RelCliDir]),
    ok = filelib:ensure_dir(filename:join([Cli,"bin","."])),
    ok = filelib:ensure_dir(filename:join([Cli,"releases","."])),
    ok = filelib:ensure_dir(filename:join([Cli,"log","."])),

    P1GOrig = filename:join([MasterDir, "releases", "P1G"]),
    ok = copy_tree(Conf,P1GOrig,filename:join(Cli,"releases")),
    
    case os:type() of
	{unix,_} ->
	    ok = subst_file(filename:join([DataDir, "start_client"]),
			    filename:join([Cli,"bin","start"]),
			    [{"ROOT",MasterDir},
			     {"CLIENTARGS",ClientArgs}],
			    [{chmod,8#0755}]);
	_ ->
	    ok
    end,
    
    StartErlData = filename:join([MasterDir, "releases", "start_erl.data"]),
    CliRelDir = filename:join([Cli, "releases"]),
    copy_file(StartErlData, CliRelDir),
    
    RR = filename:join([MasterDir, "releases", "RELEASES"]),
    copy_file(RR, CliRelDir),

    ok.


delete_release(Conf) ->
    PrivDir = priv_dir(Conf),

    {ok, OrigWd} = file:get_cwd(),

    ok = file:set_cwd(PrivDir),
    ?t:format("========  current dir ~p~n",[PrivDir]),
    {ok, Dirs} = file:list_dir(PrivDir),
    ?t:format("========  deleting  ~p~n",[Dirs]),

    ok = delete_release_os(Dirs--["save"]),
    {ok,Remaining} = file:list_dir(PrivDir),
    ?t:format("========  remaining  ~p~n",[Remaining]),

    case Remaining of
	[] ->
	    ok;
	_ ->
	    delete_release_os(Remaining),
	    Remaining2 = file:list_dir(PrivDir),
	    ?t:format("========  remaining after second try ~p~n",[Remaining2])
    end,

    ok = file:set_cwd(OrigWd),
    ok.


delete_release_os(Dirs) ->
    case os:type() of
	{unix, _} ->
	    delete_release_unix(Dirs);
	{win32, _} ->
	    delete_release_win32(Dirs);
	Os ->
	    test_server:fail({error, {not_yet_implemented_os, Os}})
    end.


delete_release_unix([]) ->
    ok;
delete_release_unix(["save"|Dirs]) ->
    delete_release_unix(Dirs);
delete_release_unix([Dir|Dirs]) ->
    Rm = string:concat("rm -rf ", Dir),
    ?t:format("============== COMMAND ~p~n",[Rm]),
    case file:list_dir(Dir) of
	{error, enotdir} ->
	    ok;
	X ->
	    ?t:format("------- Dir ~p~n       ~p~n",[Dir, X])
    end,
    case os:cmd(Rm) of
	      [] ->
		  ?t:format("------- Result of COMMAND ~p~n",[ok]);
	      Y ->
		  ?t:format("!!!!!!! delete ERROR  Dir ~p Error ~p~n",[Dir, Y]),
		  ?t:format("------- ls -al  ~p~n",[os:cmd("ls -al " ++ Dir)])
	  end,

    delete_release_unix(Dirs).

delete_release_win32([]) ->
    ok;
delete_release_win32(["save"|Dirs]) ->
    delete_release_win32(Dirs);
delete_release_win32([Dir|Dirs]) ->
    Rm =
       case filelib:is_dir(Dir) of
          true ->
             string:concat("rmdir /s /q ", Dir);
          false ->
             string:concat("del /q ", Dir)
       end,
    ?t:format("============== COMMAND ~p~n",[Rm]),
    [] = os:cmd(Rm),
    delete_release_win32(Dirs).


node_name(Sname) when is_atom(Sname) ->
    {ok,Host} = inet:gethostname(),
    list_to_atom(atom_to_list(Sname) ++ "@" ++ Host).

copy_file(Src, Dest) ->
    copy_file(Src, Dest, []).
copy_file(Src, Dest, Opts) ->
    case file:copy(Src,Dest) of
	{ok,_} ->
	    preserve(Src,Dest,Opts),
	    chmod(Dest,Opts),
	    ok;
	{error,eisdir} ->
	    NewDest = filename:join(Dest, filename:basename(Src)),
	    case file:copy(Src,NewDest) of
		{ok,_} -> 
		    preserve(Src,NewDest,Opts),
		    chmod(NewDest,Opts);
		{error,Reason} -> 
		    copy_error(Src,Dest,Reason)
	    end;
	{error,Reason} ->
	    copy_error(Src,Dest,Reason)
    end.

preserve(Src,Dest,Opts) ->
    case lists:member(preserve, Opts) of
	true ->
	    {ok, FileInfo} = file:read_file_info(Src),
	    ok = file:write_file_info(Dest, FileInfo);
	false ->
	    ok
    end.

chmod(Dest,Opts) ->
    case lists:keyfind(chmod,1,Opts) of
	{chmod,Mode} ->
	    ok = file:change_mode(Dest, Mode);
	false ->
	    ok
    end.



copy_error(Src, Dest, Reason) ->
    io:format("Copy ~s to ~s failed: ~s\n",
	      [Src,Dest,file:format_error(Reason)]),
    ?t:fail(file_copy_failed).

copy_tree(Conf, Src, DestDir) ->
    case catch copy_tree(Conf, Src, filename:basename(Src), DestDir) of
	ok ->
	    ok;
	{'EXIT', {{badmatch,Error},_Stack}} ->
	    %% Most probably, an erl_tar call has failed.
	    %% Known to happen on some platforms (symbolic_link_too_long)
	    Error;
	{'EXIT', Reason} ->
	    {error, Reason}
    end.

copy_tree(Conf, Src, NewName, DestDir) ->
    PrivDir = priv_dir(Conf),
    TempTarName = filename:join(PrivDir, "temp_tar_file.tar"),
    %% Not compressing tar file here since that would increase test
    %% suite time by almost 100%, and the tar file is deleted
    %% imediately anyway.
    {ok,Tar} = erl_tar:open(TempTarName, [write]),
    ok = erl_tar:add(Tar, Src, NewName, []),
    ok = erl_tar:close(Tar),
    ok = erl_tar:extract(TempTarName, [{cwd,DestDir}]),
    ok = file:delete(TempTarName),
    ok.

%% subst_file(Src, Dest, Vars)
%% Src = Dest = string(), filename and path
%% Vars = [{Var, Val}]
%% Var = Val = string()
%% Substitute all occurrences of %Var% for Val in Src, using the list
%% of variables in Vars. Result is written to Dest.
%%
subst_file(Src, Dest, Vars) ->
    subst_file(Src, Dest, Vars, []).
subst_file(Src, Dest, Vars, Opts) ->
    {ok, Bin} = file:read_file(Src),
    Conts = binary_to_list(Bin),
    NConts = subst(Conts, Vars),
    ok = file:write_file(Dest, NConts),
    preserve(Src,Dest,Opts),
    chmod(Dest,Opts).

subst(Str, Vars) ->
    subst(Str, Vars, []).

subst([$%, C| Rest], Vars, Result) when $A =< C, C =< $Z ->
    subst_var([C| Rest], Vars, Result, []);
subst([$%, C| Rest], Vars, Result) when $a =< C, C =< $z ->
    subst_var([C| Rest], Vars, Result, []);
subst([$%, C| Rest], Vars, Result) when  C == $_ ->
    subst_var([C| Rest], Vars, Result, []);
subst([C| Rest], Vars, Result) ->
    subst(Rest, Vars, [C| Result]);
subst([], _Vars, Result) ->
    lists:reverse(Result).

subst_var([$%| Rest], Vars, Result, VarAcc) ->
    Key = lists:reverse(VarAcc),
    case lists:keysearch(Key, 1, Vars) of
        {value, {Key, Value}} ->
            subst(Rest, Vars, lists:reverse(Value, Result));
        false ->
            subst(Rest, Vars, [$%| VarAcc ++ [$%| Result]])
    end;
subst_var([C| Rest], Vars, Result, VarAcc) ->
    subst_var(Rest, Vars, Result, [C| VarAcc]);
subst_var([], Vars, Result, VarAcc) ->
    subst([], Vars, [VarAcc ++ [$%| Result]]).


priv_dir(Conf) ->
%%    filename:absname(?config(priv_dir, Conf)). % Get rid of trailing slash
    %% Due to problem with long paths on windows => creating a new
    %% priv_dir under data_dir
    Dir = filename:absname(filename:join(?config(data_dir, Conf),priv_dir)),
    filelib:ensure_dir(filename:join(Dir,"*")),
    Dir.

latest_version(Dir) ->
    List = filelib:wildcard(Dir ++ "*"),
    lists:last(lists:sort(List)).

%% A printer process which receives messages from other nodes and
%% prints in the log
reg_print_proc() ->
    catch unregister(rh_print),
    Pid = spawn_link(?MODULE, rh_print, []),
    register(rh_print, Pid),
    ok.

rh_print() ->
    receive
	{print, {Module,Line}, [H|T]} ->
	    ?t:format("=== ~p:~p - ~p",[Module,Line,H]),
	    lists:foreach(fun(Term) -> ?t:format("    ~p",[Term]) end, T),
	    ?t:format("",[]),
	    rh_print();
	kill ->
	    exit(normal)
    end.

stop_print_proc() ->
    case whereis(rh_print) of %%removes the printer process
	undefined ->
	    ok;
	Pid when is_pid(Pid) ->
	    rh_print ! kill
    end.

%% Create the first target release, vsn P1G. This release is used for
%% all test cases in {group,release}
create_p1g(Conf,Sname) ->
    do_create_p1g(Conf,filename:join(priv_dir(Conf),Sname)).

do_create_p1g(Conf,TargetDir) ->
    PrivDir = priv_dir(Conf),
    DataDir = ?config(data_dir,Conf),
    ErtsVsn = "4.4",
    ErtsDir = "erts-"++ErtsVsn,

    %% Create dirs
    BinDir = filename:join(TargetDir,bin),
    ReleasesDir = filename:join(TargetDir,releases),
    LogDir = filename:join(TargetDir,log),
    ok = filelib:ensure_dir(filename:join(BinDir,"*")),
    ok = filelib:ensure_dir(filename:join(ReleasesDir,"*")),
    ok = filelib:ensure_dir(filename:join(LogDir,"*")),

    %% Copy stuff 
    ErtsLatest = latest_version(filename:join(code:root_dir(),"erts")),
    ok = copy_tree(Conf, ErtsLatest, ErtsDir, TargetDir),
    ErtsBinDir = filename:join([TargetDir,ErtsDir,bin]),

    case os:type() of
	{unix, _} ->
	    copy_file(filename:join([ErtsBinDir, "epmd"]), BinDir, [preserve]),
	    copy_file(filename:join([ErtsBinDir, "run_erl"]), BinDir, [preserve]),
	    copy_file(filename:join([ErtsBinDir, "to_erl"]), BinDir, [preserve]),

	    %% Create the start_erl shell script
	    ok = subst_file(filename:join([ErtsBinDir,"start_erl.src"]),
			    filename:join([BinDir,"start_erl"]),
			    [{"EMU","beam"}],
			    [{chmod,8#0755}]);
	{win32,_} ->
	    %% Add a batch file to use as HEART_COMMAND
	    ok = copy_file(filename:join(DataDir, "heart_restart.bat"),
			   ErtsBinDir,[preserve])
    end,

    copy_file(filename:join(DataDir, "../installer.beam"),
	      filename:join([DataDir,lib,"installer-1.0",ebin])),
    copy_file(filename:join(DataDir, "../rh_test_lib.beam"),
	      filename:join([DataDir,lib,"installer-1.0",ebin])),

    %% Create .rel, .script and .boot files
    RelName = "rel0",
    RelVsn = "P1G",
    RelDir = filename:join(PrivDir,RelName),
    RelFileName = filename:join(RelDir,RelName),
    RelFile = RelFileName ++ ".rel",
    ok = filelib:ensure_dir(RelFile),
    LibPath = filename:join([DataDir,lib,"*",ebin]),

    TarFile = create_basic_release(Conf, RelFile, RelVsn, {ErtsVsn,false},
				   LibPath, [], [], [], []),

    %% Extract tar file in target directory (i.e. same directory as erts etc.)
    ok = erl_tar:extract(TarFile, [{cwd, TargetDir}, compressed]),

    %% Create start_erl.data
    StartErlDataFile = filename:join([ReleasesDir, "start_erl.data"]),
    StartErlData = io_lib:fwrite("~s ~s~n", [ErtsVsn, RelVsn]),
    ok = file:write_file(StartErlDataFile, StartErlData),

    %% Create RELEASES
    ok = release_handler:create_RELEASES(TargetDir,ReleasesDir,RelFile,[]),

    ok.

%% Create version P1H - which is P1G + a-1.0
%% Must have run create_p1g first!!
create_p1h(Conf) ->
    create_upgrade_release(Conf,"rel1","P1H",{"4.4",false},[{a,"1.0"}],
			   [{a,[{key2,val2}]}],{"rel0",[new_appl]}).

%% Create version P1I - which is P1H, but with application a upgraded to a-1.1
%% Must have run create_p1h first!!
create_p1i(Conf) ->
    create_upgrade_release(Conf,"rel2","P1I",{"4.4",false},[{a,"1.1"}],
			   [{a,[{key2,newval2}]}],
			   {"rel1",[{extra,gott}]}).

%% Create version P2A - which is P1I, but with erts-<latest>
%% Must have run create_p1i first!!
create_p2a(Conf) ->
    ErtsVsn = erlang:system_info(version),
    create_upgrade_release(Conf,"rel3","P2A",{ErtsVsn,code:root_dir()},
			   [{a,"1.1"}],[{a,[{key2,newval2}]}],
			   {"rel2",[new_emu]}).

%% Create a release tar package which can be installed on top of P1G
create_upgrade_release(Conf,RelName,RelVsn,Erts,Apps,Config,{UpFromName,Descr}) ->
    PrivDir = priv_dir(Conf),
    DataDir = ?config(data_dir,Conf),

    RelDir = filename:join(PrivDir,RelName),
    RelFileName = filename:join(RelDir,RelName),
    RelFile = RelFileName ++ ".rel",
    ok = filelib:ensure_dir(RelFile),
    LibPath = filename:join([DataDir,lib,"*",ebin]),

    UpFrom = [{filename:join([PrivDir,UpFromName,UpFromName]),Descr}],

    create_basic_release(Conf, RelFile, RelVsn, Erts, LibPath,
			 Apps, Config, UpFrom, []),
    ok.

%% Create .rel, .script, .boot, sys.config and tar
create_basic_release(Conf, RelFile,RelVsn,{ErtsVsn,ErtsDir},LibPath,ExtraApps,Config,UpFrom,DownTo) ->
    RelDir = filename:dirname(RelFile),
    RelFileName = filename:rootname(RelFile),

    %% Create .rel file
    create_installer_rel_file(RelFile,RelVsn,ErtsVsn,ExtraApps),

    %% Generate .script and .boot
    ok = systools:make_script(RelFileName,
			      [{path,[LibPath]},
			       {outdir,RelDir}]),

    %% Generate relup
    ok = systools:make_relup(RelFileName,UpFrom,DownTo,[{path,[LibPath]},
							      {outdir,RelDir}]),

    %% Create sys.config
    ok = write_term_file(filename:join(RelDir,"sys.config"),Config),


    %% Create tar file (i.e. collect all lib/app-*/* and system files)
    ok = systools:make_tar(RelFileName,
			   [{path,[LibPath]},
			    {outdir,RelDir} |
			    case ErtsDir of
				false -> [];
				_ -> [{erts,ErtsDir}]
			    end]),
    
    TarFileName = RelFileName ++ ".tar.gz",

    case os:type() of
	{win32,_} when ErtsDir=/=false -> modify_tar_win32(Conf, TarFileName);
	_ -> ok
    end,

    TarFileName.

%% Create a .rel file
create_installer_rel_file(RelFile,RelVsn,ErtsVsn,ExtraApps) ->
    create_rel_file(RelFile,"SASL-test",RelVsn,ErtsVsn,
		    [{installer,"1.0"}|ExtraApps]).

create_rel_file(RelFile,RelName,RelVsn,ErtsVsn,ExtraApps) ->
    {ok,KernelVsn} = application:get_key(kernel,vsn),
    {ok,StdlibVsn} = application:get_key(stdlib,vsn),
    {ok,SaslVsn} = application:get_key(sasl,vsn),
    application:load(tools),
    {ok,ToolsVsn} = application:get_key(tools,vsn),
    application:load(runtime_tools),
    {ok,RuntimeToolsVsn} = application:get_key(runtime_tools,vsn),
    
    RelFileContent = {release,
		      {RelName, RelVsn},
		      {erts, ErtsVsn},
		      [{kernel, KernelVsn},
		       {stdlib, StdlibVsn},
		       {sasl, SaslVsn},
		       {runtime_tools, RuntimeToolsVsn},
		       {tools, ToolsVsn} |
		       ExtraApps]},
    ok = write_term_file(RelFile,RelFileContent).

%% Insert a term in a file, which can be read with file:consult/1.
write_term_file(File,Term) ->
    ok = file:write_file(File,io_lib:format("~p.~n",[Term])).
    

%% Check that global group info is correct
check_gg_info(Node,OtherAlive,OtherDead,Synced) ->
    GGI = rpc:call(Node, global_group, info, []),
    GI = rpc:call(Node, global, info,[]),
    try do_check_gg_info(OtherAlive,OtherDead,Synced,GGI,GI) 
    catch _:E ->
	    ?t:format("~ncheck_gg_info failed for ~p: ~p~nwhen GGI was: ~p~n"
		      "and GI was: ~p~n",
		      [Node,E,GGI,GI]),
	    ?t:fail("check_gg_info failed")
    end.

do_check_gg_info(OtherAlive,OtherDead,Synced,GGI,GI) ->
    {_,gg1} = lists:keyfind(own_group_name,1,GGI),
    {_,synced} = lists:keyfind(state,1,GGI),
    {_,AllNodes} = lists:keyfind(own_group_nodes,1,GGI),
    true = lists:sort(AllNodes) =:= lists:sort(OtherAlive++OtherDead),
    {_,[]} = lists:keyfind(sync_error,1,GGI),
    {_,[{gg2,[_,_]}]} = lists:keyfind(other_groups,1,GGI),

    %% There is a known bug in global_group (OTP-9177) which causes
    %% the following to fail every now and then:
    %% {_,SyncedNodes} = lists:keyfind(synced_nodes,1,GGI),
    %% true = lists:sort(SyncedNodes) =:= lists:sort(Synced),
    %% {_,NoContact} = lists:keyfind(no_contact,1,GGI),
    %% true = lists:sort(NoContact) =:= lists:sort(OtherDead),

    %% Therefore we use global:info instead for this part
    {state,_,_,SyncedNodes,_,_,_,_,_,_,_} = GI,
    true = lists:sort(SyncedNodes) =:= lists:sort(Synced),

    %% .. and we only check that all OtherDead are listed as
    %% no_contact (due to th bug there might be more nodes in this
    %% list)
    {_,NoContact} = lists:keyfind(no_contact,1,GGI),
    true =
	lists:sort(OtherDead) =:=
	lists:sort([NC || NC <- NoContact,lists:member(NC,OtherDead)]),

    ok.

%% Return the configuration (to be inserted in sys.config) for global group tests
gg_config(Snames) ->
    Nodes = [node_name(Sname) || Sname <- Snames],
    [{kernel, [{sync_nodes_optional, Nodes},
	       {sync_nodes_timeout, 10000},
	       {global_groups, 
		[{gg1, Nodes},
		 {gg2, [node_name(Sname) || Sname <- [ggq,ggw]]}]}]},
     {a, [{key2, val2}]}].

%% Start a node with short name SnameStr, and unpack P1H
unpack_p1h(Conf,Sname) ->
    PrivDir = priv_dir(Conf),
    [Node] = start_nodes(Conf,[Sname],"create_p1h"),
    ok = rpc_inst(Node, unpack_p1h, [PrivDir]),
    Node.

%% On the given node, install P1H and make it permanent
%% This function is to be called after unpack_p1h/2, with the same node.
permanent_p1h(Node) ->
    ok = rpc_inst(Node, permanent_p1h, []).

%% For each node in ToNodes, create a target installation which is
%% indentical to the target installation for FromNode.
copy_installed(Conf,FromNode,ToNodes) ->
    PrivDir = priv_dir(Conf),
    DataDir = ?config(data_dir,Conf),

    %% Instead of using copy_tree on the complete node directory, I'm
    %% splitting this in separate tar files per subdirectory so the
    %% log directory can be completely skipped. The reason for this is
    %% that the tar file might become faulty if the node is alive and
    %% writing to the log while the tar is created.
    FromDir = filename:join(PrivDir,FromNode),
    {ok,FromDirNames} = file:list_dir(FromDir),
    TempTarFiles =
	[begin
	     TempTarFile = filename:join(PrivDir,"temp_" ++ FDN ++ ".tar"),
	     {ok,Tar} = erl_tar:open(TempTarFile,[write]),
	     ok = erl_tar:add(Tar,filename:join(FromDir,FDN),FDN,[]),
	     ok = erl_tar:close(Tar),
	     TempTarFile
         end || FDN <- FromDirNames, FDN=/="log"],
    lists:foreach(
      fun(Node) ->
	      NodeDir = filename:join(PrivDir,Node),
	      ok = filelib:ensure_dir(filename:join([NodeDir,"log","*"])),
              lists:foreach(
		fun(TempTarFile) ->
                        ok = erl_tar:extract(TempTarFile,[{cwd,NodeDir}])
		end, TempTarFiles),
	      case os:type() of
		  {unix,_} ->
		      %% Create start script
		      %% Using a customized start script from DataDir
		      %% where some options (heart and nodename) are
		      %% added compared to the start.src in the erlang
		      %% distribution.
		      ok = subst_file(filename:join(DataDir, "start"),
				      filename:join([NodeDir, "bin", "start"]),
				      [{"ROOT",NodeDir}],
				      [preserve]);
		  {win32,_} ->
		      %% Write erl.ini
		      ErtsDirs =
			  filelib:wildcard(filename:join(NodeDir,"erts-*")),
		      lists:foreach(
			fun(ErtsDir) ->
				ok = subst_file(
				       filename:join(DataDir, "erl.ini.src"),
				       filename:join([ErtsDir, "bin", "erl.ini"]),
				       [{"ROOTDIR",NodeDir},
					{"BINDIR",filename:join(ErtsDir,"bin")}])
			end,
			ErtsDirs),

		      %% The service on windows runs as local
		      %% administrator (not otptest user), so we need
		      %% to chmod the release in order to allow the
		      %% executing node to install releases, write
		      %% logs etc.
		      chmod_release_win32(NodeDir)
	      end
      end,
      ToNodes),

    lists:foreach(fun(TempTarFile) -> file:delete(TempTarFile) end, TempTarFiles),
    ok.

chmod_release_win32(Dir) ->
    os:cmd("echo y|cacls " ++ Dir ++ " /T /E /G Administrators:F").

start_nodes(Conf,Snames,Tag) ->
    PrivDir = priv_dir(Conf),
    Nodes = 
	lists:map(
	  fun(Sname) ->
		  NodeDir = filename:join(PrivDir,Sname),
		  Node = node_name(Sname),

		  case os:type() of
		      {unix,_} ->
			  start_node_unix(Sname,NodeDir);
		      {win32,_} ->
			  start_node_win32(Sname,NodeDir)
		  end,
		  Node
	  end,
	  Snames),
    wait_nodes_up(Nodes,Tag),
    Nodes.
    
start_node_unix(Sname,NodeDir) ->
    Script = filename:join([NodeDir,"bin","start"]),
    Cmd = "env NODENAME="++atom_to_list(Sname) ++ " " ++ Script,
    %% {ok,StartFile} = file:read_file(Cmd),
    %% io:format("~s:\n~s~n~n",[Start,binary_to_list(StartFile)]),
    Res = os:cmd(Cmd),
    io:format("Start ~p: ~p~n=>\t~p~n", [Sname,Cmd,Res]).

start_node_win32(Sname,NodeDir) ->
    Name = atom_to_list(Sname) ++ "_P1G",
    ErtsBinDir = filename:join(NodeDir,"erts-4.4/bin"),

    StartErlArgs = rh_test_lib:get_start_erl_args(NodeDir),
    ServiceArgs = rh_test_lib:get_service_args(NodeDir, Sname, StartErlArgs),

    Erlsrv = filename:nativename(filename:join(ErtsBinDir,"erlsrv")),
    rh_test_lib:erlsrv(Erlsrv,stop,Name),
    rh_test_lib:erlsrv(Erlsrv,remove,Name),
    ok = rh_test_lib:erlsrv(Erlsrv,add,Name,ServiceArgs),
    ok = rh_test_lib:erlsrv(Erlsrv,start,Name),
    ok.

%% Create a unique node name for each test case
tc_sname(Config) ->
    tc_sname(Config,"").
tc_sname(Config,Fix) when is_atom(Fix) ->
    tc_sname(Config,atom_to_list(Fix));
tc_sname(Config,Fix) when is_list(Fix) ->
    list_to_atom(
      atom_to_list(?MODULE)
      ++ "-" ++ atom_to_list(?config(sname_prefix, Config)) ++
	  case Fix of
	      "" -> "";
	      _ -> "-" ++ Fix
	  end).

tc_full_node_name(Config) ->
    tc_full_node_name(Config,"").
tc_full_node_name(Config,Fix) ->
    node_name(tc_sname(Config,Fix)).


%% When installing a release for which the sys.config includes added
%% or changed global group(s), this node (test_sever@host) will be
%% disconnected from the test node (Node) by global_group.erl. This
%% will cause the rpc:call to terminate with {badrpc,nodedown} even if
%% the installation succeeds. This function installs the release,
%% accepts the faulty return value and then checks if the release was
%% successfully installed.
install_release_changed_gg(Node,RelVsn) ->
    stop_cover(Node),
    {badrpc,nodedown} = rpc:call(Node,release_handler,install_release,[RelVsn]),
    timer:sleep(100),
    wait_installed(Node,RelVsn,4).

wait_installed(Node,RelVsn,0) ->
    ?t:fail("install_release_changed_gg failed for " ++ RelVsn ++ 
	    " on " ++ atom_to_list(Node));
wait_installed(Node,RelVsn,N) ->
    Rels = rpc:call(Node,release_handler,which_releases,[]),
    case lists:keyfind(RelVsn, 2, Rels) of
	{"SASL-test", RelVsn, _Libs, current} -> 
	    start_cover(Node),
	    ok;
	_ -> 
	    timer:sleep(500),
	    wait_installed(Node,RelVsn,N-1)
    end.

%% Start/stop cover measurements on the given node
start_cover(Node) ->
    cover_fun(Node,start).
stop_cover(Node) ->
    cover_fun(Node,stop).

cover_fun(Node,Func) ->
    case ?t:is_cover() of
	true ->
	    cover:Func(Node);
	false ->
	    ok
    end.

%%%-----------------------------------------------------------------
%%% Create a fake release ....

%% This function will create and install a release build on the
%% current running OTP release. It includes kernel, stdlib and sasl,
%% and possibly other applications if they are listed in AppDirs =
%% [{App,Vsn,LibDir}]
create_and_install_fake_first_release(Dir,AppDirs) ->
    %% Create the first release
    {RelName,RelVsn} = init:script_id(),
    {Rel,_} = create_fake_release(Dir,RelName,RelVsn,AppDirs),
    ReleasesDir = filename:join(Dir, "releases"),
    RelDir = filename:dirname(Rel),

    %% And install it
    RelVsnDir = filename:join(ReleasesDir, RelVsn),
    ok = filelib:ensure_dir(filename:join(RelVsnDir,"*")),

    ok = copy_file(Rel++".rel",RelVsnDir),
    ok = copy_file(Rel++".boot",filename:join(RelVsnDir, "start.boot")),
    ok = copy_file(filename:join(RelDir,"sys.config"),RelVsnDir),

    ok = release_handler:create_RELEASES(code:root_dir(),
					 ReleasesDir,
					 Rel++".rel",
					 AppDirs),

    Rel.
    
%% This function create a new release, including a relup file. It can
%% be upgraded to from the release created by
%% create_and_install_fake_first_release/2. Unpack first by calls to
%% release_handler:set_unpacked and release_handler:install_file.
create_fake_upgrade_release(Dir,RelVsn,AppDirs,{UpFrom,DownTo,ExtraLibs}) ->
    %% Create a new release
    {RelName,_} = init:script_id(),
    {Rel,Paths} = create_fake_release(Dir,RelName,RelVsn,AppDirs),
    RelDir = filename:dirname(Rel),

    %% And a relup file so it can be upgraded to
    RelupPath = Paths ++ [filename:join([Lib,"*","ebin"]) || Lib <- ExtraLibs],
    ok = systools:make_relup(Rel,UpFrom,DownTo,[{path,RelupPath},
						{outdir,RelDir}]),
    
    Rel.


create_fake_release(Dir,RelName,RelVsn,AppDirs) ->
    %% Create .rel files
    RelDir = filename:join(Dir,"rel_" ++ RelVsn),
    Rel = filename:join([RelDir,"rel_" ++ RelVsn]),
    ok = filelib:ensure_dir(Rel),
    ErtsVsn = erlang:system_info(version),

    {Apps,Paths} = 
	lists:foldl(fun({App,Vsn,Lib},{As,Ps}) ->
			    {[{App,Vsn}|As],
			     lists:umerge([filename:join([Lib,"*",ebin])],Ps)}
		    end,
		    {[],[]},
		    AppDirs),

    create_rel_file(Rel++".rel",RelName,RelVsn,ErtsVsn,Apps),
    
    %% Generate boot scripts
    ok = systools:make_script(Rel,[local,
				   {path, Paths},
				   {outdir,RelDir}]),
    ok = copy_file(Rel++".boot", filename:join(RelDir,"start.boot")),

    %% Use an own 'releases' directory - we don't want to change the
    %% contents of $OTP_ROOT/releases
    %% Inform SASL about this via sys.config
    ReleasesDir = filename:join(Dir, "releases"),
    Config = [{sasl,[{releases_dir,ReleasesDir}]}],
    ok = write_term_file(filename:join(RelDir,"sys.config"), Config),
    
    {Rel,Paths}.
    

rpc_inst(Node,Func,Args) ->
    rpc:call(Node,installer,Func,[node()|Args]).

delete_all_services() ->
    ErlSrv = erlsrv:erlsrv(erlang:system_info(version)),
    [_|Serviceinfo] = string:tokens(os:cmd(ErlSrv ++ " list"),"\n"),
    Services =
	[lists:takewhile(fun($\t) -> false; (_) -> true end,S)
	 || S <- Serviceinfo],
    ?t:format("Services to remove: ~p~n",[Services]),
    lists:foreach(fun(S) ->
			  rh_test_lib:erlsrv(ErlSrv,stop,S),
			  rh_test_lib:erlsrv(ErlSrv,remove,S)
		  end,
		  Services).

modify_tar_win32(Conf, TarFileName) ->
    DataDir = ?config(data_dir,Conf),
    PrivDir = priv_dir(Conf),
    TmpDir = filename:join(PrivDir,"tmp_modify_tar_win32"),
    ok = erl_tar:extract(TarFileName,[{cwd,TmpDir},compressed]),

    ErtsBinDir = filelib:wildcard(filename:join([TmpDir,"erts-*","bin"])),
    ok = copy_file(filename:join(DataDir, "heart_restart.bat"),
		   ErtsBinDir,[preserve]),

    {ok,Fs} = file:list_dir(TmpDir),
    {ok,T} = erl_tar:open(TarFileName,[write,compressed]),
    [ok = erl_tar:add(T,filename:join(TmpDir,F),F,[]) || F <- Fs],
    ok = erl_tar:close(T),
    ok.