Merge branch 'siri/test-ct_release_test/OTP-13390' into maint

* siri/test-ct_release_test/OTP-13390:
  Remove upgrade tests from stdlib_SUITE
  Add test of ct_release_test

3 files changed, 310 insertions, 1 deletions

diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index ff4495b104..a1cead4550 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -65,7 +65,8 @@ MODULES= \
 	ct_cover_nomerge_SUITE \
 	ct_groups_search_SUITE \
 	ct_surefire_SUITE \
-	ct_telnet_SUITE
+	ct_telnet_SUITE \
+	ct_release_test_SUITE
 
 ERL_FILES= $(MODULES:%=%.erl)
 
diff --git a/lib/common_test/test/ct_release_test_SUITE.erl b/lib/common_test/test/ct_release_test_SUITE.erl
new file mode 100644
index 0000000000..66d07155ac
--- /dev/null
+++ b/lib/common_test/test/ct_release_test_SUITE.erl
@@ -0,0 +1,190 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2014. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_release_test_SUITE
+%%%
+%%% Description:
+%%% Test ct_release_test module
+%%%
+%%%-------------------------------------------------------------------
+-module(ct_release_test_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+-define(suite, release_test_SUITE).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+    case os:type() of
+	{win32,_} ->
+	    {skipped, "Upgrade tests do currently not work on windows"};
+	_ ->
+	    ct_test_support:init_per_suite(Config)
+    end.
+
+end_per_suite(Config) ->
+    ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+    ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+    ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+    [
+     minor,
+     major,
+     major_fail_init,
+     major_fail_upgraded,
+     major_fail_downgraded,
+     major_fail_no_init
+    ].
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+minor(Config) when is_list(Config) ->
+    {Suite,Cfg} = setup1(Config),
+    {Opts,ERPid} = setup([{suite,Suite},
+			  {testcase,minor},
+			  {label,minor}|Cfg], Config),
+    execute(minor, Opts, ERPid, Config).
+
+major(Config) when is_list(Config) ->
+    {Suite,Cfg} = setup1(Config),
+    {Opts,ERPid} = setup([{suite,Suite},
+			  {testcase,major},
+			  {label,major}|Cfg], Config),
+    execute(major, Opts, ERPid, Config).
+
+major_fail_init(Config) when is_list(Config) ->
+    {Suite,Cfg} = setup1(Config),
+    {Opts,ERPid} = setup([{suite,Suite},
+			  {testcase,major_fail_init},
+			  {label,major_fail_init}|Cfg], Config),
+    execute(major_fail_init, Opts, ERPid, Config).
+
+major_fail_upgraded(Config) when is_list(Config) ->
+    {Suite,Cfg} = setup1(Config),
+    {Opts,ERPid} = setup([{suite,Suite},
+			  {testcase,major_fail_upgraded},
+			  {label,major_fail_upgraded}|Cfg], Config),
+    execute(major_fail_upgraded, Opts, ERPid, Config).
+
+major_fail_downgraded(Config) when is_list(Config) ->
+    {Suite,Cfg} = setup1(Config),
+    {Opts,ERPid} = setup([{suite,Suite},
+			  {testcase,major_fail_downgraded},
+			  {label,major_fail_downgraded}|Cfg], Config),
+    execute(major_fail_downgraded, Opts, ERPid, Config).
+
+major_fail_no_init(Config) when is_list(Config) ->
+    {Suite,Cfg} = setup1(Config),
+    {Opts,ERPid} = setup([{suite,Suite},
+			  {testcase,major_fail_no_init},
+			  {label,major_fail_no_init}|Cfg], Config),
+    execute(major_fail_no_init, Opts, ERPid, Config).
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+setup1(Config) ->
+    DataDir = ?config(data_dir, Config),
+    Suite = filename:join(DataDir, atom_to_list(?suite)),
+    Cfg = case ct:get_config(otp_releases) of
+	      undefined ->
+		  [];
+	      Rels ->
+		  CfgFile = filename:join(DataDir, "release_test.cfg"),
+		  file:write_file(CfgFile,
+				  io_lib:format("{otp_releases,~p}.",[Rels])),
+		  [{config,CfgFile}]
+	  end,
+    {Suite,Cfg}.
+
+setup(Test, Config) ->
+    Opts0 = ct_test_support:get_opts(Config),
+    Level = ?config(trace_level, Config),
+    EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+    Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+    ERPid = ct_test_support:start_event_receiver(Config),
+    {Opts,ERPid}.
+
+execute(Name, Opts, ERPid, Config) ->
+    ok = ct_test_support:run(Opts, Config),
+    Events = ct_test_support:get_events(ERPid, Config),
+
+    ct_test_support:log_events(Name,
+			       reformat(Events, ?eh),
+			       ?config(priv_dir, Config),
+			       Opts),
+
+    verify_events(Name,Events,Config).
+
+reformat(Events, EH) ->
+    ct_test_support:reformat(Events, EH).
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+verify_events(TC,Events,Config) ->
+    Ok = expected_events(TC,ok),
+    case ct_test_support:verify_events(Ok, Events, Config) of
+	ok ->
+	    ok;
+	{event_not_found,{?eh,tc_done,{_Suite,TC,ok}}}=R1 ->
+	    ct:log("Did not find 'ok', checking if skipped...",[]),
+	    Skipped = expected_events(TC,{skipped,"Old release not available"}),
+	    case ct_test_support:verify_events(Skipped, Events, Config) of
+		ok ->
+		    {skipped,"Old release not available"};
+		R2 ->
+		    ct:log("Did not find skipped case either: ~n~p",[R2]),
+		    exit(R1)
+	    end
+    end.
+
+expected_events(TC,Result) ->
+    OneTest =
+	[{?eh,start_logging,{'DEF','RUNDIR'}},
+	 {?eh,tc_done,{?suite,TC,Result}},
+	 {?eh,stop_logging,[]}],
+    %% 2 tests (ct:run_test + script_start) is default
+    OneTest ++ OneTest.
diff --git a/lib/common_test/test/ct_release_test_SUITE_data/release_test_SUITE.erl b/lib/common_test/test/ct_release_test_SUITE_data/release_test_SUITE.erl
new file mode 100644
index 0000000000..04c92be0d1
--- /dev/null
+++ b/lib/common_test/test/ct_release_test_SUITE_data/release_test_SUITE.erl
@@ -0,0 +1,118 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 1997-2015. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%%----------------------------------------------------------------
+%%% Purpose: Test the support for application upgrade/code_change test
+%%%-----------------------------------------------------------------
+-module(release_test_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+-define(APP,runtime_tools). % "randomly" selected 'application under test'
+
+%%
+%% all/1
+%%
+all() ->
+    [minor,
+     major,
+     major_fail_init,
+     major_fail_upgraded,
+     major_fail_downgraded,
+     major_fail_no_init].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_testcase(major_fail_no_init, Config) ->
+    Config;
+init_per_testcase(_Case, Config) ->
+    ct_release_test:init(Config).
+end_per_testcase(_Case, Config) ->
+    ct_release_test:cleanup(Config).
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+minor(Config) ->
+    ct_release_test:upgrade(?APP,minor,{?MODULE,[]},Config).
+
+major(Config) ->
+    ct_release_test:upgrade(?APP,major,{?MODULE,[]},Config).
+
+major_fail_init(Config) ->
+    try ct_release_test:upgrade(?APP,major,{?MODULE,fail_init},Config)
+    catch exit:{test_case_failed,
+		{test_upgrade_callback,_Mod,_Func,_Args,
+		 {'EXIT',{test_case_failed,upgrade_init_failed}}}} ->
+	    ok
+    end.
+
+major_fail_upgraded(Config) ->
+    try ct_release_test:upgrade(?APP,major,{?MODULE,fail_upgraded},Config)
+    catch exit:{test_case_failed,
+		{test_upgrade_callback,_Mod,_Func,_Args,
+		 {'EXIT',{test_case_failed,upgrade_upgraded_failed}}}} ->
+	    ok
+    end.
+
+major_fail_downgraded(Config) ->
+    try ct_release_test:upgrade(?APP,major,{?MODULE,fail_downgraded},Config)
+    catch exit:{test_case_failed,
+		{test_upgrade_callback,_Mod,_Func,_Args,
+		 {'EXIT',{test_case_failed,upgrade_downgraded_failed}}}} ->
+	    ok
+    end.
+
+major_fail_no_init(Config) ->
+    try ct_release_test:upgrade(?APP,major,[],Config)
+    catch exit:{test_case_failed,"ct_release_test:init/1 not run"} ->
+	    ok
+    end.
+
+%%%-----------------------------------------------------------------
+%%% ct_release_test callbacks
+
+%% Version numbers are checked by ct_release_test, so there is nothing
+%% more to check here...
+upgrade_init(CtData,fail_init) ->
+    ct:fail(upgrade_init_failed);
+upgrade_init(CtData,State) ->
+    {ok,{FromVsn,ToVsn}} = ct_release_test:get_app_vsns(CtData,?APP),
+    case ct_release_test:get_appup(CtData,?APP) of
+	{ok,{FromVsn,ToVsn,UpInstrs,DownInstrs}} ->
+	    io:format("Upgrade/downgrade ~p: ~p <--> ~p~n"
+		      "Upgrade instructions:   ~p~n"
+		      "Downgrade instructions: ~p",
+		      [?APP,FromVsn,ToVsn,UpInstrs,DownInstrs]);
+	{error,{vsn_not_found,_}} when FromVsn==ToVsn ->
+	    io:format("No upgrade test for ~p, same version",[?APP])
+    end,
+    State.
+upgrade_upgraded(CtData,fail_upgraded) ->
+    ct:fail(upgrade_upgraded_failed);
+upgrade_upgraded(_CtData,State) ->
+    State.
+upgrade_downgraded(CtData,fail_downgraded) ->
+    ct:fail(upgrade_downgraded_failed);
+upgrade_downgraded(_CtData,State) ->
+    State.