Add support for executing pre-loaded suites (e.g. modules loaded as binaries)

Also fixed bug in test_server that calls end_per_testcase after test case timeout or abortion.

5 files changed, 184 insertions, 51 deletions

diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index ab900734f2..3d9b579a0c 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -1106,14 +1106,15 @@ do_run(Tests, Skip, Opts, Args) ->
 		    log_ts_names(Opts1#opts.testspecs),
 		    TestSuites = suite_tuples(Tests),
 
-		    {SuiteMakeErrors,AllMakeErrors} =
+		    {_TestSuites1,SuiteMakeErrors,AllMakeErrors} =
 			case application:get_env(common_test, auto_compile) of
 			    {ok,false} ->
-				SuitesNotFound = verify_suites(TestSuites),
-				{SuitesNotFound,SuitesNotFound};
+				{TestSuites1,SuitesNotFound} =
+				    verify_suites(TestSuites),
+				{TestSuites1,SuitesNotFound,SuitesNotFound};
 			    _ ->
 				{SuiteErrs,HelpErrs} = auto_compile(TestSuites),
-				{SuiteErrs,SuiteErrs++HelpErrs}
+				{TestSuites,SuiteErrs,SuiteErrs++HelpErrs}
 			end,
 
 		    case continue(AllMakeErrors) of
@@ -1190,30 +1191,55 @@ auto_compile(TestSuites) ->
 verify_suites(TestSuites) ->
     io:nl(),
     Verify =
-	fun({Dir,Suite},NotFound) ->
+	fun({Dir,Suite}=DS,{Found,NotFound}) ->
 		case locate_test_dir(Dir, Suite) of
 		    {ok,TestDir} ->
 			if Suite == all ->
-				NotFound;
+				{[DS|Found],NotFound};
 			   true ->
-				Beam = filename:join(TestDir, atom_to_list(Suite)++".beam"),
+				Beam = filename:join(TestDir,
+						     atom_to_list(Suite)++".beam"),
 				case filelib:is_regular(Beam) of
 				    true  ->
-					NotFound;
+					{[DS|Found],NotFound};
 				    false ->
-					Name = filename:join(TestDir, atom_to_list(Suite)),
-					io:format("Suite ~w not found in directory ~s~n",
-						  [Suite,TestDir]),
-					[{{Dir,Suite},[Name]} | NotFound]
+					case code:is_loaded(Suite) of
+					    {file,SuiteFile} ->
+						%% test suite is already loaded and
+						%% since auto_compile == false,
+						%% let's assume the user has
+						%% loaded the beam file explicitly
+						ActualDir = filename:dirname(SuiteFile),
+						{[{ActualDir,Suite}|Found],NotFound};
+					    false ->
+						Name =
+						    filename:join(TestDir,
+								  atom_to_list(Suite)),
+						io:format(user,
+							  "Suite ~w not found"
+							  "in directory ~s~n",
+							  [Suite,TestDir]),
+						{Found,[{DS,[Name]}|NotFound]}
+					end
 				end
 			end;
 		    {error,_Reason} ->
-			io:format("Directory ~s is invalid~n", [Dir]),
-			Name = filename:join(Dir, atom_to_list(Suite)),
-			[{{Dir,Suite},[Name]} | NotFound]
+			case code:is_loaded(Suite) of
+			    {file,SuiteFile} ->
+				%% test suite is already loaded and since
+				%% auto_compile == false, let's assume the
+				%% user has loaded the beam file explicitly
+				ActualDir = filename:dirname(SuiteFile),
+				{[{ActualDir,Suite}|Found],NotFound};
+			    false ->
+				io:format(user, "Directory ~s is invalid~n", [Dir]),
+				Name = filename:join(Dir, atom_to_list(Suite)),
+				{Found,[{DS,[Name]}|NotFound]}
+			end
 		end
 	end,
-    lists:reverse(lists:foldl(Verify, [], TestSuites)).
+    {ActualFound,Missing} = lists:foldl(Verify, {[],[]}, TestSuites),
+    {lists:reverse(ActualFound),lists:reverse(Missing)}.
 
 save_make_errors([]) ->
     [];
diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl
index 199a1ad97a..b479d4133c 100644
--- a/lib/common_test/test/ct_error_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE.erl
@@ -655,21 +655,33 @@ test_events(timetrap_end_conf) ->
     [
      {?eh,start_logging,{'DEF','RUNDIR'}},
      {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
-     {?eh,start_info,{1,1,3}},
+     {?eh,start_info,{1,1,6}},
      {?eh,tc_start,{timetrap_1_SUITE,init_per_suite}},
      {?eh,tc_done,{timetrap_1_SUITE,init_per_suite,ok}},
      {?eh,tc_start,{timetrap_1_SUITE,tc1}},
      {?eh,tc_done,
-       {timetrap_1_SUITE,tc1,{failed,{timetrap_timeout,1000}}}},
+      {timetrap_1_SUITE,tc1,{failed,{timetrap_timeout,1000}}}},
      {?eh,test_stats,{0,1,{0,0}}},
      {?eh,tc_start,{timetrap_1_SUITE,tc2}},
      {?eh,tc_done,
-       {timetrap_1_SUITE,tc2,{failed,{testcase_aborted,testing_end_conf}}}},
+      {timetrap_1_SUITE,tc2,{failed,{timetrap_timeout,1000}}}},
      {?eh,test_stats,{0,2,{0,0}}},
      {?eh,tc_start,{timetrap_1_SUITE,tc3}},
      {?eh,tc_done,
-       {timetrap_1_SUITE,tc3,{failed,{testcase_aborted,testing_end_conf}}}},
+      {timetrap_1_SUITE,tc3,{failed,{testcase_aborted,testing_end_conf}}}},
      {?eh,test_stats,{0,3,{0,0}}},
+     {?eh,tc_start,{timetrap_1_SUITE,tc4}},
+     {?eh,tc_done,
+      {timetrap_1_SUITE,tc4,{failed,{testcase_aborted,testing_end_conf}}}},
+     {?eh,test_stats,{0,4,{0,0}}},
+     {?eh,tc_start,{timetrap_1_SUITE,tc5}},
+     {?eh,tc_done,
+      {timetrap_1_SUITE,tc5,{failed,{timetrap_timeout,1000}}}},
+     {?eh,test_stats,{0,5,{0,0}}},
+     {?eh,tc_start,{timetrap_1_SUITE,tc6}},
+     {?eh,tc_done,
+      {timetrap_1_SUITE,tc6,{failed,{testcase_aborted,testing_end_conf}}}},
+     {?eh,test_stats,{0,6,{0,0}}},
      {?eh,tc_start,{timetrap_1_SUITE,end_per_suite}},
      {?eh,tc_done,{timetrap_1_SUITE,end_per_suite,ok}},
      {?eh,test_done,{'DEF','STOP_TIME'}},
diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_1_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_1_SUITE.erl
index 2e6432d05d..7a13a7c8a5 100644
--- a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_1_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_1_SUITE.erl
@@ -36,13 +36,19 @@ suite() ->
 %% Reason = term()
 %%--------------------------------------------------------------------
 init_per_suite(Config) ->
-    Config.
+    TabPid = spawn(fun() ->
+			   ets:new(?MODULE, [named_table, set, public]),
+			   ets:insert(?MODULE, {last_case,ok}),
+			   receive _ -> ok end
+		   end),
+    [{tab,TabPid} | Config].
 
 %%--------------------------------------------------------------------
 %% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
 %% Config0 = Config1 = [tuple()]
 %%--------------------------------------------------------------------
-end_per_suite(_Config) ->
+end_per_suite(Config) ->
+    exit(?config(tab, Config), kill),
     ok.
 
 %%--------------------------------------------------------------------
@@ -71,10 +77,29 @@ end_per_group(_GroupName, _Config) ->
 %% Config0 = Config1 = [tuple()]
 %% Reason = term()
 %%--------------------------------------------------------------------
-init_per_testcase(tc3, Config) ->
-    [{default_timeout,5000}|Config];
-init_per_testcase(_TestCase, Config) ->
-    Config.
+init_per_testcase(TC, Config) ->
+    {_,_} = process_info(?config(tab, Config), priority),
+    [{_,ok}] = ets:lookup(?MODULE, last_case),
+    ets:insert(?MODULE, {last_case,fail}),
+    init_per_testcase1(TC, Config).
+
+init_per_testcase1(tc1, Config) ->
+    [{tc,tc1}|Config];
+
+init_per_testcase1(tc2, Config) ->
+    [{tc,tc2}|Config];
+
+init_per_testcase1(tc3, Config) ->
+    [{tc,tc3}|Config];
+
+init_per_testcase1(tc4, Config) ->
+    [{tc,tc4},{default_timeout,5000}|Config];
+
+init_per_testcase1(tc5, Config) ->
+    [{tc,tc5}|Config];
+
+init_per_testcase1(tc6, Config) ->
+    [{tc,tc6}|Config].
 
 %%--------------------------------------------------------------------
 %% Function: end_per_testcase(TestCase, Config0) ->
@@ -82,18 +107,45 @@ init_per_testcase(_TestCase, Config) ->
 %% TestCase = atom()
 %% Config0 = Config1 = [tuple()]
 %%--------------------------------------------------------------------
-end_per_testcase(tc1, Config) ->
-    ct:pal("tc1: ~p", [Config]),
+end_per_testcase(TC, Config) ->
+    {_,_} = process_info(?config(tab, Config), priority),
+    [{_,fail}] = ets:lookup(?MODULE, last_case),
+    ets:insert(?MODULE, {last_case,ok}),
+    end_per_testcase1(TC, Config).
+
+end_per_testcase1(tc1, Config) ->
+    ct:pal("end_per_testcase(tc1): ~p", [Config]),
+    tc1 = ?config(tc, Config),
+    {failed,timetrap_timeout} = ?config(tc_status, Config),
     ok;
 
-end_per_testcase(tc2, Config) ->
-    ct:pal("tc2: ~p", [Config]),
+end_per_testcase1(tc2, Config) ->
+    ct:pal("end_per_testcase(tc2): ~p", [Config]),
+    tc2 = ?config(tc, Config),
+    {failed,timetrap_timeout} = ?config(tc_status, Config),
+    timer:sleep(2000);
+
+end_per_testcase1(tc3, Config) ->
+    ct:pal("end_per_testcase(tc3): ~p", [Config]),
+    tc3 = ?config(tc, Config),
+    {failed,{testcase_aborted,testing_end_conf}} = ?config(tc_status, Config),
     ok;
 
-end_per_testcase(tc3, Config) ->
-    ct:pal("tc3: ~p", [Config]),
-    ct:sleep(10000),
-    ok.
+end_per_testcase1(tc4, Config) ->
+    ct:pal("end_per_testcase(tc4): ~p", [Config]),
+    tc4 = ?config(tc, Config),
+    {failed,{testcase_aborted,testing_end_conf}} = ?config(tc_status, Config),
+    timer:sleep(2000);
+
+end_per_testcase1(tc5, Config) ->
+    ct:pal("end_per_testcase(tc5): ~p", [Config]),
+    tc5 = ?config(tc, Config),
+    exit(end_per_tc_fail_after_timeout);
+
+end_per_testcase1(tc6, Config) ->
+    ct:pal("end_per_testcase(tc6): ~p", [Config]),
+    tc6 = ?config(tc, Config),
+    exit(end_per_tc_fail_after_abort).
 
 %%--------------------------------------------------------------------
 %% Function: groups() -> [Group]
@@ -118,18 +170,25 @@ groups() ->
 %% Reason = term()
 %%--------------------------------------------------------------------
 all() ->
-    [tc1,tc2,tc3].
+    [tc1, tc2, tc3, tc4, tc5, tc6].
 
 tc1(_) ->
-    ct:sleep(3000),
-    ok.
+    timer:sleep(2000).
 
 tc2(_) ->
-    spawn(ct, abort_current_testcase, [testing_end_conf]),
-    timer:sleep(3000),
-    ok.
+    timer:sleep(2000).
 
 tc3(_) ->
     spawn(ct, abort_current_testcase, [testing_end_conf]),
-    timer:sleep(3000),
-    ok.
+    timer:sleep(2000).
+
+tc4(_) ->
+    spawn(ct, abort_current_testcase, [testing_end_conf]),
+    timer:sleep(2000).
+
+tc5(_) ->
+    timer:sleep(2000).
+
+tc6(_) ->
+    spawn(ct, abort_current_testcase, [testing_end_conf]),
+    timer:sleep(2000).
diff --git a/lib/common_test/test/ct_misc_1_SUITE.erl b/lib/common_test/test/ct_misc_1_SUITE.erl
index f42081ee7e..e311876fb1 100644
--- a/lib/common_test/test/ct_misc_1_SUITE.erl
+++ b/lib/common_test/test/ct_misc_1_SUITE.erl
@@ -76,15 +76,15 @@ beam_me_up(Config) when is_list(Config) ->
     CTNode = ?config(ct_node, Config),
 
     %% Path = rpc:call(CTNode, code, get_path, []),
-    [_ | Parts] = lists:reverse(filename:split(DataDir)),
-    TSDir = filename:join(lists:reverse(Parts)),
-    true = rpc:call(CTNode, code, del_path, [TSDir]),
+    %% [_ | Parts] = lists:reverse(filename:split(DataDir)),
+    %% TSDir = filename:join(lists:reverse(Parts)),
+    %% true = rpc:call(CTNode, code, del_path, [TSDir]),
 
     Mods = [beam_1_SUITE, beam_2_SUITE],
     Suites = [atom_to_list(M) || M <- Mods],
     [{error,_} = rpc:call(CTNode, code, load_file, [M]) || M <- Mods],
 
-    code:add_path(TSDir),
+    code:add_path(DataDir),
     CRes =
 	[compile:file(filename:join(DataDir,F),
 		      [verbose,report_errors,
@@ -95,6 +95,7 @@ beam_me_up(Config) when is_list(Config) ->
 	{ok,Mod,Bin} <- CRes],
 
     {Opts,ERPid} = setup([{suite,Suites},{auto_compile,false}], Config),
+
     ok = ct_test_support:run(ct, run_test, [Opts], Config),
     Events = ct_test_support:get_events(ERPid, Config),
 
@@ -135,4 +136,30 @@ events_to_check(Test, N) ->
     test_events(Test) ++ events_to_check(Test, N-1).
 
 test_events(beam_me_up) ->
-    [].
+    [
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,start_info,{2,2,4}},
+     {?eh,tc_start,{beam_1_SUITE,init_per_suite}},
+     {?eh,tc_done,{beam_1_SUITE,init_per_suite,ok}},
+     {?eh,tc_start,{beam_1_SUITE,tc1}},
+     {?eh,tc_done,{beam_1_SUITE,tc1,ok}},
+     {?eh,test_stats,{1,0,{0,0}}},
+     {?eh,tc_start,{beam_1_SUITE,tc2}},
+     {?eh,tc_done,{beam_1_SUITE,tc2,{failed,{error,'tc2 failed'}}}},
+     {?eh,test_stats,{1,1,{0,0}}},
+     {?eh,tc_start,{beam_1_SUITE,end_per_suite}},
+     {?eh,tc_done,{beam_1_SUITE,end_per_suite,ok}},
+     {?eh,tc_start,{beam_2_SUITE,init_per_suite}},
+     {?eh,tc_done,{beam_2_SUITE,init_per_suite,ok}},
+     {?eh,tc_start,{beam_2_SUITE,tc1}},
+     {?eh,tc_done,{beam_2_SUITE,tc1,ok}},
+     {?eh,test_stats,{2,1,{0,0}}},
+     {?eh,tc_start,{beam_2_SUITE,tc2}},
+     {?eh,tc_done,{beam_2_SUITE,tc2,{failed,{error,'tc2 failed'}}}},
+     {?eh,test_stats,{2,2,{0,0}}},
+     {?eh,tc_start,{beam_2_SUITE,end_per_suite}},
+     {?eh,tc_done,{beam_2_SUITE,end_per_suite,ok}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]}
+    ].
diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl
index b70ac08622..d6de907a1b 100644
--- a/lib/test_server/src/test_server.erl
+++ b/lib/test_server/src/test_server.erl
@@ -876,10 +876,19 @@ call_end_conf(Mod,Func,TCPid,TCExitReason,Loc,Conf,TVal) ->
     EndConfProc =
 	fun() ->
 		Supervisor = self(),
-		EndConfApply = fun() ->
-				       apply(Mod,end_per_testcase,[Func,Conf]),
-				       Supervisor ! {self(),end_conf}
-			       end,
+		EndConfApply =
+		    fun() ->
+			    case catch apply(Mod,end_per_testcase,[Func,Conf]) of
+				{'EXIT',Why} ->
+				    group_leader() ! {printout,12,
+						      "ERROR! ~p:end_per_testcase(~p, ~p)"
+						      " crashed!\n\tReason: ~p\n",
+						      [Mod,Func,Conf,Why]};
+				_ ->
+				    ok
+			    end,
+			    Supervisor ! {self(),end_conf}
+		       end,
 		Pid = spawn_link(EndConfApply),
 		receive
 		    {Pid,end_conf} ->