%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 2004-2010. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
%% compliance with the License. You should have received a copy of the
%% Erlang Public License along with this software. If not, it can be
%% retrieved online at http://www.erlang.org/.
%%
%% Software distributed under the License is distributed on an "AS IS"
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
%% the License for the specific language governing rights and limitations
%% under the License.
%%
%% %CopyrightEnd%
%%

%%% @doc Common Test Framework test execution control module.
%%%
%%% <p>This module exports functions for installing and running tests
%%% withing the Common Test Framework.</p>

-module(ct_run).


%% Script interface
-export([script_start/0,script_usage/0]).

%% User interface
-export([install/1,install/2,run/1,run/2,run/3,run_test/1,
	 run_testspec/1,step/3,step/4,refresh_logs/1]).


%% Exported for VTS
-export([run_make/3,do_run/3,tests/1,tests/2,tests/3]).


%% Misc internal functions
-export([variables_file_name/1,script_start1/2,run_test1/1]).

-include("ct_event.hrl").
-include("ct_util.hrl").

-define(abs(Name), filename:absname(Name)).
-define(testdir(Name, Suite), ct_util:get_testdir(Name, Suite)).

%%%-----------------------------------------------------------------
%%% @spec script_start() -> void()
%%%
%%% @doc Start tests via the run_test program or script.
%%% 
%%% <p>Example:<br/><code>./run_test -config config.ctc -dir
%%% $TEST_DIR</code></p>
%%%
%%% <p>Example:<br/><code>./run_test -config config.ctc -suite
%%% $SUITE_PATH/$SUITE_NAME [-case $CASE_NAME]</code></p>
%%%
script_start() ->
    process_flag(trap_exit, true),
    Args = merge_arguments(init:get_arguments()),
    Tracing = start_trace(Args),
    Res = 
	case ct_repeat:loop_test(script, Args) of
	    false ->	    
		{ok,Cwd} = file:get_cwd(),
		CTVsn = 
		    case filename:basename(code:lib_dir(common_test)) of
			CTBase when is_list(CTBase) ->
			    case string:tokens(CTBase, "-") of
				["common_test",Vsn] -> " v"++Vsn;
				_ -> ""
			    end
		    end,
		io:format("~nCommon Test~s starting (cwd is ~s)~n~n", [CTVsn,Cwd]),
		Self = self(),
		Pid = spawn_link(fun() -> script_start1(Self, Args) end),
	        receive 
		    {'EXIT',Pid,Reason} ->
			case Reason of
			    {user_error,What} ->
				io:format("\nTest run failed!\nReason: ~p\n\n", [What]),
				{error,What};
			    _ ->
				io:format("Test run crashed! This could be an internal error "
					  "- please report!\n\n"
					  "~p\n\n", [Reason]),
				{error,Reason}				
			end;
		    {Pid,{error,Reason}} ->
			io:format("\nTest run failed! Reason:\n~p\n\n",[Reason]),
			{error,Reason};
		    {Pid,Result} ->
			Result
		end;
	    Result ->
		Result
	end,
    stop_trace(Tracing),
    Res.

script_start1(Parent, Args) ->
    VtsOrShell = 
	case lists:keymember(vts, 1, Args) of
	    true -> 
		vts;
	    false ->
		case lists:keymember(shell, 1, Args) of
		    true -> shell;
		    false -> false
		end
	end,
    LogDir =
	case lists:keysearch(logdir, 1, Args) of
	    {value,{logdir,[LogD]}} -> LogD;
	    false -> "."
	end,
    EvHandlers =	
	case lists:keysearch(event_handler, 1, Args) of
	    {value,{event_handler,Handlers}} -> 
		lists:map(fun(H) -> {list_to_atom(H),[]} end, Handlers);
	    false -> 
		[]
	end,
    Cover = 
	case lists:keysearch(cover, 1, Args) of
	    {value,{cover,CoverFile}} -> 
		{cover,?abs(CoverFile)};
	    false -> 
		false
	end,

    case lists:keysearch(ct_decrypt_key, 1, Args) of
	{value,{_,[DecryptKey]}} ->
	    application:set_env(common_test, decrypt, {key,DecryptKey});
	false ->
	    case lists:keysearch(ct_decrypt_file, 1, Args) of
		{value,{_,[DecryptFile]}} ->
		    application:set_env(common_test, decrypt, 
					{file,filename:absname(DecryptFile)});
		false ->
		    application:unset_env(common_test, decrypt)
	    end
    end,

    case lists:keysearch(no_auto_compile, 1, Args) of
	{value,_} ->
	    application:set_env(common_test, auto_compile, false);
	false ->
	    application:set_env(common_test, auto_compile, true),

	    InclDirs =
		case lists:keysearch(include,1,Args) of
		    {value,{include,Incl}} when is_list(hd(Incl)) ->
			Incl;		    
		    {value,{include,Incl}} when is_list(Incl) ->
			[Incl];
		    false ->
			[]
		end,
	    case os:getenv("CT_INCLUDE_PATH") of
		false ->
		    application:set_env(common_test, include, InclDirs);
		CtInclPath ->
		    InclDirs1 = string:tokens(CtInclPath,[$:,$ ,$,]),
		    application:set_env(common_test, include, InclDirs1++InclDirs)
	    end
    end,

    case lists:keysearch(basic_html, 1, Args) of
	{value,_} ->
	    application:set_env(common_test, basic_html, true);
	false ->
	    application:set_env(common_test, basic_html, false)
    end,

    Result =
	case lists:keysearch(refresh_logs, 1, Args) of
	    {value,{refresh_logs,Refresh}} ->
		LogDir1 = case Refresh of
			      [] -> LogDir;
			      [RefreshDir] -> ?abs(RefreshDir)
			  end,
		{ok,Cwd} = file:get_cwd(),
		file:set_cwd(LogDir1),
		timer:sleep(500),		% give the shell time to print version etc
		io:nl(),
		case catch ct_logs:make_all_suites_index(refresh) of
		    {'EXIT',ASReason} ->
			file:set_cwd(Cwd),
			{error,{all_suites_index,ASReason}};
		    _ ->
			case catch ct_logs:make_all_runs_index(refresh) of
			    {'EXIT',ARReason} ->
				file:set_cwd(Cwd),
				{error,{all_runs_index,ARReason}};
			    _ ->
				file:set_cwd(Cwd),
				io:format("Logs in ~s refreshed!~n~n", [LogDir1]),
				timer:sleep(500), % time to flush io before quitting
				ok
			end
		end;
	    false ->    
		Config = ct_config:prepare_config_list(Args),
		case Config of
		    [] ->
			case install([{config,[]},
				      {event_handler,EvHandlers}],
				     LogDir) of
			    ok ->
				script_start2(VtsOrShell, [], EvHandlers,
					      Args, LogDir, Cover);
			    Error ->
				Error
			end;
		    Config ->
			case lists:keysearch(spec, 1, Args) of
			    false ->
				case check_and_install_configfiles(Config,
						LogDir, EvHandlers) of
				    ok ->
					script_start2(VtsOrShell, Config,
						      EvHandlers, Args, LogDir,
						      Cover);
				    Error ->
					Error
				end;
			    _ ->
				script_start2(VtsOrShell, Config,
					      EvHandlers, Args, LogDir, Cover)
			end
		end
	end,
    Parent ! {self(), Result}.

check_and_install_configfiles(Configs, LogDir, EvHandlers) ->
    case ct_config:check_config_files(Configs) of
    false->
	install([{config,Configs},
		 {event_handler,EvHandlers}], LogDir);
    {value, {error, {nofile, File}}} ->
	{error,{cant_read_config_file,File}};
    {value, {error, {wrong_config, Message}}}->
	{error,{wrong_config, Message}};
    {value, {error, {callback, File}}} ->
	{error,{cant_load_callback_module,File}}
    end.

script_start2(false, ConfigFiles, EvHandlers, Args, LogDir, Cover) ->
    case lists:keysearch(spec, 1, Args) of
	{value,{spec,[]}} ->
	    {error,no_testspec_specified};
	{value,{spec,Specs}} ->
	    Relaxed = lists:keymember(allow_user_terms, 1, Args),
	    %% using testspec as input for test
	    case catch ct_testspec:collect_tests_from_file(Specs, Relaxed) of
		{error,Reason} ->
		    {error,Reason};
		TS ->
		    {LogDir1,TSCoverFile,ConfigFiles1,EvHandlers1,Include1} = 
			get_data_for_node(TS,node()),
		    UserInclude =
			case application:get_env(common_test, include) of
			    {ok,Include} -> Include++Include1;
			    _ -> Include1
			end,
		    application:set_env(common_test, include, UserInclude),
		    LogDir2 = which_logdir(LogDir,LogDir1),
		    CoverOpt = case {Cover,TSCoverFile} of
				   {false,undef} -> [];
				   {_,undef} ->     [Cover];
				   {false,_} ->     [{cover,TSCoverFile}]
			       end,
		    case check_and_install_configfiles(
					 ConfigFiles++ConfigFiles1, LogDir2,
					 EvHandlers++EvHandlers1) of
			ok ->
			    {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
			    do_run(Run, Skip, CoverOpt, Args, LogDir2);
			Error ->
			    Error
		    end
	    end;
	false ->
	    script_start3(false, ConfigFiles, EvHandlers, Args, LogDir, Cover)
    end;
script_start2(VtsOrShell, ConfigFiles, EvHandlers, Args, LogDir, Cover) ->
    script_start3(VtsOrShell, ConfigFiles, EvHandlers, Args, LogDir, Cover).

script_start3(VtsOrShell, ConfigFiles, EvHandlers, Args, LogDir, Cover) ->
    case lists:keysearch(dir, 1, Args) of
	{value,{dir,[]}} ->
	    {error,no_dir_specified};
	{value,{dir,Dirs}} ->
	    script_start4(VtsOrShell, ConfigFiles, EvHandlers, tests(Dirs),
			  Cover, Args, LogDir);
	false ->
	    case lists:keysearch(suite, 1, Args) of
		{value,{suite,[]}} ->
		    {error,no_suite_specified};
		{value,{suite,Suites}} ->
		    StepOrCover =
			case lists:keysearch(step, 1, Args) of
			    {value,Step} -> Step;
			    false -> Cover
			end,
		    S2M = fun(S) ->
				  {filename:dirname(S),
				   list_to_atom(
				     filename:rootname(filename:basename(S)))}
			  end,
		    DirMods = lists:map(S2M, Suites),
		    {Specified,GroupsAndCases} =
			case {lists:keysearch(group, 1, Args),
			      lists:keysearch('case', 1, Args)} of
			    {{value,{_,Gs}},{value,{_,Cs}}} -> {true,Gs++Cs};
			    {{value,{_,Gs}},_}              -> {true,Gs};
			    {_,{value,{_,Cs}}}              -> {true,Cs};
			    _                               -> {false,[]}
			end,
		    if Specified, length(GroupsAndCases) == 0 ->
			    {error,no_case_or_group_specified};
		       Specified, length(DirMods) > 1 ->
			    {error,multiple_suites_and_cases};
		       length(GroupsAndCases) > 0, length(DirMods) == 1 ->
			    GsAndCs = lists:map(fun(C) -> list_to_atom(C) end, 
						GroupsAndCases),
			    script_start4(VtsOrShell, ConfigFiles, EvHandlers,
					  tests(DirMods, GsAndCs),
					  StepOrCover, Args, LogDir);
		       not Specified, length(DirMods) > 0 ->
			    script_start4(VtsOrShell, ConfigFiles, EvHandlers,
					  tests(DirMods), 
					  StepOrCover, Args, LogDir);
		       true ->
			    {error,incorrect_suite_and_case_options}
		    end;
		false when VtsOrShell=/=false ->
		    script_start4(VtsOrShell, ConfigFiles, EvHandlers,
				  [], Cover, Args, LogDir);
		false ->
		    script_usage(),
		    {error,incorrect_usage}
	    end
    end.

script_start4(vts, ConfigFiles, EvHandlers, Tests, false, _Args, LogDir) ->
    vts:init_data(ConfigFiles, EvHandlers, ?abs(LogDir), Tests);
script_start4(shell, ConfigFiles, EvHandlers, _Tests, false, Args, LogDir) ->
    Opts = [{config,ConfigFiles},{event_handler,EvHandlers}],
    if ConfigFiles == [] ->
	    ok;
       true ->
	    io:format("\nInstalling: ~p\n\n", [ConfigFiles])
    end,
    case install(Opts) of
	ok ->
	    ct_util:start(interactive, LogDir),
	    log_ts_names(Args),
	    io:nl(),
	    ok;
	Error ->
	    Error
    end;
script_start4(vts, _CfgFs, _EvHs, _Tests, _Cover={cover,_}, _Args, _LogDir) ->
    %% Add support later (maybe).
    script_usage(),
    erlang:halt();
script_start4(shell, _CfgFs, _EvHs, _Tests, _Cover={cover,_}, _Args, _LogDir) ->
    %% Add support later (maybe).
    script_usage();
script_start4(false, _CfgFs, _EvHs, Tests, Cover={cover,_}, Args, LogDir) ->
    do_run(Tests, [], [Cover], Args, LogDir);
script_start4(false, _ConfigFiles, _EvHandlers, Tests, false, Args, LogDir) ->
    do_run(Tests, [], [], Args, LogDir);
script_start4(false, _ConfigFiles, _EvHandlers, Test, Step, Args, LogDir) ->
    do_run(Test, [], [Step], Args, LogDir);
script_start4(vts, _ConfigFiles, _EvHandlers, _Test, _Step, _Args, _LogDir) ->
    script_usage(),
    erlang:halt();
script_start4(shell, _ConfigFiles, _EvHandlers, _Test, _Step, _Args, _LogDir) ->
    script_usage().

%%%-----------------------------------------------------------------
%%% @spec script_usage() -> ok
%%% @doc Print usage information for <code>run_test</code>.
script_usage() ->
    io:format("\n\nUsage:\n\n"),
    io:format("Run tests in web based GUI:\n\n"
	      "\trun_test -vts [-browser Browser]"
	      "\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]"
	      "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]"
	      "\n\t[-dir TestDir1 TestDir2 .. TestDirN] |"
	      "\n\t[-suite Suite [-case Case]]"
	      "\n\t[-include InclDir1 InclDir2 .. InclDirN]" 
	      "\n\t[-no_auto_compile]"
	      "\n\t[-basic_html]\n\n"),
    io:format("Run tests from command line:\n\n"
	      "\trun_test [-dir TestDir1 TestDir2 .. TestDirN] |"
	      "\n\t[-suite Suite1 Suite2 .. SuiteN [-case Case1 Case2 .. CaseN]]"
	      "\n\t[-step [config | keep_inactive]]"
	      "\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]"
	      "\n\t[-userconfig CallbackModule ConfigFile1 .. ConfigFileN]"
	      "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]"
	      "\n\t[-logdir LogDir]"
	      "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]"
	      "\n\t[-stylesheet CSSFile]"
	      "\n\t[-cover CoverCfgFile]"
	      "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]"
	      "\n\t[-include InclDir1 InclDir2 .. InclDirN]" 
	      "\n\t[-no_auto_compile]"
	      "\n\t[-basic_html]" 
	      "\n\t[-repeat N [-force_stop]] |" 
	      "\n\t[-duration HHMMSS [-force_stop]] |"
	      "\n\t[-until [YYMoMoDD]HHMMSS [-force_stop]]\n\n"),
    io:format("Run tests using test specification:\n\n"
	      "\trun_test -spec TestSpec1 TestSpec2 .. TestSpecN"
	      "\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]"
	      "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]"
	      "\n\t[-logdir LogDir]"
	      "\n\t[-allow_user_terms]"
	      "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]"
	      "\n\t[-stylesheet CSSFile]"
	      "\n\t[-cover CoverCfgFile]"
	      "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]"
	      "\n\t[-include InclDir1 InclDir2 .. InclDirN]" 
	      "\n\t[-no_auto_compile]"
	      "\n\t[-basic_html]" 
	      "\n\t[-repeat N [-force_stop]] |" 
	      "\n\t[-duration HHMMSS [-force_stop]] |"
	      "\n\t[-until [YYMoMoDD]HHMMSS [-force_stop]]\n\n"),
    io:format("Refresh the HTML index files:\n\n"
	      "\trun_test -refresh_logs [LogDir]"
	      "[-logdir LogDir] "
	      "[-basic_html]\n\n"),
    io:format("Run CT in interactive mode:\n\n"
	      "\trun_test -shell"
	      "\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]"
	      "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]\n\n").
	      

%%%-----------------------------------------------------------------
%%% @hidden
%%% @equiv ct:install/1
install(Opts) ->
    install(Opts, ".").

install(Opts, LogDir) ->
    case application:get_env(common_test, decrypt) of
	{ok,_} ->
	    ok;
	_ ->
	    case lists:keysearch(decrypt, 1, Opts) of
		{value,{_,KeyOrFile}} ->
		    application:set_env(common_test, decrypt, KeyOrFile);
		false ->
		    application:unset_env(common_test, decrypt)
	    end
    end,
    case whereis(ct_util_server) of
	undefined ->
	    VarFile = variables_file_name(LogDir),
	    %% io:format("Varfile=~p~n", [VarFile]),
	    case file:open(VarFile, [write]) of
		{ok,Fd} ->
		    [io:format(Fd, "~p.\n", [Opt]) || Opt <- Opts],
		    file:close(Fd),
		    ok;
		{error,Reason} -> 
		    io:format("CT failed to install configuration data. Please "
			      "verify that the log directory exists and that "
			      "write permission is set.\n\n", []),
		    {error,{VarFile,Reason}}
	    end;
	_ ->
	    io:format("It is not possible to install CT while running "
		      "in interactive mode.\n"
		      "To exit this mode, run ct:stop_interactive().\n"
		      "To enter the interactive mode again, "
		      "run ct:start_interactive()\n\n", []),
	    {error,interactive_mode}
    end.

variables_file_name(Dir) ->
    filename:join(Dir, "variables-"++atom_to_list(node())).

%%%-----------------------------------------------------------------
%%% @hidden
%%% @equiv ct:run_test/1

%%   Opts = [OptTuples]
%%   OptTuples = {config,CfgFiles} | {dir,TestDirs} | {suite,Suites} |
%%               {testcase,Cases} | {spec,TestSpecs} | {allow_user_terms,Bool} |
%%               {logdir,LogDir} | {cover,CoverSpecFile} | {step,StepOpts} |
%%               {silent_connections,Conns} | {event_handler,EventHandlers} | 
%%               {include,InclDirs} | {auto_compile,Bool} | 
%%               {repeat,N} | {duration,DurTime} | {until,StopTime} | {force_stop,Bool} |
%%               {decrypt,KeyOrFile}

run_test(Opt) when is_tuple(Opt) ->
    run_test([Opt]);

run_test(Opts) when is_list(Opts) ->
    case lists:keysearch(refresh_logs, 1, Opts) of
	{value,{_,RefreshDir}} ->
	    refresh_logs(?abs(RefreshDir)),
	    ok;
	false ->
	    Tracing = start_trace(Opts),
	    {ok,Cwd} = file:get_cwd(),
	    io:format("~nCommon Test starting (cwd is ~s)~n~n", [Cwd]),	    
	    Res =
		case ct_repeat:loop_test(func, Opts) of
		    false ->
			case catch run_test1(Opts) of
			    {'EXIT',Reason} -> 
				file:set_cwd(Cwd),
				{error,Reason};
			    Result -> 
				Result
			end;
		    Result ->
			Result
		end,
	    stop_trace(Tracing),
	    Res
    end.

run_test1(Opts) ->
    LogDir =
	case lists:keysearch(logdir, 1, Opts) of
	    {value,{_,LD}} when is_list(LD) -> LD;
	    false -> "."
	end,
    CfgFiles = ct_config:get_config_file_list(Opts),
    EvHandlers =
	case lists:keysearch(event_handler, 1, Opts) of
	    {value,{_,H}} when is_atom(H) ->
		[{H,[]}];
	    {value,{_,H}} ->
		Hs =
		    if is_tuple(H) -> [H];
		       is_list(H) -> H;
		       true -> []
		    end,
		lists:flatten(
		  lists:map(fun(EH) when is_atom(EH) ->
				    {EH,[]};
			       ({HL,Args}) when is_list(HL) ->
				    [{EH,Args} || EH <- HL];
			       ({EH,Args}) when is_atom(EH) ->
				    {EH,Args};
			       (_) ->
				    []
			    end, Hs));
	    _ ->
		[]
	end,
    SilentConns =
	case lists:keysearch(silent_connections, 1, Opts) of
	    {value,{_,all}} ->
		[];
	    {value,{_,Conns}} ->
		Conns;
	    _ ->
		undefined
	end,
    Cover = 
	case lists:keysearch(cover, 1, Opts) of
	    {value,{_,CoverFile}} ->
		[{cover,?abs(CoverFile)}];
	    _ ->
		[]
	end,
    Include =
	case lists:keysearch(auto_compile, 1, Opts) of
	    {value,{auto_compile,ACBool}} ->
		application:set_env(common_test, auto_compile, ACBool),
		[];
	    _ ->
		application:set_env(common_test, auto_compile, true),		
		InclDirs =
		    case lists:keysearch(include, 1, Opts) of
			{value,{include,Incl}}  when is_list(hd(Incl)) ->
			    Incl;		    
			{value,{include,Incl}} when is_list(Incl) ->
			    [Incl];
			false ->
			    []
		    end,
		case os:getenv("CT_INCLUDE_PATH") of
		    false ->
			application:set_env(common_test, include, InclDirs),
			InclDirs;
		    CtInclPath ->
			InclDirs1 = string:tokens(CtInclPath, [$:,$ ,$,]),
			AllInclDirs = InclDirs1++InclDirs,
			application:set_env(common_test, include, AllInclDirs),
			AllInclDirs
		end
	end,

    case lists:keysearch(decrypt, 1, Opts) of
	{value,{_,Key={key,_}}} ->
	    application:set_env(common_test, decrypt, Key);
	{value,{_,{file,KeyFile}}} ->
	    application:set_env(common_test, decrypt, {file,filename:absname(KeyFile)});
	false ->
	    application:unset_env(common_test, decrypt)
    end,

    case lists:keysearch(basic_html, 1, Opts) of
	{value,{basic_html,BasicHtmlBool}} ->
	    application:set_env(common_test, basic_html, BasicHtmlBool);
	_ ->
	    application:set_env(common_test, basic_html, false)
    end,

    case lists:keysearch(spec, 1, Opts) of
	{value,{_,Specs}} ->
	    Relaxed =
		case lists:keysearch(allow_user_terms, 1, Opts) of
		    {value,{_,true}} -> true;
		    _ -> false
		end,	    
	    %% using testspec(s) as input for test
	    run_spec_file(LogDir, CfgFiles, EvHandlers, Include, Specs, Relaxed, Cover,
			 replace_opt([{silent_connections,SilentConns}], Opts));
	false ->
	    case lists:keysearch(prepared_tests, 1, Opts) of
		{value,{_,{Run,Skip},Specs}} ->	% use prepared tests
		    run_prepared(LogDir, CfgFiles, EvHandlers,
				 Run, Skip, Cover,
				 replace_opt([{silent_connections,SilentConns},
					      {spec,Specs}],Opts));
		false ->		% use dir|suite|case
		    StepOrCover =
			case lists:keysearch(step, 1, Opts) of
			    {value,Step} -> [Step];
			    false -> Cover
			end,
		    run_dir(LogDir, CfgFiles, EvHandlers, StepOrCover,
			    replace_opt([{silent_connections,SilentConns}], Opts))
	    end
    end.

replace_opt([O={Key,_Val}|Os], Opts) ->
    [O | replace_opt(Os, lists:keydelete(Key, 1, Opts))];
replace_opt([], Opts) ->
    Opts.

run_spec_file(LogDir, CfgFiles, EvHandlers, Include, Specs, Relaxed, Cover, Opts) ->
    Specs1 = case Specs of
		 [X|_] when is_integer(X) -> [Specs];
		 _ -> Specs
	     end,
    AbsSpecs = lists:map(fun(SF) -> ?abs(SF) end, Specs1), 
    log_ts_names(AbsSpecs),
    case catch ct_testspec:collect_tests_from_file(AbsSpecs, Relaxed) of
	{error,CTReason} ->
	    exit(CTReason);
	TS ->
	    {LogDir1,TSCoverFile,CfgFiles1,EvHandlers1,Include1} = 
		get_data_for_node(TS, node()),
	    application:set_env(common_test, include, Include++Include1),
	    LogDir2 = which_logdir(LogDir, LogDir1),
	    CoverOpt = case {Cover,TSCoverFile} of
			   {[],undef} -> [];
			   {_,undef} ->  Cover;
			   {[],_} ->     [{cover,TSCoverFile}]
		       end,
	    case check_and_install_configfiles(CfgFiles++CfgFiles1, LogDir2,
				 EvHandlers++EvHandlers1) of
		ok ->
		    {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
		    do_run(Run, Skip, CoverOpt,
			   replace_opt([{spec,AbsSpecs}], Opts),
			   LogDir2);
		{error,GCFReason} ->
		    exit(GCFReason)
	    end
    end.

run_prepared(LogDir, CfgFiles, EvHandlers, Run, Skip, Cover, Opts) ->
    case check_and_install_configfiles(CfgFiles, LogDir, EvHandlers) of
	ok ->
	    do_run(Run, Skip, Cover, Opts, LogDir);
	{error,Reason} ->
	    exit(Reason)
    end.    

check_config_file(Callback, File)->
    case Callback:check_parameter(File) of
	{ok, {file, File}}->
	    ?abs(File);
	{ok, {config, _}}->
	    File;
	{error, {wrong_config, Message}}->
	    exit({wrong_config, {Callback, Message}});
	{error, {nofile, File}}->
	    exit({no_such_file, ?abs(File)})
    end.

run_dir(LogDir, CfgFiles, EvHandlers, StepOrCover, Opts) ->
    AbsCfgFiles = 
	lists:map(fun({Callback, FileList})->
	    case code:is_loaded(Callback) of
		{file, _Path}->
		    ok;
		false->
		    case code:load_file(Callback) of
			{module, Callback}->
			    ok;
			{error, _}->
			    exit({no_such_module, Callback})
		    end
	    end,
	    {Callback,
		lists:map(fun(File)->
		    check_config_file(Callback, File)
		end, FileList)}
	    end,
	CfgFiles),
    case install([{config,AbsCfgFiles},{event_handler,EvHandlers}], LogDir) of
	ok -> ok;
	{error,IReason} -> exit(IReason)
    end,
    case lists:keysearch(dir,1,Opts) of
	{value,{_,Dirs=[Dir|_]}} when not is_integer(Dir),
	                              length(Dirs)>1 ->
	    %% multiple dirs (no suite)
	    do_run(tests(Dirs), [], StepOrCover, Opts, LogDir);
	false ->				% no dir
	    %% fun for converting suite name to {Dir,Mod} tuple
	    S2M = fun(S) when is_list(S) ->
			  {filename:dirname(S),
			   list_to_atom(filename:rootname(filename:basename(S)))};
		     (A) ->
			  {".",A}
		  end,
	    case lists:keysearch(suite, 1, Opts) of
		{value,{_,Suite}} when is_integer(hd(Suite)) ; is_atom(Suite) ->
		    {Dir,Mod} = S2M(Suite),
		    case listify(proplists:get_value(group, Opts, [])) ++
			 listify(proplists:get_value(testcase, Opts, [])) of
			[] ->
			    do_run(tests(Dir, listify(Mod)), [], StepOrCover, Opts, LogDir);
			GsAndCs ->
			    do_run(tests(Dir, Mod, GsAndCs), [], StepOrCover, Opts, LogDir)
		    end;
		{value,{_,Suites}} ->
		    do_run(tests(lists:map(S2M, Suites)), [], StepOrCover, Opts, LogDir);
		_ ->
		    exit(no_tests_specified)
	    end;		   
	{value,{_,Dir}} ->
	    case lists:keysearch(suite, 1, Opts) of
		{value,{_,Suite}} when is_integer(hd(Suite)) ; is_atom(Suite) ->
		    Mod = if is_atom(Suite) -> Suite; 
			     true -> list_to_atom(Suite) 
			  end,
		    case listify(proplists:get_value(group, Opts, [])) ++
			 listify(proplists:get_value(testcase, Opts, [])) of
			[] ->
			    do_run(tests(Dir, listify(Mod)), [], StepOrCover, Opts, LogDir);
			GsAndCs ->
			    do_run(tests(Dir, Mod, GsAndCs), [], StepOrCover, Opts, LogDir)
		    end;
		{value,{_,Suites=[Suite|_]}} when is_list(Suite) ->
		    Mods = lists:map(fun(Str) -> list_to_atom(Str) end, Suites),
		    do_run(tests(delistify(Dir), Mods), [], StepOrCover, Opts, LogDir);		    
		{value,{_,Suites}} ->
		    do_run(tests(delistify(Dir), Suites), [], StepOrCover, Opts, LogDir);		    
	        false ->			% no suite, only dir
		    do_run(tests(listify(Dir)), [], StepOrCover, Opts, LogDir)
	    end   
    end.

%%%-----------------------------------------------------------------
%%% @hidden
%%%

%% using testspec(s) as input for test
run_testspec(TestSpec) ->
    {ok,Cwd} = file:get_cwd(),
    io:format("~nCommon Test starting (cwd is ~s)~n~n", [Cwd]),
    case catch run_testspec1(TestSpec) of
	{'EXIT',Reason} -> 
	    file:set_cwd(Cwd),
	    {error,Reason};
	Result -> 
	    Result
    end.

run_testspec1(TestSpec) ->
    case ct_testspec:collect_tests_from_list(TestSpec,false) of
	{error,CTReason} ->
	    exit(CTReason);
	TS ->
	    {LogDir,TSCoverFile,CfgFiles,EvHandlers,Include} = 
		get_data_for_node(TS,node()),
	    case os:getenv("CT_INCLUDE_PATH") of
		false ->
		    application:set_env(common_test, include, Include);
		CtInclPath ->
		    EnvInclude = string:tokens(CtInclPath, [$:,$ ,$,]),
		    application:set_env(common_test, include, EnvInclude++Include)
	    end,
	    CoverOpt = if TSCoverFile == undef -> [];
			  true -> [{cover,TSCoverFile}]
		       end,
	    case check_and_install_configfiles(CfgFiles,LogDir,EvHandlers) of
		ok ->
		    {Run,Skip} = ct_testspec:prepare_tests(TS,node()),
		    do_run(Run,Skip,CoverOpt,[],LogDir);
		{error,GCFReason} ->
		    exit(GCFReason)
	    end
    end.


get_data_for_node(#testspec{logdir=LogDirs,
			    cover=CoverFs,
			    config=Cfgs,
			    userconfig=UsrCfgs,
			    event_handler=EvHs,
			    include=Incl}, Node) ->
    LogDir = case lists:keysearch(Node,1,LogDirs) of
		 {value,{Node,Dir}} -> Dir;
		 false -> "."
	     end,
    Cover = case lists:keysearch(Node,1,CoverFs) of
		{value,{Node,CovFile}} -> CovFile;
		false -> undef
	    end,
    ConfigFiles = [{?ct_config_txt, F} || {N,F} <- Cfgs, N==Node] ++
		  [CBF || {N, CBF} <- UsrCfgs, N==Node],
    EvHandlers =  [{H,A} || {N,H,A} <- EvHs, N==Node],
    Include =  [I || {N,I} <- Incl, N==Node],
    {LogDir,Cover,ConfigFiles,EvHandlers,Include}.


refresh_logs(LogDir) ->
    {ok,Cwd} = file:get_cwd(),
    case file:set_cwd(LogDir) of
	E = {error,_Reason} ->
	    E;
	_ ->
	    case catch ct_logs:make_all_suites_index(refresh) of
		{'EXIT',ASReason} ->
		    file:set_cwd(Cwd),
		    {error,{all_suites_index,ASReason}};
		_ ->
		    case catch ct_logs:make_all_runs_index(refresh) of
			{'EXIT',ARReason} ->
			    file:set_cwd(Cwd),
			    {error,{all_runs_index,ARReason}};
			_ ->
			    file:set_cwd(Cwd),
			    io:format("Logs in ~s refreshed!~n",[LogDir]),
			    ok
		    end
	    end
    end.

which_logdir(".",Dir) ->
    Dir;
which_logdir(Dir,_) ->
    Dir.
	   
listify([C|_]=Str) when is_integer(C) -> [Str];
listify(L) when is_list(L) -> L;
listify(E) -> [E].

delistify([E]) -> E;
delistify(E)   -> E.


%%%-----------------------------------------------------------------
%%% @hidden
%%% @equiv ct:run/3
run(TestDir, Suite, Cases) ->
    install([]),
    do_run(tests(TestDir, Suite, Cases), []).

%%%-----------------------------------------------------------------
%%% @hidden
%%% @equiv ct:run/2
run(TestDir, Suite) when is_list(TestDir), is_integer(hd(TestDir)) ->
    install([]),
    do_run(tests(TestDir, Suite), []).

%%%-----------------------------------------------------------------
%%% @hidden
%%% @equiv ct:run/1
run(TestDirs) ->
    install([]),
    do_run(tests(TestDirs), []).


tests(TestDir, Suites, []) when is_list(TestDir), is_integer(hd(TestDir)) ->
    [{?testdir(TestDir,Suites),ensure_atom(Suites),all}];
tests(TestDir, Suite, Cases) when is_list(TestDir), is_integer(hd(TestDir)) ->
    [{?testdir(TestDir,Suite),ensure_atom(Suite),Cases}].
tests([{Dir,Suite}],Cases) ->
    [{?testdir(Dir,Suite),ensure_atom(Suite),Cases}];
tests(TestDir, Suite) when is_list(TestDir), is_integer(hd(TestDir)) ->
    tests(TestDir, ensure_atom(Suite), all).
tests(DirSuites) when is_list(DirSuites), is_tuple(hd(DirSuites)) ->
    [{?testdir(Dir,Suite),ensure_atom(Suite),all} || {Dir,Suite} <- DirSuites];
tests(TestDir) when is_list(TestDir), is_integer(hd(TestDir)) ->
    tests([TestDir]);
tests(TestDirs) when is_list(TestDirs), is_list(hd(TestDirs)) ->
    [{?testdir(TestDir,all),all,all} || TestDir <- TestDirs].

do_run(Tests, Opt) ->
    do_run(Tests, [], Opt, [], ".").

do_run(Tests, Opt, LogDir) ->
    do_run(Tests, [], Opt, [], LogDir).

do_run(Tests, Skip, Opt, Args, LogDir) ->
    case code:which(test_server) of
	non_existing ->
	    exit({error,no_path_to_test_server});
	_ ->
	    Opt1 = 
		case lists:keysearch(cover, 1, Opt) of
		    {value,{_,CoverFile}} ->
			case ct_cover:get_spec(CoverFile) of
			    {error,Reason} ->
				exit({error,Reason});
			    Spec ->
				[{cover_spec,Spec} |
				 lists:keydelete(cover, 1, Opt)]
			end;
		    _ ->
			Opt
		end,
	    %% This env variable is used by test_server to determine
	    %% which framework it runs under.
	    case os:getenv("TEST_SERVER_FRAMEWORK") of
		false ->
		    os:putenv("TEST_SERVER_FRAMEWORK", "ct_framework");
		"ct_framework" ->
		    ok;
		Other ->
		    erlang:display(list_to_atom("Note: TEST_SERVER_FRAMEWORK = " ++ Other))
	    end,
	    case ct_util:start(LogDir) of
		{error,interactive_mode} ->
		    io:format("CT is started in interactive mode. "
			      "To exit this mode, run ct:stop_interactive().\n"
			      "To enter the interactive mode again, "
			      "run ct:start_interactive()\n\n",[]),
		    {error,interactive_mode};

		_Pid ->
		    %% save style sheet info
		    case lists:keysearch(stylesheet, 1, Args) of
			{value,{_,SSFile}} ->
			    ct_util:set_testdata({stylesheet,SSFile});
			_ ->
			    ct_util:set_testdata({stylesheet,undefined})
		    end,

		    case lists:keysearch(silent_connections, 1, Args) of
			{value,{silent_connections,undefined}} ->
			    ok;
			{value,{silent_connections,[]}} ->
			    Conns = ct_util:override_silence_all_connections(),
			    ct_logs:log("Silent connections", "~p", [Conns]);
			{value,{silent_connections,Cs}} ->
			    Conns = lists:map(fun(S) when is_list(S) ->
						      list_to_atom(S);
						 (A) -> A
					      end, Cs),
			    ct_util:override_silence_connections(Conns),
			    ct_logs:log("Silent connections", "~p", [Conns]);
			_ ->
			    ok
		    end,
		    log_ts_names(Args),
		    TestSuites = suite_tuples(Tests),

		    {SuiteMakeErrors,AllMakeErrors} =
			case application:get_env(common_test, auto_compile) of
			    {ok,false} ->
				SuitesNotFound = verify_suites(TestSuites),
				{SuitesNotFound,SuitesNotFound};
			    _ ->
				{SuiteErrs,HelpErrs} = auto_compile(TestSuites),
				{SuiteErrs,SuiteErrs++HelpErrs}
			end,

		    case continue(AllMakeErrors) of
			true ->
			    SavedErrors = save_make_errors(SuiteMakeErrors),
			    ct_repeat:log_loop_info(Args),
			    {Tests1,Skip1} = final_tests(Tests,[],Skip,SavedErrors),
			    R = do_run_test(Tests1, Skip1, Opt1),
			    ct_util:stop(normal),
			    R;
			false ->
			    io:nl(),
			    ct_util:stop(clean),
			    BadMods =
				lists:foldr(
				  fun({{_,_},Ms}, Acc) ->
					  Ms ++ lists:foldl(
						  fun(M, Acc1) ->
							  lists:delete(M, Acc1)
						  end, Acc, Ms)
				  end, [], AllMakeErrors),
			    {error,{make_failed,BadMods}}
		    end
	    end
    end.

%% attempt to compile the modules specified in TestSuites
auto_compile(TestSuites) ->
    io:format("~nCommon Test: Running make in test directories...~n"),
    UserInclude =
	case application:get_env(common_test, include) of
	    {ok,UserInclDirs} when length(UserInclDirs) > 0 ->
		io:format("Including the following directories:~n"),
		[begin io:format("~p~n",[UserInclDir]), {i,UserInclDir} end || 
		 UserInclDir <- UserInclDirs];
	    _ ->
		[]
	end,
    SuiteMakeErrors =
	lists:flatmap(fun({TestDir,Suite} = TS) ->
			      case run_make(suites, TestDir, Suite, UserInclude) of
				  {error,{make_failed,Bad}} -> 
				      [{TS,Bad}];
				  {error,_} -> 
				      [{TS,[filename:join(TestDir,"*_SUITE")]}];
				  _ -> 
				      []
			      end
		      end, TestSuites),

    %% try to compile other modules than SUITEs in the test directories
    {_,HelpMakeErrors} =
	lists:foldl(
	  fun({Dir,Suite}, {Done,Failed}) ->
		  case lists:member(Dir, Done) of
		      false ->
			  Failed1 =
			      case run_make(helpmods, Dir, Suite, UserInclude) of
				  {error,{make_failed,BadMods}} ->
				      [{{Dir,all},BadMods}|Failed];
				  {error,_} ->
				      [{{Dir,all},[Dir]}|Failed];
				  _ ->
				      Failed
			      end,
			  {[Dir|Done],Failed1};
		      true ->		    % already visited
			  {Done,Failed}
		  end
	  end, {[],[]}, TestSuites),    
    {SuiteMakeErrors,lists:reverse(HelpMakeErrors)}.

%% verify that specified test suites exist (if auto compile is disabled)
verify_suites(TestSuites) ->
    io:nl(),
    Verify = 
	fun({Dir,Suite},NotFound) ->
		case locate_test_dir(Dir, Suite) of
		    {ok,TestDir} ->
			if Suite == all ->
				NotFound;
			   true ->
				Beam = filename:join(TestDir, atom_to_list(Suite)++".beam"),
				case filelib:is_regular(Beam) of
				    true  -> 
					NotFound;
				    false -> 
					Name = filename:join(TestDir, atom_to_list(Suite)),
					io:format("Suite ~w not found in directory ~s~n", 
						  [Suite,TestDir]),
					[{{Dir,Suite},[Name]} | NotFound]
				end
			end;
		    {error,_Reason} ->
			io:format("Directory ~s is invalid~n", [Dir]),
			Name = filename:join(Dir, atom_to_list(Suite)),
			[{{Dir,Suite},[Name]} | NotFound]
		end
	end,
    lists:reverse(lists:foldl(Verify, [], TestSuites)).
		     
    
save_make_errors([]) ->
    [];
save_make_errors(Errors) ->
    Suites = get_bad_suites(Errors,[]),
    ct_logs:log("MAKE RESULTS",
		"Error compiling or locating the following suites: ~n~p",[Suites]),
    %% save the info for logger
    file:write_file(?missing_suites_info,term_to_binary(Errors)),
    Errors.

get_bad_suites([{{_TestDir,_Suite},Failed}|Errors], BadSuites) ->
    get_bad_suites(Errors,BadSuites++Failed);
get_bad_suites([], BadSuites) ->
    BadSuites.

    

%%%-----------------------------------------------------------------
%%% @hidden
%%% @equiv ct:step/3
step(TestDir, Suite, Case) ->
    step(TestDir, Suite, Case, []).

%%%-----------------------------------------------------------------
%%% @hidden
%%% @equiv ct:step/4
step(TestDir, Suite, Case, Opts) when is_list(TestDir), is_atom(Suite), is_atom(Case), 
				   Suite =/= all, Case =/= all ->
    do_run([{TestDir,Suite,Case}], [{step,Opts}]).


%%%-----------------------------------------------------------------
%%% Internal
suite_tuples([{TestDir,Suites,_} | Tests]) when is_list(Suites) ->
    lists:map(fun(S) -> {TestDir,S} end, Suites) ++ suite_tuples(Tests);
suite_tuples([{TestDir,Suite,_} | Tests]) when is_atom(Suite) ->
    [{TestDir,Suite} | suite_tuples(Tests)];
suite_tuples([]) ->
    [].

final_tests([{TestDir,Suites,_}|Tests],
	    Final, Skip, Bad) when is_list(Suites), is_atom(hd(Suites)) ->
%     Separate =
% 	fun(S,{DoSuite,Dont}) ->		
% 		case lists:keymember({TestDir,S},1,Bad) of
% 		    false ->	
% 			{[S|DoSuite],Dont};
% 		    true ->	
% 			SkipIt = {TestDir,S,"Make failed"},
% 			{DoSuite,Dont++[SkipIt]}
% 		end
% 	end,
	
%     {DoSuites,Skip1} =
% 	lists:foldl(Separate,{[],Skip},Suites),
%     Do = {TestDir,lists:reverse(DoSuites),all},

    Skip1 = [{TD,S,"Make failed"} || {{TD,S},_} <- Bad, S1 <- Suites,
				     S == S1, TD == TestDir],
    Final1 = [{TestDir,S,all} || S <- Suites], 
    final_tests(Tests, lists:reverse(Final1)++Final, Skip++Skip1, Bad);

final_tests([{TestDir,all,all}|Tests], Final, Skip, Bad) ->
    MissingSuites =
	case lists:keysearch({TestDir,all}, 1, Bad) of
	    {value,{_,Failed}} ->
		[list_to_atom(filename:basename(F)) || F <- Failed];
	    false ->
		[]
	end,
    Missing = [{TestDir,S,"Make failed"} || S <- MissingSuites],
    Final1 = [{TestDir,all,all}|Final],
    final_tests(Tests, Final1, Skip++Missing, Bad);

final_tests([{TestDir,Suite,Cases}|Tests],
	    Final, Skip, Bad) when Cases==[]; Cases==all  ->
    final_tests([{TestDir,[Suite],all}|Tests], Final, Skip, Bad);

final_tests([{TestDir,Suite,Cases}|Tests], Final, Skip, Bad) ->
    case lists:keymember({TestDir,Suite}, 1, Bad) of
	false ->
	    Do = {TestDir,Suite,Cases},
	    final_tests(Tests, [Do|Final], Skip, Bad);
	true ->
	    Do = {TestDir,Suite,Cases},
	    Skip1 = Skip ++ [{TestDir,Suite,Cases,"Make failed"}],
	    final_tests(Tests, [Do|Final], Skip1, Bad)
    end;

final_tests([], Final, Skip, _Bad) ->
    {lists:reverse(Final),Skip}.

continue([]) ->    
    true;
continue(_MakeErrors) ->
    io:nl(),
    OldGl = group_leader(),
    case set_group_leader_same_as_shell() of
	true ->
	    S = self(),
	    io:format("Failed to compile or locate one or more test suites\n"
		      "Press \'c\' to continue or \'a\' to abort.\n"
		      "Will continue in 15 seconds if no answer is given!\n"),
	    Pid = spawn(fun() ->
				case io:get_line('(c/a) ') of
				    "c\n" ->
					S ! true;
				    _ ->
					S ! false
				end
			end),
	    group_leader(OldGl, self()),
	    receive R when R==true; R==false ->
		    R
	    after 15000 ->
		    exit(Pid, kill),
		    io:format("... timeout - continuing!!\n"),
		    true
	    end;
	false ->				% no shell process to use
	    true
    end.

set_group_leader_same_as_shell() ->
    %%! Locate the shell process... UGLY!!!
    GS2or3 = fun(P) ->
		     case process_info(P,initial_call) of
			 {initial_call,{group,server,X}} when X == 2 ; X == 3 ->
			     true;
			 _ ->
			     false
		     end
	     end,	
    case [P || P <- processes(), GS2or3(P), 
	       true == lists:keymember(shell,1,element(2,process_info(P,dictionary)))] of
	[GL|_] ->
	    group_leader(GL, self());
	[] ->
	    false
    end.

check_and_add([{TestDir0,M,_} | Tests], Added) ->
    case locate_test_dir(TestDir0, M) of
	{ok,TestDir} ->
	    case lists:member(TestDir, Added) of
		true ->
		    check_and_add(Tests, Added);
		false ->
		    true = code:add_patha(TestDir),
		    check_and_add(Tests, [TestDir|Added])
	    end;
	{error,_} ->
	    {error,{invalid_directory,TestDir0}}
    end;
check_and_add([], _) ->
    ok.

do_run_test(Tests, Skip, Opt) ->
    case check_and_add(Tests, []) of
	ok ->
	    ct_util:set_testdata({stats,{0,0,{0,0}}}),
	    ct_util:set_testdata({cover,undefined}),
	    test_server_ctrl:start_link(local),
	    case lists:keysearch(cover_spec, 1, Opt) of
		{value,{_,CovData={CovFile,
				   CovNodes,
				   _CovImport,
				   CovExport,
				   #cover{app        = CovApp,
					  level      = CovLevel,
					  excl_mods  = CovExcl,
					  incl_mods  = CovIncl,
					  cross      = CovCross,
					  src        = _CovSrc}}}} ->
		    ct_logs:log("COVER INFO","Using cover specification file: ~s~n"
				"App: ~w~n"
				"Cross cover: ~w~n"
				"Including ~w modules~n"
				"Excluding ~w modules",
				[CovFile,CovApp,CovCross,length(CovIncl),length(CovExcl)]), 

		    %% cover export file will be used for export and import
		    %% between tests so make sure it doesn't exist initially
		    case filelib:is_file(CovExport) of
			true ->
			    DelResult = file:delete(CovExport),
			    ct_logs:log("COVER INFO",
					"Warning! Export file ~s already exists. "
					"Deleting with result: ~p",
					[CovExport,DelResult]);
			false ->
			    ok
		    end,

		    %% tell test_server which modules should be cover compiled
		    %% note that actual compilation is done when tests start
		    test_server_ctrl:cover(CovApp, CovFile, CovExcl, CovIncl,
					   CovCross, CovExport, CovLevel),
		    %% save cover data (used e.g. to add nodes dynamically)
		    ct_util:set_testdata({cover,CovData}),
		    %% start cover on specified nodes
		    if (CovNodes /= []) and (CovNodes /= undefined) ->
			    ct_logs:log("COVER INFO",
					"Nodes included in cover session: ~w",
					[CovNodes]),
			    cover:start(CovNodes);
		       true ->
			    ok
		    end,
		    true;
		_ ->
		    false
	    end,	 
	    %% let test_server expand the test tuples and count no of cases
	    {Suites,NoOfCases} = count_test_cases(Tests, Skip),
	    Suites1 = delete_dups(Suites),
	    NoOfTests = length(Tests),
	    NoOfSuites = length(Suites1),
	    ct_util:warn_duplicates(Suites1),	    
	    {ok,Cwd} = file:get_cwd(),
	    io:format("~nCWD set to: ~p~n", [Cwd]),
	    if NoOfCases == unknown ->
		    io:format("~nTEST INFO: ~w test(s), ~w suite(s)~n~n", 
			      [NoOfTests,NoOfSuites]),
		    ct_logs:log("TEST INFO","~w test(s), ~w suite(s)", 
				[NoOfTests,NoOfSuites]);
	       true ->
		    io:format("~nTEST INFO: ~w test(s), ~w case(s) in ~w suite(s)~n~n", 
			      [NoOfTests,NoOfCases,NoOfSuites]),
		    ct_logs:log("TEST INFO","~w test(s), ~w case(s) in ~w suite(s)", 
				[NoOfTests,NoOfCases,NoOfSuites])
	    end,
	    ct_event:notify(#event{name=start_info,
				   node=node(),
				   data={NoOfTests,NoOfSuites,NoOfCases}}),
	    CleanUp = add_jobs(Tests, Skip, Opt, []),
	    unlink(whereis(test_server_ctrl)),
	    catch test_server_ctrl:wait_finish(), 
	    %% check if last testcase has left a "dead" trace window 
	    %% behind, and if so, kill it
	    case ct_util:get_testdata(interpret) of
		{_What,kill,{TCPid,AttPid}} ->
		    ct_util:kill_attached(TCPid, AttPid);
		_ ->
		    ok
	    end,
	    lists:foreach(fun(Suite) -> 
				  maybe_cleanup_interpret(Suite, Opt) 
			  end, CleanUp);
	Error ->
	    Error
    end.

delete_dups([S | Suites]) ->
    Suites1 = lists:delete(S, Suites),
    [S | delete_dups(Suites1)];
delete_dups([]) ->
    [].

count_test_cases(Tests, Skip) ->
    SendResult = fun(Me, Result) -> Me ! {no_of_cases,Result} end,
    TSPid = test_server_ctrl:start_get_totals(SendResult),
    Ref = erlang:monitor(process, TSPid),
    add_jobs(Tests, Skip, [], []),
    {Suites,NoOfCases} = count_test_cases1(length(Tests), 0, [], Ref),
    erlang:demonitor(Ref),
    test_server_ctrl:stop_get_totals(),
    {Suites,NoOfCases}.

count_test_cases1(0, N, Suites, _) ->
    {lists:flatten(Suites), N};
count_test_cases1(Jobs, N, Suites, Ref) ->
    receive
	{no_of_cases,{Ss,N1}} -> 
	    count_test_cases1(Jobs-1, add_known(N,N1), [Ss|Suites], Ref);
	{'DOWN', Ref, _, _, _} -> 
	    {[],0}
    end.				      

add_known(unknown, _) ->
    unknown;
add_known(_, unknown) ->
    unknown;
add_known(N, N1) ->
    N+N1.

add_jobs([{TestDir,all,_}|Tests], Skip, Opt, CleanUp) ->
    Name = get_name(TestDir),
    case catch test_server_ctrl:add_dir_with_skip(Name, TestDir,
						  skiplist(TestDir,Skip)) of
	{'EXIT',_} -> 
	    CleanUp;
	_ ->
	    wait_for_idle(),
	    add_jobs(Tests, Skip, Opt, CleanUp)
    end;
add_jobs([{TestDir,[Suite],all}|Tests], Skip, Opt, CleanUp) when is_atom(Suite) ->
    add_jobs([{TestDir,Suite,all}|Tests], Skip, Opt, CleanUp);
add_jobs([{TestDir,Suites,all}|Tests], Skip, Opt, CleanUp) when is_list(Suites) ->
    Name = get_name(TestDir) ++ ".suites",
    case catch test_server_ctrl:add_module_with_skip(Name, Suites,
						     skiplist(TestDir,Skip)) of
	{'EXIT',_} -> 
	    CleanUp;
	_ ->
	    wait_for_idle(),
	    add_jobs(Tests, Skip, Opt, CleanUp)
    end;
add_jobs([{TestDir,Suite,all}|Tests], Skip, Opt, CleanUp) ->
    case maybe_interpret(Suite, all, Opt) of
	ok ->
	    Name =  get_name(TestDir) ++ "." ++ atom_to_list(Suite),
	    case catch test_server_ctrl:add_module_with_skip(Name, [Suite],
							     skiplist(TestDir,Skip)) of
		{'EXIT',_} -> 
		    CleanUp;
		_ ->
		    wait_for_idle(),
		    add_jobs(Tests, Skip, Opt, [Suite|CleanUp])
	    end;
	Error ->
	    Error
    end;
add_jobs([{TestDir,Suite,[Case]}|Tests], Skip, Opt, CleanUp) when is_atom(Case) ->
    add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opt, CleanUp);
add_jobs([{TestDir,Suite,Cases}|Tests], Skip, Opt, CleanUp) when is_list(Cases) ->
    case maybe_interpret(Suite, Cases, Opt) of
	ok ->
	    Name =  get_name(TestDir) ++ "." ++	atom_to_list(Suite) ++ ".cases",
	    case catch test_server_ctrl:add_cases_with_skip(Name, Suite, Cases,
							    skiplist(TestDir,Skip)) of
		{'EXIT',_} -> 
		    CleanUp;
		_ ->
		    wait_for_idle(),
		    add_jobs(Tests, Skip, Opt, [Suite|CleanUp])
	    end;
	Error ->
	    Error
    end;
add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opt, CleanUp) when is_atom(Case) ->
    case maybe_interpret(Suite, Case, Opt) of
	ok ->
	    Name = get_name(TestDir) ++	"." ++ atom_to_list(Suite) ++ "." ++ 
		atom_to_list(Case),
	    case catch test_server_ctrl:add_case_with_skip(Name, Suite, Case,
							   skiplist(TestDir,Skip)) of
		{'EXIT',_} -> 
		    CleanUp;
		_ ->
		    wait_for_idle(),
		    add_jobs(Tests, Skip, Opt, [Suite|CleanUp])
	    end;
	Error ->
	    Error
    end;
add_jobs([], _, _, CleanUp) ->
    CleanUp.

wait_for_idle() ->
    ct_util:update_last_run_index(),
    Notify = fun(Me) -> Me ! idle end,
    case catch test_server_ctrl:idle_notify(Notify) of
	{'EXIT',_} ->
	    error;
	TSPid ->
	    %% so we don't hang forever if test_server dies
	    Ref = erlang:monitor(process, TSPid),
	    Result = receive
			 idle -> ok;
			 {'DOWN', Ref, _, _, _} -> error
		     end,
	    erlang:demonitor(Ref),
	    ct_util:update_last_run_index(),
	    Result
    end.

skiplist(Dir, [{Dir,all,Cmt}|Skip]) ->
    %% we need to turn 'all' into list of modules since
    %% test_server doesn't do skips on Dir level
    Ss = filelib:wildcard(filename:join(Dir, "*_SUITE.beam")),
    [{list_to_atom(filename:basename(S,".beam")),Cmt} || S <- Ss] ++ skiplist(Dir,Skip);
skiplist(Dir, [{Dir,S,Cmt}|Skip]) ->
    [{S,Cmt} | skiplist(Dir, Skip)];
skiplist(Dir, [{Dir,S,C,Cmt}|Skip]) ->
    [{S,C,Cmt} | skiplist(Dir, Skip)];
skiplist(Dir, [_|Skip]) ->
    skiplist(Dir, Skip);
skiplist(_Dir, []) ->
    [].

get_name(Dir) ->
    TestDir =
	case filename:basename(Dir) of
	    "test" ->
		filename:dirname(Dir);
	    _ ->
		Dir
	end,
    Base = filename:basename(TestDir),
    case filename:basename(filename:dirname(TestDir)) of
	"" -> 
	    Base;
	TopDir ->
	    TopDir ++ "." ++ Base
    end.


run_make(TestDir, Mod, UserInclude) ->
    run_make(suites, TestDir, Mod, UserInclude).

run_make(Targets, TestDir0, Mod, UserInclude) when is_list(Mod) ->
    run_make(Targets, TestDir0, list_to_atom(Mod), UserInclude);

run_make(Targets, TestDir0, Mod, UserInclude) ->
    case locate_test_dir(TestDir0, Mod) of
	{ok,TestDir} ->
	    %% send a start_make notification which may suspend
	    %% the process if some other node is compiling files
	    %% in the same directory
	    ct_event:sync_notify(#event{name=start_make,
					node=node(),
					data=TestDir}),
	    {ok,Cwd} = file:get_cwd(),
	    ok = file:set_cwd(TestDir),
	    TestServerInclude = get_dir(test_server, "include"),
	    CtInclude = get_dir(common_test, "include"),
	    XmerlInclude = get_dir(xmerl, "include"),
	    ErlFlags = UserInclude ++ [{i,TestServerInclude},
				       {i,CtInclude},
				       {i,XmerlInclude},
				       debug_info],
	    Result = 
		if Mod == all ; Targets == helpmods ->
			case (catch ct_make:all([noexec|ErlFlags])) of
			    {'EXIT',_} = Failure -> 
				Failure;
			    MakeInfo ->
				FileTest = fun(F, suites) -> is_suite(F);
					      (F, helpmods) -> not is_suite(F)
					   end,
				Files = lists:flatmap(fun({F,out_of_date}) ->
							      case FileTest(F, Targets) of
								  true -> [F];
								  false -> []
							      end;
							 (_) ->
							      []
						      end, MakeInfo),
				(catch ct_make:files(Files, [load|ErlFlags]))
			end;
		   true ->
			(catch ct_make:files([Mod], [load|ErlFlags]))
		end,
	    
	    ok = file:set_cwd(Cwd),
	    %% send finished_make notification
	    ct_event:notify(#event{name=finished_make,
				   node=node(),
				   data=TestDir}),
	    case Result of
		{up_to_date,_} ->
		    ok;
		{'EXIT',Reason} ->
		    io:format("{error,{make_crashed,~p}\n", [Reason]),
		    {error,{make_crashed,TestDir,Reason}};
		{error,ModInfo} ->
		    io:format("{error,make_failed}\n", []),
		    Bad = [filename:join(TestDir, M) || {M,R} <- ModInfo, 
							R == error],
		    {error,{make_failed,Bad}}
	    end;
	{error,_} ->
	    io:format("{error,{invalid_directory,~p}}\n", [TestDir0]),
	    {error,{invalid_directory,TestDir0}}
    end.

get_dir(App, Dir) ->
    filename:join(code:lib_dir(App), Dir).

maybe_interpret(Suite, Cases, [{step,StepOpts}]) ->
    %% if other suite has run before this one, check if last testcase 
    %% has left a "dead" trace window behind, and if so, kill it
    case ct_util:get_testdata(interpret) of
	{_What,kill,{TCPid,AttPid}} ->
	    ct_util:kill_attached(TCPid, AttPid);
	_ ->
	    ok
    end,
    maybe_interpret1(Suite, Cases, StepOpts);
maybe_interpret(_, _, _) ->
    ok.

maybe_interpret1(Suite, all, StepOpts) ->
    case i:ii(Suite) of
	{module,_} ->
	    i:iaa([break]),
	    case get_all_testcases(Suite) of
		{error,_} ->
		    {error,no_testcases_found};
		Cases ->
		    maybe_interpret2(Suite, Cases, StepOpts)
	    end;
	error ->
	    {error,could_not_interpret_module}
    end;
maybe_interpret1(Suite, Case, StepOpts) when is_atom(Case) ->
    maybe_interpret1(Suite, [Case], StepOpts);
maybe_interpret1(Suite, Cases, StepOpts) when is_list(Cases) ->
    case i:ii(Suite) of
	{module,_} ->
	    i:iaa([break]),
	    maybe_interpret2(Suite, Cases, StepOpts);
	error ->
	    {error,could_not_interpret_module}
    end.

maybe_interpret2(Suite, Cases, StepOpts) ->
    set_break_on_config(Suite, StepOpts),
    [i:ib(Suite, Case, 1) || Case <- Cases],
    test_server_ctrl:multiply_timetraps(infinity),
    WinOp = case lists:member(keep_inactive, ensure_atom(StepOpts)) of
		true -> no_kill;
		false -> kill
	    end,		    
    ct_util:set_testdata({interpret,{{Suite,Cases},WinOp,
				     {undefined,undefined}}}),
    ok.

set_break_on_config(Suite, StepOpts) ->
    case lists:member(config, ensure_atom(StepOpts)) of
	true ->
	    i:ib(Suite, init_per_suite, 1),
	    i:ib(Suite, init_per_testcase, 2),
	    i:ib(Suite, end_per_testcase, 2),
	    i:ib(Suite, end_per_suite, 1);
	false ->
	    ok
    end.

maybe_cleanup_interpret(Suite, [{step,_}]) ->
    i:iq(Suite);
maybe_cleanup_interpret(_, _) ->
    ok.

log_ts_names(Args) ->
    case lists:keysearch(spec, 1, Args) of
	{value,{_,Specs}} ->
	    List = lists:map(fun(Name) ->
				     Name ++ " "
			     end, Specs),
	    ct_logs:log("Test Specification file(s)", "~s",
			[lists:flatten(List)]);
	_ ->
	    ok
    end.
			  
merge_arguments(Args) ->
    merge_arguments(Args, []).

merge_arguments([LogDir={logdir,_}|Args], Merged) ->
    merge_arguments(Args, handle_arg(replace, LogDir, Merged));
merge_arguments([CoverFile={cover,_}|Args], Merged) ->
    merge_arguments(Args, handle_arg(replace, CoverFile, Merged));
merge_arguments([Arg={_,_}|Args], Merged) ->
    merge_arguments(Args, handle_arg(merge, Arg, Merged));
merge_arguments([], Merged) ->
    Merged.

handle_arg(replace, {Key,Elems}, [{Key,_}|Merged]) ->
    [{Key,Elems}|Merged];
handle_arg(merge, {Key,Elems}, [{Key,PrevElems}|Merged]) ->
    [{Key,PrevElems++Elems}|Merged];
handle_arg(Op, Arg, [Other|Merged]) ->
    [Other|handle_arg(Op, Arg, Merged)];
handle_arg(_,Arg,[]) ->
    [Arg].

locate_test_dir(Dir, Suite) ->
    TestDir = case ct_util:is_test_dir(Dir) of
		  true  -> Dir;
		  false -> ct_util:get_testdir(Dir, Suite)
	      end,
    case filelib:is_dir(TestDir) of
	true  -> {ok,TestDir};
	false -> {error,invalid}
    end.

is_suite(Mod) when is_atom(Mod) ->
    is_suite(atom_to_list(Mod));
is_suite(ModOrFile) when is_list(ModOrFile) ->
    case lists:reverse(filename:basename(ModOrFile, ".erl")) of
	[$E,$T,$I,$U,$S,$_|_] ->
	    true;
	_ ->
	    case lists:reverse(filename:basename(ModOrFile, ".beam")) of
		[$E,$T,$I,$U,$S,$_|_] ->
		    true;
		_ ->
		    false
	    end
    end.

get_all_testcases(Suite) ->
    %%! this needs to be updated to handle testcase groups later!!
    case catch Suite:all() of
	{'EXIT',Why} ->
	    {error,Why};
	{skip,_} ->
	    [];
	Cases ->
	    AllCases =
		lists:foldl(fun({sequence,SeqName}, All) ->
				    case catch Suite:sequences() of
					{'EXIT',_} ->
					    All;
					Seqs ->
					    case proplists:get_value(SeqName, Seqs) of
						undefined ->
						    All;
						SeqCases ->
						    lists:reverse(SeqCases) ++ All
					    end
				    end;
			       (Case,All) ->
				    [Case|All]
			    end, [], Cases),
	    lists:reverse(AllCases)
    end.
					

%% Internal tracing support. If {ct_trace,TraceSpec} is present, the
%% TraceSpec file will be consulted and dbg used to trace function
%% calls during test run. Expected terms in TraceSpec:
%% {m,Mod} or {f,Mod,Func}.
start_trace(Args) ->
    case lists:keysearch(ct_trace,1,Args) of
	{value,{ct_trace,File}} ->
	    TraceSpec = delistify(File),
	    case file:consult(TraceSpec) of
		{ok,Terms} ->
		    case catch do_trace(Terms) of
			ok -> 
			    true;
			{_,Error} ->
			    io:format("Warning! Tracing not started. Reason: ~p~n~n",
				      [Error]),
			    false
		    end;		           
		{_,Error} ->
		    io:format("Warning! Tracing not started. Reason: ~p~n~n",
			      [Error]),
		    false
	    end;	    			
	false ->
	    false		
    end.

do_trace(Terms) ->
    dbg:tracer(),
    dbg:p(self(), [sos,call]),
    lists:foreach(fun({m,M}) ->
			  case dbg:tpl(M,[{'_',[],[{return_trace}]}]) of
			      {error,What} -> exit({error,{tracing_failed,What}});
			      _ -> ok
			  end;			  
		     ({f,M,F}) ->
			  case dbg:tpl(M,F,[{'_',[],[{return_trace}]}]) of
			      {error,What} -> exit({error,{tracing_failed,What}});
			      _ -> ok
			  end;			  
		     (Huh) ->
			  exit({error,{unrecognized_trace_term,Huh}})
		  end, Terms),
    ok.
			 
stop_trace(true) ->
    dbg:stop_clear();
stop_trace(false) ->
    ok.
    
ensure_atom(Atom) when is_atom(Atom) ->
    Atom;
ensure_atom(String) when is_list(String), is_integer(hd(String)) ->
    list_to_atom(String);
ensure_atom(List) when is_list(List) ->
    [ensure_atom(Item) || Item <- List];
ensure_atom(Other) ->				
    Other.