Add tests for test specifications

4 files changed, 83 insertions, 16 deletions

diff --git a/lib/common_test/src/ct_config.erl b/lib/common_test/src/ct_config.erl
index bc57930381..4b99d8db88 100755
--- a/lib/common_test/src/ct_config.erl
+++ b/lib/common_test/src/ct_config.erl
@@ -204,7 +204,6 @@ get_config_file_list(Opts)->
     CfgFiles.
 
 read_config_files(Opts) ->
-    %ct:pal("ct_config:read_config_files/1:~nOpts:~n~p", [Opts]),
     AddCallback = fun(CallBack, [])->
 			[{CallBack, []}];
 		     (CallBack, Files)->
@@ -220,7 +219,6 @@ read_config_files(Opts) ->
 	false->
 	    []
     end,
-    %ct:pal("ct_config:read_config_files/1:~nConfigFiles:~n~p", [ConfigFiles]),
     read_config_files_int(ConfigFiles, fun store_config/3).
 
 read_config_files_int([{Callback, File}|Files], FunToSave)->
@@ -687,7 +685,6 @@ check_callback_load(Callback)->
     end.
 
 check_config_files(Configs)->
-    ct:pal("ct_config:check_config_files/1~nConfigs:~n~p", [Configs]),
     lists:keysearch(nok, 1,
 	lists:flatten(
 	    lists:map(fun({Callback, Files})->
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 8a384a1b3a..ea30ccc13b 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -305,6 +305,28 @@ get_global([{node,Ref,Node}|Ts],Spec=#testspec{nodes=Refs}) ->
 get_global([_|Ts],Spec) -> get_global(Ts,Spec);
 get_global([],Spec) -> Spec.
 
+% TODO probably we can terminate here, if any problem with the filename
+% anyway, later ct_run will do it for us :-)
+get_absfile(Callback, FullName,#testspec{spec_dir=SpecDir}) ->
+    % we need to temporary switch to new cwd here, because
+    % otherwise config files cannot be found
+    {ok, OldWd} = file:get_cwd(),
+    ok = file:set_cwd(SpecDir),
+    R =  Callback:check_parameter(FullName),
+    ok = file:set_cwd(OldWd),
+    case R of
+	{ok, {file, FullName}}->
+	    File = filename:basename(FullName),
+	    Dir = get_absname(filename:dirname(FullName),SpecDir),
+	    filename:join(Dir,File);
+	{ok, {config, FullName}}->
+	    FullName;
+	{nok, {nofile, FullName}}->
+	    FullName;
+	{nok, {wrong_config, FullName}}->
+	    FullName
+    end.
+
 get_absfile(FullName,#testspec{spec_dir=SpecDir}) ->
     File = filename:basename(FullName),
     Dir = get_absname(filename:dirname(FullName),SpecDir),
@@ -430,12 +452,7 @@ add_tests([{config,Node,[{Callback,F}|Fs]}|Ts],Spec) when is_list(F) ->
     Cfgs = Spec#testspec.config,
     Node1 = ref2node(Node,Spec#testspec.nodes),
     add_tests([{config,Node,Fs}|Ts],
-% TODO FIX IT SOMEHOW! There SHOULD be absolute paths,
-% but it can't be applied to the config parameters
-% probably, that's a good idea to call Callback:check_parameter/1
-% and proceed according to the results
-% 	      Spec#testspec{config=[{Node1,{Callback, get_absfile(F,Spec)}}|Cfgs]});
-	      Spec#testspec{config=[{Node1,{Callback,[F]}}|Cfgs]});
+	      Spec#testspec{config=[{Node1,{Callback, [get_absfile(Callback, F,Spec)]}}|Cfgs]});
 add_tests([{config,_Node,[]}|Ts],Spec) ->
     %io:format("4: add_tests([{config,_,[]}|~p],~p)~n", [Ts, Spec]),
     add_tests(Ts,Spec);
diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl
index 4ae68fbfa8..ad7c521946 100644
--- a/lib/common_test/test/ct_config_SUITE.erl
+++ b/lib/common_test/test/ct_config_SUITE.erl
@@ -63,7 +63,10 @@ all(suite) ->
     [
 	require,
 	userconfig_static,
-	userconfig_dynamic
+	userconfig_dynamic,
+	testspec_legacy,
+	testspec_static,
+	testspec_dynamic
     ].
 
 
@@ -93,9 +96,52 @@ userconfig_dynamic(Config) when is_list(Config) ->
 	     {userconfig, {config_driver, "config_server"}},
              ["config_2_SUITE"]).
 
+testspec_legacy(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+    make_spec(DataDir,
+		"config/spec_legacy.spec",
+		[config_1_SUITE],
+		[{config, filename:join(DataDir, "config/config.txt")}]),
+    run_test(testspec_legacy,
+	     Config,
+	     {spec, filename:join(DataDir, "config/spec_legacy.spec")},
+             []),
+    file:delete(filename:join(DataDir, "config/spec_legacy.spec")).
+
+testspec_static(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+    make_spec(DataDir,
+		"config/spec_static.spec",
+		[config_1_SUITE],
+		[{userconfig, {ct_config_xml, filename:join(DataDir, "config/config.xml")}}]),
+    run_test(testspec_static,
+	     Config,
+	     {spec, filename:join(DataDir, "config/spec_static.spec")},
+             []),
+    file:delete(filename:join(DataDir, "config/spec_static.spec")).
+
+testspec_dynamic(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+    make_spec(DataDir, "config/spec_dynamic.spec",
+		[config_2_SUITE],
+		[{userconfig, {config_driver, "config_server"}}]),
+    run_test(testspec_dynamic,
+	     Config,
+	     {spec, filename:join(DataDir, "config/spec_dynamic.spec")},
+             []),
+    file:delete(filename:join(DataDir, "config/spec_dynamic.spec")).
+
 %%%-----------------------------------------------------------------
 %%% HELP FUNCTIONS
 %%%-----------------------------------------------------------------
+% {suites, "ct_config_SUITE_data/config/test", config_2_SUITE}.
+make_spec(DataDir, Filename, Suites, Config)->
+    {ok, Fd} = file:open(filename:join(DataDir, Filename), [write]),
+    ok = file:write(Fd,
+		io_lib:format("{suites, \"~sconfig/test/\", ~p}.~n", [DataDir, Suites])),
+    lists:foreach(fun(C)-> ok=file:write(Fd, io_lib:format("~p.~n", [C])) end, Config),
+    ok = file:close(Fd).
+
 run_test(Name, Config, CTConfig, SuiteNames)->
     DataDir = ?config(data_dir, Config),
     Joiner = fun(Suite) -> filename:join(DataDir, "config/test/"++Suite) end,
@@ -123,7 +169,9 @@ reformat_events(Events, EH) ->
 %%%-----------------------------------------------------------------
 %%% TEST EVENTS
 %%%-----------------------------------------------------------------
-expected_events(ReqOrUCS) when ReqOrUCS==require;  ReqOrUCS==userconfig_static->
+expected_events(Static) when
+	Static == require; Static == testspec_legacy;
+	Static == userconfig_static; Static == testspec_static->
 [
  {?eh,start_logging,{'DEF','RUNDIR'}},
  {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
@@ -163,7 +211,7 @@ expected_events(ReqOrUCS) when ReqOrUCS==require;  ReqOrUCS==userconfig_static->
  {?eh,stop_logging,[]}
 ];
 
-expected_events(userconfig_dynamic)->
+expected_events(Dynamic) when Dynamic == testspec_dynamic; Dynamic == userconfig_dynamic->
 [
  {ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
  {ct_test_support_eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
diff --git a/lib/common_test/test/ct_config_SUITE_data/config/test/config_driver.erl b/lib/common_test/test/ct_config_SUITE_data/config/test/config_driver.erl
index 37572500c7..670639f7c7 100644
--- a/lib/common_test/test/ct_config_SUITE_data/config/test/config_driver.erl
+++ b/lib/common_test/test/ct_config_SUITE_data/config/test/config_driver.erl
@@ -33,9 +33,14 @@ read_config(ServerName)->
 
 check_parameter(ServerName)->
     ServerModule = list_to_atom(ServerName),
-    case code:load_file(ServerModule) of
-	{module, ServerModule}->
+    case code:is_loaded(ServerModule) of
+	{file, _}->
 	    {ok, {config, ServerName}};
-	{error, nofile}->
-	    {nok, {wrong_config, "File not found: " ++ ServerName ++ ".beam"}}
+	false->
+	    case code:load_file(ServerModule) of
+		{module, ServerModule}->
+		    {ok, {config, ServerName}};
+		{error, nofile}->
+		    {nok, {wrong_config, "File not found: " ++ ServerName ++ ".beam"}}
+	    end
     end.