Merge branch 'siri/ct/get_dirs_from_testspec/OTP-14132' into maint

* siri/ct/get_dirs_from_testspec/OTP-14132:
  [ct] Add ct_testspec:get_tests/1

5 files changed, 192 insertions, 6 deletions

diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile
index e495f587a3..152ece5d25 100644
--- a/lib/common_test/doc/src/Makefile
+++ b/lib/common_test/doc/src/Makefile
@@ -53,7 +53,8 @@ XML_REF3_FILES = ct.xml \
 	ct_slave.xml \
 	ct_property_test.xml \
 	ct_netconfc.xml \
-	ct_hooks.xml
+	ct_hooks.xml \
+        ct_testspec.xml
 XML_REF6_FILES = common_test_app.xml
 
 XML_PART_FILES = part.xml 
diff --git a/lib/common_test/doc/src/ct_testspec.xml b/lib/common_test/doc/src/ct_testspec.xml
new file mode 100644
index 0000000000..36893f66cf
--- /dev/null
+++ b/lib/common_test/doc/src/ct_testspec.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<!DOCTYPE erlref SYSTEM "erlref.dtd">
+
+<erlref>
+  <header>
+    <copyright>
+      <year>2016</year>
+      <holder>Ericsson AB. All Rights Reserved.</holder>
+    </copyright>
+    <legalnotice>
+      Licensed under the Apache License, Version 2.0 (the "License");
+      you may not use this file except in compliance with the License.
+      You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+      See the License for the specific language governing permissions and
+      limitations under the License.
+
+    </legalnotice>
+
+    <title>ct_testspec</title>
+    <prepared></prepared>
+    <responsible></responsible>
+    <docno></docno>
+    <approved></approved>
+    <checked></checked>
+    <date></date>
+    <rev>A</rev>
+    <file>ct_testspec.xml</file>
+  </header>
+  <module>ct_testspec</module>
+  <modulesummary>Parsing of test specifications for Common Test.
+  </modulesummary>
+
+<description>
+
+    <p>Parsing of test specifications for <c>Common Test</c>.</p>
+
+    <p>This module exports help functions for parsing of test specifications.</p>
+
+</description>
+
+  <funcs>
+    <func>
+    <name>get_tests(SpecsIn) -&gt; {ok, [{Specs,Tests}]} | {error, Reason}</name>
+    <fsummary>Parse the given test specification files and return the tests to run and skip.</fsummary>
+      <type>
+        <v>SpecsIn = [string()] | [[string()]]</v>
+        <v>Specs = [string()]</v>
+        <v>Test = [{Node,Run,Skip}]</v>
+        <v>Node = atom()</v>
+        <v>Run = {Dir,Suites,Cases}</v>
+        <v>Skip = {Dir,Suites,Comment} | {Dir,Suites,Cases,Comment}</v>
+        <v>Dir = string()</v>
+        <v>Suites = atom | [atom()] | all</v>
+        <v>Cases = atom | [atom()] | all</v>
+        <v>Comment = string()</v>
+        <v>Reason = term()</v>
+      </type>
+      <desc><marker id="add_nodes-1"/>
+        <p>Parse the given test specification files and return the
+        tests to run and skip.</p>
+
+        <p>If <c>SpecsIn=[Spec1,Spec2,...]</c>, separate tests will be
+        created per specification. If
+        <c>SpecsIn=[[Spec1,Spec2,...]]</c>, all specifications will be
+        merge into one test.</p>
+
+        <p>For each test, a <c>{Specs,Tests}</c> element is returned,
+        where <c>Specs</c> is a list of all included test
+        specifications, and <c>Tests</c> specifies actual tests to
+        run/skip per node.</p>
+      </desc>
+    </func>
+
+  </funcs>
+
+</erlref>
+
+
diff --git a/lib/common_test/doc/src/ref_man.xml b/lib/common_test/doc/src/ref_man.xml
index d1567e2d3c..1ac20db5c2 100644
--- a/lib/common_test/doc/src/ref_man.xml
+++ b/lib/common_test/doc/src/ref_man.xml
@@ -47,6 +47,7 @@
   <xi:include href="ct_slave.xml"/>
   <xi:include href="ct_hooks.xml"/>
   <xi:include href="ct_property_test.xml"/>
+  <xi:include href="ct_testspec.xml"/>
 </application>
 
 
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 991abb0666..16001ce4c8 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -26,7 +26,8 @@
 
 -export([prepare_tests/1, prepare_tests/2, 
 	 collect_tests_from_list/2, collect_tests_from_list/3,
-	 collect_tests_from_file/2, collect_tests_from_file/3]).
+	 collect_tests_from_file/2, collect_tests_from_file/3,
+         get_tests/1]).
 
 -export([testspec_rec2list/1, testspec_rec2list/2]).
 
@@ -803,6 +804,31 @@ list_nodes(#testspec{nodes=NodeRefs}) ->
     lists:map(fun({_Ref,Node}) -> Node end, NodeRefs).		      
 
 
+%%%-----------------------------------------------------------------
+%%% Parse the given test specs and return the complete set of specs
+%%% and tests to run/skip.
+%%% [Spec1,Spec2,...] means create separate tests per spec
+%%% [[Spec1,Spec2,...]] means merge all specs into one
+-spec get_tests(Specs) -> {ok,[{Specs,Tests}]} | {error,Reason} when
+      Specs :: [string()] | [[string()]],
+      Tests :: {Node,Run,Skip},
+      Node :: atom(),
+      Run :: {Dir,Suites,Cases},
+      Skip :: {Dir,Suites,Comment} | {Dir,Suites,Cases,Comment},
+      Dir :: string(),
+      Suites :: atom | [atom()] | all,
+      Cases :: atom | [atom()] | all,
+      Comment :: string(),
+      Reason :: term().
+
+get_tests(Specs) ->
+    case collect_tests_from_file(Specs,true) of
+        Tests when is_list(Tests) ->
+            {ok,[{S,prepare_tests(R)} || {S,R} <- Tests]};
+        Error ->
+            Error
+    end.
+
 %%     -----------------------------------------------------
 %%   /                                                       \
 %%  |  When adding test/config terms, remember to update      |
diff --git a/lib/common_test/test/ct_testspec_2_SUITE.erl b/lib/common_test/test/ct_testspec_2_SUITE.erl
index 1a941df185..1bab80942a 100644
--- a/lib/common_test/test/ct_testspec_2_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_2_SUITE.erl
@@ -220,7 +220,24 @@ basic_compatible_no_nodes(_Config) ->
 					    {tc2,{skip,"skipped"}}]}]}],
 		       merge_tests = true},
 
-    verify_result(Verify,ListResult,FileResult).
+    verify_result(Verify,ListResult,FileResult),
+
+    {ok,Tests} = ct_testspec:get_tests([SpecFile]),
+    ct:pal("ct_testspec:get_tests/1:~n~p~n", [Tests]),
+    [{[SpecFile],[{Node,Run,Skip}]}] = Tests,
+    [{Alias1V,x_SUITE,all},
+     {Alias1V,y_SUITE,[{g1,all},{g2,all},tc1,tc2]},
+     {Alias1V,z_SUITE,all},
+     {Alias2V,x_SUITE,all},
+     {Alias2V,y_SUITE,all}] = lists:sort(Run),
+    [{Alias1V,z_SUITE,"skipped"},
+     {Alias2V,x_SUITE,{g1,all},"skipped"},
+     {Alias2V,x_SUITE,{g2,all},"skipped"},
+     {Alias2V,y_SUITE,tc1,"skipped"},
+     {Alias2V,y_SUITE,tc2,"skipped"}] = lists:sort(Skip),
+
+    ok.
+
 
 %%%-----------------------------------------------------------------
 %%%
@@ -346,7 +363,25 @@ basic_compatible_nodes(_Config) ->
 					    {tc2,{skip,"skipped"}}]}]}],
 		       merge_tests = true},
 
-    verify_result(Verify,ListResult,FileResult).
+    verify_result(Verify,ListResult,FileResult),
+
+    {ok,Tests} = ct_testspec:get_tests([SpecFile]),
+    ct:pal("ct_testspec:get_tests/1:~n~p~n", [Tests]),
+    [{[SpecFile],[{Node,[],[]},
+                  {Node1,Run1,Skip1},
+                  {Node2,Run2,Skip2}]}] = Tests,
+    [{TO1V,x_SUITE,all},
+     {TO1V,y_SUITE,[{g1,all},{g2,all},tc1,tc2]},
+     {TO1V,z_SUITE,all}] = lists:sort(Run1),
+    [{TO2V,x_SUITE,all},
+     {TO2V,y_SUITE,all}] = lists:sort(Run2),
+    [{TO1V,z_SUITE,"skipped"}] = lists:sort(Skip1),
+    [{TO2V,x_SUITE,{g1,all},"skipped"},
+     {TO2V,x_SUITE,{g2,all},"skipped"},
+     {TO2V,y_SUITE,tc1,"skipped"},
+     {TO2V,y_SUITE,tc2,"skipped"}] = lists:sort(Skip2),
+
+    ok.
 
 %%%-----------------------------------------------------------------
 %%%
@@ -439,7 +474,28 @@ no_merging(_Config) ->
 				 [{y_SUITE,[{tc1,{skip,"skipped"}},
 					    {tc2,{skip,"skipped"}}]}]}]},
 		       
-    verify_result(Verify,ListResult,FileResult).
+    verify_result(Verify,ListResult,FileResult),
+
+    {ok,Tests} = ct_testspec:get_tests([SpecFile]),
+    ct:pal("ct_testspec:get_tests/1:~n~p~n", [Tests]),
+    [{[SpecFile],[{Node,[],[]},
+                  {Node1,Run1,Skip1},
+                  {Node2,Run2,Skip2}]}] = Tests,
+    [{TO1V,x_SUITE,all},
+     {TO1V,y_SUITE,[tc1,tc2]},
+     {TO1V,y_SUITE,[{g1,all},{g2,all}]},
+     {TO1V,z_SUITE,all}] = lists:sort(Run1),
+    [{TO2V,x_SUITE,all},
+     {TO2V,x_SUITE,[{skipped,g1,all},{skipped,g2,all}]},
+     {TO2V,y_SUITE,all},
+     {TO2V,y_SUITE,[{skipped,tc1},{skipped,tc2}]}] = lists:sort(Run2),
+    [{TO1V,z_SUITE,"skipped"}] = lists:sort(Skip1),
+    [{TO2V,x_SUITE,{g1,all},"skipped"},
+     {TO2V,x_SUITE,{g2,all},"skipped"},
+     {TO2V,y_SUITE,tc1,"skipped"},
+     {TO2V,y_SUITE,tc2,"skipped"}] = lists:sort(Skip2),
+
+    ok.
 
 %%%-----------------------------------------------------------------
 %%%
@@ -510,7 +566,25 @@ multiple_specs(_Config) ->
 				  {y_SUITE,[all,{tc1,{skip,"skipped"}},
 					    {tc2,{skip,"skipped"}}]}]}]},
 		       
-    verify_result(Verify,FileResult,FileResult).
+    verify_result(Verify,FileResult,FileResult),
+
+    {ok,Tests} = ct_testspec:get_tests([[SpecFile1,SpecFile2]]),
+    ct:pal("ct_testspec:get_tests/1:~n~p~n", [Tests]),
+    [{[SpecFile1,SpecFile2],[{Node,[],[]},
+                             {Node1,Run1,Skip1},
+                             {Node2,Run2,Skip2}]}] = Tests,
+    [{TO1V,x_SUITE,all},
+     {TO1V,y_SUITE,[{g1,all},{g2,all},tc1,tc2]},
+     {TO1V,z_SUITE,all}] = lists:sort(Run1),
+    [{TO2V,x_SUITE,all},
+     {TO2V,y_SUITE,all}] = lists:sort(Run2),
+    [{TO1V,z_SUITE,"skipped"}] = lists:sort(Skip1),
+    [{TO2V,x_SUITE,{g1,all},"skipped"},
+     {TO2V,x_SUITE,{g2,all},"skipped"},
+     {TO2V,y_SUITE,tc1,"skipped"},
+     {TO2V,y_SUITE,tc2,"skipped"}] = lists:sort(Skip2),
+
+    ok.
 
 %%%-----------------------------------------------------------------
 %%%