aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/test
diff options
context:
space:
mode:
authorPeter Andersson <[email protected]>2013-06-05 12:38:53 +0200
committerPeter Andersson <[email protected]>2013-06-05 16:14:24 +0200
commit0681d5fa0b41210a114df57faabcd84fa7bc7e01 (patch)
tree970aa2507d887e5ca44ac828b9a9e86d724c24d6 /lib/common_test/test
parentb715feddc88bd8f6bdb664cc2e87425de7d31898 (diff)
downloadotp-0681d5fa0b41210a114df57faabcd84fa7bc7e01.tar.gz
otp-0681d5fa0b41210a114df57faabcd84fa7bc7e01.tar.bz2
otp-0681d5fa0b41210a114df57faabcd84fa7bc7e01.zip
Fix error with testspec terms not being merged properly
Diffstat (limited to 'lib/common_test/test')
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE.erl46
1 files changed, 18 insertions, 28 deletions
diff --git a/lib/common_test/test/ct_testspec_3_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE.erl
index d8a59b982c..5fa187e5b4 100644
--- a/lib/common_test/test/ct_testspec_3_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_3_SUITE.erl
@@ -284,6 +284,24 @@ events_to_check(_, 0) ->
events_to_check(Test, N) ->
test_events(Test) ++ events_to_check(Test, N-1).
+
+%%%!
+%%%! IMPORTANT NOTE ABOUT THE TEST ORDER:
+%%%!
+%%%! When merging testspec terms, CT will group the tests by TestDir and
+%%%! Suite, before term order (in testspec). That means that if tests
+%%%! are ordered like e.g:
+%%%! {Dir1,Suite11}, {Dir2,Suite21}, {Dir1,Suite12},
+%%%! the execution order after merge (even if no merge takes place),
+%%%! will be:
+%%%! {Dir1,[Suite11,Suite12]}, {Dir2,Suite21}
+%%%!
+%%%! Also, tests in a tree of included testspecs are always collected
+%%%! and merged in depth-first manner, meaning even if a particular test is
+%%%! on a higher level in the tree, it may be executed later than a test on a
+%%%! lower level.
+%%%!
+
test_events(start_separate) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
@@ -552,8 +570,6 @@ test_events(incl_join1) ->
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]},
-%%! WHY NOT 12,22,11,21?
-
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
{?eh,start_info,{4,4,20}},
@@ -580,8 +596,6 @@ test_events(incl_join1) ->
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}];
-%%! WHY NOT 22,11,21,12?
-
test_events(incl_join2) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
@@ -609,8 +623,6 @@ test_events(incl_join2) ->
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}];
-%%! WHY NOT 22,12,11,21?
-
test_events(incl_both1) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
@@ -709,8 +721,6 @@ test_events(incl_both2) ->
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]},
-%%! WHY NOT 11,21,12,22?
-
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
{?eh,start_info,{3,2,15}},
@@ -828,8 +838,6 @@ test_events(incl_both_and_join1) ->
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]},
-%%! WHY NOT 11,21,12,22?
-
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
{?eh,start_info,{2,2,10}},
@@ -874,8 +882,6 @@ test_events(incl_both_and_join2) ->
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]},
-%%! WHY NOT 11,21,12,22?
-
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
{?eh,start_info,{3,2,15}},
@@ -1078,8 +1084,6 @@ test_events(rec_incl_separate2) ->
[{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}];
-%%! OK
-
test_events(rec_incl_join1) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
@@ -1105,8 +1109,6 @@ test_events(rec_incl_join1) ->
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]},
-%%! WHY NOT 12,22,11,21?
-
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
{?eh,start_info,{5,5,25}},
@@ -1134,8 +1136,6 @@ test_events(rec_incl_join1) ->
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}];
-%%! WHY NOT 23,11,21,12,22?
-
test_events(rec_incl_join2) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
@@ -1165,8 +1165,6 @@ test_events(rec_incl_join2) ->
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}];
-%%! WHY NOT 23,12,22,11,21?
-
test_events(rec_incl_separate_join1) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
@@ -1192,8 +1190,6 @@ test_events(rec_incl_separate_join1) ->
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]},
-%%! WHY NOT 12,22,11,21?
-
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
{?eh,start_info,{4,4,20}},
@@ -1217,8 +1213,6 @@ test_events(rec_incl_separate_join1) ->
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]},
-%%! WHY NOT 22,11,21,12?
-
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
{?eh,start_info,{1,1,5}},
@@ -1253,8 +1247,6 @@ test_events(rec_incl_separate_join1) ->
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]},
-%%! WHY NOT 22,11,21,12?
-
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
{?eh,start_info,{4,4,20}},
@@ -1278,8 +1270,6 @@ test_events(rec_incl_separate_join1) ->
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}];
-%%! WHY NOT 12,22,11,21?
-
test_events(rec_incl_separate_join2) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},