aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/test/ct_groups_spec_SUITE.erl
blob: 363a3c6bf0edac4474760bdfd795365d6c9748b9 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 2009-2011. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%     http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
%% %CopyrightEnd%
%%

%%%-------------------------------------------------------------------
%%% File: ct_groups_spec_SUITE
%%%
%%% Description: 
%%% Test that overriding default group properties with group terms
%%% in all/0 and in test specifications works as expected.
%%%
%%%-------------------------------------------------------------------
-module(ct_groups_spec_SUITE).

-compile(export_all).

-include_lib("common_test/include/ct.hrl").
-include_lib("common_test/include/ct_event.hrl").

-define(eh, ct_test_support_eh).

%%--------------------------------------------------------------------
%% TEST SERVER CALLBACK FUNCTIONS
%%--------------------------------------------------------------------

%%--------------------------------------------------------------------
%% Description: Since Common Test starts another Test Server
%% instance, the tests need to be performed on a separate node (or
%% there will be clashes with logging processes etc).
%%--------------------------------------------------------------------
init_per_suite(Config) ->
    Config1 = ct_test_support:init_per_suite(Config),
    Config1.

end_per_suite(Config) ->
    ct_test_support:end_per_suite(Config).

init_per_testcase(TestCase, Config) ->
    ct_test_support:init_per_testcase(TestCase, Config).

end_per_testcase(TestCase, Config) ->
    ct_test_support:end_per_testcase(TestCase, Config).

suite() -> [{ct_hooks,[ts_install_cth]}].

all() -> 
    [
     simple_group_opt,
     simple_group_case_opt,
     override_with_all,
     override_with_spec
    ].

%%--------------------------------------------------------------------
%% TEST CASES
%%--------------------------------------------------------------------

%%%-----------------------------------------------------------------
%%% 
simple_group_opt(Config) when is_list(Config) -> 
    DataDir = ?config(data_dir, Config),
    Suite = filename:join(DataDir, "groups_spec_1_SUITE"),
    {Opts,ERPid} = setup([{suite,Suite},{group,[g1,g5]},
			  {label,simple_group_opt}], Config),
    ok = execute(simple_group_opt, Opts, ERPid, Config).

%%%-----------------------------------------------------------------
%%% 
simple_group_case_opt(Config) when is_list(Config) -> 
    DataDir = ?config(data_dir, Config),
    Suite = filename:join(DataDir, "groups_spec_1_SUITE"),
    {Opts,ERPid} = setup([{suite,Suite},{group,g5},{testcase,[t52,t54]},
			  {label,simple_group_case_opt}], Config),
    ok = execute(simple_group_case_opt, Opts, ERPid, Config).

%%%-----------------------------------------------------------------
%%% 
override_with_all(Config) when is_list(Config) -> 
    DataDir = ?config(data_dir, Config),
    Suite = filename:join(DataDir, "groups_spec_1_SUITE"),
    {Opts,ERPid} = setup([{suite,Suite},{label,override_with_all}], Config),
    ok = execute(override_with_all, Opts, ERPid, Config).

%%%-----------------------------------------------------------------
%%% 
override_with_spec(Config) when is_list(Config) -> 
    DataDir = ?config(data_dir, Config),
    Spec = filename:join(DataDir, "override.spec"),
    {Opts,ERPid} = setup([{spec,Spec},{label,override_with_spec}], Config),
    ok = execute(override_with_spec, Opts, ERPid, Config).

%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
%%%-----------------------------------------------------------------

setup(Test, Config) ->
    Opts0 = ct_test_support:get_opts(Config),
    Level = ?config(trace_level, Config),
    EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
    Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
    ERPid = ct_test_support:start_event_receiver(Config),
    {Opts,ERPid}.

execute(Name, Opts, ERPid, Config) ->
    ok = ct_test_support:run(Opts, Config),
    Events = ct_test_support:get_events(ERPid, Config),

    ct_test_support:log_events(Name, 
			       reformat(Events, ?eh),
			       ?config(priv_dir, Config),
			       Opts),

    TestEvents = events_to_check(Name),
    ct_test_support:verify_events(TestEvents, Events, Config).

reformat(Events, EH) ->
    ct_test_support:reformat(Events, EH).

%%%-----------------------------------------------------------------
%%% TEST EVENTS
%%%-----------------------------------------------------------------
events_to_check(Test) ->
    %% 2 tests (ct:run_test + script_start) is default
    events_to_check(Test, 2).

events_to_check(_, 0) ->
    [];
events_to_check(Test, N) ->
    test_events(Test) ++ events_to_check(Test, N-1).


test_events(simple_group_opt) ->
    [
     {?eh,start_logging,{'DEF','RUNDIR'}},
     {?eh,start_info,{1,1,7}},
     {?eh,tc_done,{groups_spec_1_SUITE,init_per_suite,ok}},

     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g1,[]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g1,[]},ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t11,ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t12,{failed,{error,crashes}}}},
      {?eh,tc_done,{groups_spec_1_SUITE,t13,ok}},
      {?eh,test_stats,{2,1,{0,0}}},
      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g1,[]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g1,[]},ok}}],

     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g2,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g2,[sequence]},ok}},

      [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g4,[]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g4,[]},ok}},

       {parallel,
	[{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g5,[parallel]}}},
	 {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g5,[parallel]},ok}},
	 {?eh,tc_start,{groups_spec_1_SUITE,t51}},
	 {?eh,tc_done,{groups_spec_1_SUITE,t51,ok}},
	 {?eh,tc_start,{groups_spec_1_SUITE,t52}},
	 {?eh,tc_done,{groups_spec_1_SUITE,t52,{failed,{timetrap_timeout,2000}}}},
	 {?eh,tc_start,{groups_spec_1_SUITE,t53}},
	 {?eh,tc_done,{groups_spec_1_SUITE,t53,{failed,{error,crashes}}}},
	 {?eh,tc_start,{groups_spec_1_SUITE,t54}},
	 {?eh,tc_done,{groups_spec_1_SUITE,t54,ok}},
	 {?eh,test_stats,{4,3,{0,0}}},
	 {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g5,[parallel]}}},
	 {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g5,[parallel]},ok}}]},

       {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g4,[]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g4,[]},ok}}],

      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g2,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g2,[sequence]},ok}}],

     {?eh,tc_done,{groups_spec_1_SUITE,end_per_suite,ok}},
     {?eh,test_done,{'DEF','STOP_TIME'}},
     {?eh,stop_logging,[]}
    ];

test_events(simple_group_case_opt) ->
    [
     {?eh,start_logging,{'DEF','RUNDIR'}},
     {?eh,start_info,{1,1,2}},
     {?eh,tc_done,{groups_spec_1_SUITE,init_per_suite,ok}},

     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g2,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g2,[sequence]},ok}},

      [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g4,[]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g4,[]},ok}},

       {parallel,
	[{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g5,[parallel]}}},
	 {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g5,[parallel]},ok}},
	 {?eh,tc_start,{groups_spec_1_SUITE,t52}},
	 {?eh,tc_done,{groups_spec_1_SUITE,t52,{failed,{timetrap_timeout,2000}}}},
	 {?eh,tc_start,{groups_spec_1_SUITE,t54}},
	 {?eh,tc_done,{groups_spec_1_SUITE,t54,ok}},
	 {?eh,test_stats,{1,1,{0,0}}},
	 {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g5,[parallel]}}},
	 {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g5,[parallel]},ok}}]},

       {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g4,[]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g4,[]},ok}}],

      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g2,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g2,[sequence]},ok}}],

     {?eh,tc_done,{groups_spec_1_SUITE,end_per_suite,ok}},
     {?eh,test_done,{'DEF','STOP_TIME'}},
     {?eh,stop_logging,[]}
    ];

test_events(override_with_all) ->
    [
     {?eh,start_logging,{'DEF','RUNDIR'}},
     {?eh,start_info,{1,1,45}},
     {?eh,tc_done,{groups_spec_1_SUITE,init_per_suite,ok}},

     %% TEST: {group,g1,default}
     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g1,[]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g1,[]},ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t11,ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t12,{failed,{error,crashes}}}},
      {?eh,tc_done,{groups_spec_1_SUITE,t13,ok}},
      {?eh,test_stats,{2,1,{0,0}}},
      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g1,[]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g1,[]},ok}}],

     %% TEST: {group,g1,[sequence]}
     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g1,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g1,[sequence]},ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t11,ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t12,{failed,{error,crashes}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t13,g1},
			 {failed,{groups_spec_1_SUITE,t12}}}},
      {?eh,test_stats,{3,2,{0,1}}},
      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g1,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g1,[sequence]},ok}}],

     %% TEST: {group,g1,[parallel],[]}
     {parallel,
      [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g1,[parallel]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g1,[parallel]},ok}},
       {?eh,tc_start,{groups_spec_1_SUITE,t11}},
       {?eh,tc_done,{groups_spec_1_SUITE,t11,ok}},
       {?eh,tc_start,{groups_spec_1_SUITE,t12}},
       {?eh,tc_done,{groups_spec_1_SUITE,t12,{failed,{error,crashes}}}},
       {?eh,tc_start,{groups_spec_1_SUITE,t13}},
       {?eh,tc_done,{groups_spec_1_SUITE,t13,ok}},
       {?eh,test_stats,{5,3,{0,1}}},
       {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g1,[parallel]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g1,[parallel]},ok}}]},

     %% TEST: {group,g2,[],[]}
     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g2,[]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g2,[]},ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t21,ok}},
      {?eh,test_stats,{6,3,{0,1}}},

      {parallel,
       [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g3,[parallel]}}},
	{?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g3,[parallel]},ok}},
	{?eh,tc_start,{groups_spec_1_SUITE,t31}},
	{?eh,tc_done,{groups_spec_1_SUITE,t31,ok}},
	{?eh,tc_start,{groups_spec_1_SUITE,t32}},
	{?eh,tc_done,{groups_spec_1_SUITE,t32,{failed,{error,crashes}}}},
	{?eh,tc_start,{groups_spec_1_SUITE,t33}},
	{?eh,tc_done,{groups_spec_1_SUITE,t33,ok}},
	{?eh,test_stats,{8,4,{0,1}}},
	{?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g3,[parallel]}}},
	{?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g3,[parallel]},ok}}]},

      {?eh,tc_done,{groups_spec_1_SUITE,t22,{failed,{error,crashes}}}},
      {?eh,test_stats,{8,5,{0,1}}},

      [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g4,[]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g4,[]},ok}},
       {?eh,tc_done,{groups_spec_1_SUITE,t41,ok}},
       {?eh,test_stats,{9,5,{0,1}}},

       {parallel,
	[{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g5,[parallel]}}},
	 {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g5,[parallel]},ok}},
	 {?eh,tc_start,{groups_spec_1_SUITE,t51}},
	 {?eh,tc_done,{groups_spec_1_SUITE,t51,ok}},
	 {?eh,tc_start,{groups_spec_1_SUITE,t52}},
	 {?eh,tc_done,{groups_spec_1_SUITE,t52,{failed,{timetrap_timeout,2000}}}},
	 {?eh,tc_start,{groups_spec_1_SUITE,t53}},
	 {?eh,tc_done,{groups_spec_1_SUITE,t53,{failed,{error,crashes}}}},
	 {?eh,tc_start,{groups_spec_1_SUITE,t54}},
	 {?eh,tc_done,{groups_spec_1_SUITE,t54,ok}},
	 {?eh,test_stats,{11,7,{0,1}}},
	 {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g5,[parallel]}}},
	 {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g5,[parallel]},ok}}]},

       {?eh,tc_done,{groups_spec_1_SUITE,t42,{failed,{error,crashes}}}},
       {?eh,test_stats,{11,8,{0,1}}},
       {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g4,[]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g4,[]},ok}}],

      {?eh,tc_done,{groups_spec_1_SUITE,t23,ok}},
      {?eh,test_stats,{12,8,{0,1}}},
      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g2,[]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g2,[]},ok}}],

     %% TEST: {group,g2,default,[{g3,[sequence]}]}
     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g2,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g2,[sequence]},ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t21,ok}},
      {?eh,test_stats,{13,8,{0,1}}},

      [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g3,[sequence]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g3,[sequence]},ok}},
       {?eh,tc_done,{groups_spec_1_SUITE,t31,ok}},
       {?eh,tc_done,{groups_spec_1_SUITE,t32,{failed,{error,crashes}}}},
       {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t33,g3},
			  {failed,{groups_spec_1_SUITE,t32}}}},
       {?eh,test_stats,{14,9,{0,2}}},
       {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g3,[sequence]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g3,[sequence]},ok}}],

      {?eh,tc_done,{groups_spec_1_SUITE,t22,{failed,{error,crashes}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t41,g4},
			 {failed,{groups_spec_1_SUITE,t22}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t51,g5},
			 {failed,{groups_spec_1_SUITE,t22}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t52,g5},
			 {failed,{groups_spec_1_SUITE,t22}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t53,g5},
			 {failed,{groups_spec_1_SUITE,t22}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t54,g5},
			 {failed,{groups_spec_1_SUITE,t22}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t42,g4},
			 {failed,{groups_spec_1_SUITE,t22}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t23,g2},
			 {failed,{groups_spec_1_SUITE,t22}}}},
      {?eh,test_stats,{14,10,{0,9}}},

      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g2,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g2,[sequence]},ok}}],

     %% TEST: {group,g2,[],[{g4,[sequence],[{g5,[sequence]}]},{g3,[sequence]}]}
     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g2,[]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g2,[]},ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t21,ok}},
      {?eh,test_stats,{15,10,{0,9}}},

      [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g3,[sequence]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g3,[sequence]},ok}},
       {?eh,tc_done,{groups_spec_1_SUITE,t31,ok}},
       {?eh,tc_done,{groups_spec_1_SUITE,t32,{failed,{error,crashes}}}},
       {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t33,g3},
			  {failed,{groups_spec_1_SUITE,t32}}}},
       {?eh,test_stats,{16,11,{0,10}}},
       {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g3,[sequence]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g3,[sequence]},ok}}],

      {?eh,tc_done,{groups_spec_1_SUITE,t22,{failed,{error,crashes}}}},
      {?eh,test_stats,{16,12,{0,10}}},

      [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g4,[sequence]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g4,[sequence]},ok}},
       {?eh,tc_done,{groups_spec_1_SUITE,t41,ok}},
       {?eh,test_stats,{17,12,{0,10}}},

       [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g5,[sequence]}}},
	{?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g5,[sequence]},ok}},
	{?eh,tc_done,{groups_spec_1_SUITE,t51,ok}},
	{?eh,tc_done,{groups_spec_1_SUITE,t52,{failed,{timetrap_timeout,2000}}}},
	{?eh,tc_auto_skip,{groups_spec_1_SUITE,{t53,g5},
			   {failed,{groups_spec_1_SUITE,t52}}}},
	{?eh,tc_auto_skip,{groups_spec_1_SUITE,{t54,g5},
			   {failed,{groups_spec_1_SUITE,t52}}}},
	{?eh,test_stats,{18,13,{0,12}}},
	{?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g5,[sequence]}}},
	{?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g5,[sequence]},ok}}],

       {?eh,tc_done,{groups_spec_1_SUITE,t42,{failed,{error,crashes}}}},
       {?eh,test_stats,{18,14,{0,12}}},
       {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g4,[sequence]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g4,[sequence]},ok}}],

      {?eh,tc_done,{groups_spec_1_SUITE,t23,ok}},
      {?eh,test_stats,{19,14,{0,12}}},
      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g2,[]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g2,[]},ok}}],

     {?eh,tc_done,{groups_spec_1_SUITE,end_per_suite,ok}},
     {?eh,test_done,{'DEF','STOP_TIME'}},
     {?eh,stop_logging,[]}
    ];

test_events(override_with_spec) ->
    [
     {?eh,start_logging,{'DEF','RUNDIR'}},
     {?eh,start_info,{7,4,49}},

     %% TEST: {groups, dir, groups_spec_1_SUITE, {g1,default}}.
     {?eh,tc_done,{groups_spec_1_SUITE,init_per_suite,ok}},
     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g1,[]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g1,[]},ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t11,ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t12,{failed,{error,crashes}}}},
      {?eh,tc_done,{groups_spec_1_SUITE,t13,ok}},
      {?eh,test_stats,{2,1,{0,0}}},
      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g1,[]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g1,[]},ok}}],
     {?eh,tc_done,{groups_spec_1_SUITE,end_per_suite,ok}},

     %% TEST: {groups, dir, groups_spec_1_SUITE, [{g1,[sequence]},
     %%                                           {g1,[parallel],[]}]}.
     {?eh,tc_done,{groups_spec_1_SUITE,init_per_suite,ok}},
     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g1,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g1,[sequence]},ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t11,ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t12,{failed,{error,crashes}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t13,g1},
			 {failed,{groups_spec_1_SUITE,t12}}}},
      {?eh,test_stats,{3,2,{0,1}}},
      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g1,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g1,[sequence]},ok}}],
     {parallel,
      [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g1,[parallel]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g1,[parallel]},ok}},
       {?eh,tc_start,{groups_spec_1_SUITE,t11}},
       {?eh,tc_done,{groups_spec_1_SUITE,t11,ok}},
       {?eh,tc_start,{groups_spec_1_SUITE,t12}},
       {?eh,tc_done,{groups_spec_1_SUITE,t12,{failed,{error,crashes}}}},
       {?eh,tc_start,{groups_spec_1_SUITE,t13}},
       {?eh,tc_done,{groups_spec_1_SUITE,t13,ok}},
       {?eh,test_stats,{5,3,{0,1}}},
       {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g1,[parallel]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g1,[parallel]},ok}}]},
     {?eh,tc_done,{groups_spec_1_SUITE,end_per_suite,ok}},

     %% TEST: {groups, dir, groups_spec_1_SUITE, {g2,[],[]}}.
     {?eh,tc_done,{groups_spec_1_SUITE,init_per_suite,ok}},
     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g2,[]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g2,[]},ok}},
      {?eh,tc_start,{groups_spec_1_SUITE,t21}},
      {?eh,test_stats,{6,3,{0,1}}},
      {parallel,
       [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g3,[parallel]}}},
	{?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g3,[parallel]},ok}},
	{?eh,tc_start,{groups_spec_1_SUITE,t31}},
	{?eh,tc_done,{groups_spec_1_SUITE,t31,ok}},
	{?eh,tc_start,{groups_spec_1_SUITE,t32}},
	{?eh,tc_done,{groups_spec_1_SUITE,t32,{failed,{error,crashes}}}},
	{?eh,tc_start,{groups_spec_1_SUITE,t33}},
	{?eh,tc_done,{groups_spec_1_SUITE,t33,ok}},
	{?eh,test_stats,{8,4,{0,1}}},
	{?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g3,[parallel]}}},
	{?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g3,[parallel]},ok}}]},
      {?eh,tc_done,{groups_spec_1_SUITE,t22,{failed,{error,crashes}}}},
      {?eh,test_stats,{8,5,{0,1}}},
      [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g4,[]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g4,[]},ok}},
       {?eh,tc_done,{groups_spec_1_SUITE,t41,ok}},
       {?eh,test_stats,{9,5,{0,1}}},
       {parallel,
	[{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g5,[parallel]}}},
	 {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g5,[parallel]},ok}},
	 {?eh,tc_start,{groups_spec_1_SUITE,t51}},
	 {?eh,tc_done,{groups_spec_1_SUITE,t51,ok}},
	 {?eh,tc_start,{groups_spec_1_SUITE,t52}},
	 {?eh,tc_done,{groups_spec_1_SUITE,t52,{failed,{timetrap_timeout,2000}}}},
	 {?eh,tc_start,{groups_spec_1_SUITE,t53}},
	 {?eh,tc_done,{groups_spec_1_SUITE,t53,{failed,{error,crashes}}}},
	 {?eh,tc_start,{groups_spec_1_SUITE,t54}},
	 {?eh,tc_done,{groups_spec_1_SUITE,t54,ok}},
	 {?eh,test_stats,{11,7,{0,1}}},
	 {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g5,[parallel]}}},
	 {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g5,[parallel]},ok}}]},
       {?eh,tc_done,{groups_spec_1_SUITE,t42,{failed,{error,crashes}}}},
       {?eh,test_stats,{11,8,{0,1}}},
       {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g4,[]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g4,[]},ok}}],
      {?eh,tc_done,{groups_spec_1_SUITE,t23,ok}},
      {?eh,test_stats,{12,8,{0,1}}},
      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g2,[]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g2,[]},ok}}],
     {?eh,tc_done,{groups_spec_1_SUITE,end_per_suite,ok}},

     %% TEST: {groups, dir, groups_spec_1_SUITE, {g2,default,[{g3,[sequence]}]}}
     {?eh,tc_done,{groups_spec_1_SUITE,init_per_suite,ok}},
     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g2,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g2,[sequence]},ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t21,ok}},
      {?eh,test_stats,{13,8,{0,1}}},
      [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g3,[sequence]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g3,[sequence]},ok}},
       {?eh,tc_done,{groups_spec_1_SUITE,t31,ok}},
       {?eh,tc_done,{groups_spec_1_SUITE,t32,{failed,{error,crashes}}}},
       {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t33,g3},
			  {failed,{groups_spec_1_SUITE,t32}}}},
       {?eh,test_stats,{14,9,{0,2}}},
       {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g3,[sequence]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g3,[sequence]},ok}}],
      {?eh,tc_done,{groups_spec_1_SUITE,t22,{failed,{error,crashes}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t41,g4},
			 {failed,{groups_spec_1_SUITE,t22}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t51,g5},
			 {failed,{groups_spec_1_SUITE,t22}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t52,g5},
			 {failed,{groups_spec_1_SUITE,t22}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t53,g5},
			 {failed,{groups_spec_1_SUITE,t22}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t54,g5},
			 {failed,{groups_spec_1_SUITE,t22}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t42,g4},
			 {failed,{groups_spec_1_SUITE,t22}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t23,g2},
			 {failed,{groups_spec_1_SUITE,t22}}}},
      {?eh,test_stats,{14,10,{0,9}}},
      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g2,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g2,[sequence]},ok}}],
     {?eh,tc_done,{groups_spec_1_SUITE,end_per_suite,ok}},

     %% TEST: {groups, dir, groups_spec_1_SUITE,
     %%        {g2,[],[{g4,[sequence],[{g5,[sequence]}]},{g3,[sequence]}]}}.
     {?eh,tc_done,{groups_spec_1_SUITE,init_per_suite,ok}},
     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g2,[]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g2,[]},ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t21,ok}},
      {?eh,test_stats,{15,10,{0,9}}},
      [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g3,[sequence]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g3,[sequence]},ok}},
       {?eh,tc_done,{groups_spec_1_SUITE,t31,ok}},
       {?eh,tc_done,{groups_spec_1_SUITE,t32,{failed,{error,crashes}}}},
       {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t33,g3},
			  {failed,{groups_spec_1_SUITE,t32}}}},
       {?eh,test_stats,{16,11,{0,10}}},
       {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g3,[sequence]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g3,[sequence]},ok}}],
      {?eh,tc_done,{groups_spec_1_SUITE,t22,{failed,{error,crashes}}}},
      {?eh,test_stats,{16,12,{0,10}}},
      [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g4,[sequence]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g4,[sequence]},ok}},
       {?eh,tc_done,{groups_spec_1_SUITE,t41,ok}},
       {?eh,test_stats,{17,12,{0,10}}},
       [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g5,[sequence]}}},
	{?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g5,[sequence]},ok}},
	{?eh,tc_done,{groups_spec_1_SUITE,t51,ok}},
	{?eh,tc_done,{groups_spec_1_SUITE,t52,{failed,{timetrap_timeout,2000}}}},
	{?eh,tc_auto_skip,{groups_spec_1_SUITE,{t53,g5},
			   {failed,{groups_spec_1_SUITE,t52}}}},
	{?eh,tc_auto_skip,{groups_spec_1_SUITE,{t54,g5},
			   {failed,{groups_spec_1_SUITE,t52}}}},
	{?eh,test_stats,{18,13,{0,12}}},
	{?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g5,[sequence]}}},
	{?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g5,[sequence]},ok}}],
       {?eh,tc_done,{groups_spec_1_SUITE,t42,{failed,{error,crashes}}}},
       {?eh,test_stats,{18,14,{0,12}}},
       {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g4,[sequence]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g4,[sequence]},ok}}],
      {?eh,tc_done,{groups_spec_1_SUITE,t23,ok}},
      {?eh,test_stats,{19,14,{0,12}}},
      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g2,[]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g2,[]},ok}}],
     {?eh,tc_done,{groups_spec_1_SUITE,end_per_suite,ok}},

     %% TEST: {groups, dir, groups_spec_1_SUITE, {g1,[sequence]}, {cases,[t12,t13]}}
     {?eh,tc_done,{groups_spec_1_SUITE,init_per_suite,ok}},
     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g1,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g1,[sequence]},ok}},
      {?eh,tc_done,{groups_spec_1_SUITE,t12,{failed,{error,crashes}}}},
      {?eh,tc_auto_skip,{groups_spec_1_SUITE,{t13,g1},
			 {failed,{groups_spec_1_SUITE,t12}}}},
      {?eh,test_stats,{19,15,{0,13}}},
      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g1,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g1,[sequence]},ok}}],
     {?eh,tc_done,{groups_spec_1_SUITE,end_per_suite,ok}},

     %% TEST: {groups, dir, groups_spec_1_SUITE, {g5,[]}, {cases,[t53,t54]}}
     {?eh,tc_done,{groups_spec_1_SUITE,init_per_suite,ok}},
     [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g2,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g2,[sequence]},ok}},
      [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g4,[]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g4,[]},ok}},
       [{?eh,tc_start,{groups_spec_1_SUITE,{init_per_group,g5,[]}}},
	{?eh,tc_done,{groups_spec_1_SUITE,{init_per_group,g5,[]},ok}},
	{?eh,tc_done,{groups_spec_1_SUITE,t53,{failed,{error,crashes}}}},
	{?eh,tc_done,{groups_spec_1_SUITE,t54,ok}},
	{?eh,test_stats,{20,16,{0,13}}},
	{?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g5,[]}}},
	{?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g5,[]},ok}}],
       {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g4,[]}}},
       {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g4,[]},ok}}],
      {?eh,tc_start,{groups_spec_1_SUITE,{end_per_group,g2,[sequence]}}},
      {?eh,tc_done,{groups_spec_1_SUITE,{end_per_group,g2,[sequence]},ok}}],
     {?eh,tc_done,{groups_spec_1_SUITE,end_per_suite,ok}},

     {?eh,test_done,{'DEF','STOP_TIME'}},
     {?eh,stop_logging,[]}
    ].