1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
|
%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 2003-2016. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
%% %CopyrightEnd%
%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% %%%
%%% WARNING %%%
%%% %%%
%%% This is experimental code which may be changed or removed %%%
%%% anytime without any warning. %%%
%%% %%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%% @doc EXPERIMENTAL support in common-test for calling property based tests.
%%%
%%% <p>This module is a first step towards running Property Based testing in the
%%% Common Test framework. A property testing tool like QuickCheck or PropEr is
%%% assumed to be installed.</p>
%%%
%%% <p>The idea is to have a common_test testsuite calling a property testing
%%% tool with special property test suites as defined by that tool. In this manual
%%% we assume the usual Erlang Application directory structure. The tests are
%%% collected in the application's <c>test</c> directory. The test directory
%%% has a sub-directory called <c>property_test</c> where everything needed for
%%% the property tests are collected.</p>
%%%
%%% <p>A typical ct test suite using <c>ct_property_test</c> is organized as follows:
%%% </p>
%%% ```
%%% -include_lib("common_test/include/ct.hrl").
%%%
%%% all() -> [prop_ftp_case].
%%%
%%% init_per_suite(Config) ->
%%% ct_property_test:init_per_suite(Config).
%%%
%%% %%%---- test case
%%% prop_ftp_case(Config) ->
%%% ct_property_test:quickcheck(
%%% ftp_simple_client_server:prop_ftp(Config),
%%% Config
%%% ).
%%% '''
%%%
%%% <warning>
%%% <p>
%%% This is experimental code which may be changed or removed
%%% anytime without any warning.
%%% </p>
%%% </warning>
%%%
%%% @end
-module(ct_property_test).
%% API
-export([init_per_suite/1,
quickcheck/2]).
-include_lib("common_test/include/ct.hrl").
%%%-----------------------------------------------------------------
%%% @spec init_per_suite(Config) -> Config | {skip,Reason}
%%%
%%% @doc Initializes Config for property testing.
%%%
%%% <p>The function investigates if support is available for either Quickcheck, PropEr,
%%% or Triq.
%%% The options <c>{property_dir,AbsPath}</c> and
%%% <c>{property_test_tool,Tool}</c> is set in the Config returned.</p>
%%% <p>The function is intended to be called in the init_per_suite in the test suite.</p>
%%% <p>The property tests are assumed to be in the subdirectory <c>property_test</c>.</p>
%%% @end
init_per_suite(Config) ->
case which_module_exists([eqc,proper,triq]) of
{ok,ToolModule} ->
ct:pal("Found property tester ~p",[ToolModule]),
Path = property_tests_path("property_test", Config),
case compile_tests(Path,ToolModule) of
error ->
{fail, "Property test compilation failed in "++Path};
up_to_date ->
add_code_pathz(Path),
[{property_dir,Path},
{property_test_tool,ToolModule} | Config]
end;
not_found ->
ct:pal("No property tester found",[]),
{skip, "No property testing tool found"}
end.
%%%-----------------------------------------------------------------
%%% @spec quickcheck(Property, Config) -> true | {fail,Reason}
%%%
%%% @doc Call quickcheck and return the result in a form suitable for common_test.
%%%
%%% <p>The function is intended to be called in the test cases in the test suite.</p>
%%% @end
quickcheck(Property, Config) ->
Tool = proplists:get_value(property_test_tool,Config),
F = function_name(quickcheck, Tool),
mk_ct_return( Tool:F(Property), Tool ).
%%%================================================================
%%%
%%% Local functions
%%%
%%% Make return values back to the calling Common Test suite
mk_ct_return(true, _Tool) ->
true;
mk_ct_return(Other, Tool) ->
try lists:last(hd(Tool:counterexample()))
of
{set,{var,_},{call,M,F,Args}} ->
{fail, io_lib:format("~p:~tp/~p returned bad result",[M,F,length(Args)])}
catch
_:_ ->
{fail, Other}
end.
%%% Check if a property testing tool is found
which_module_exists([Module|Modules]) ->
case module_exists(Module) of
true -> {ok,Module};
false -> which_module_exists(Modules)
end;
which_module_exists(_) ->
not_found.
module_exists(Module) ->
is_list(catch Module:module_info()).
%%% The path to the property tests
property_tests_path(Dir, Config) ->
DataDir = proplists:get_value(data_dir, Config),
filename:join(lists:droplast(filename:split(DataDir))++[Dir]).
%%% Extend the code path with Dir if it not already present
add_code_pathz(Dir) ->
case lists:member(Dir, code:get_path()) of
true -> ok;
false ->
true = code:add_pathz(Dir),
ok
end.
compile_tests(Path, ToolModule) ->
MacroDefs = macro_def(ToolModule),
{ok,Cwd} = file:get_cwd(),
ok = file:set_cwd(Path),
{ok,FileNames} = file:list_dir("."),
BeamFiles = [F || F<-FileNames,
filename:extension(F) == ".beam"],
_ = [file:delete(F) || F<-BeamFiles],
ct:pal("Compiling in ~tp:~n Deleted ~p~n MacroDefs=~p",[Path,BeamFiles,MacroDefs]),
Result = make:all([load|MacroDefs]),
ok = file:set_cwd(Cwd),
Result.
macro_def(eqc) -> [{d, 'EQC'}];
macro_def(proper) -> [{d, 'PROPER'}];
macro_def(triq) -> [{d, 'TRIQ'}].
function_name(quickcheck, triq) -> check;
function_name(F, _) -> F.
|
