aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--doc/src/guide/common_test.asciidoc89
-rw-r--r--plugins/ct.mk24
-rw-r--r--test/Makefile40
-rw-r--r--test/plugin_ct.mk219
4 files changed, 328 insertions, 44 deletions
diff --git a/doc/src/guide/common_test.asciidoc b/doc/src/guide/common_test.asciidoc
index 7daeae5..3f00baf 100644
--- a/doc/src/guide/common_test.asciidoc
+++ b/doc/src/guide/common_test.asciidoc
@@ -1,5 +1,90 @@
== Common Test
-// @todo Write it.
+Common Test is Erlang's functional testing framework.
+Erlang.mk automates the discovery and running of Common
+Test suites.
-Placeholder chapter.
+=== Writing tests
+
+The http://www.erlang.org/doc/apps/common_test/write_test_chapter.html[Common Test user guide]
+is the best place to learn how to write tests. Erlang.mk
+requires that file names for test suites end with '_SUITE.erl'
+and that the files be located in the '$(TEST_DIR)' directory.
+This defaults to 'test/'.
+
+=== Configuration
+
+The `CT_OPTS` variable allows you to set extra Common Test
+options. Options are documented in the
+http://www.erlang.org/doc/apps/common_test/run_test_chapter.html[Common Test user guide].
+You can use it to set Common Test hooks, for example:
+
+[source,make]
+CT_OPTS = -ct_hooks cowboy_ct_hook
+
+The `CT_SUITES` variable can be used to override what
+Common Test suites Erlang.mk will be aware of. It does
+not normally need to be set as Erlang.mk will find the
+test suites automatically.
+
+The name of the suite is the part before `_SUITE.erl`.
+If the file is named 'http_SUITE.erl', the test suite
+is `http`:
+
+[source,make]
+CT_SUITES = http ws
+
+=== Usage
+
+To run all tests (including Common Test):
+
+[source,bash]
+$ make tests
+
+To run all tests and static checks (including Common Test):
+
+[source,bash]
+$ make check
+
+You can also run Common Test separately:
+
+[source,bash]
+$ make ct
+
+Erlang.mk will create targets for all test suites it finds.
+If you have a file named 'test/http_SUITE.erl', then the
+target `ct-http` will run that specific test suite:
+
+[source,bash]
+$ make ct-http
+
+Erlang.mk provides a convenient way to run a specific
+group or a specific test case within a specific group,
+using the variable `t`. Note that this only applies to
+suite-specific targets, like the `ct-http` example above.
+
+To run all tests from the `http_compress` group in the
+`http_SUITE` test suite, write:
+
+[source,bash]
+$ make ct-http t=http_compress
+
+Similarly, to run a specific test case in that group:
+
+[source,bash]
+$ make ct-http t=http_compress:headers_dupe
+
+To do the same against a multi-application repository,
+you can use the `-C` option:
+
+[source,bash]
+$ make -C apps/my_app ct-http t=my_group:my_case
+
+Note that this also applies to dependencies. When using Cowboy
+as a dependency, you can run the following directly:
+
+[source,bash]
+$ make -C deps/cowboy ct-http t=http_compress
+
+Finally, link:coverage.asciidoc[code coverage] is available,
+but covered in its own chapter.
diff --git a/plugins/ct.mk b/plugins/ct.mk
index fb4b1ca..2d45032 100644
--- a/plugins/ct.mk
+++ b/plugins/ct.mk
@@ -1,7 +1,7 @@
# Copyright (c) 2013-2015, Loïc Hoguin <[email protected]>
# This file is part of erlang.mk and subject to the terms of the ISC License.
-.PHONY: ct distclean-ct
+.PHONY: ct apps-ct distclean-ct
# Configuration.
@@ -36,17 +36,33 @@ CT_RUN = ct_run \
-logdir $(CURDIR)/logs
ifeq ($(CT_SUITES),)
-ct:
+ct: $(if $(IS_APP),,apps-ct)
else
-ct: test-build
+ct: test-build $(if $(IS_APP),,apps-ct)
$(verbose) mkdir -p $(CURDIR)/logs/
$(gen_verbose) $(CT_RUN) -suite $(addsuffix _SUITE,$(CT_SUITES)) $(CT_OPTS)
endif
+ifneq ($(ALL_APPS_DIRS),)
+apps-ct:
+ $(verbose) for app in $(ALL_APPS_DIRS); do $(MAKE) -C $$app ct IS_APP=1; done
+endif
+
+ifndef t
+CT_EXTRA =
+else
+ifeq (,$(findstring :,$t))
+CT_EXTRA = -group $t
+else
+t_words = $(subst :, ,$t)
+CT_EXTRA = -group $(firstword $(t_words)) -case $(lastword $(t_words))
+endif
+endif
+
define ct_suite_target
ct-$(1): test-build
$(verbose) mkdir -p $(CURDIR)/logs/
- $(gen_verbose) $(CT_RUN) -suite $(addsuffix _SUITE,$(1)) $(CT_OPTS)
+ $(gen_verbose) $(CT_RUN) -suite $(addsuffix _SUITE,$(1)) $(CT_EXTRA) $(CT_OPTS)
endef
$(foreach test,$(CT_SUITES),$(eval $(call ct_suite_target,$(test))))
diff --git a/test/Makefile b/test/Makefile
index f7ecf64..728d751 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -303,49 +303,13 @@ endef
# The following tests are slowly being converted.
# Do NOT use -j with legacy tests.
-.PHONY: legacy clean-legacy ct tests-cover docs
+.PHONY: legacy clean-legacy tests-cover docs
-legacy: clean-legacy ct tests-cover docs pkgs
+legacy: clean-legacy tests-cover docs pkgs
clean-legacy:
$t rm -rf app1
-ct: app1
- $i "ct: Testing ct and related targets."
- $i "Setting up test suite."
- $t mkdir -p app1/test
- $t printf "%s\n" \
- "-module(m_SUITE)." \
- "-export([all/0, testcase1/1])." \
- "all() -> [testcase1]." \
- "testcase1(_) -> 2 = m:succ(1)." \
- > app1/test/m_SUITE.erl
- $t $(MAKE) -C app1 ct $v
- $i "Checking files created by '$(MAKE) ct'."
- $t [ -e app1/test/m_SUITE.beam ]
- $t [ -e app1/ebin/m.beam ]
- $t [ -e app1/logs ]
- $i "Checking that '$(MAKE) clean' does not delete logs."
- $t $(MAKE) -C app1 clean $v
- $t [ -e app1/logs ]
- $i "Testing target 'ct-mysuite' where mysuite_SUITE is a test suite."
- $t $(MAKE) -C app1 ct-m $v
- $i "Checking that '$(MAKE) ct' returns non-zero for a failing suite."
- $t printf "%s\n" \
- "-module(failing_SUITE)." \
- "-export([all/0, testcase1/1])." \
- "all() -> [testcase1]." \
- "testcase1(_) -> 42 = m:succ(1)." \
- > app1/test/failing_SUITE.erl
- $t ! $(MAKE) -C app1 ct-failing $v
- $i "Checking that '$(MAKE) distclean-ct' deletes logs."
- $t $(MAKE) -C app1 distclean-ct $v
- $t [ ! -e app1/logs ]
- $t [ -e app1/ebin/m.beam ]
- $i "Cleaning up test data."
- $t rm -rf app1/test
- $i "Test 'ct' passed."
-
# TODO: do coverage for 'tests' instead of 'eunit ct' when triq is fixed
tests-cover: app1
$i "tests-cover: Testing 'eunit' and 'ct' with COVER=1"
diff --git a/test/plugin_ct.mk b/test/plugin_ct.mk
new file mode 100644
index 0000000..dd36801
--- /dev/null
+++ b/test/plugin_ct.mk
@@ -0,0 +1,219 @@
+# Common Test plugin.
+
+CT_CASES = all apps-only case check group opts suite tests
+CT_TARGETS = $(addprefix ct-,$(CT_CASES))
+
+.PHONY: ct $(CT_TARGETS)
+
+ct: $(CT_TARGETS)
+
+ct-all: build clean
+
+ $i "Bootstrap a new OTP application named $(APP)"
+ $t mkdir $(APP)/
+ $t cp ../erlang.mk $(APP)/
+ $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v
+
+ $i "Check that Common Test detects no tests"
+ $t $(MAKE) -C $(APP) ct | grep -q "Nothing to be done for 'ct'."
+
+ $i "Generate a Common Test suite"
+ $t mkdir $(APP)/test
+ $t printf "%s\n" \
+ "-module($(APP)_SUITE)." \
+ "-export([all/0, ok/1])." \
+ "all() -> [ok]." \
+ "ok(_) -> ok." > $(APP)/test/$(APP)_SUITE.erl
+
+ $i "Check that Common Test runs tests"
+# We can't pipe CT's output without it crashing, so let's check that
+# the command succeeds and log files are created instead.
+ $t test ! -e $(APP)/logs/index.html
+ $t $(MAKE) -C $(APP) ct $v
+ $t test -f $(APP)/logs/index.html
+
+ $i "Generate a Common Test suite with a failing test case"
+ $t printf "%s\n" \
+ "-module($(APP)_fail_SUITE)." \
+ "-export([all/0, fail/1])." \
+ "all() -> [fail]." \
+ "fail(_) -> throw(fail)." > $(APP)/test/$(APP)_fail_SUITE.erl
+
+ $i "Check that Common Test errors out"
+ $t ! $(MAKE) -C $(APP) ct $v
+
+ $i "Check that logs are kept on clean"
+ $t $(MAKE) -C $(APP) clean $v
+ $t test -f $(APP)/logs/index.html
+
+ $i "Check that logs are deleted on distclean"
+ $t $(MAKE) -C $(APP) distclean $v
+ $t test ! -e $(APP)/logs/index.html
+
+ct-apps-only: build clean
+
+ $i "Create a multi application repository with no root application"
+ $t mkdir $(APP)/
+ $t cp ../erlang.mk $(APP)/
+ $t echo "include erlang.mk" > $(APP)/Makefile
+
+ $i "Create a new application named my_app"
+ $t $(MAKE) -C $(APP) new-app in=my_app $v
+
+ $i "Create a new library named my_lib"
+ $t $(MAKE) -C $(APP) new-lib in=my_lib $v
+
+ $i "Check that Common Test detects no tests"
+ $t $(MAKE) -C $(APP) ct | grep -q "Nothing to be done for 'ct'."
+
+ $i "Generate a Common Test suite in my_app"
+ $t mkdir $(APP)/apps/my_app/test
+ $t printf "%s\n" \
+ "-module(my_app_SUITE)." \
+ "-export([all/0, ok/1])." \
+ "all() -> [ok]." \
+ "ok(_) -> ok." > $(APP)/apps/my_app/test/my_app_SUITE.erl
+
+ $i "Generate a Common Test suite in my_lib"
+ $t mkdir $(APP)/apps/my_lib/test
+ $t printf "%s\n" \
+ "-module(my_lib_SUITE)." \
+ "-export([all/0, ok/1])." \
+ "all() -> [ok]." \
+ "ok(_) -> ok." > $(APP)/apps/my_lib/test/my_lib_SUITE.erl
+
+ $i "Check that Common Test runs tests"
+# We can't pipe CT's output without it crashing, so let's check that
+# the command succeeds and log files are created instead.
+ $t test ! -e $(APP)/apps/my_app/logs/index.html
+ $t test ! -e $(APP)/apps/my_lib/logs/index.html
+ $t $(MAKE) -C $(APP) ct $v
+ $t test -f $(APP)/apps/my_app/logs/index.html
+ $t test -f $(APP)/apps/my_lib/logs/index.html
+
+ct-case: build clean
+
+ $i "Bootstrap a new OTP application named $(APP)"
+ $t mkdir $(APP)/
+ $t cp ../erlang.mk $(APP)/
+ $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v
+
+ $i "Generate a Common Test suite with two cases"
+ $t mkdir $(APP)/test
+ $t printf "%s\n" \
+ "-module($(APP)_SUITE)." \
+ "-export([all/0, groups/0, ok/1, bad/1])." \
+ "all() -> [{group, mygroup}]." \
+ "groups() -> [{mygroup, [ok, bad]}]." \
+ "ok(_) -> ok." \
+ "bad(_) -> throw(fail)." > $(APP)/test/$(APP)_SUITE.erl
+
+ $i "Check that we can run Common Test on a specific test case"
+ $t $(MAKE) -C $(APP) ct-$(APP) t=mygroup:ok $v
+
+ct-check: build clean
+
+ $i "Bootstrap a new OTP application named $(APP)"
+ $t mkdir $(APP)/
+ $t cp ../erlang.mk $(APP)/
+ $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v
+
+ $i "Generate a Common Test suite"
+ $t mkdir $(APP)/test
+ $t printf "%s\n" \
+ "-module($(APP)_SUITE)." \
+ "-export([all/0, ok/1])." \
+ "all() -> [ok]." \
+ "ok(_) -> ok." > $(APP)/test/$(APP)_SUITE.erl
+
+ $i "Check that Common Test runs on 'make check'"
+ $t test ! -e $(APP)/logs/index.html
+ $t $(MAKE) -C $(APP) check $v
+ $t test -f $(APP)/logs/index.html
+
+ct-group: build clean
+
+ $i "Bootstrap a new OTP application named $(APP)"
+ $t mkdir $(APP)/
+ $t cp ../erlang.mk $(APP)/
+ $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v
+
+ $i "Generate a Common Test suite with two groups"
+ $t mkdir $(APP)/test
+ $t printf "%s\n" \
+ "-module($(APP)_SUITE)." \
+ "-export([all/0, groups/0, ok/1, bad/1])." \
+ "all() -> [{group, okgroup}, {group, badgroup}]." \
+ "groups() -> [{okgroup, [ok]}, {badgroup, [bad]}]." \
+ "ok(_) -> ok." \
+ "bad(_) -> throw(fail)." > $(APP)/test/$(APP)_SUITE.erl
+
+ $i "Check that we can run Common Test on a specific group"
+ $t $(MAKE) -C $(APP) ct-$(APP) t=okgroup $v
+
+ct-opts: build clean
+
+ $i "Bootstrap a new OTP application named $(APP)"
+ $t mkdir $(APP)/
+ $t cp ../erlang.mk $(APP)/
+ $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v
+
+ $i "Set CT_OPTS in the Makefile"
+ $t perl -ni.bak -e 'print;if ($$.==1) {print "CT_OPTS = -label hello_ct_opts\n"}' $(APP)/Makefile
+
+ $i "Generate a Common Test suite"
+ $t mkdir $(APP)/test
+ $t printf "%s\n" \
+ "-module($(APP)_SUITE)." \
+ "-export([all/0, ok/1])." \
+ "all() -> [ok]." \
+ "ok(_) -> ok." > $(APP)/test/$(APP)_SUITE.erl
+
+ $i "Run Common Test"
+ $t $(MAKE) -C $(APP) ct $v
+
+ $i "Check that Common Test uses options from CT_OPTS"
+ $t grep -q hello_ct_opts $(APP)/logs/index.html
+
+ct-suite: build clean
+
+ $i "Bootstrap a new OTP application named $(APP)"
+ $t mkdir $(APP)/
+ $t cp ../erlang.mk $(APP)/
+ $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v
+
+ $i "Generate two Common Test suites"
+ $t mkdir $(APP)/test
+ $t printf "%s\n" \
+ "-module($(APP)_ok_SUITE)." \
+ "-export([all/0, ok/1])." \
+ "all() -> [ok]." \
+ "ok(_) -> ok." > $(APP)/test/$(APP)_ok_SUITE.erl
+ $t printf "%s\n" \
+ "-module($(APP)_fail_SUITE)." \
+ "-export([all/0, bad/1])." \
+ "all() -> [bad]." \
+ "bad(_) -> throw(fail)." > $(APP)/test/$(APP)_fail_SUITE.erl
+
+ $i "Check that we can run Common Test on a specific test suite"
+ $t $(MAKE) -C $(APP) ct-$(APP)_ok $v
+
+ct-tests: build clean
+
+ $i "Bootstrap a new OTP application named $(APP)"
+ $t mkdir $(APP)/
+ $t cp ../erlang.mk $(APP)/
+ $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v
+
+ $i "Generate a Common Test suite"
+ $t mkdir $(APP)/test
+ $t printf "%s\n" \
+ "-module($(APP)_SUITE)." \
+ "-export([all/0, ok/1])." \
+ "all() -> [ok]." \
+ "ok(_) -> ok." > $(APP)/test/$(APP)_SUITE.erl
+
+ $i "Check that Common Test runs on 'make tests'"
+ $t test ! -e $(APP)/logs/index.html
+ $t $(MAKE) -C $(APP) tests $v
+ $t test -f $(APP)/logs/index.html