diff options
| -rw-r--r-- | doc/src/guide/common_test.asciidoc | 89 | ||||
| -rw-r--r-- | plugins/ct.mk | 24 | ||||
| -rw-r--r-- | test/Makefile | 40 | ||||
| -rw-r--r-- | test/plugin_ct.mk | 219 |
4 files changed, 328 insertions, 44 deletions
diff --git a/doc/src/guide/common_test.asciidoc b/doc/src/guide/common_test.asciidoc index 7daeae5..3f00baf 100644 --- a/doc/src/guide/common_test.asciidoc +++ b/doc/src/guide/common_test.asciidoc @@ -1,5 +1,90 @@ == Common Test -// @todo Write it. +Common Test is Erlang's functional testing framework. +Erlang.mk automates the discovery and running of Common +Test suites. -Placeholder chapter. +=== Writing tests + +The http://www.erlang.org/doc/apps/common_test/write_test_chapter.html[Common Test user guide] +is the best place to learn how to write tests. Erlang.mk +requires that file names for test suites end with '_SUITE.erl' +and that the files be located in the '$(TEST_DIR)' directory. +This defaults to 'test/'. + +=== Configuration + +The `CT_OPTS` variable allows you to set extra Common Test +options. Options are documented in the +http://www.erlang.org/doc/apps/common_test/run_test_chapter.html[Common Test user guide]. +You can use it to set Common Test hooks, for example: + +[source,make] +CT_OPTS = -ct_hooks cowboy_ct_hook + +The `CT_SUITES` variable can be used to override what +Common Test suites Erlang.mk will be aware of. It does +not normally need to be set as Erlang.mk will find the +test suites automatically. + +The name of the suite is the part before `_SUITE.erl`. +If the file is named 'http_SUITE.erl', the test suite +is `http`: + +[source,make] +CT_SUITES = http ws + +=== Usage + +To run all tests (including Common Test): + +[source,bash] +$ make tests + +To run all tests and static checks (including Common Test): + +[source,bash] +$ make check + +You can also run Common Test separately: + +[source,bash] +$ make ct + +Erlang.mk will create targets for all test suites it finds. +If you have a file named 'test/http_SUITE.erl', then the +target `ct-http` will run that specific test suite: + +[source,bash] +$ make ct-http + +Erlang.mk provides a convenient way to run a specific +group or a specific test case within a specific group, +using the variable `t`. Note that this only applies to +suite-specific targets, like the `ct-http` example above. + +To run all tests from the `http_compress` group in the +`http_SUITE` test suite, write: + +[source,bash] +$ make ct-http t=http_compress + +Similarly, to run a specific test case in that group: + +[source,bash] +$ make ct-http t=http_compress:headers_dupe + +To do the same against a multi-application repository, +you can use the `-C` option: + +[source,bash] +$ make -C apps/my_app ct-http t=my_group:my_case + +Note that this also applies to dependencies. When using Cowboy +as a dependency, you can run the following directly: + +[source,bash] +$ make -C deps/cowboy ct-http t=http_compress + +Finally, link:coverage.asciidoc[code coverage] is available, +but covered in its own chapter. diff --git a/plugins/ct.mk b/plugins/ct.mk index fb4b1ca..2d45032 100644 --- a/plugins/ct.mk +++ b/plugins/ct.mk @@ -1,7 +1,7 @@ # Copyright (c) 2013-2015, Loïc Hoguin <[email protected]> # This file is part of erlang.mk and subject to the terms of the ISC License. -.PHONY: ct distclean-ct +.PHONY: ct apps-ct distclean-ct # Configuration. @@ -36,17 +36,33 @@ CT_RUN = ct_run \ -logdir $(CURDIR)/logs ifeq ($(CT_SUITES),) -ct: +ct: $(if $(IS_APP),,apps-ct) else -ct: test-build +ct: test-build $(if $(IS_APP),,apps-ct) $(verbose) mkdir -p $(CURDIR)/logs/ $(gen_verbose) $(CT_RUN) -suite $(addsuffix _SUITE,$(CT_SUITES)) $(CT_OPTS) endif +ifneq ($(ALL_APPS_DIRS),) +apps-ct: + $(verbose) for app in $(ALL_APPS_DIRS); do $(MAKE) -C $$app ct IS_APP=1; done +endif + +ifndef t +CT_EXTRA = +else +ifeq (,$(findstring :,$t)) +CT_EXTRA = -group $t +else +t_words = $(subst :, ,$t) +CT_EXTRA = -group $(firstword $(t_words)) -case $(lastword $(t_words)) +endif +endif + define ct_suite_target ct-$(1): test-build $(verbose) mkdir -p $(CURDIR)/logs/ - $(gen_verbose) $(CT_RUN) -suite $(addsuffix _SUITE,$(1)) $(CT_OPTS) + $(gen_verbose) $(CT_RUN) -suite $(addsuffix _SUITE,$(1)) $(CT_EXTRA) $(CT_OPTS) endef $(foreach test,$(CT_SUITES),$(eval $(call ct_suite_target,$(test)))) diff --git a/test/Makefile b/test/Makefile index f7ecf64..728d751 100644 --- a/test/Makefile +++ b/test/Makefile @@ -303,49 +303,13 @@ endef # The following tests are slowly being converted. # Do NOT use -j with legacy tests. -.PHONY: legacy clean-legacy ct tests-cover docs +.PHONY: legacy clean-legacy tests-cover docs -legacy: clean-legacy ct tests-cover docs pkgs +legacy: clean-legacy tests-cover docs pkgs clean-legacy: $t rm -rf app1 -ct: app1 - $i "ct: Testing ct and related targets." - $i "Setting up test suite." - $t mkdir -p app1/test - $t printf "%s\n" \ - "-module(m_SUITE)." \ - "-export([all/0, testcase1/1])." \ - "all() -> [testcase1]." \ - "testcase1(_) -> 2 = m:succ(1)." \ - > app1/test/m_SUITE.erl - $t $(MAKE) -C app1 ct $v - $i "Checking files created by '$(MAKE) ct'." - $t [ -e app1/test/m_SUITE.beam ] - $t [ -e app1/ebin/m.beam ] - $t [ -e app1/logs ] - $i "Checking that '$(MAKE) clean' does not delete logs." - $t $(MAKE) -C app1 clean $v - $t [ -e app1/logs ] - $i "Testing target 'ct-mysuite' where mysuite_SUITE is a test suite." - $t $(MAKE) -C app1 ct-m $v - $i "Checking that '$(MAKE) ct' returns non-zero for a failing suite." - $t printf "%s\n" \ - "-module(failing_SUITE)." \ - "-export([all/0, testcase1/1])." \ - "all() -> [testcase1]." \ - "testcase1(_) -> 42 = m:succ(1)." \ - > app1/test/failing_SUITE.erl - $t ! $(MAKE) -C app1 ct-failing $v - $i "Checking that '$(MAKE) distclean-ct' deletes logs." - $t $(MAKE) -C app1 distclean-ct $v - $t [ ! -e app1/logs ] - $t [ -e app1/ebin/m.beam ] - $i "Cleaning up test data." - $t rm -rf app1/test - $i "Test 'ct' passed." - # TODO: do coverage for 'tests' instead of 'eunit ct' when triq is fixed tests-cover: app1 $i "tests-cover: Testing 'eunit' and 'ct' with COVER=1" diff --git a/test/plugin_ct.mk b/test/plugin_ct.mk new file mode 100644 index 0000000..dd36801 --- /dev/null +++ b/test/plugin_ct.mk @@ -0,0 +1,219 @@ +# Common Test plugin. + +CT_CASES = all apps-only case check group opts suite tests +CT_TARGETS = $(addprefix ct-,$(CT_CASES)) + +.PHONY: ct $(CT_TARGETS) + +ct: $(CT_TARGETS) + +ct-all: build clean + + $i "Bootstrap a new OTP application named $(APP)" + $t mkdir $(APP)/ + $t cp ../erlang.mk $(APP)/ + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v + + $i "Check that Common Test detects no tests" + $t $(MAKE) -C $(APP) ct | grep -q "Nothing to be done for 'ct'." + + $i "Generate a Common Test suite" + $t mkdir $(APP)/test + $t printf "%s\n" \ + "-module($(APP)_SUITE)." \ + "-export([all/0, ok/1])." \ + "all() -> [ok]." \ + "ok(_) -> ok." > $(APP)/test/$(APP)_SUITE.erl + + $i "Check that Common Test runs tests" +# We can't pipe CT's output without it crashing, so let's check that +# the command succeeds and log files are created instead. + $t test ! -e $(APP)/logs/index.html + $t $(MAKE) -C $(APP) ct $v + $t test -f $(APP)/logs/index.html + + $i "Generate a Common Test suite with a failing test case" + $t printf "%s\n" \ + "-module($(APP)_fail_SUITE)." \ + "-export([all/0, fail/1])." \ + "all() -> [fail]." \ + "fail(_) -> throw(fail)." > $(APP)/test/$(APP)_fail_SUITE.erl + + $i "Check that Common Test errors out" + $t ! $(MAKE) -C $(APP) ct $v + + $i "Check that logs are kept on clean" + $t $(MAKE) -C $(APP) clean $v + $t test -f $(APP)/logs/index.html + + $i "Check that logs are deleted on distclean" + $t $(MAKE) -C $(APP) distclean $v + $t test ! -e $(APP)/logs/index.html + +ct-apps-only: build clean + + $i "Create a multi application repository with no root application" + $t mkdir $(APP)/ + $t cp ../erlang.mk $(APP)/ + $t echo "include erlang.mk" > $(APP)/Makefile + + $i "Create a new application named my_app" + $t $(MAKE) -C $(APP) new-app in=my_app $v + + $i "Create a new library named my_lib" + $t $(MAKE) -C $(APP) new-lib in=my_lib $v + + $i "Check that Common Test detects no tests" + $t $(MAKE) -C $(APP) ct | grep -q "Nothing to be done for 'ct'." + + $i "Generate a Common Test suite in my_app" + $t mkdir $(APP)/apps/my_app/test + $t printf "%s\n" \ + "-module(my_app_SUITE)." \ + "-export([all/0, ok/1])." \ + "all() -> [ok]." \ + "ok(_) -> ok." > $(APP)/apps/my_app/test/my_app_SUITE.erl + + $i "Generate a Common Test suite in my_lib" + $t mkdir $(APP)/apps/my_lib/test + $t printf "%s\n" \ + "-module(my_lib_SUITE)." \ + "-export([all/0, ok/1])." \ + "all() -> [ok]." \ + "ok(_) -> ok." > $(APP)/apps/my_lib/test/my_lib_SUITE.erl + + $i "Check that Common Test runs tests" +# We can't pipe CT's output without it crashing, so let's check that +# the command succeeds and log files are created instead. + $t test ! -e $(APP)/apps/my_app/logs/index.html + $t test ! -e $(APP)/apps/my_lib/logs/index.html + $t $(MAKE) -C $(APP) ct $v + $t test -f $(APP)/apps/my_app/logs/index.html + $t test -f $(APP)/apps/my_lib/logs/index.html + +ct-case: build clean + + $i "Bootstrap a new OTP application named $(APP)" + $t mkdir $(APP)/ + $t cp ../erlang.mk $(APP)/ + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v + + $i "Generate a Common Test suite with two cases" + $t mkdir $(APP)/test + $t printf "%s\n" \ + "-module($(APP)_SUITE)." \ + "-export([all/0, groups/0, ok/1, bad/1])." \ + "all() -> [{group, mygroup}]." \ + "groups() -> [{mygroup, [ok, bad]}]." \ + "ok(_) -> ok." \ + "bad(_) -> throw(fail)." > $(APP)/test/$(APP)_SUITE.erl + + $i "Check that we can run Common Test on a specific test case" + $t $(MAKE) -C $(APP) ct-$(APP) t=mygroup:ok $v + +ct-check: build clean + + $i "Bootstrap a new OTP application named $(APP)" + $t mkdir $(APP)/ + $t cp ../erlang.mk $(APP)/ + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v + + $i "Generate a Common Test suite" + $t mkdir $(APP)/test + $t printf "%s\n" \ + "-module($(APP)_SUITE)." \ + "-export([all/0, ok/1])." \ + "all() -> [ok]." \ + "ok(_) -> ok." > $(APP)/test/$(APP)_SUITE.erl + + $i "Check that Common Test runs on 'make check'" + $t test ! -e $(APP)/logs/index.html + $t $(MAKE) -C $(APP) check $v + $t test -f $(APP)/logs/index.html + +ct-group: build clean + + $i "Bootstrap a new OTP application named $(APP)" + $t mkdir $(APP)/ + $t cp ../erlang.mk $(APP)/ + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v + + $i "Generate a Common Test suite with two groups" + $t mkdir $(APP)/test + $t printf "%s\n" \ + "-module($(APP)_SUITE)." \ + "-export([all/0, groups/0, ok/1, bad/1])." \ + "all() -> [{group, okgroup}, {group, badgroup}]." \ + "groups() -> [{okgroup, [ok]}, {badgroup, [bad]}]." \ + "ok(_) -> ok." \ + "bad(_) -> throw(fail)." > $(APP)/test/$(APP)_SUITE.erl + + $i "Check that we can run Common Test on a specific group" + $t $(MAKE) -C $(APP) ct-$(APP) t=okgroup $v + +ct-opts: build clean + + $i "Bootstrap a new OTP application named $(APP)" + $t mkdir $(APP)/ + $t cp ../erlang.mk $(APP)/ + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v + + $i "Set CT_OPTS in the Makefile" + $t perl -ni.bak -e 'print;if ($$.==1) {print "CT_OPTS = -label hello_ct_opts\n"}' $(APP)/Makefile + + $i "Generate a Common Test suite" + $t mkdir $(APP)/test + $t printf "%s\n" \ + "-module($(APP)_SUITE)." \ + "-export([all/0, ok/1])." \ + "all() -> [ok]." \ + "ok(_) -> ok." > $(APP)/test/$(APP)_SUITE.erl + + $i "Run Common Test" + $t $(MAKE) -C $(APP) ct $v + + $i "Check that Common Test uses options from CT_OPTS" + $t grep -q hello_ct_opts $(APP)/logs/index.html + +ct-suite: build clean + + $i "Bootstrap a new OTP application named $(APP)" + $t mkdir $(APP)/ + $t cp ../erlang.mk $(APP)/ + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v + + $i "Generate two Common Test suites" + $t mkdir $(APP)/test + $t printf "%s\n" \ + "-module($(APP)_ok_SUITE)." \ + "-export([all/0, ok/1])." \ + "all() -> [ok]." \ + "ok(_) -> ok." > $(APP)/test/$(APP)_ok_SUITE.erl + $t printf "%s\n" \ + "-module($(APP)_fail_SUITE)." \ + "-export([all/0, bad/1])." \ + "all() -> [bad]." \ + "bad(_) -> throw(fail)." > $(APP)/test/$(APP)_fail_SUITE.erl + + $i "Check that we can run Common Test on a specific test suite" + $t $(MAKE) -C $(APP) ct-$(APP)_ok $v + +ct-tests: build clean + + $i "Bootstrap a new OTP application named $(APP)" + $t mkdir $(APP)/ + $t cp ../erlang.mk $(APP)/ + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v + + $i "Generate a Common Test suite" + $t mkdir $(APP)/test + $t printf "%s\n" \ + "-module($(APP)_SUITE)." \ + "-export([all/0, ok/1])." \ + "all() -> [ok]." \ + "ok(_) -> ok." > $(APP)/test/$(APP)_SUITE.erl + + $i "Check that Common Test runs on 'make tests'" + $t test ! -e $(APP)/logs/index.html + $t $(MAKE) -C $(APP) tests $v + $t test -f $(APP)/logs/index.html |