diff options
| -rw-r--r-- | build.config | 3 | ||||
| -rw-r--r-- | core/test.mk | 8 | ||||
| -rw-r--r-- | plugins/cover.mk | 136 | ||||
| -rw-r--r-- | plugins/eunit.mk | 21 | ||||
| -rw-r--r-- | test/Makefile | 63 |
5 files changed, 205 insertions, 26 deletions
diff --git a/build.config b/build.config index d4b22db..d68eaff 100644 --- a/build.config +++ b/build.config @@ -26,3 +26,6 @@ plugins/relx plugins/shell plugins/triq plugins/xref + +# Plugins enhancing the functionality of other plugins. +plugins/cover diff --git a/core/test.mk b/core/test.mk index 475afa2..385ec85 100644 --- a/core/test.mk +++ b/core/test.mk @@ -26,13 +26,13 @@ test-dir: endif ifeq ($(wildcard ebin/test),) -test-build: ERLC_OPTS=$(TEST_ERLC_OPTS) -test-build: clean deps test-deps +test-build:: ERLC_OPTS=$(TEST_ERLC_OPTS) +test-build:: clean deps test-deps @$(MAKE) --no-print-directory app-build test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)" $(gen_verbose) touch ebin/test else -test-build: ERLC_OPTS=$(TEST_ERLC_OPTS) -test-build: deps test-deps +test-build:: ERLC_OPTS=$(TEST_ERLC_OPTS) +test-build:: deps test-deps @$(MAKE) --no-print-directory app-build test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)" endif diff --git a/plugins/cover.mk b/plugins/cover.mk new file mode 100644 index 0000000..3397dca --- /dev/null +++ b/plugins/cover.mk @@ -0,0 +1,136 @@ +# Copyright 2015, Viktor Söderqvist <[email protected]> +# This file is part of erlang.mk and subject to the terms of the ISC License. + +COVER_REPORT_DIR = cover + +# utility variables for representing special symbols +empty := +space := $(empty) $(empty) +comma := , + +# Hook in coverage to eunit + +ifdef COVER +ifdef EUNIT_RUN +EUNIT_RUN_BEFORE += -eval \ + 'case cover:compile_beam_directory("ebin") of \ + {error, _} -> halt(1); \ + _ -> ok \ + end.' +EUNIT_RUN_AFTER += -eval 'cover:export("eunit.coverdata").' +endif +endif + +# Hook in coverage to ct + +ifdef COVER +ifdef CT_RUN + +# All modules in 'ebin' +COVER_MODS = $(notdir $(basename $(shell echo ebin/*.beam))) + +test-build:: $(TEST_DIR)/ct.cover.spec + +$(TEST_DIR)/ct.cover.spec: + @echo Cover mods: $(COVER_MODS) + $(gen_verbose) printf "%s\n" \ + '{incl_mods,[$(subst $(space),$(comma),$(COVER_MODS))]}.' \ + '{export,"$(CURDIR)/ct.coverdata"}.' > $@ + +CT_RUN += -cover $(TEST_DIR)/ct.cover.spec +endif +endif + +# Core targets + +ifdef COVER +ifneq ($(COVER_REPORT_DIR),) +tests:: + @$(MAKE) make --no-print-directory cover-report +endif +endif + +clean:: coverdata-clean + +ifneq ($(COVER_REPORT_DIR),) +distclean:: cover-report-clean +endif + +help:: + @printf "%s\n" "" \ + "Cover targets:" \ + " cover-report Generate a HTML coverage report from previously collected" \ + " cover data." \ + " all.coverdata Merge {eunit,ct}.coverdata into one coverdata file." \ + "" \ + "If COVER=1 is set, coverage data is generated by the targets eunit and ct. The" \ + "target tests additionally generates a HTML coverage report from the combined" \ + "coverdata files from each of these testing tools. HTML reports can be disabled" \ + "by setting COVER_REPORT_DIR to empty." + +# Plugin specific targets + +COVERDATA = $(filter-out all.coverdata,$(wildcard *.coverdata)) + +.PHONY: coverdata-clean +coverdata-clean: + $(gen_verbose) rm -f *.coverdata ct.cover.spec + +# Merge all coverdata files into one. +all.coverdata: $(COVERDATA) + $(gen_verbose) $(ERL) -eval ' \ + $(foreach f,$(COVERDATA),cover:import("$(f)") == ok orelse halt(1),) \ + cover:export("$@"), halt(0).' + +# These are only defined if COVER_REPORT_DIR is non-empty. Set COVER_REPORT_DIR to +# empty if you want the coverdata files but not the HTML report. +ifneq ($(COVER_REPORT_DIR),) + +.PHONY: cover-report-clean cover-report + +cover-report-clean: + $(gen_verbose) rm -rf $(COVER_REPORT_DIR) + +ifeq ($(COVERDATA),) +cover-report: +else + +# Modules which include eunit.hrl always contain one line without coverage +# because eunit defines test/0 which is never called. We compensate for this. +EUNIT_HRL_MODS = $(subst $(space),$(comma),$(shell \ + grep -e '^\s*-include.*include/eunit\.hrl"' src/*.erl \ + | sed "s/^src\/\(.*\)\.erl:.*/'\1'/" | uniq)) + +cover-report: + $(gen_verbose) mkdir -p $(COVER_REPORT_DIR) + $(gen_verbose) $(ERL) -eval ' \ + $(foreach f,$(COVERDATA),cover:import("$(f)") == ok orelse halt(1),) \ + Ms = cover:imported_modules(), \ + [cover:analyse_to_file(M, "$(COVER_REPORT_DIR)/" ++ atom_to_list(M) \ + ++ ".COVER.html", [html]) || M <- Ms], \ + Report = [begin {ok, R} = cover:analyse(M, module), R end || M <- Ms], \ + EunitHrlMods = [$(EUNIT_HRL_MODS)], \ + Report1 = [{M, {Y, case lists:member(M, EunitHrlMods) of \ + true -> N - 1; false -> N end}} || {M, {Y, N}} <- Report], \ + TotalY = lists:sum([Y || {_, {Y, _}} <- Report1]), \ + TotalN = lists:sum([N || {_, {_, N}} <- Report1]), \ + TotalPerc = round(100 * TotalY / (TotalY + TotalN)), \ + {ok, F} = file:open("$(COVER_REPORT_DIR)/index.html", [write]), \ + io:format(F, "<!DOCTYPE html><html>~n" \ + "<head><meta charset=\"UTF-8\">~n" \ + "<title>Coverage report</title></head>~n" \ + "<body>~n", []), \ + io:format(F, "<h1>Coverage</h1>~n<p>Total: ~p%</p>~n", [TotalPerc]),\ + io:format(F, "<table><tr><th>Module</th><th>Coverage</th></tr>~n", []), \ + [io:format(F, "<tr><td><a href=\"~p.COVER.html\">~p</a></td>" \ + "<td>~p%</td></tr>~n", \ + [M, M, round(100 * Y / (Y + N))]) || {M, {Y, N}} <- Report1], \ + How = "$(subst $(space),$(comma)$(space),$(basename $(COVERDATA)))", \ + Date = "$(shell date -u "+%Y-%m-%dT%H:%M:%SZ")", \ + io:format(F, "</table>~n" \ + "<p>Generated using ~s and erlang.mk on ~s.</p>~n" \ + "</body></html>", [How, Date]), \ + halt().' + +endif +endif # ifneq ($(COVER_REPORT_DIR),) diff --git a/plugins/eunit.mk b/plugins/eunit.mk index ec3228e..b9f2856 100644 --- a/plugins/eunit.mk +++ b/plugins/eunit.mk @@ -6,22 +6,19 @@ # Configuration +# All modules in TEST_DIR ifeq ($(strip $(TEST_DIR)),) -TAGGED_EUNIT_TESTS = {dir,"ebin"} -else -ifeq ($(wildcard $(TEST_DIR)),) -TAGGED_EUNIT_TESTS = {dir,"ebin"} +TEST_DIR_MODS = else -# All modules in TEST_DIR TEST_DIR_MODS = $(notdir $(basename $(shell find $(TEST_DIR) -type f -name *.beam))) +endif + # All modules in 'ebin' EUNIT_EBIN_MODS = $(notdir $(basename $(shell find ebin -type f -name *.beam))) # Only those modules in TEST_DIR with no matching module in 'ebin'. # This is done to avoid some tests being executed twice. EUNIT_MODS = $(filter-out $(patsubst %,%_tests,$(EUNIT_EBIN_MODS)),$(TEST_DIR_MODS)) -TAGGED_EUNIT_TESTS = {dir,"ebin"} $(foreach mod,$(EUNIT_MODS),$(shell echo $(mod) | sed -e 's/\(.*\)/{module,\1}/g')) -endif -endif +TAGGED_EUNIT_TESTS = $(foreach mod,$(EUNIT_EBIN_MODS) $(EUNIT_MODS),{module,$(mod)}) EUNIT_OPTS ?= @@ -42,10 +39,16 @@ help:: # Plugin-specific targets. +EUNIT_RUN_BEFORE ?= +EUNIT_RUN_AFTER ?= EUNIT_RUN = $(ERL) \ -pa $(TEST_DIR) $(DEPS_DIR)/*/ebin \ -pz ebin \ - -eval 'case eunit:test([$(call str-join,$(TAGGED_EUNIT_TESTS))], [$(EUNIT_OPTS)]) of ok -> halt(0); error -> halt(1) end.' + $(EUNIT_RUN_BEFORE) \ + -eval 'case eunit:test([$(call str-join,$(TAGGED_EUNIT_TESTS))],\ + [$(EUNIT_OPTS)]) of ok -> ok; error -> halt(1) end.' \ + $(EUNIT_RUN_AFTER) \ + -eval 'halt(0).' eunit: test-build $(gen_verbose) $(EUNIT_RUN) diff --git a/test/Makefile b/test/Makefile index 265c67c..180b5a1 100644 --- a/test/Makefile +++ b/test/Makefile @@ -33,9 +33,9 @@ else i = @echo == endif -.PHONY: all clean app ct eunit docs +.PHONY: all clean app ct eunit tests-cover docs -all: app ct eunit docs clean +all: app ct eunit tests-cover docs clean $i '+---------------------+' $i '| All tests passed. |' $i '+---------------------+' @@ -100,17 +100,7 @@ ct: app1 eunit: app1 $i "eunit: Testing the 'eunit' target." $i "Running eunit test case inside module src/t.erl" - $t printf '%s\n' \ - '-module(t).' \ - '-export([succ/1]).' \ - 'succ(N) -> N + 1.' \ - '-ifdef(TEST).' \ - '-include_lib("eunit/include/eunit.hrl").' \ - 'succ_test() ->' \ - ' ?assertEqual(2, succ(1)),' \ - ' os:cmd("echo t >> test-eunit.log").' \ - '-endif.' \ - > app1/src/t.erl + $t $(call create-module-t) $t make -C app1 eunit $v $i "Checking that the eunit test in module t." $t echo t | cmp app1/test-eunit.log - @@ -147,6 +137,38 @@ eunit: app1 $t rm -rf app1/eunit app1/src/t.erl app1/test-eunit.log $i "Test 'eunit' passed." +# TODO: do coverage for 'tests' instead of 'eunit ct' when triq is fixed +tests-cover: app1 + $i "tests-cover: Testing 'eunit' and 'ct' with COVER=1" + $i "Setting up eunit and ct suites." + $t $(call create-module-t) + $t mkdir -p app1/test + $t printf "%s\n" \ + "-module(m_SUITE)." \ + "-export([all/0, testcase1/1])." \ + "all() -> [testcase1]." \ + "testcase1(_) -> 2 = m:succ(1)." \ + > app1/test/m_SUITE.erl + $i "Running tests with coverage analysis." + $t make -C app1 eunit ct COVER=1 $v + $t [ -e app1/test-eunit.log ] + $t [ -e app1/eunit.coverdata ] + $t [ -e app1/ct.coverdata ] + $i "Generating coverage report." + $t make -C app1 cover-report COVER=1 $v + $t [ -e app1/cover/m.COVER.html ] + $t [ -e app1/cover/t.COVER.html ] + $t [ -e app1/cover/index.html ] + $i "Checking combined coverage from eunit and ct." + $t [ `grep 'Total: 100%' app1/cover/index.html | wc -l` -eq 1 ] + $i "Checking that cover-clean removes cover data and report." + $t make -C app1 cover-clean $v + $t [ ! -e app1/cover ] && [ ! -e app1/eunit.coverdata ] + @# clean up + $t rm -rf app1/src/t.erl app1/test app1/test-eunit.log + $t make -C app1 clean $v + $i "Test 'tests-cover' passed." + docs: app1 $i "docs: Testing EDoc including DOC_DEPS." $t printf "%s\n" \ @@ -181,3 +203,18 @@ app1: "-export([succ/1])." \ "succ(N) -> N + 1." \ > app1/src/m.erl + +# Extra module in app1 used for testing eunit +define create-module-t +printf '%s\n' \ + '-module(t).' \ + '-export([succ/1]).' \ + 'succ(N) -> N + 1.' \ + '-ifdef(TEST).' \ + '-include_lib("eunit/include/eunit.hrl").' \ + 'succ_test() ->' \ + ' ?assertEqual(2, succ(1)),' \ + ' os:cmd("echo t >> test-eunit.log").' \ + '-endif.' \ + > app1/src/t.erl +endef |