From f35f0f2c84b88d993e47afa218c989fdaee42688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 12 Oct 2015 00:03:05 +0200 Subject: Rework package testing Better error reporting and can now build everything in parallel. Failed builds are kept; others are deleted. The following command builds everything and then gives a diff of what got fixed/broken since last time ("> " is newly broken, "< " is newly fixed). make packages -j 32 -k; make summary --- .gitignore | 6 +- Makefile | 24 +++++- test/Makefile | 229 ++++++++++++++++++++++++++++++++++------------------------ 3 files changed, 159 insertions(+), 100 deletions(-) diff --git a/.gitignore b/.gitignore index 3d5a519..c1573e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,4 @@ doc/guide.pdf doc/html -test/app1/ -test/pkgs.log -test/temp-python/ -test/tmp/ +test/logs/ +test/packages/ diff --git a/Makefile b/Makefile index 5eff5d6..1af155d 100644 --- a/Makefile +++ b/Makefile @@ -25,18 +25,36 @@ all: | sed 's/^ERLANG_MK_VERSION = .*/ERLANG_MK_VERSION = $(ERLANG_MK_VERSION)/' > $(ERLANG_MK) ifdef p +# Remove p from the list of variables since that conflicts with bootstrapping. +MAKEOVERRIDES := $(filter-out p=$p,$(MAKEOVERRIDES)) + check: - $(MAKE) -C test pkg-$p + $(MAKE) -C test pkg-$p KEEP_BUILDS=1 else ifdef c check: - $(MAKE) -C test $c LEGACY=$(LEGACY) + $(MAKE) -C test $c else check: - $(MAKE) -C test LEGACY=$(LEGACY) + $(MAKE) -C test endif endif +packages: + $(MAKE) -C test packages + +summary: + @mkdir -p test/logs/ + @touch test/logs/latest.log test/packages/errors.log + -@sort test/packages/errors.log | diff test/logs/latest.log - + @sort test/packages/errors.log > test/logs/latest.log + @cp test/logs/latest.log "test/logs/$(subst $(empty) $(empty),_,$(shell date --rfc-3339 seconds))" + +search: + @$(MAKE) --no-print-directory \ + -f core/core.mk $(addprefix -f,$(wildcard index/*.mk)) -f core/index.mk \ + search + clean: $(MAKE) -C test clean rm -rf doc/guide.pdf doc/html diff --git a/test/Makefile b/test/Makefile index 20b2328..dc11ccd 100644 --- a/test/Makefile +++ b/test/Makefile @@ -79,11 +79,11 @@ else ifeq ($V,1) else ifeq ($V,2) t = @echo " TEST " $@; v = V=0 - i = @echo "== $@:" + i = @echo == $@: else t = v = V=1 - i = @echo "== $@:" + i = @echo == $@: endif # Main targets. @@ -93,7 +93,7 @@ endif all:: core clean:: clean-core - $t rm -f erl_crash.dump + $t rm -rf erl_crash.dump packages/ build: $i "Generate a bleeding edge Erlang.mk" @@ -177,6 +177,138 @@ core-help: build clean-core-help $i "Run 'make help' and check that it prints help" $t test -n "`$(MAKE) -C $(APP) help` | grep Usage" +# Packages. + +PACKAGES = $(foreach pkg,$(sort $(wildcard ../index/*.mk)),$(notdir $(basename $(pkg)))) + +packages: $(addprefix pkg-,$(PACKAGES)) + +define pkg_target +.PHONY: pkg-$1 + +pkg-$1: clean build + +# Make sure $@ is defined inside the define. + $(eval @ = pkg-$1) + +# Get the real application's name. + $(eval APP_NAME = `sed '2!d;s/pkg_$1_name = //' ../index/$1.mk`) + + $i "Bootstrap a new OTP library in packages/$1_pkg" + $t mkdir -p packages/$1_pkg/ + $t cp ../erlang.mk packages/$1_pkg/ + $t cd packages/$1_pkg/ && $(MAKE) -f erlang.mk bootstrap-lib $v + + $i "Add package $1 to the Makefile" + $t perl -ni.bak -e 'print;if ($$$$.==1) {print "DEPS = $1\n"}' packages/$1_pkg/Makefile + + $(if $(filter amqp_client,$1), + $i "Set RABBITMQ_CLIENT_PATCH" + $(eval PATCHES := RABBITMQ_CLIENT_PATCH=1)) + + $(if $(filter rabbit,$1), + $i "Set RABBITMQ_SERVER_PATCH" + $(eval PATCHES := RABBITMQ_SERVER_PATCH=1)) + + $i "Compile package $1" + $t if ! ( cd packages/$1_pkg/ && $(MAKE) $(PATCHES) $v ); then \ + echo "$1: compile error" >> packages/errors.log; \ + false; \ + fi + + $i "Check that $1 has a .app file" + $t if ! test -f packages/$1_pkg/deps/$(APP_NAME)/ebin/$(APP_NAME).app; then \ + echo "$1: no .app file" >> packages/errors.log; \ + false; \ + fi + + $i "Check that all applications and their modules can be loaded" + $t if ! ( cd packages/$1_pkg/ && $(ERL) -pa deps/*/ebin/ -eval " \ + Apps = [list_to_atom(App) || \"deps/\" ++ App \ + <- filelib:wildcard(\"deps/*\")], \ + [begin \ + io:format(\"Loading application ~p~n\", [App]), \ + case application:load(App) of \ + ok -> ok; \ + {error, {already_loaded, App}} -> ok \ + end, \ + {ok, Mods} = application:get_key(App, modules), \ + [try io:format(\" Loading module ~p~n\", [Mod]), \ + {module, Mod} = code:load_file(Mod) \ + catch C:R -> timer:sleep(500), erlang:C(R) \ + end || Mod <- Mods] \ + end || App <- Apps], \ + halt()." ); then \ + echo "$1: load error" >> packages/errors.log; \ + false; \ + fi + + $i "Recompile package $1" + $t if ! ( cd packages/$1_pkg/ && $(MAKE) $(PATCHES) $v ); then \ + echo "$(1): recompile error" >> packages/errors.log; \ + false; \ + fi + + $i "Check that $1 has a .app file" + $t if ! test -f packages/$1_pkg/deps/$(APP_NAME)/ebin/$(APP_NAME).app; then \ + echo "$1: no .app file" >> packages/errors.log; \ + false; \ + fi + + $i "Check that all applications and their modules can still be loaded" + $t if ! ( cd packages/$1_pkg/ && $(ERL) -pa deps/*/ebin/ -eval " \ + Apps = [list_to_atom(App) || \"deps/\" ++ App \ + <- filelib:wildcard(\"deps/*\")], \ + [begin \ + io:format(\"Loading application ~p~n\", [App]), \ + case application:load(App) of \ + ok -> ok; \ + {error, {already_loaded, App}} -> ok \ + end, \ + {ok, Mods} = application:get_key(App, modules), \ + [try io:format(\" Loading module ~p~n\", [Mod]), \ + {module, Mod} = code:load_file(Mod) \ + catch C:R -> timer:sleep(500), erlang:C(R) \ + end || Mod <- Mods] \ + end || App <- Apps], \ + halt()." ); then \ + echo "$1: recompile+load error" >> packages/errors.log; \ + false; \ + fi + + $i "Check that no erl_crash.dump file exists" + $t if ( ! find packages/$1_pkg/ -type f -name erl_crash.dump ); then \ + echo "$(1): erl_crash.dump found" >> packages/errors.log; \ + fi + + $(if $(KEEP_BUILDS),, + $i "OK; delete the build directory" + $t rm -rf packages/$1_pkg/) +endef + +$(foreach pkg,$(PACKAGES),$(eval $(call pkg_target,$(pkg)))) + +################## + +# Test application used for testing. +app1: + $(call app1_setup) + +# Extra module in app1 used for testing eunit +define create-module-t +printf '%s\n' \ + '-module(t).' \ + '-export([succ/1]).' \ + 'succ(N) -> N + 1.' \ + '-ifdef(TEST).' \ + '-include_lib("eunit/include/eunit.hrl").' \ + 'succ_test() ->' \ + ' ?assertEqual(2, succ(1)),' \ + ' os:cmd("echo t >> test-eunit.log").' \ + '-endif.' \ + > app1/src/t.erl +endef + # Legacy tests. # # The following tests are slowly being converted. @@ -187,7 +319,7 @@ core-help: build clean-core-help legacy: clean-legacy ct eunit tests-cover docs pkgs clean-legacy: - $t rm -rf app1 pkgs.log + $t rm -rf app1 ct: app1 $i "ct: Testing ct and related targets." @@ -338,92 +470,3 @@ define app1_setup "succ(N) -> N + 1." \ > app1/src/m.erl endef - -define pkg_test_target -pkg-$(1)-clean: - $t rm -rf app1 erl_crash.dump - -pkg-$(1)-app1: - $(call app1_setup) - -# Running 'make' twice to make sure it recompiles fine. -pkg-$(1): pkg-$(1)-clean pkg-$(1)-app1 - $i - $i " pkgs: Checking that '$(1)' builds correctly" - $i - $t printf "%s\n" \ - "PROJECT = app1" \ - "DEPS = $(1)" \ - "include erlang.mk" \ - > app1/Makefile - $t \ - if [ "$(1)" = "amqp_client" ]; then \ - $(MAKE) -C app1 RABBITMQ_CLIENT_PATCH=1; \ - elif [ "$(1)" = "rabbit" ]; then \ - $(MAKE) -C app1 RABBITMQ_SERVER_PATCH=1; \ - else \ - $(MAKE) -C app1; \ - fi; \ - if [ $$$$? -ne 0 ]; then \ - echo "$(1): make error" >> pkgs.log; \ - else \ - $(MAKE) -C app1; if [ $$$$? -ne 0 ]; then \ - echo "$(1): re-make error" >> pkgs.log; \ - else \ - find . -type f -name erl_crash.dump; if [ $$$$? -ne 0 ]; then \ - echo "$(1): erl_crash.dump found" >> pkgs.log; \ - else \ - erl +A0 -noinput -boot start_clean -pa app1/deps/*/ebin -eval " \ - Apps = [list_to_atom(App) || \"app1/deps/\" ++ App \ - <- filelib:wildcard(\"app1/deps/*\")], \ - [begin \ - io:format(\"Loading application ~p~n\", [App]), \ - case application:load(App) of \ - {error, _} -> ok; \ - ok -> \ - {ok, Mods} = application:get_key(App, modules), \ - [try io:format(\" Loading module ~p~n\", [Mod]), \ - {module, Mod} = code:load_file(Mod) \ - catch C:R -> timer:sleep(500), erlang:C(R) \ - end || Mod <- Mods] \ - end \ - end || App <- Apps], \ - halt()."; if [ $$$$? -ne 0 ]; then \ - echo "$(1): load error" >> pkgs.log; \ - fi \ - fi \ - fi \ - fi -endef - -PACKAGES = $(foreach pkg,$(sort $(wildcard ../index/*.mk)),$(notdir $(basename $(pkg)))) - -$(foreach pkg,$(PACKAGES),$(eval $(call pkg_test_target,$(pkg)))) - -pkgs: $(addprefix pkg-,$(PACKAGES)) - @if [ -f pkgs.log ]; then \ - echo "+-------------------------------+"; \ - echo "| ERRORS WHILE TESTING PACKAGES |"; \ - echo "+-------------------------------+"; \ - cat pkgs.log; \ - exit 33; \ - fi - -# Test application used for testing. -app1: - $(call app1_setup) - -# Extra module in app1 used for testing eunit -define create-module-t -printf '%s\n' \ - '-module(t).' \ - '-export([succ/1]).' \ - 'succ(N) -> N + 1.' \ - '-ifdef(TEST).' \ - '-include_lib("eunit/include/eunit.hrl").' \ - 'succ_test() ->' \ - ' ?assertEqual(2, succ(1)),' \ - ' os:cmd("echo t >> test-eunit.log").' \ - '-endif.' \ - > app1/src/t.erl -endef -- cgit v1.2.3