Browse Source

Merge branch 'cover' of https://github.com/zuiderkwast/erlang.mk

Rebased and moved ct.cover.spec into the test/ directory.
Loïc Hoguin 10 years ago
parent
commit
7d28bd8d9e
5 changed files with 205 additions and 26 deletions
  1. 3 0
      build.config
  2. 4 4
      core/test.mk
  3. 136 0
      plugins/cover.mk
  4. 12 9
      plugins/eunit.mk
  5. 50 13
      test/Makefile

+ 3 - 0
build.config

@@ -26,3 +26,6 @@ plugins/relx
 plugins/shell
 plugins/triq
 plugins/xref
+
+# Plugins enhancing the functionality of other plugins.
+plugins/cover

+ 4 - 4
core/test.mk

@@ -26,13 +26,13 @@ test-dir:
 endif
 
 ifeq ($(wildcard ebin/test),)
-test-build: ERLC_OPTS=$(TEST_ERLC_OPTS)
-test-build: clean deps test-deps
+test-build:: ERLC_OPTS=$(TEST_ERLC_OPTS)
+test-build:: clean deps test-deps
 	@$(MAKE) --no-print-directory app-build test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)"
 	$(gen_verbose) touch ebin/test
 else
-test-build: ERLC_OPTS=$(TEST_ERLC_OPTS)
-test-build: deps test-deps
+test-build:: ERLC_OPTS=$(TEST_ERLC_OPTS)
+test-build:: deps test-deps
 	@$(MAKE) --no-print-directory app-build test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)"
 endif
 

+ 136 - 0
plugins/cover.mk

@@ -0,0 +1,136 @@
+# Copyright 2015, Viktor Söderqvist <viktor@zuiderkwast.se>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+COVER_REPORT_DIR = cover
+
+# utility variables for representing special symbols
+empty :=
+space := $(empty) $(empty)
+comma := ,
+
+# Hook in coverage to eunit
+
+ifdef COVER
+ifdef EUNIT_RUN
+EUNIT_RUN_BEFORE += -eval \
+	'case cover:compile_beam_directory("ebin") of \
+		{error, _} -> halt(1); \
+		_ -> ok \
+	end.'
+EUNIT_RUN_AFTER += -eval 'cover:export("eunit.coverdata").'
+endif
+endif
+
+# Hook in coverage to ct
+
+ifdef COVER
+ifdef CT_RUN
+
+# All modules in 'ebin'
+COVER_MODS = $(notdir $(basename $(shell echo ebin/*.beam)))
+
+test-build:: $(TEST_DIR)/ct.cover.spec
+
+$(TEST_DIR)/ct.cover.spec:
+	@echo Cover mods: $(COVER_MODS)
+	$(gen_verbose) printf "%s\n" \
+		'{incl_mods,[$(subst $(space),$(comma),$(COVER_MODS))]}.' \
+		'{export,"$(CURDIR)/ct.coverdata"}.' > $@
+
+CT_RUN += -cover $(TEST_DIR)/ct.cover.spec
+endif
+endif
+
+# Core targets
+
+ifdef COVER
+ifneq ($(COVER_REPORT_DIR),)
+tests::
+	@$(MAKE) make --no-print-directory cover-report
+endif
+endif
+
+clean:: coverdata-clean
+
+ifneq ($(COVER_REPORT_DIR),)
+distclean:: cover-report-clean
+endif
+
+help::
+	@printf "%s\n" "" \
+		"Cover targets:" \
+		"  cover-report  Generate a HTML coverage report from previously collected" \
+		"                cover data." \
+		"  all.coverdata Merge {eunit,ct}.coverdata into one coverdata file." \
+		"" \
+		"If COVER=1 is set, coverage data is generated by the targets eunit and ct. The" \
+		"target tests additionally generates a HTML coverage report from the combined" \
+		"coverdata files from each of these testing tools. HTML reports can be disabled" \
+		"by setting COVER_REPORT_DIR to empty."
+
+# Plugin specific targets
+
+COVERDATA = $(filter-out all.coverdata,$(wildcard *.coverdata))
+
+.PHONY: coverdata-clean
+coverdata-clean:
+	$(gen_verbose) rm -f *.coverdata ct.cover.spec
+
+# Merge all coverdata files into one.
+all.coverdata: $(COVERDATA)
+	$(gen_verbose) $(ERL) -eval ' \
+		$(foreach f,$(COVERDATA),cover:import("$(f)") == ok orelse halt(1),) \
+		cover:export("$@"), halt(0).'
+
+# These are only defined if COVER_REPORT_DIR is non-empty. Set COVER_REPORT_DIR to
+# empty if you want the coverdata files but not the HTML report.
+ifneq ($(COVER_REPORT_DIR),)
+
+.PHONY: cover-report-clean cover-report
+
+cover-report-clean:
+	$(gen_verbose) rm -rf $(COVER_REPORT_DIR)
+
+ifeq ($(COVERDATA),)
+cover-report:
+else
+
+# Modules which include eunit.hrl always contain one line without coverage
+# because eunit defines test/0 which is never called. We compensate for this.
+EUNIT_HRL_MODS = $(subst $(space),$(comma),$(shell \
+	grep -e '^\s*-include.*include/eunit\.hrl"' src/*.erl \
+	| sed "s/^src\/\(.*\)\.erl:.*/'\1'/" | uniq))
+
+cover-report:
+	$(gen_verbose) mkdir -p $(COVER_REPORT_DIR)
+	$(gen_verbose) $(ERL) -eval ' \
+	$(foreach f,$(COVERDATA),cover:import("$(f)") == ok orelse halt(1),) \
+	Ms = cover:imported_modules(), \
+	[cover:analyse_to_file(M, "$(COVER_REPORT_DIR)/" ++ atom_to_list(M) \
+		++ ".COVER.html", [html])  || M <- Ms], \
+	Report = [begin {ok, R} = cover:analyse(M, module), R end || M <- Ms], \
+	EunitHrlMods = [$(EUNIT_HRL_MODS)], \
+	Report1 = [{M, {Y, case lists:member(M, EunitHrlMods) of \
+		true -> N - 1; false -> N end}} || {M, {Y, N}} <- Report], \
+	TotalY = lists:sum([Y || {_, {Y, _}} <- Report1]), \
+	TotalN = lists:sum([N || {_, {_, N}} <- Report1]), \
+	TotalPerc = round(100 * TotalY / (TotalY + TotalN)), \
+	{ok, F} = file:open("$(COVER_REPORT_DIR)/index.html", [write]), \
+	io:format(F, "<!DOCTYPE html><html>~n" \
+		"<head><meta charset=\"UTF-8\">~n" \
+		"<title>Coverage report</title></head>~n" \
+		"<body>~n", []), \
+	io:format(F, "<h1>Coverage</h1>~n<p>Total: ~p%</p>~n", [TotalPerc]),\
+	io:format(F, "<table><tr><th>Module</th><th>Coverage</th></tr>~n", []), \
+	[io:format(F, "<tr><td><a href=\"~p.COVER.html\">~p</a></td>" \
+		"<td>~p%</td></tr>~n", \
+		[M, M, round(100 * Y / (Y + N))]) || {M, {Y, N}} <- Report1], \
+	How = "$(subst $(space),$(comma)$(space),$(basename $(COVERDATA)))", \
+	Date = "$(shell date -u "+%Y-%m-%dT%H:%M:%SZ")", \
+	io:format(F, "</table>~n" \
+		"<p>Generated using ~s and erlang.mk on ~s.</p>~n" \
+		"</body></html>", [How, Date]), \
+	halt().'
+
+endif
+endif # ifneq ($(COVER_REPORT_DIR),)

+ 12 - 9
plugins/eunit.mk

@@ -6,22 +6,19 @@
 
 # Configuration
 
+# All modules in TEST_DIR
 ifeq ($(strip $(TEST_DIR)),)
-TAGGED_EUNIT_TESTS = {dir,"ebin"}
-else
-ifeq ($(wildcard $(TEST_DIR)),)
-TAGGED_EUNIT_TESTS = {dir,"ebin"}
+TEST_DIR_MODS = 
 else
-# All modules in TEST_DIR
 TEST_DIR_MODS = $(notdir $(basename $(shell find $(TEST_DIR) -type f -name *.beam)))
+endif
+
 # All modules in 'ebin'
 EUNIT_EBIN_MODS = $(notdir $(basename $(shell find ebin -type f -name *.beam)))
 # Only those modules in TEST_DIR with no matching module in 'ebin'.
 # This is done to avoid some tests being executed twice.
 EUNIT_MODS = $(filter-out $(patsubst %,%_tests,$(EUNIT_EBIN_MODS)),$(TEST_DIR_MODS))
-TAGGED_EUNIT_TESTS = {dir,"ebin"} $(foreach mod,$(EUNIT_MODS),$(shell echo $(mod) | sed -e 's/\(.*\)/{module,\1}/g'))
-endif
-endif
+TAGGED_EUNIT_TESTS = $(foreach mod,$(EUNIT_EBIN_MODS) $(EUNIT_MODS),{module,$(mod)})
 
 EUNIT_OPTS ?=
 
@@ -42,10 +39,16 @@ help::
 
 # Plugin-specific targets.
 
+EUNIT_RUN_BEFORE ?=
+EUNIT_RUN_AFTER ?=
 EUNIT_RUN = $(ERL) \
 	-pa $(TEST_DIR) $(DEPS_DIR)/*/ebin \
 	-pz ebin \
-	-eval 'case eunit:test([$(call str-join,$(TAGGED_EUNIT_TESTS))], [$(EUNIT_OPTS)]) of ok -> halt(0); error -> halt(1) end.'
+	$(EUNIT_RUN_BEFORE) \
+	-eval 'case eunit:test([$(call str-join,$(TAGGED_EUNIT_TESTS))],\
+		[$(EUNIT_OPTS)]) of ok -> ok; error -> halt(1) end.' \
+	$(EUNIT_RUN_AFTER) \
+	-eval 'halt(0).'
 
 eunit: test-build
 	$(gen_verbose) $(EUNIT_RUN)

+ 50 - 13
test/Makefile

@@ -33,9 +33,9 @@ else
 	i = @echo ==
 endif
 
-.PHONY: all clean app ct eunit docs
+.PHONY: all clean app ct eunit tests-cover docs
 
-all: app ct eunit docs clean
+all: app ct eunit tests-cover docs clean
 	$i '+---------------------+'
 	$i '|  All tests passed.  |'
 	$i '+---------------------+'
@@ -100,17 +100,7 @@ ct: app1
 eunit: app1
 	$i "eunit: Testing the 'eunit' target."
 	$i "Running eunit test case inside module src/t.erl"
-	$t printf '%s\n' \
-		'-module(t).' \
-		'-export([succ/1]).' \
-		'succ(N) -> N + 1.' \
-		'-ifdef(TEST).' \
-		'-include_lib("eunit/include/eunit.hrl").' \
-		'succ_test() ->' \
-		'	?assertEqual(2, succ(1)),' \
-		'	os:cmd("echo t >> test-eunit.log").' \
-		'-endif.' \
-		> app1/src/t.erl
+	$t $(call create-module-t)
 	$t make -C app1 eunit $v
 	$i "Checking that the eunit test in module t."
 	$t echo t | cmp app1/test-eunit.log -
@@ -147,6 +137,38 @@ eunit: app1
 	$t rm -rf app1/eunit app1/src/t.erl app1/test-eunit.log
 	$i "Test 'eunit' passed."
 
+# TODO: do coverage for 'tests' instead of 'eunit ct' when triq is fixed
+tests-cover: app1
+	$i "tests-cover: Testing 'eunit' and 'ct' with COVER=1"
+	$i "Setting up eunit and ct suites."
+	$t $(call create-module-t)
+	$t mkdir -p app1/test
+	$t printf "%s\n" \
+		"-module(m_SUITE)." \
+		"-export([all/0, testcase1/1])." \
+		"all() -> [testcase1]." \
+		"testcase1(_) -> 2 = m:succ(1)." \
+	 > app1/test/m_SUITE.erl
+	$i "Running tests with coverage analysis."
+	$t make -C app1 eunit ct COVER=1 $v
+	$t [ -e app1/test-eunit.log ]
+	$t [ -e app1/eunit.coverdata ]
+	$t [ -e app1/ct.coverdata ]
+	$i "Generating coverage report."
+	$t make -C app1 cover-report COVER=1 $v
+	$t [ -e app1/cover/m.COVER.html ]
+	$t [ -e app1/cover/t.COVER.html ]
+	$t [ -e app1/cover/index.html ]
+	$i "Checking combined coverage from eunit and ct."
+	$t [ `grep 'Total: 100%' app1/cover/index.html | wc -l` -eq 1 ]
+	$i "Checking that cover-clean removes cover data and report."
+	$t make -C app1 cover-clean $v
+	$t [ ! -e app1/cover ] && [ ! -e app1/eunit.coverdata ]
+	@# clean up
+	$t rm -rf app1/src/t.erl app1/test app1/test-eunit.log
+	$t make -C app1 clean $v
+	$i "Test 'tests-cover' passed."
+
 docs: app1
 	$i "docs: Testing EDoc including DOC_DEPS."
 	$t printf "%s\n" \
@@ -181,3 +203,18 @@ app1:
 		"-export([succ/1])." \
 		"succ(N) -> N + 1." \
 		> app1/src/m.erl
+
+# Extra module in app1 used for testing eunit
+define create-module-t
+printf '%s\n' \
+	'-module(t).' \
+	'-export([succ/1]).' \
+	'succ(N) -> N + 1.' \
+	'-ifdef(TEST).' \
+	'-include_lib("eunit/include/eunit.hrl").' \
+	'succ_test() ->' \
+	'	?assertEqual(2, succ(1)),' \
+	'	os:cmd("echo t >> test-eunit.log").' \
+	'-endif.' \
+	> app1/src/t.erl
+endef