erlank.mk/test/Makefile

# Copyright (c) 2014, Viktor Söderqvist <viktor@zuiderkwast.se>
# This file is part of erlang.mk and subject to the terms of the ISC License.

# Tests for erlang.mk targets. If any test fails or if you run a target other
# than 'all', you must probably do 'make clean' before you can test again.

# Verbosity.

V ?= 0

# Temporary files directory.

ERLANG_MK_TMP=$(CURDIR)/tmp
export ERLANG_MK_TMP

# Temporary application name, taken from rule name.

APP = $(subst -,_,$@)
APP_TO_CLEAN = $(subst -,_,$(patsubst clean-%,%,$@))

# Erlang, quickly!

ERL = erl +A0 -noinput -boot start_clean

# t = Verbosity control for tests
# v = Verbosity control for erlang.mk
# i = Command to display (or suppress) info messages
ifeq ($V,0)
	# Show info messages only
	t = @
	v = V=0 >/dev/null 2>&1
	i = @echo $@:
else ifeq ($V,1)
	# Show test commands
	t =
	v = V=0 >/dev/null 2>&1
	i = @echo == $@:
else ifeq ($V,2)
	# Show briefly what erlang.mk is doing
	t = @echo " TEST  " $@;
	v = V=0
	i = @echo == $@:
else
	# Show all commands with maximum verbosity
	t =
	v = V=1
	i = @echo == $@:
endif

.PHONY: all clean app ct eunit tests-cover docs

.NOTPARALLEL:

all: clean core bootstrap app ct eunit tests-cover docs pkgs
	$i '+---------------------+'
	$i '|  All tests passed.  |'
	$i '+---------------------+'

clean: clean-core clean-bootstrap
	$t rm -rf app1 pkgs.log $(ERLANG_MK_TMP)

build:
	$i "Generate a bleeding edge Erlang.mk"
	$t cd .. && $(MAKE) $v

# Core.

.PHONY: core clean-core

core: core-upgrade

clean-core: clean-core-upgrade

# Core: Erlang.mk upgrade.

CORE_UPGRADE_CASES = no-config custom-config renamed-config custom-build-dir
CORE_UPGRADE_TARGETS = $(addprefix core-upgrade-,$(CORE_UPGRADE_CASES))
CORE_UPGRADE_CLEAN_TARGETS = $(addprefix clean-,$(CORE_UPGRADE_TARGETS))

.PHONY: core-upgrade $(CORE_UPGRADE_TARGETS) $(CORE_UPGRADE_CLEAN_TARGETS)

core-upgrade: $(CORE_UPGRADE_TARGETS)

core-upgrade-no-config: build clean-core-upgrade-no-config

	$i "Bootstrap a new OTP library named $(APP)"
	$t mkdir $(APP)/
	$t cp ../erlang.mk $(APP)/
	$t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib $v

	$i "Append a rule to the Erlang.mk file for testing purposes"
	$t echo "erlang_mk_upgrade_test_rule: ; @echo FAIL" >> $(APP)/erlang.mk

	$i "Check that the test rule works as intended"
	$t test "FAIL" = "`$(MAKE) -C $(APP) --no-print-directory erlang_mk_upgrade_test_rule V=0`"

	$i "Upgrade Erlang.mk"
	$t $(MAKE) -C $(APP) erlang-mk $v

	$i "Check that the rule is gone"
	$t if $(MAKE) -C $(APP) erlang_mk_upgrade_test_rule $v; then false; fi

core-upgrade-custom-config: build clean-core-upgrade-custom-config

	$i "Bootstrap a new OTP library named $(APP)"
	$t mkdir $(APP)/
	$t cp ../erlang.mk $(APP)/
	$t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib $v

	$i "Create a custom build.config file without plugins"
	$t echo "core/*" > $(APP)/build.config

	$i "Upgrade Erlang.mk"
	$t $(MAKE) -C $(APP) erlang-mk $v

	$i "Check that the bootstrap plugin is gone"
	$t if $(MAKE) -C $(APP) list-templates $v; then false; fi

core-upgrade-renamed-config: build clean-core-upgrade-renamed-config

	$i "Bootstrap a new OTP library named $(APP)"
	$t mkdir $(APP)/
	$t cp ../erlang.mk $(APP)/
	$t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib $v

	$i "Create a custom build.config file without plugins; name it my.build.config"
	$t echo "core/*" > $(APP)/my.build.config

	$i "Set ERLANG_MK_BUILD_CONFIG=my.build.config in the Makefile"
	$t echo "ERLANG_MK_BUILD_CONFIG = my.build.config" >> $(APP)/Makefile

	$i "Upgrade Erlang.mk"
	$t $(MAKE) -C $(APP) erlang-mk $v

	$i "Check that the bootstrap plugin is gone"
	$t if $(MAKE) -C $(APP) list-templates $v; then false; fi

core-upgrade-custom-build-dir: build clean-core-upgrade-custom-build-dir

	$i "Bootstrap a new OTP library named $(APP)"
	$t mkdir $(APP)/
	$t cp ../erlang.mk $(APP)/
	$t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib $v

	$i "Append a rule to the Erlang.mk file for testing purposes"
	$t echo "erlang_mk_upgrade_test_rule: ; @echo FAIL" >> $(APP)/erlang.mk

	$i "Check that the test rule works as intended"
	$t test "FAIL" = "`$(MAKE) -C $(APP) --no-print-directory erlang_mk_upgrade_test_rule V=0`"

	$i "Create the custom build directory"
	$t mkdir $(APP)/custom/
	$t test -d $(APP)/custom/

	$i "Upgrade Erlang.mk with a custom build directory"
	$t ERLANG_MK_BUILD_DIR=custom $(MAKE) -C $(APP) erlang-mk $v

	$i "Check that the rule is gone"
	$t if $(MAKE) -C $(APP) erlang_mk_upgrade_test_rule $v; then false; fi

	$i "Check that the custom build directory is gone"
	$t test ! -d $(APP)/custom/

clean-core-upgrade: $(CORE_UPGRADE_CLEAN_TARGETS)

$(CORE_UPGRADE_CLEAN_TARGETS):
	$t rm -rf $(APP_TO_CLEAN)/

# Bootstrap plugin.

BOOTSTRAP_CASES = app lib rel templates
BOOTSTRAP_TARGETS = $(addprefix bootstrap-,$(BOOTSTRAP_CASES))
BOOTSTRAP_CLEAN_TARGETS = $(addprefix clean-,$(BOOTSTRAP_TARGETS))

.PHONY: bootstrap $(BOOTSTRAP_TARGETS) $(BOOTSTRAP_CLEAN_TARGETS)

bootstrap: $(BOOTSTRAP_TARGETS)

bootstrap-app: build clean-bootstrap-app

	$i "Bootstrap a new OTP application named $(APP)"
	$t mkdir $(APP)/
	$t cp ../erlang.mk $(APP)/
	$t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v

	$i "Check that all bootstrapped files exist"
	$t test -f $(APP)/Makefile
	$t test -f $(APP)/src/$(APP).app.src
	$t test -f $(APP)/src/$(APP)_app.erl
	$t test -f $(APP)/src/$(APP)_sup.erl

	$i "Build the application"
	$t $(MAKE) -C $(APP) $v

	$i "Check that all compiled files exist"
	$t test -f $(APP)/ebin/$(APP).app
	$t test -f $(APP)/ebin/$(APP)_app.beam
	$t test -f $(APP)/ebin/$(APP)_sup.beam

	$i "Check that the application was compiled correctly"
	$t $(ERL) -pa $(APP)/ebin/ -eval " \
		ok = application:start($(APP)), \
		{ok, [$(APP)_app, $(APP)_sup]} = application:get_key($(APP), modules), \
		{module, $(APP)_app} = code:load_file($(APP)_app), \
		{module, $(APP)_sup} = code:load_file($(APP)_sup), \
		halt()"

bootstrap-lib: build clean-bootstrap-lib

	$i "Bootstrap a new OTP library named $(APP)"
	$t mkdir $(APP)/
	$t cp ../erlang.mk $(APP)/
	$t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib $v

	$i "Check that all bootstrapped files exist"
	$t test -f $(APP)/Makefile
	$t test -f $(APP)/src/$(APP).app.src

	$i "Build the application"
	$t $(MAKE) -C $(APP) $v

	$i "Check that all compiled files exist"
	$t test -f $(APP)/ebin/$(APP).app

	$i "Check that the application was compiled correctly"
	$t $(ERL) -pa $(APP)/ebin/ -eval " \
		ok = application:start($(APP)), \
		{ok, []} = application:get_key($(APP), modules), \
		halt()"

bootstrap-rel: build clean-bootstrap-rel

	$i "Bootstrap a new release-enabled OTP application named $(APP)"
	$t mkdir $(APP)/
	$t cp ../erlang.mk $(APP)/
	$t $(MAKE) -C $(APP) -f erlang.mk bootstrap bootstrap-rel $v

	$i "Check that all bootstrapped files exist"
	$t test -f $(APP)/Makefile
	$t test -f $(APP)/relx.config
	$t test -f $(APP)/rel/sys.config
	$t test -f $(APP)/rel/vm.args
	$t test -f $(APP)/src/$(APP).app.src
	$t test -f $(APP)/src/$(APP)_app.erl
	$t test -f $(APP)/src/$(APP)_sup.erl

	$i "Build the application and the release"
	$t $(MAKE) -C $(APP) $v

	$i "Check that all compiled files exist"
	$t test -f $(APP)/ebin/$(APP).app
	$t test -f $(APP)/ebin/$(APP)_app.beam
	$t test -f $(APP)/ebin/$(APP)_sup.beam

	$i "Check that the release was generated"
	$t test -f $(APP)/_rel/$(APP)_release/bin/$(APP)_release

	$i "Check that the release can be started and stopped"
	$t $(APP)/_rel/$(APP)_release/bin/$(APP)_release start $v
	$t $(APP)/_rel/$(APP)_release/bin/$(APP)_release stop $v

	$i "Check that there's no erl_crash.dump file"
	$t test ! -f $(APP)/_rel/$(APP)_release/erl_crash.dump

bootstrap-templates: build clean-bootstrap-templates

	$i "Bootstrap a new OTP library named $(APP)"
	$t mkdir $(APP)/
	$t cp ../erlang.mk $(APP)/
	$t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib $v

	$i "Check that we can get the list of templates"
	$t test `$(MAKE) -C $(APP) --no-print-directory list-templates V=0 | wc -l` -eq 1

	$i "Generate one of each template"
	$t $(MAKE) -C $(APP) --no-print-directory new t=gen_fsm n=my_fsm
	$t $(MAKE) -C $(APP) --no-print-directory new t=gen_server n=my_server
	$t $(MAKE) -C $(APP) --no-print-directory new t=supervisor n=my_sup
	$t $(MAKE) -C $(APP) --no-print-directory new t=cowboy_http n=my_http
	$t $(MAKE) -C $(APP) --no-print-directory new t=cowboy_loop n=my_loop
	$t $(MAKE) -C $(APP) --no-print-directory new t=cowboy_rest n=my_rest
	$t $(MAKE) -C $(APP) --no-print-directory new t=cowboy_ws n=my_ws
	$t $(MAKE) -C $(APP) --no-print-directory new t=ranch_protocol n=my_protocol

# Here we disable warnings because templates contain missing behaviors.
	$i "Build the application"
	$t $(MAKE) -C $(APP) ERLC_OPTS=+debug_info $v

	$i "Check that all compiled files exist"
	$t test -f $(APP)/ebin/$(APP).app
	$t test -f $(APP)/ebin/my_fsm.beam
	$t test -f $(APP)/ebin/my_server.beam
	$t test -f $(APP)/ebin/my_sup.beam

	$i "Check that all the modules can be loaded"
	$t $(ERL) -pa $(APP)/ebin/ -eval " \
		ok = application:start($(APP)), \
		{ok, Mods = [my_fsm, my_http, my_loop, my_protocol, my_rest, my_server, my_sup, my_ws]} \
			= application:get_key($(APP), modules), \
		[{module, M} = code:load_file(M) || M <- Mods], \
		halt()"

clean-bootstrap: $(BOOTSTRAP_CLEAN_TARGETS)

$(BOOTSTRAP_CLEAN_TARGETS):
	$t rm -rf $(APP_TO_CLEAN)/

########################################
# Legacy tests.

app: app1
	$i "app: Testing the 'app' target."
	$t $(MAKE) -C app1 app $v
	$i "Checking the modules line in the generated .app file."
	$t [ `grep -E "{modules, *\['m'\]}" app1/ebin/app1.app | wc -l` -eq 1 ]
	$t [ -e app1/ebin/m.beam ]
	$i "Checking that '$(MAKE) clean-app' deletes ebin."
	$t $(MAKE) -C app1 clean-app $v
	$t [ ! -e app1/ebin ]
	$i "Checking that '$(MAKE) app' returns non-zero on compile errors."
	$t printf "%s\n" \
		"-module(syntax_error)." \
		"foo lorem_ipsum dolor sit amet." \
	 > app1/src/syntax_error.erl
	$t if $(MAKE) -C app1 app $v ; then false ; fi
	$t rm app1/src/syntax_error.erl
	$i "Test 'app' passed."

ct: app1
	$i "ct: Testing ct and related targets."
	$i "Setting up test suite."
	$t mkdir -p app1/test
	$t printf "%s\n" \
		"-module(m_SUITE)." \
		"-export([all/0, testcase1/1])." \
		"all() -> [testcase1]." \
		"testcase1(_) -> 2 = m:succ(1)." \
	 > app1/test/m_SUITE.erl
	$t $(MAKE) -C app1 ct $v
	$i "Checking files created by '$(MAKE) ct'."
	$t [ -e app1/test/m_SUITE.beam ]
	$t [ -e app1/ebin/m.beam ]
	$t [ -e app1/logs ]
	$i "Checking that '$(MAKE) clean' does not delete logs."
	$t $(MAKE) -C app1 clean $v
	$t [ -e app1/logs ]
	$i "Testing target 'ct-mysuite' where mysuite_SUITE is a test suite."
	$t $(MAKE) -C app1 ct-m $v
	$i "Checking that '$(MAKE) ct' returns non-zero for a failing suite."
	$t printf "%s\n" \
		"-module(failing_SUITE)." \
		"-export([all/0, testcase1/1])." \
		"all() -> [testcase1]." \
		"testcase1(_) -> 42 = m:succ(1)." \
	 > app1/test/failing_SUITE.erl
	$t if $(MAKE) -C app1 ct-failing $v ; then false ; fi
	$i "Checking that '$(MAKE) distclean-ct' deletes logs."
	$t $(MAKE) -C app1 distclean-ct $v
	$t [ ! -e app1/logs ]
	$t [ -e app1/ebin/m.beam ]
	$i "Cleaning up test data."
	$t rm -rf app1/test
	$i "Test 'ct' passed."

eunit: app1
	$i "eunit: Testing the 'eunit' target."
	$i "Running eunit test case inside module src/t.erl"
	$t $(call create-module-t)
	$t $(MAKE) -C app1 distclean $v
	$t $(MAKE) -C app1 eunit $v
	$i "Checking that the eunit test in module t."
	$t echo t | cmp app1/test-eunit.log -
	$t rm app1/test-eunit.log
	$i "Running eunit tests in a separate directory."
	$t mkdir -p app1/eunit
	$t printf '%s\n' \
		'-module(t_tests).' \
		'-include_lib("eunit/include/eunit.hrl").' \
		'succ_test() ->' \
		'	?assertEqual(2, t:succ(1)),' \
		'	os:cmd("echo t_tests >> test-eunit.log").' \
		> app1/eunit/t_tests.erl
	$t printf '%s\n' \
		'-module(x_tests).' \
		'-include_lib("eunit/include/eunit.hrl").' \
		'succ_test() ->' \
		'	?assertEqual(2, t:succ(1)),' \
		'	os:cmd("echo x_tests >> test-eunit.log").' \
		> app1/eunit/x_tests.erl
	$t $(MAKE) -C app1 distclean TEST_DIR=eunit $v
	$t $(MAKE) -C app1 eunit TEST_DIR=eunit $v
	$i "Checking that '$(MAKE) eunit' didn't run the tests in t_tests twice, etc."
	$t printf "%s\n" t t_tests x_tests | cmp app1/test-eunit.log -
	$t rm app1/test-eunit.log
	$i "Checking that '$(MAKE) eunit' returns non-zero for a failing test."
	$t rm -f app1/eunit/*
	$t printf "%s\n" \
		"-module(t_tests)." \
		'-include_lib("eunit/include/eunit.hrl").' \
		"succ_test() ->" \
		"	?assertEqual(42, t:succ(1))." \
		> app1/eunit/t_tests.erl
	$t $(MAKE) -C app1 distclean TEST_DIR=eunit $v
	$t if $(MAKE) -C app1 eunit TEST_DIR=eunit $v ; then false ; fi
	$t rm -rf app1/eunit app1/src/t.erl app1/test-eunit.log
	$i "Test 'eunit' passed."

# TODO: do coverage for 'tests' instead of 'eunit ct' when triq is fixed
tests-cover: app1
	$i "tests-cover: Testing 'eunit' and 'ct' with COVER=1"
	$i "Setting up eunit and ct suites."
	$t $(call create-module-t)
	$t mkdir -p app1/test
	$t printf "%s\n" \
		"-module(m_SUITE)." \
		"-export([all/0, testcase1/1])." \
		"all() -> [testcase1]." \
		"testcase1(_) -> 2 = m:succ(1)." \
	 > app1/test/m_SUITE.erl
	$i "Running tests with coverage analysis."
	$t $(MAKE) -C app1 eunit ct COVER=1 $v
	$t [ -e app1/test-eunit.log ]
	$t [ -e app1/eunit.coverdata ]
	$t [ -e app1/ct.coverdata ]
	$i "Generating coverage report."
	$t $(MAKE) -C app1 cover-report COVER=1 $v
	$t [ -e app1/cover/m.COVER.html ]
	$t [ -e app1/cover/t.COVER.html ]
	$t [ -e app1/cover/index.html ]
	$i "Checking combined coverage from eunit and ct."
	$t [ `grep 'Total: 100%' app1/cover/index.html | wc -l` -eq 1 ]
	$i "Checking that cover-report-clean removes cover report."
	$t $(MAKE) -C app1 cover-report-clean $v
	$t [ ! -e app1/cover ]
	$i "Checking that coverdata-clean removes cover data."
	$t $(MAKE) -C app1 coverdata-clean $v
	$t [ ! -e app1/eunit.coverdata ]
	@# clean up
	$t rm -rf app1/src/t.erl app1/test app1/test-eunit.log
	$t $(MAKE) -C app1 clean $v
	$i "Test 'tests-cover' passed."

docs: app1
	$i "docs: Testing EDoc including DOC_DEPS."
	$t printf "%s\n" \
		"PROJECT = app1" \
		"DOC_DEPS = edown" \
		"dep_edown = git https://github.com/uwiger/edown.git 0.7" \
		"EDOC_OPTS = {doclet, edown_doclet}" \
		"include erlang.mk" \
		"distclean:: distclean-doc-md" \
		"distclean-doc-md:" \
		"	rm -rf doc/*.md" \
		> app1/Makefile-doc
	$i "Downloading doc deps (edown) and building docs."
	$t $(MAKE) -C app1 -f Makefile-doc docs $v
	$i "Checking that '$(MAKE) docs' using edown generated a markdown file."
	$t [ -e app1/doc/m.md ]
	$i "Checking that '$(MAKE) distclean' deletes all generated doc files."
	$t $(MAKE) -C app1 -f Makefile-doc distclean $v
	$t [ "`ls app1/doc/`" = "" ]
	$i "Cleaning up test data."
	$t rm app1/Makefile-doc
	$i "Test 'docs' passed."

define app1_setup
	$i "Setting up app."
	$t mkdir -p app1
	$t cd .. && $(MAKE)
	$t cp ../erlang.mk app1/
	$t $(MAKE) -C app1 -f erlang.mk bootstrap-lib
	$t printf "%s\n" \
		"-module(m)." \
		"-export([succ/1])." \
		"succ(N) -> N + 1." \
		> app1/src/m.erl
endef

define pkg_test_target
pkg-$(1)-clean:
	$t rm -rf app1 erl_crash.dump

pkg-$(1)-app1:
	$(call app1_setup)

# Running 'make' twice to make sure it recompiles fine.
pkg-$(1): pkg-$(1)-clean pkg-$(1)-app1
	$i
	$i "  pkgs: Checking that '$(1)' builds correctly"
	$i
	$t printf "%s\n" \
		"PROJECT = app1" \
		"DEPS = $(1)" \
		"include erlang.mk" \
		> app1/Makefile
	$t \
	if [ "$(1)" = "amqp_client" ]; then \
		$(MAKE) -C app1 RABBITMQ_CLIENT_PATCH=1; \
	elif [ "$(1)" = "rabbit" ]; then \
		$(MAKE) -C app1 RABBITMQ_SERVER_PATCH=1; \
	else \
		$(MAKE) -C app1; \
	fi; \
	if [ $$$$? -ne 0 ]; then \
		echo "$(1): make error" >> pkgs.log; \
	else \
		$(MAKE) -C app1; if [ $$$$? -ne 0 ]; then \
			echo "$(1): re-make error" >> pkgs.log; \
		else \
			find . -type f -name erl_crash.dump; if [ $$$$? -ne 0 ]; then \
				echo "$(1): erl_crash.dump found" >> pkgs.log; \
			else \
				erl +A0 -noinput -boot start_clean -pa app1/deps/*/ebin -eval " \
						Apps = [list_to_atom(App) || \"app1/deps/\" ++ App \
							<- filelib:wildcard(\"app1/deps/*\")], \
						[begin \
							io:format(\"Loading application ~p~n\", [App]), \
							case application:load(App) of \
								{error, _} -> ok; \
								ok -> \
									{ok, Mods} = application:get_key(App, modules), \
									[try io:format(\"  Loading module ~p~n\", [Mod]), \
										{module, Mod} = code:load_file(Mod) \
									catch C:R -> timer:sleep(500), erlang:C(R) \
									end || Mod <- Mods] \
							end \
						end || App <- Apps], \
						halt()."; if [ $$$$? -ne 0 ]; then \
					echo "$(1): load error" >> pkgs.log; \
				fi \
			fi \
		fi \
	fi
endef

PACKAGES = $(foreach pkg,$(sort $(wildcard ../index/*.mk)),$(notdir $(basename $(pkg))))

$(foreach pkg,$(PACKAGES),$(eval $(call pkg_test_target,$(pkg))))

pkgs: $(addprefix pkg-,$(PACKAGES))
	@if [ -f pkgs.log ]; then \
		echo "+-------------------------------+"; \
		echo "| ERRORS WHILE TESTING PACKAGES |"; \
		echo "+-------------------------------+"; \
		cat pkgs.log; \
		exit 33; \
	fi

# Test application used for testing.
app1:
	$(call app1_setup)

# Extra module in app1 used for testing eunit
define create-module-t
printf '%s\n' \
	'-module(t).' \
	'-export([succ/1]).' \
	'succ(N) -> N + 1.' \
	'-ifdef(TEST).' \
	'-include_lib("eunit/include/eunit.hrl").' \
	'succ_test() ->' \
	'	?assertEqual(2, succ(1)),' \
	'	os:cmd("echo t >> test-eunit.log").' \
	'-endif.' \
	> app1/src/t.erl
endef