core/deps-tools.mk: New targets `fetch-deps` and `list-deps`

.. to fetch and list deps recurvively.

Therefore, they work on direct dependencies, dependencies' dependencies,
and so on. Nothing is built with those targets.

The following targets are also available to work on other kinds of
dependencies:
    make fetch-doc-deps
    make fetch-rel-deps
    make fetch-test-deps
    make fetch-shell-deps
    make list-doc-deps
    make list-rel-deps
    make list-test-deps
    make list-shell-deps

In all cases, they include "normal" and eg. "test" dependencies of the
top-level project, then only "normal" dependencies' dependencies.

It's possible to include several kinds in one go:
    make fetch-deps DEP_TYPES='doc test'
    make list-deps DEP_TYPES='doc test'

As it may be difficult to use the output of `make list-*` because
the list could appear after other targets output (like dependencies
fetching), the list is available in files pointed by the following
variables:
    $(ERLANG_MK_RECURSIVE_DEPS_LIST)
    $(ERLANG_MK_RECURSIVE_DOC_DEPS_LIST)
    $(ERLANG_MK_RECURSIVE_REL_DEPS_LIST)
    $(ERLANG_MK_RECURSIVE_TEST_DEPS_LIST)
    $(ERLANG_MK_RECURSIVE_SHELL_DEPS_LIST)

Internally, `list-deps` is simply implemented on top of `fetch-deps`:
the latter fills a sorted log with all the directories it traversed.
`list-deps` finally just prints the log file to stdout.

Fixes #560.

build.config

@@ -37,3 +37,6 @@ plugins/xref
 
 # Plugins enhancing the functionality of other plugins.
 plugins/cover
+
+# Core modules which can use variables from plugins.
+core/deps-tools

core/core.mk

@@ -121,6 +121,8 @@ help::
 		"  all           Run deps, app and rel targets in that order" \
 		"  app           Compile the project" \
 		"  deps          Fetch dependencies (if needed) and compile them" \
+		"  fetch-deps    Fetch dependencies recursively (if needed) without compiling them" \
+		"  list-deps     List dependencies recursively on stdout" \
 		"  search q=...  Search for a package in the built-in index" \
 		"  rel           Build a release for this project, if applicable" \
 		"  docs          Build the documentation for this project" \

core/deps-tools.mk

@@ -0,0 +1,105 @@
+# Copyright (c) 2013-2015, Loïc Hoguin <essen@ninenines.eu>
+# Copyright (c) 2015-2016, Jean-Sébastien Pédron <jean-sebastien@rabbitmq.com>
+# This file is part of erlang.mk and subject to the terms of the ISC License.
+
+# Fetch dependencies recursively (without building them).
+
+.PHONY: fetch-deps fetch-doc-deps fetch-rel-deps fetch-test-deps \
+	fetch-shell-deps
+
+.PHONY: $(ERLANG_MK_RECURSIVE_DEPS_LIST) \
+	$(ERLANG_MK_RECURSIVE_DOC_DEPS_LIST) \
+	$(ERLANG_MK_RECURSIVE_REL_DEPS_LIST) \
+	$(ERLANG_MK_RECURSIVE_TEST_DEPS_LIST) \
+	$(ERLANG_MK_RECURSIVE_SHELL_DEPS_LIST)
+
+fetch-deps: $(ERLANG_MK_RECURSIVE_DEPS_LIST)
+fetch-doc-deps: $(ERLANG_MK_RECURSIVE_DOC_DEPS_LIST)
+fetch-rel-deps: $(ERLANG_MK_RECURSIVE_REL_DEPS_LIST)
+fetch-test-deps: $(ERLANG_MK_RECURSIVE_TEST_DEPS_LIST)
+fetch-shell-deps: $(ERLANG_MK_RECURSIVE_SHELL_DEPS_LIST)
+
+ifneq ($(SKIP_DEPS),)
+$(ERLANG_MK_RECURSIVE_DEPS_LIST) \
+$(ERLANG_MK_RECURSIVE_DOC_DEPS_LIST) \
+$(ERLANG_MK_RECURSIVE_REL_DEPS_LIST) \
+$(ERLANG_MK_RECURSIVE_TEST_DEPS_LIST) \
+$(ERLANG_MK_RECURSIVE_SHELL_DEPS_LIST):
+	$(verbose) :> $@
+else
+# By default, we fetch "normal" dependencies. They are also included no
+# matter the type of requested dependencies.
+#
+# $(ALL_DEPS_DIRS) includes $(BUILD_DEPS).
+
+$(ERLANG_MK_RECURSIVE_DEPS_LIST): $(ALL_DEPS_DIRS)
+$(ERLANG_MK_RECURSIVE_DOC_DEPS_LIST): $(ALL_DEPS_DIRS) $(ALL_DOC_DEPS_DIRS)
+$(ERLANG_MK_RECURSIVE_REL_DEPS_LIST): $(ALL_DEPS_DIRS) $(ALL_REL_DEPS_DIRS)
+$(ERLANG_MK_RECURSIVE_TEST_DEPS_LIST): $(ALL_DEPS_DIRS) $(ALL_TEST_DEPS_DIRS)
+$(ERLANG_MK_RECURSIVE_SHELL_DEPS_LIST): $(ALL_DEPS_DIRS) $(ALL_SHELL_DEPS_DIRS)
+
+# Allow to use fetch-deps and $(DEP_TYPES) to fetch multiple types of
+# dependencies with a single target.
+ifneq ($(filter doc,$(DEP_TYPES)),)
+$(ERLANG_MK_RECURSIVE_DEPS_LIST): $(ALL_DOC_DEPS_DIRS)
+endif
+ifneq ($(filter rel,$(DEP_TYPES)),)
+$(ERLANG_MK_RECURSIVE_DEPS_LIST): $(ALL_REL_DEPS_DIRS)
+endif
+ifneq ($(filter test,$(DEP_TYPES)),)
+$(ERLANG_MK_RECURSIVE_DEPS_LIST): $(ALL_TEST_DEPS_DIRS)
+endif
+ifneq ($(filter shell,$(DEP_TYPES)),)
+$(ERLANG_MK_RECURSIVE_DEPS_LIST): $(ALL_SHELL_DEPS_DIRS)
+endif
+
+ERLANG_MK_RECURSIVE_TMP_LIST := $(abspath $(ERLANG_MK_TMP)/recursive-tmp-deps.log)
+
+$(ERLANG_MK_RECURSIVE_DEPS_LIST) \
+$(ERLANG_MK_RECURSIVE_DOC_DEPS_LIST) \
+$(ERLANG_MK_RECURSIVE_REL_DEPS_LIST) \
+$(ERLANG_MK_RECURSIVE_TEST_DEPS_LIST) \
+$(ERLANG_MK_RECURSIVE_SHELL_DEPS_LIST):
+ifeq ($(IS_APP)$(IS_DEP),)
+	$(verbose) mkdir -p $(ERLANG_MK_TMP)
+	$(verbose) rm -f $(ERLANG_MK_RECURSIVE_TMP_LIST)
+endif
+ifndef IS_APP
+	$(verbose) for dep in $(ALL_APPS_DIRS) ; do \
+		$(MAKE) -C $$dep $@ \
+		 IS_APP=1 \
+		 ERLANG_MK_RECURSIVE_TMP_LIST=$(ERLANG_MK_RECURSIVE_TMP_LIST) \
+		 || exit $$?; \
+	done
+endif
+	$(verbose) for dep in $^ ; do \
+		if ! grep -qs ^$$dep$$ $(ERLANG_MK_RECURSIVE_TMP_LIST); then \
+			echo $$dep >> $(ERLANG_MK_RECURSIVE_TMP_LIST); \
+			if grep -qs -E "^[[:blank:]]*include[[:blank:]]+(erlang\.mk|.*/erlang\.mk)$$" \
+			 $$dep/GNUmakefile $$dep/makefile $$dep/Makefile; then \
+				$(MAKE) -C $$dep fetch-deps \
+				 IS_DEP=1 \
+				 ERLANG_MK_RECURSIVE_TMP_LIST=$(ERLANG_MK_RECURSIVE_TMP_LIST) \
+				 || exit $$?; \
+			fi \
+		fi \
+	done
+ifeq ($(IS_APP)$(IS_DEP),)
+	$(verbose) sort < $(ERLANG_MK_RECURSIVE_TMP_LIST) | uniq > $@
+	$(verbose) rm $(ERLANG_MK_RECURSIVE_TMP_LIST)
+endif
+endif # ifneq ($(SKIP_DEPS),)
+
+# List dependencies recursively.
+
+.PHONY: list-deps list-doc-deps list-rel-deps list-test-deps \
+	list-shell-deps
+
+list-deps: $(ERLANG_MK_RECURSIVE_DEPS_LIST)
+list-doc-deps: $(ERLANG_MK_RECURSIVE_DOC_DEPS_LIST)
+list-rel-deps: $(ERLANG_MK_RECURSIVE_REL_DEPS_LIST)
+list-test-deps: $(ERLANG_MK_RECURSIVE_TEST_DEPS_LIST)
+list-shell-deps: $(ERLANG_MK_RECURSIVE_SHELL_DEPS_LIST)
+
+list-deps list-doc-deps list-rel-deps list-test-deps list-shell-deps:
+	$(verbose) cat $^

core/deps.mk

@@ -603,6 +603,15 @@ distclean-deps:
 	$(gen_verbose) rm -rf $(DEPS_DIR)
 endif
 
+# Forward-declare variables used in core/deps-tools.mk. This is required
+# in case plugins use them.
+
+ERLANG_MK_RECURSIVE_DEPS_LIST = $(ERLANG_MK_TMP)/recursive-deps-list.log
+ERLANG_MK_RECURSIVE_DOC_DEPS_LIST = $(ERLANG_MK_TMP)/recursive-doc-deps-list.log
+ERLANG_MK_RECURSIVE_REL_DEPS_LIST = $(ERLANG_MK_TMP)/recursive-rel-deps-list.log
+ERLANG_MK_RECURSIVE_TEST_DEPS_LIST = $(ERLANG_MK_TMP)/recursive-test-deps-list.log
+ERLANG_MK_RECURSIVE_SHELL_DEPS_LIST = $(ERLANG_MK_TMP)/recursive-shell-deps-list.log
+
 # External plugins.
 
 DEP_PLUGINS ?=

doc/src/guide/deps.asciidoc

@@ -286,6 +286,55 @@ different version of D, it will always be A's version
 that wins, because we fetch all dependencies of A before
 fetching those from B or C.
 
+=== Fetching and listing dependencies only
+
+You can fetch all dependencies recursively without building anything,
+with the `make fetch-deps` command. It follows the same rules described
+in the section above.
+
+You can list all dependencies recursively, again without building
+anything, with the `make list-deps` command. It will obviously need
+to fetch all dependencies exactly like `make fetch-deps`. Once
+everything is fetched, it prints a sorted list of absolute paths to the
+dependencies.
+
+By default, `fetch-deps` and `list-deps` work on the `BUILD_DEPS`
+and `DEPS` lists only. To also fetch/list `TEST_DEPS`, `DOC_DEPS`,
+`REL_DEPS` and/or `SHELL_DEPS`, you have two possibilities:
+
+* You can use `make fetch-test-deps`, `make fetch-doc-deps`, `make
+  fetch-rel-deps` and `make fetch-shell-deps` commands respectively.
+  If you want to list them, you can use `make list-test-deps`, `make
+  list-doc-deps`, `make list-rel-deps` and `make list-shell-deps`
+  respectively.
+* You can use `make fetch-deps` or `make list-deps` with the Makefile
+  variable `DEP_TYPES` set to a list of dependency types you want.
+  The types are `test`, `doc`, `rel` and `shell` respectively. For
+  example, you can list test and doc dependencies with `make list-deps
+  DEP_TYPES='test doc'`.
+
+Note that only first level `TEST_DEPS`, `DOC_DEPS`, `REL_DEPS` and
+`SHELL_DEPS` are included, not dependencies' one. In other word,
+`make list-test-deps` lists the `TEST_DEPS` of your project, but not
+`TEST_DEPS` of the projects yours depend on.
+
+No matter which method you use, `BUILD_DEPS` and `DEPS` are always
+included.
+
+Internally, the `make fetch-*` commands store the complete list of
+dependencies in files named `$(ERLANG_MK_RECURSIVE_DEPS_LIST)`,
+`$(ERLANG_MK_RECURSIVE_TEST_DEPS_LIST)`,
+`$(ERLANG_MK_RECURSIVE_DOC_DEPS_LIST)`,
+`$(ERLANG_MK_RECURSIVE_REL_DEPS_LIST)` and
+`$(ERLANG_MK_RECURSIVE_SHELL_DEPS_LIST)`. Those files are simply printed
+by the `make list-*` commands.
+
+`make list-*` commands are made for human beings. If you need the list
+of dependencies in a Makefile or a script, you should use the content
+of those files directly instead. The reason is that `make fetch-*` and
+`make list-*` may have unwanted content in their output, such as actual
+fetching of dependencies.
+
 === Ignoring unwanted dependencies
 
 Sometimes, you may want to ignore dependencies entirely.

test/core_deps.mk

@@ -1,6 +1,6 @@
 # Core: Packages and dependencies.
 
-CORE_DEPS_CASES = apps apps-build-count apps-conflict apps-deep-conflict apps-dir apps-dir-include-lib apps-new-app apps-new-lib apps-new-tpl apps-only autopatch-rebar build-c-8cc build-c-imagejs build-erl build-js dep-commit dir doc fetch-cp fetch-custom fetch-fail-bad fetch-fail-unknown fetch-git fetch-git-submodule fetch-hex fetch-hg fetch-legacy fetch-svn ignore mv mv-rebar no-autopatch no-autopatch-erlang-mk no-autopatch-rebar order-first order-top otp pkg rel search shell skip test
+CORE_DEPS_CASES = apps apps-build-count apps-conflict apps-deep-conflict apps-dir apps-dir-include-lib apps-new-app apps-new-lib apps-new-tpl apps-only autopatch-rebar build-c-8cc build-c-imagejs build-erl build-js dep-commit dir doc fetch-cp fetch-custom fetch-fail-bad fetch-fail-unknown fetch-git fetch-git-submodule fetch-hex fetch-hg fetch-legacy fetch-svn ignore list-deps mv mv-rebar no-autopatch no-autopatch-erlang-mk no-autopatch-rebar order-first order-top otp pkg rel search shell skip test
 CORE_DEPS_TARGETS = $(addprefix core-deps-,$(CORE_DEPS_CASES))
 
 .PHONY: core-deps $(CORE_DEPS_TARGETS)
@@ -983,6 +983,141 @@ endif
 	$i "Check that the correct dependencies were fetched"
 	$t test -d $(APP)/deps/ranch
 
+define add_dep_and_subdep
+	$i "Bootstrap a new OTP library named $(APP)-$(1)subdep"
+	$t mkdir $(APP)-$(1)subdep/
+	$t cp ../erlang.mk $(APP)-$(1)subdep/
+	$t $(MAKE) -C $(APP)-$(1)subdep --no-print-directory -f erlang.mk bootstrap-lib $$v
+
+	$i "Create a Git repository for $(APP)-$(1)subdep"
+	$t (cd $(APP)-$(1)subdep && \
+		git init -q && \
+		git config user.name "Testsuite" && \
+		git config user.email "testsuite@erlang.mk" && \
+		git add . && \
+		git commit -q -m "Initial commit")
+
+	$i "Bootstrap a new OTP library named $(APP)-$(1)dep"
+	$t mkdir $(APP)-$(1)dep/
+	$t cp ../erlang.mk $(APP)-$(1)dep/
+	$t $(MAKE) -C $(APP)-$(1)dep --no-print-directory -f erlang.mk bootstrap-lib $$v
+
+	$i "Add $(APP)-$(1)subdep as a dependency"
+	$t perl -ni.bak -e \
+		'print;if ($$.==1) {print "DEPS = $(1)subdep\ndep_$(1)subdep = git file://$(abspath $(APP)-$(1)subdep) master\n"}' \
+		$(APP)-$(1)dep/Makefile
+	$t rm $(APP)-$(1)dep/Makefile.bak
+
+	$i "Create a Git repository for $(APP)-$(1)dep"
+	$t (cd $(APP)-$(1)dep && \
+		git init -q && \
+		git config user.name "Testsuite" && \
+		git config user.email "testsuite@erlang.mk" && \
+		git add . && \
+		git commit -q -m "Initial commit")
+endef
+
+core-deps-list-deps: build clean
+
+	$(call add_dep_and_subdep,)
+	$(call add_dep_and_subdep,doc)
+	$(call add_dep_and_subdep,rel)
+	$(call add_dep_and_subdep,test)
+	$(call add_dep_and_subdep,shell)
+
+	$i "Bootstrap a new OTP library named $(APP)"
+	$t mkdir $(APP)/
+	$t cp ../erlang.mk $(APP)/
+	$t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib $v
+
+	$i "Add $(APP)-dep and $(APP)-testdep as a dependency"
+	$t sed -i.bak '2i\
+DEPS = dep\
+DOC_DEPS = docdep\
+REL_DEPS = reldep\
+TEST_DEPS = testdep\
+SHELL_DEPS = shelldep\
+dep_dep = git file://$(abspath $(APP)-dep) master\
+dep_docdep = git file://$(abspath $(APP)-docdep) master\
+dep_reldep = git file://$(abspath $(APP)-reldep) master\
+dep_testdep = git file://$(abspath $(APP)-testdep) master\
+dep_shelldep = git file://$(abspath $(APP)-shelldep) master\
+' $(APP)/Makefile
+	$t rm $(APP)/Makefile.bak
+
+	$i "Create a Git repository for $(APP)"
+	$t (cd $(APP) && \
+		git init -q && \
+		git config user.name "Testsuite" && \
+		git config user.email "testsuite@erlang.mk" && \
+		git add . && \
+		git commit -q -m "Initial commit")
+
+	$i "List application dependencies"
+	$t $(MAKE) -C $(APP) --no-print-directory list-deps $v
+	$t test -d $(APP)/deps/subdep
+	$t printf "%s\n%s\n" $(abspath $(APP)/deps/dep $(APP)/deps/subdep) > $(APP)/expected-deps.txt
+	$t cmp $(APP)/expected-deps.txt $(APP)/.erlang.mk/recursive-deps-list.log
+	$t $(MAKE) -C $(APP) --no-print-directory distclean $v
+
+	$i "List application doc dependencies"
+	$t $(MAKE) -C $(APP) --no-print-directory list-doc-deps $v
+	$t test -d $(APP)/deps/subdep
+	$t test -d $(APP)/deps/docsubdep
+	$t printf "%s\n%s\n%s\n%s\n" \
+		$(abspath $(APP)/deps/dep $(APP)/deps/subdep $(APP)/deps/docdep $(APP)/deps/docsubdep) \
+		| sort > $(APP)/expected-doc-deps.txt
+	$t cmp $(APP)/expected-doc-deps.txt $(APP)/.erlang.mk/recursive-doc-deps-list.log
+	$t $(MAKE) -C $(APP) --no-print-directory distclean $v
+
+	$i "List application rel dependencies"
+	$t $(MAKE) -C $(APP) --no-print-directory list-rel-deps $v
+	$t test -d $(APP)/deps/subdep
+	$t test -d $(APP)/deps/relsubdep
+	$t printf "%s\n%s\n%s\n%s\n" \
+		$(abspath $(APP)/deps/dep $(APP)/deps/subdep $(APP)/deps/reldep $(APP)/deps/relsubdep) \
+		| sort > $(APP)/expected-rel-deps.txt
+	$t cmp $(APP)/expected-rel-deps.txt $(APP)/.erlang.mk/recursive-rel-deps-list.log
+	$t $(MAKE) -C $(APP) --no-print-directory distclean $v
+
+	$i "List application test dependencies"
+	$t $(MAKE) -C $(APP) --no-print-directory list-test-deps $v
+	$t test -d $(APP)/deps/subdep
+	$t test -d $(APP)/deps/testsubdep
+	$t printf "%s\n%s\n%s\n%s\n" \
+		$(abspath $(APP)/deps/dep $(APP)/deps/subdep $(APP)/deps/testdep $(APP)/deps/testsubdep) \
+		| sort > $(APP)/expected-test-deps.txt
+	$t cmp $(APP)/expected-test-deps.txt $(APP)/.erlang.mk/recursive-test-deps-list.log
+	$t $(MAKE) -C $(APP) --no-print-directory distclean $v
+
+	$i "List application shell dependencies"
+	$t $(MAKE) -C $(APP) --no-print-directory list-shell-deps $v
+	$t test -d $(APP)/deps/subdep
+	$t test -d $(APP)/deps/shellsubdep
+	$t printf "%s\n%s\n%s\n%s\n" \
+		$(abspath $(APP)/deps/dep $(APP)/deps/subdep $(APP)/deps/shelldep $(APP)/deps/shellsubdep) \
+		| sort > $(APP)/expected-shell-deps.txt
+	$t cmp $(APP)/expected-shell-deps.txt $(APP)/.erlang.mk/recursive-shell-deps-list.log
+	$t $(MAKE) -C $(APP) --no-print-directory distclean $v
+
+	$i "List application all dependencies (all kinds)"
+	$t $(MAKE) -C $(APP) --no-print-directory list-deps DEP_TYPES='doc rel test shell' $v
+	$t test -d $(APP)/deps/subdep
+	$t test -d $(APP)/deps/docsubdep
+	$t test -d $(APP)/deps/relsubdep
+	$t test -d $(APP)/deps/testsubdep
+	$t test -d $(APP)/deps/shellsubdep
+	$t printf "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n" \
+		$(abspath \
+			$(APP)/deps/dep $(APP)/deps/subdep \
+			$(APP)/deps/docdep $(APP)/deps/docsubdep \
+			$(APP)/deps/reldep $(APP)/deps/relsubdep \
+			$(APP)/deps/testdep $(APP)/deps/testsubdep \
+			$(APP)/deps/shelldep $(APP)/deps/shellsubdep) \
+		| sort > $(APP)/expected-all-deps.txt
+	$t cmp $(APP)/expected-all-deps.txt $(APP)/.erlang.mk/recursive-deps-list.log
+	$t $(MAKE) -C $(APP) --no-print-directory distclean $v
+
 core-deps-mv: build clean
 
 	$i "Bootstrap a new OTP library named $(APP)"