Don't rebuild dependencies by default

Unless it's a symbolic link, it's built directly, FULL=1 is set
or the file ebin/dep_built in the dependency is removed.

See the documentation changes for more details.

This provides immense build speed gains, for example on a
RabbitMQ project it went from 10s to 1s for the 2nd+ builds.

core/deps.mk

@@ -75,6 +75,12 @@ dep_verbose_0 = @echo " DEP    $1 ($(call dep_commit,$1))";
 dep_verbose_2 = set -x;
 dep_verbose = $(dep_verbose_$(V))
 
+# Optimization: don't recompile deps unless truly necessary.
+
+ifndef IS_DEP
+$(shell rm -f ebin/dep_built)
+endif
+
 # Core targets.
 
 apps:: $(ALL_APPS_DIRS) clean-tmp-deps.log
@@ -117,8 +123,11 @@ deps:: $(ALL_DEPS_DIRS) apps clean-tmp-deps.log
 			:; \
 		else \
 			echo $$dep >> $(ERLANG_MK_TMP)/deps.log; \
-			if [ -f $$dep/GNUmakefile ] || [ -f $$dep/makefile ] || [ -f $$dep/Makefile ]; then \
+			if [ -z "$(strip $(FULL))" ] && [ ! -L $$dep ] && [ -f $$dep/ebin/dep_built ]; then \
+				:; \
+			elif [ -f $$dep/GNUmakefile ] || [ -f $$dep/makefile ] || [ -f $$dep/Makefile ]; then \
 				$(MAKE) -C $$dep IS_DEP=1; \
+				if [ ! -L $$dep ] && [ -d $$dep/ebin ]; then touch $$dep/ebin/dep_built; fi; \
 			else \
 				echo "Error: No Makefile to build dependency $$dep." >&2; \
 				exit 2; \

doc/src/guide/deps.asciidoc

@@ -296,6 +296,24 @@ different version of D, it will always be A's version
 that wins, because we fetch all dependencies of A before
 fetching those from B or C.
 
+Once a dependency is built, it will not be built again by
+default. Typically dependencies do not need to be recompiled
+and this speeds up building immensely. There are a few ways
+to force recompiling a dependency however:
+
+* The dependency directory is a symbolic link; the dependency
+  will always be recompiled.
+
+* The dependency is built directly, for example with a command
+  like `make -C deps/cowlib`, or `make` in the dependency's
+  directory.
+
+* The variable `FULL` is set, for example `make FULL=1`. This
+  will force building of all dependencies. This can be added
+  to your Makefile before including 'erlang.mk'.
+
+* The file `ebin/dep_built` in the dependency is removed.
+
 === Fetching and listing dependencies only
 
 You can fetch all dependencies recursively without building anything,

test/Makefile

@@ -190,7 +190,7 @@ pkg-$1: build clean
 	fi
 
 	$i "Recompile package $1"
-	$t if ! ( cd packages/$1_pkg/ && $(MAKE) $(PATCHES) $v ); then \
+	$t if ! ( cd packages/$1_pkg/ && $(MAKE) $(PATCHES) FULL=1 $v ); then \
 		echo "$(1): recompile error" >> packages/errors.log; \
 		false; \
 	fi

test/core_deps.mk

@@ -106,6 +106,109 @@ core-deps-build-js: build clean
 		false = lists:member(jquery, Deps), \
 		halt()"
 
+core-deps-dep-built: build clean
+
+	$i "Bootstrap a new OTP library named $(APP)"
+	$t mkdir $(APP)/
+	$t cp ../erlang.mk $(APP)/
+	$t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib $v
+
+	$i "Add cowlib to the list of dependencies"
+	$t perl -ni.bak -e 'print;if ($$.==1) {print "DEPS = cowlib\n"}' $(APP)/Makefile
+
+	$i "Build the application"
+	$t $(MAKE) -C $(APP) $v
+
+	$i "Touch one cowlib file to mark it for recompilation"
+	$t $(SLEEP)
+	$t touch $(APP)/deps/cowlib/src/cow_http.erl
+
+	$i "Check that cowlib is not rebuilt"
+	$t touch $(APP)/EXPECT
+	$t $(SLEEP)
+	$t $(MAKE) -C $(APP) $v
+	$t find $(APP)/deps/cowlib -type f -newer $(APP)/EXPECT | sort | diff $(APP)/EXPECT -
+	$t rm $(APP)/EXPECT
+
+	$i "Delete the dep_built file"
+	$t rm $(APP)/deps/cowlib/ebin/dep_built
+
+	$i "Check that cowlib was rebuilt"
+	$t printf "%s\n" \
+		$(APP)/deps/cowlib/cowlib.d \
+		$(APP)/deps/cowlib/ebin/cowlib.app \
+		$(APP)/deps/cowlib/ebin/cow_http.beam \
+		$(APP)/deps/cowlib/ebin/dep_built | sort > $(APP)/EXPECT
+	$t $(SLEEP)
+	$t $(MAKE) -C $(APP) $v
+# Files in .git might end up modified due to the id generation in the .app file.
+	$t find $(APP)/deps/cowlib -type f -newer $(APP)/EXPECT | grep -v ".git" | sort | diff $(APP)/EXPECT -
+	$t rm $(APP)/EXPECT
+
+core-deps-dep-built-full: build clean
+
+	$i "Bootstrap a new OTP library named $(APP)"
+	$t mkdir $(APP)/
+	$t cp ../erlang.mk $(APP)/
+	$t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib $v
+
+	$i "Add cowlib to the list of dependencies"
+	$t perl -ni.bak -e 'print;if ($$.==1) {print "DEPS = cowlib\n"}' $(APP)/Makefile
+
+	$i "Build the application"
+	$t $(MAKE) -C $(APP) $v
+
+	$i "Touch one cowlib file to mark it for recompilation"
+	$t $(SLEEP)
+	$t touch $(APP)/deps/cowlib/src/cow_http.erl
+
+	$i "Check that cowlib is rebuilt with FULL=1"
+	$t printf "%s\n" \
+		$(APP)/deps/cowlib/cowlib.d \
+		$(APP)/deps/cowlib/ebin/cowlib.app \
+		$(APP)/deps/cowlib/ebin/cow_http.beam \
+		$(APP)/deps/cowlib/ebin/dep_built | sort > $(APP)/EXPECT
+	$t $(SLEEP)
+	$t $(MAKE) -C $(APP) FULL=1 $v
+# Files in .git might end up modified due to the id generation in the .app file.
+	$t find $(APP)/deps/cowlib -type f -newer $(APP)/EXPECT | grep -v ".git" | sort | diff $(APP)/EXPECT -
+	$t rm $(APP)/EXPECT
+
+core-deps-dep-built-ln: build clean
+
+	$i "Bootstrap a new OTP library named $(APP)"
+	$t mkdir $(APP)/
+	$t cp ../erlang.mk $(APP)/
+	$t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib $v
+
+	$i "Add cowlib to the list of dependencies"
+	$t perl -ni.bak -e 'print;if ($$.==1) {print "DEPS = cowlib\n"}' $(APP)/Makefile
+
+	$i "Clone cowlib manually inside $(APP)"
+	$t git clone -q https://github.com/ninenines/cowlib $(APP)/cowlib
+
+	$i "Link to cowlib instead of fetching the dependency"
+	$t mkdir -p $(APP)/deps
+	$t ln -s ../cowlib $(APP)/deps/cowlib
+
+	$i "Build the application"
+	$t $(MAKE) -C $(APP) $v
+
+	$i "Touch one cowlib file to mark it for recompilation"
+	$t $(SLEEP)
+	$t touch $(APP)/deps/cowlib/src/cow_http.erl
+
+	$i "Check that cowlib is rebuilt; symlinked deps don't create dep_built"
+	$t printf "%s\n" \
+		$(APP)/cowlib/cowlib.d \
+		$(APP)/cowlib/ebin/cowlib.app \
+		$(APP)/cowlib/ebin/cow_http.beam | sort > $(APP)/EXPECT
+	$t $(SLEEP)
+	$t $(MAKE) -C $(APP) $v
+# Files in .git might end up modified due to the id generation in the .app file.
+	$t find $(APP)/cowlib -type f -newer $(APP)/EXPECT | grep -v ".git" | sort | diff $(APP)/EXPECT -
+	$t rm $(APP)/EXPECT
+
 core-deps-dep-commit: build clean
 
 	$i "Bootstrap a new OTP library named $(APP)"