Allow git + hex deps to be cached to XDG_CACHE_HOME

core/deps.mk

@@ -24,6 +24,11 @@ export REBAR_DEPS_DIR
 REBAR3_GIT ?= https://github.com/erlang/rebar3
 REBAR3_COMMIT ?= 3f563feaf1091a1980241adefa83a32dd2eebf7c # 3.20.0
 
+CACHE_DEPS ?= 0
+
+CACHE_DIR ?= $(if $(XDG_CACHE_HOME),$(XDG_CACHE_HOME),$(HOME)/.cache)/erlang.mk
+export CACHE_DIR
+
 # External "early" plugins (see core/plugins.mk for regular plugins).
 # They both use the core_dep_plugin macro.
 
@@ -703,6 +708,36 @@ define dep_autopatch_appsrc.erl
 	halt()
 endef
 
+ifeq ($(CACHE_DEPS),1)
+
+define __fetch_git
+	mkdir -p $(CACHE_DIR)/gits; \
+	if test -d "$(join $(CACHE_DIR)/gits/,$(call dep_name,$(1)))"; then \
+		cd $(join $(CACHE_DIR)/gits/,$(call dep_name,$(1))); \
+		if ! git checkout -q $(call dep_commit,$(1)); then \
+			git remote set-url origin $(call dep_repo,$(1)) && \
+			git pull --all && \
+			git cat-file -e $(call dep_commit,$(1)) 2>/dev/null; \
+		fi; \
+	else \
+		git clone -q -n -- $(call dep_repo,$(1)) $(join $(CACHE_DIR)/gits/,$(call dep_name,$(1))); \
+	fi; \
+	git clone -q --branch $(call dep_commit,$(1)) --single-branch -- $(join $(CACHE_DIR)/gits/,$(call dep_name,$(1))) $(2)
+endef
+
+define dep_fetch_git
+	$(call __fetch_git,$(1),$(DEPS_DIR)/$(call dep_name,$(1)));
+endef
+
+define dep_fetch_git-subfolder
+	mkdir -p $(ERLANG_MK_TMP)/git-subfolder; \
+	$(call __fetch_git,$(1),$(ERLANG_MK_TMP)/git-subfolder/$(call dep_name,$(1))); \
+	ln -s $(ERLANG_MK_TMP)/git-subfolder/$(call dep_name,$1)/$(word 4,$(dep_$(1))) \
+		$(DEPS_DIR)/$(call dep_name,$1);
+endef
+
+else
+
 define dep_fetch_git
 	git clone -q -n -- $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); \
 	cd $(DEPS_DIR)/$(call dep_name,$(1)) && git checkout -q $(call dep_commit,$(1));
@@ -718,6 +753,8 @@ define dep_fetch_git-subfolder
 		$(DEPS_DIR)/$(call dep_name,$1);
 endef
 
+endif
+
 define dep_fetch_git-submodule
 	git submodule update --init -- $(DEPS_DIR)/$1;
 endef
@@ -739,6 +776,19 @@ define dep_fetch_ln
 	ln -s $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1));
 endef
 
+ifeq ($(CACHE_DEPS),1)
+
+# Hex only has a package version. No need to look in the Erlang.mk packages.
+define dep_fetch_hex
+	mkdir -p $(CACHE_DIR)/hex $(DEPS_DIR)/$1; \
+	$(eval hex_tar_name=$(if $(word 3,$(dep_$1)),$(word 3,$(dep_$1)),$1)-$(strip $(word 2,$(dep_$1))).tar) \
+	$(if $(wildcard $(CACHE_DIR)/hex/$(hex_tar_name)),,$(call core_http_get,$(CACHE_DIR)/hex/$(hex_tar_name),\
+		https://repo.hex.pm/tarballs/$(hex_tar_name))); \
+	tar -xOf $(CACHE_DIR)/hex/$(hex_tar_name) contents.tar.gz | tar -C $(DEPS_DIR)/$1 -xzf -;
+endef
+
+else
+
 # Hex only has a package version. No need to look in the Erlang.mk packages.
 define dep_fetch_hex
 	mkdir -p $(ERLANG_MK_TMP)/hex $(DEPS_DIR)/$1; \
@@ -747,6 +797,8 @@ define dep_fetch_hex
 	tar -xOf $(ERLANG_MK_TMP)/hex/$1.tar contents.tar.gz | tar -C $(DEPS_DIR)/$1 -xzf -;
 endef
 
+endif
+
 define dep_fetch_fail
 	echo "Error: Unknown or invalid dependency: $(1)." >&2; \
 	exit 78;
@@ -817,6 +869,16 @@ distclean-deps:
 	$(gen_verbose) rm -rf $(DEPS_DIR)
 endif
 
+ifeq ($(CACHE_DEPS),1)
+cacheclean:: cacheclean-git cacheclean-hex
+
+cacheclean-git:
+	$(gen_verbose) rm -rf $(CACHE_DIR)/git
+
+cacheclean-hex:
+	$(gen_verbose) rm -rf $(CACHE_DIR)/hex
+endif
+
 # Forward-declare variables used in core/deps-tools.mk. This is required
 # in case plugins use them.
 

test/Makefile

@@ -13,6 +13,8 @@ endif
 # Temporary application name, taken from rule name.
 
 APP = test_$(subst -,_,$@)
+CACHE_DIR = $(CURDIR)/$(APP).cache
+export CACHE_DIR
 
 # Erlang, quickly!
 
@@ -126,7 +128,7 @@ endef
 all:: core
 
 clean::
-	$t rm -rf erl_crash.dump packages/ $(filter-out test_rebar_git/,$(wildcard test_*/))
+	$t rm -rf erl_crash.dump packages/ $(filter-out test_rebar_git/,$(wildcard test_*/)) $(CACHE_DIR)
 
 init: clean
 	$i "Prefetch Rebar if necessary"

test/core_deps.mk

@@ -1476,3 +1476,54 @@ core-deps-test: init
 		{ok, Deps} = application:get_key($(APP), applications), \
 		false = lists:member(triq, Deps), \
 		halt()"
+
+# Checks that the cache is populate and reused
+core-deps-cache-creation: init
+	$i "Bootstrap a new OTP library named $(APP)"
+	$t mkdir $(APP)/
+	$t cp ../erlang.mk $(APP)/
+	$t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib $v
+	$t tmp=`mktemp` && { echo 'DEPS += cowlib' && echo 'CACHE_DEPS=1' && cat $(APP)/Makefile; } >> $$tmp && mv $$tmp $(APP)/Makefile
+	$i "Check that the Cache Directory doesn't exist yet"
+	$t test ! -d $(CACHE_DIR)
+	$i "Pull down & Cache the deps"
+	$t $(MAKE) -C $(APP) deps
+	$i "Check that the Cache Directory has been created"
+	$t test -d $(CACHE_DIR)
+	$i "Check that cowlib was cloned"
+	$t test -d $(CACHE_DIR)/gits/cowlib
+	$t $(MAKE) -C $(APP) distclean
+	$i "Check that cowlib is still in the cache"
+	$t test -d $(CACHE_DIR)/gits/cowlib
+	$i "Break cowlib git link so we're forced to use the cache"
+	$t echo 'dep_cowlib = git bad_url master' >> $(APP)/Makefile
+	$i "Pull down the deps from the cache"
+	$t $(MAKE) -C $(APP) deps
+	$i "Check that cowlib was cloned"
+	$t test -d $(CACHE_DIR)/gits/cowlib
+
+# Checks that 2 apps can reuse the cache
+#  with different versions of the same dep
+core-deps-cache-reuse: init
+	$i "Bootstrap a new OTP library named $(APP)_1"
+	$t mkdir $(APP)_1/
+	$t cp ../erlang.mk $(APP)_1/
+	$t $(MAKE) -C $(APP)_1 -f erlang.mk bootstrap-lib $v
+	$t tmp=`mktemp` && { echo 'DEPS += cowlib' && echo 'dep_cowlib = git $$(pkg_cowlib_repo) 1.0.0' && echo 'CACHE_DEPS=1' && cat $(APP)_1/Makefile; } >> $$tmp && mv $$tmp $(APP)_1/Makefile
+	$i "Bootstrap a new OTP library named $(APP)_2"
+	$t mkdir $(APP)_2/
+	$t cp ../erlang.mk $(APP)_2/
+	$t $(MAKE) -C $(APP)_2 -f erlang.mk bootstrap-lib $v
+	$t tmp=`mktemp` && { echo 'DEPS += cowlib' && echo 'dep_cowlib = git $$(pkg_cowlib_repo) 2.0.0' && echo 'CACHE_DEPS=1' && cat $(APP)_2/Makefile; } >> $$tmp && mv $$tmp $(APP)_2/Makefile
+	$i "Clone & Cache deps in $(APP)_1"
+	$t $(MAKE) -C $(APP)_1 deps
+	$i "Check that the Cache Directory has been created"
+	$t test -d $(CACHE_DIR)
+	$i "Check that cowlib was cloned"
+	$t test -d $(CACHE_DIR)/gits/cowlib
+	$i "Clone & Re-use cached deps in $(APP)_2"
+	$t $(MAKE) -C $(APP)_2 deps
+	$i "Check that $(APP)_1 cloned cowlib 1.x.x"
+	$t test "$$(cat $(APP)_1/deps/cowlib/.git/HEAD)" = "d544a494af4dbc810fc9c15eaf5cc050cced1501"
+	$i "Check that $(APP)_2 cloned cowlib 2.x.x"
+	$t test "$$(cat $(APP)_2/deps/cowlib/.git/HEAD)" = "bd37be4d3b065600c3b76b492535e76e5d413fc1"