Add dependency tracking (makedep)

This is a very large change of a central part of Erlang.mk.
I will admit that I am not quite confident on that one. If
you do have issues following this change, please open a
ticket and I will look at it immediately.

At this point, it works for me, but I wouldn't be surprised
to hear about a few minor issues.

This commit introduces a dependency file $(PROJECT).d which
contains Makefile rules between Erlang source files and
headers, behaviors and parse_transforms. This allows us
to rebuild only the files that are needed.

The $(PROJECT).d is generated automatically when missing,
and when any of the files change.

It is possible to hook before and after this generation,
by defining a $(PROJECT).d:: rule. This allows users to
generate Erlang files which are then compiled by Erlang.mk
automatically (and to track their dependencies, of course).

Here goes nothing...

core/erlc.mk

@@ -24,6 +24,9 @@ app_verbose = $(app_verbose_$(V))
 appsrc_verbose_0 = @echo " APP   " $(PROJECT).app.src;
 appsrc_verbose = $(appsrc_verbose_$(V))
 
+makedep_verbose_0 = @echo " DEPEND" $(PROJECT).d;
+makedep_verbose = $(makedep_verbose_$(V))
+
 erlc_verbose_0 = @echo " ERLC  " $(filter-out $(patsubst %,%.erl,$(ERLC_EXCLUDE)),\
 	$(filter %.erl %.core,$(?F)));
 erlc_verbose = $(erlc_verbose_$(V))
@@ -40,9 +43,11 @@ mib_verbose = $(mib_verbose_$(V))
 # Targets.
 
 ifeq ($(wildcard ebin/test),)
-app:: app-build
+app::
+	$(verbose) $(MAKE) --no-print-directory app-build
 else
-app:: clean app-build
+app:: clean
+	$(verbose) $(MAKE) --no-print-directory app-build
 endif
 
 ifeq ($(wildcard src/$(PROJECT)_app.erl),)
@@ -70,7 +75,7 @@ define app_file
 endef
 endif
 
-app-build: erlc-include ebin/$(PROJECT).app
+app-build: ebin/$(PROJECT).app
 	$(eval GITDESCRIBE := $(shell git describe --dirty --abbrev=7 --tags --always --first-parent 2>/dev/null || true))
 	$(eval MODULES := $(patsubst %,'%',$(sort $(notdir $(basename $(shell find ebin -type f -name *.beam))))))
 ifeq ($(wildcard src/$(PROJECT).app.src),)
@@ -87,61 +92,127 @@ else
 		> ebin/$(PROJECT).app
 endif
 
-erlc-include:
-	- $(verbose) if [ -d ebin/ ]; then \
-		find include/ src/ -type f -name \*.hrl -newer ebin -exec touch $(shell find src/ -type f -name "*.erl") \; 2>/dev/null || printf ''; \
-	fi
+# Source files.
 
-define compile_erl
-	$(erlc_verbose) erlc -v $(if $(IS_DEP),$(filter-out -Werror,$(ERLC_OPTS)),$(ERLC_OPTS)) -o ebin/ \
-		-pa ebin/ -I include/ $(filter-out $(ERLC_EXCLUDE_PATHS),\
-		$(COMPILE_FIRST_PATHS) $(1))
-endef
+ifneq ($(wildcard src/),)
+ERL_FILES = $(sort $(call core_find,src/,*.erl))
+CORE_FILES = $(sort $(call core_find,src/,*.core))
 
-define compile_xyrl
-	$(xyrl_verbose) erlc -v -o ebin/ $(1)
-	$(xyrl_verbose) erlc $(ERLC_OPTS) -o ebin/ ebin/*.erl
-	$(verbose) rm ebin/*.erl
-endef
+# ASN.1 files.
 
-define compile_asn1
-	$(asn1_verbose) erlc -v -I include/ -o ebin/ $(1)
-	$(verbose) mv ebin/*.hrl include/
-	$(verbose) mv ebin/*.asn1db include/
-	$(verbose) rm ebin/*.erl
-endef
+ifneq ($(wildcard asn1/),)
+ASN1_FILES = $(sort $(call core_find,asn1/,*.asn1))
+ERL_FILES += $(addprefix src/,$(patsubst %.asn1,%.erl,$(notdir $(ASN1_FILES))))
 
-define compile_mib
-	$(mib_verbose) erlc -v $(ERLC_MIB_OPTS) -o priv/mibs/ \
-		-I priv/mibs/ $(COMPILE_MIB_FIRST_PATHS) $(1)
-	$(mib_verbose) erlc -o include/ -- priv/mibs/*.bin
+define compile_asn1
+	$(verbose) mkdir -p include/
+	$(asn1_verbose) erlc -v -I include/ -o asn1/ +noobj $(1)
+	$(verbose) mv asn1/*.erl src/
+	$(verbose) mv asn1/*.hrl include/
+	$(verbose) mv asn1/*.asn1db include/
 endef
 
-ifneq ($(wildcard src/),)
-ebin/$(PROJECT).app::
-	$(verbose) mkdir -p ebin/
-
-ifneq ($(wildcard asn1/),)
-ebin/$(PROJECT).app:: $(sort $(call core_find,asn1/,*.asn1))
-	$(verbose) mkdir -p include
+$(PROJECT).d:: $(ASN1_FILES)
 	$(if $(strip $?),$(call compile_asn1,$?))
 endif
 
+# SNMP MIB files.
+
 ifneq ($(wildcard mibs/),)
-ebin/$(PROJECT).app:: $(sort $(call core_find,mibs/,*.mib))
-	$(verbose) mkdir -p priv/mibs/ include
-	$(if $(strip $?),$(call compile_mib,$?))
+MIB_FILES = $(sort $(call core_find,mibs/,*.mib))
+
+$(PROJECT).d:: $(MIB_FILES)
+	$(verbose) mkdir -p include/ priv/mibs/
+	$(mib_verbose) erlc -v $(ERLC_MIB_OPTS) -o priv/mibs/ -I priv/mibs/ $(COMPILE_MIB_FIRST_PATHS) $(MIB_FILES)
+	$(mib_verbose) erlc -o include/ -- priv/mibs/*.bin
 endif
 
-ebin/$(PROJECT).app:: $(sort $(call core_find,src/,*.erl *.core))
+# Leex and Yecc files.
+
+XRL_FILES = $(sort $(call core_find,src/,*.xrl))
+XRL_ERL_FILES = $(addprefix src/,$(patsubst %.xrl,%.erl,$(notdir $(XRL_FILES))))
+ERL_FILES += $(XRL_ERL_FILES)
+
+YRL_FILES = $(sort $(call core_find,src/,*.yrl))
+YRL_ERL_FILES = $(addprefix src/,$(patsubst %.yrl,%.erl,$(notdir $(YRL_FILES))))
+ERL_FILES += $(YRL_ERL_FILES)
+
+$(PROJECT).d:: $(XRL_FILES) $(YRL_FILES)
+	$(if $(strip $?),$(xyrl_verbose) erlc -v -o src/ $?)
+
+# Erlang and Core Erlang files.
+
+define makedep.erl
+	ErlFiles = lists:usort(string:tokens("$(ERL_FILES)", " ")),
+	Modules = [{filename:basename(F, ".erl"), F} || F <- ErlFiles],
+	Add = fun (Dep, Acc) ->
+		case lists:keyfind(atom_to_list(Dep), 1, Modules) of
+			{_, DepFile} -> [DepFile|Acc];
+			false -> Acc
+		end
+	end,
+	AddHd = fun (Dep, Acc) ->
+		case {Dep, lists:keymember(Dep, 2, Modules)} of
+			{"src/" ++ _, false} -> [Dep|Acc];
+			{"include/" ++ _, false} -> [Dep|Acc];
+			_ -> Acc
+		end
+	end,
+	CompileFirst = fun (Deps) ->
+		First0 = [case filename:extension(D) of
+			".erl" -> filename:basename(D, ".erl");
+			_ -> []
+		end || D <- Deps],
+		case lists:usort(First0) of
+			[] -> [];
+			[[]] -> [];
+			First -> ["COMPILE_FIRST +=", [[" ", F] || F <- First], "\n"]
+		end
+	end,
+	Depend = [begin
+		case epp:parse_file(F, [{includes, ["include/"]}]) of
+			{ok, Forms} ->
+				Deps = lists:usort(lists:foldl(fun
+					({attribute, _, behavior, Dep}, Acc) -> Add(Dep, Acc);
+					({attribute, _, behaviour, Dep}, Acc) -> Add(Dep, Acc);
+					({attribute, _, compile, {parse_transform, Dep}}, Acc) -> Add(Dep, Acc);
+					({attribute, _, file, {Dep, _}}, Acc) -> AddHd(Dep, Acc);
+					(_, Acc) -> Acc
+				end, [], Forms)),
+				[F, ":", [[" ", D] || D <- Deps], "\n", CompileFirst(Deps)];
+			{error, enoent} ->
+				[]
+		end
+	end || F <- ErlFiles],
+	ok = file:write_file("$(1)", Depend),
+	halt()
+endef
+
+$(PROJECT).d:: $(ERL_FILES) $(call core_find,include/,*.hrl)
+	$(makedep_verbose) $(call erlang,$(call makedep.erl,$@))
+
+-include $(PROJECT).d
+
+ebin/$(PROJECT).app:: $(PROJECT).d
+	$(verbose) mkdir -p ebin/
+
+define compile_erl
+	$(erlc_verbose) erlc -v $(if $(IS_DEP),$(filter-out -Werror,$(ERLC_OPTS)),$(ERLC_OPTS)) -o ebin/ \
+		-pa ebin/ -I include/ $(filter-out $(ERLC_EXCLUDE_PATHS),$(COMPILE_FIRST_PATHS) $(1))
+endef
+
+ebin/$(PROJECT).app:: $(ERL_FILES) $(CORE_FILES)
 	$(if $(strip $?),$(call compile_erl,$?))
 
-ebin/$(PROJECT).app:: $(sort $(call core_find,src/,*.xrl *.yrl))
-	$(if $(strip $?),$(call compile_xyrl,$?))
+$(sort $(ERL_FILES) $(CORE_FILES)):
+	@touch $@
 endif
 
 clean:: clean-app
 
 clean-app:
-	$(gen_verbose) rm -rf ebin/ priv/mibs/ \
-		$(addprefix include/,$(addsuffix .hrl,$(notdir $(basename $(call core_find,mibs/,*.mib)))))
+	$(gen_verbose) rm -rf $(PROJECT).d ebin/ priv/mibs/ $(XRL_ERL_FILES) $(YRL_ERL_FILES) \
+		$(addprefix include/,$(patsubst %.mib,.hrl,$(notdir $(MIB_FILES)))) \
+		$(addprefix include/,$(patsubst %.asn1,.hrl,$(notdir $(ASN1_FILES)))) \
+		$(addprefix include/,$(patsubst %.asn1,.asn1db,$(notdir $(ASN1_FILES)))) \
+		$(addprefix src/,$(patsubst %.erl,.asn1db,$(notdir $(ASN1_FILES))))

doc/src/guide/app.asciidoc

@@ -0,0 +1,267 @@
+== Building
+
+Erlang.mk can do a lot of things, but it is, first and
+foremost, a build tool. In this chapter we will cover
+the basics of building a project with Erlang.mk.
+
+For most of this chapter, we will assume that you are
+using a project link:getting_started.asciidoc[generated by Erlang.mk].
+
+=== How to build
+
+To build a project, all you have to do is type `make`:
+
+[source,bash]
+$ make
+
+It will work regardless of your project: OTP applications,
+library applications, NIFs, port drivers or even releases.
+Erlang.mk also automatically downloads and compiles the
+dependencies for your project.
+
+All this is possible thanks to a combination of configuration
+and conventions. Most of the conventions come from Erlang/OTP
+itself so any seasoned Erlang developers should feel right at
+home.
+
+=== What to build
+
+Erlang.mk gives you control over three steps of the build
+process, allowing you to do a partial build if needed.
+
+A build has three phases: first any dependency is fetched
+and built, then the project itself is built and finally a
+release may be generated when applicable. A release is only
+generated for projects specifically configured to do so.
+
+Erlang.mk handles those three phases automatically when you
+type `make`. But sometimes you just want to repeat one or
+two of them.
+
+The commands detailed in this section are most useful after
+you have a successful build as they allow you to quickly
+redo a step instead of going through everything. This is
+especially useful for large projects or projects that end
+up generating releases.
+
+==== Application
+
+You can build your application specifically, without
+looking at handling dependencies or generating a release,
+by running the following command:
+
+[source,bash]
+$ make app
+
+This command is very useful if you have a lot of dependencies
+and develop on a machine with slow file access, like the
+Raspberry Pi and many other embedded devices.
+
+Note that this command may fail if a required dependency
+is missing.
+
+==== Dependencies
+
+You can build all dependencies, and nothing else, by
+running the following command:
+
+[source,bash]
+$ make deps
+
+This will fetch and compile all dependencies and their
+dependencies, recursively.
+
+link:deps.asciidoc[Packages and dependencies] are covered
+in the next chapter.
+
+==== Release
+
+You can generate the release, skipping the steps for building
+the application and dependencies, by running the following
+command:
+
+[source,bash]
+$ make rel
+
+This command can be useful if nothing changed except the
+release configuration files.
+
+Consult the link:relx.asciidoc[Releases] chapter for more
+information about what releases are and how they are generated.
+
+Note that this command may fail if a required dependency
+is missing.
+
+=== Application resource file
+
+When building your application, Erlang.mk will generate the
+http://www.erlang.org/doc/man/app.html[application resource file].
+This file is mandatory for all Erlang applications and is
+found in 'ebin/$(PROJECT).app'.
+
+`PROJECT` is a variable defined in your Makefile and taken
+from the name of the directory when Erlang.mk bootstraps
+your project.
+
+Erlang.mk can build the 'ebin/$(PROJECT).app' in two different
+ways: from the configuration found in the Makefile, or from
+the 'src/$(PROJECT).app.src' file.
+
+==== Application configuration
+
+Erlang.mk automatically fills the `PROJECT` variable when
+bootstrapping a new project, but everything else is up to
+you. None of the values are required to build your project,
+although it is recommended to fill everything relevant to
+your situation.
+
+`PROJECT`::
+	The name of the OTP application or library.
+`PROJECT_DESCRIPTION`::
+	Short description of the project.
+`PROJECT_VERSION`::
+	Current version of the project.
+`PROJECT_REGISTERED`::
+	List of the names of all registered processes.
+`OTP_DEPS`::
+	List of Erlang/OTP applications this project depends on,
+	excluding `erts`, `kernel` and `stdlib`.
+`DEPS`::
+	List of applications this project depends on that need
+	to be fetched by Erlang.mk.
+
+There's no need for quotes or anything. The relevant part of
+the Cowboy Makefile follows, if you need an example:
+
+[source,make]
+----
+PROJECT = cowboy
+PROJECT_DESCRIPTION = Small, fast, modular HTTP server.
+PROJECT_VERSION = 2.0.0-pre.2
+PROJECT_REGISTERED = cowboy_clock
+
+OTP_DEPS = crypto
+DEPS = cowlib ranch
+----
+
+Any space before and after the value is dropped.
+
+link:deps.asciidoc[Dependencies] are covered in details in
+the next chapter.
+
+==== Legacy method
+
+The 'src/$(PROJECT).app.src' file is a legacy method of
+building Erlang applications. It was introduced by the original
+`rebar` build tool, of which Erlang.mk owes a great deal as it
+is its main inspiration.
+
+The '.app.src' file serves as a template to generate the '.app'
+file. Erlang.mk will take it, fill in the `modules` value
+dynamically, and save the result in 'ebin/$(PROJECT).app'.
+
+When using this method, Erlang.mk cannot fill the `applications`
+key from dependencies automatically, which means you need to
+add them to Erlang.mk and to the '.app.src' at the same time,
+duplicating the work.
+
+=== File formats
+
+Erlang.mk supports a variety of different source file formats.
+The following formats are supported natively:
+
+[cols="<,3*^",options="header"]
+|===
+| Extension | Location | Description        | Output
+| .erl      | src/     | Erlang source      | ebin/*.beam
+| .core     | src/     | Core Erlang source | ebin/*.beam
+| .xrl      | src/     | Leex source        | src/*.erl
+| .yrl      | src/     | Yecc source        | src/*.erl
+| .asn1     | asn1/    | ASN.1 files        | include/*.hrl include/*.asn1db src/*.erl
+| .mib      | mibs/    | SNMP MIB files     | include/*.hrl priv/mibs/*.bin
+|===
+
+Files are always searched recursively.
+
+The build is ordered, so that files that generate Erlang source
+files are run before, and the resulting Erlang source files are
+then built normally.
+
+In addition, Erlang.mk keeps track of header files (`.hrl`)
+as described at the end of this chapter. It can also compile
+C code, as described in the link:ports.asciidoc[NIFs and port drivers]
+chapter.
+
+Erlang.mk also comes with plugins for the following formats:
+
+[cols="<,3*^",options="header"]
+|===
+| Extension | Location   | Description      | Output
+| .dtl      | templates/ | Django templates | ebin/*.beam
+| .proto    | src/       | Protocol buffers | ebin/*.beam
+|===
+
+=== Cold and hot builds
+
+The first time you run `make`, Erlang.mk will build everything.
+
+The second time you run `make`, and all subsequent times, Erlang.mk
+will only rebuild what changed. Erlang.mk has been optimized for
+this use case, as it is the most common during development.
+
+Erlang.mk figures out what changed by using the dependency tracking
+feature of Make. Make automatically rebuilds a target if one of its
+dependency has changed (for example if a header file has changed,
+all the source files that include it will be rebuilt), and Erlang.mk
+leverages this feature to cut down on rebuild times.
+
+Note that this applies only to building; some other features of
+Erlang.mk will run every time they are called regardless of files
+changed.
+
+=== Dependency tracking
+
+NOTE: This section is about the dependency tracking between files
+inside your project, not application dependencies.
+
+Erlang.mk keeps track of the dependencies between the different
+files in your project. This information is kept in the '$(PROJECT).d'
+file in your directory. It is generated if missing, and will be
+generated again after every file change, by default.
+
+Dependency tracking is what allows Erlang.mk to know when to
+rebuild Erlang files when header files, behaviors or parse
+transforms have changed. Erlang.mk also automatically keeps
+track of which files should be compiled first, for example
+when you have behaviors used by other modules in your project.
+
+=== Cleaning
+
+Building typically involves creating a lot of new files. Some
+are reused in rebuilds, some are simply replaced. All can be
+removed safely.
+
+Erlang.mk provides two commands to remove them: `clean` and
+`distclean`. `clean` removes all the intermediate files that
+were created as a result of building, including the BEAM files,
+the dependency tracking file and the generated documentation.
+`distclean` removes these and more, including the downloaded
+dependencies, Dialyzer's PLT file and the generated release,
+putting your directory back to the state it was before you
+started working on it.
+
+To clean:
+
+[source,bash]
+$ make clean
+
+Or distclean:
+
+[source,bash]
+$ make distclean
+
+That is the question.
+
+Note that Erlang.mk will automatically clean some files as
+part of other targets, but it will never run `distclean` if
+you don't explicitly use it.