Check template checksums also for binaries.

When compiling a template not from file (e.g. from a binary value),
keep track of the checksum the same way as is done when compiling templates on file.

Moved default value for compiler_options back to the record definition.

include/erlydtl_ext.hrl

@@ -14,7 +14,7 @@
           custom_tags_modules = [],
           reader = {file, read_file},
           module = [],
-          compiler_options = [],
+          compiler_options = [verbose, report_errors],
           binary_strings = true,
           force_recompile = false,
           locale = none,

src/erlydtl_compiler.erl

@@ -4,6 +4,7 @@
 %%% @author    Evan Miller <emmiller@gmail.com>
 %%% @author    Andreas Stenius <kaos@astekk.se>
 %%% @copyright 2008 Roberto Saccon, Evan Miller
+%%% @copyright 2014 Andreas Stenius
 %%% @doc
 %%% ErlyDTL template compiler
 %%% @end
@@ -11,6 +12,7 @@
 %%% The MIT License
 %%%
 %%% Copyright (c) 2007 Roberto Saccon, Evan Miller
+%%% Copyright (c) 2014 Andreas Stenius
 %%%
 %%% Permission is hereby granted, free of charge, to any person obtaining a copy
 %%% of this software and associated documentation files (the "Software"), to deal
@@ -31,6 +33,7 @@
 %%% THE SOFTWARE.
 %%%
 %%% @since 2007-12-16 by Roberto Saccon, Evan Miller
+%%% @since 2014 by Andreas Stenius
 %%%-------------------------------------------------------------------
 -module(erlydtl_compiler).
 -author('rsaccon@gmail.com').
@@ -59,24 +62,24 @@ compile(FileOrBinary, Module) ->
 
 compile(Binary, Module, Options0) when is_binary(Binary) ->
     File = "",
-    CheckSum = binary_to_list(erlang:md5(Binary)),
     Options = [{compiler_options, [{source, "/<text>"}]}
                |process_opts(Options0)],
     Context = init_dtl_context(File, Module, Options),
     case parse(Binary, Context) of
-        {ok, DjangoParseTree} ->
+        up_to_date -> ok;
+        {ok, DjangoParseTree, CheckSum} ->
             compile_to_binary(File, DjangoParseTree, Context, CheckSum);
-        Err -> Err
+        Other -> Other
     end;
 
 compile(File, Module, Options0) ->
     Options = process_opts(Options0),
     Context = init_dtl_context(File, Module, Options),
     case parse(File, Context) of
-        ok -> ok; %% up to date, not recompiled, maybe change return value to reflect this better?
+        up_to_date -> ok;
         {ok, DjangoParseTree, CheckSum} ->
             compile_to_binary(File, DjangoParseTree, Context, CheckSum);
-        Err -> Err
+        Other -> Other
     end.
 
 
@@ -102,7 +105,7 @@ compile_dir(Dir, Module, Options0) ->
                                     {ResultAcc, ErrorAcc};
                                 false ->
                                     case parse(FilePath, Context) of
-                                        ok -> {ResultAcc, ErrorAcc};
+                                        up_to_date -> {ResultAcc, ErrorAcc};
                                         {ok, DjangoParseTree, CheckSum} ->
                                             {[{File, DjangoParseTree, CheckSum}|ResultAcc], ErrorAcc};
                                         Err -> {ResultAcc, [Err|ErrorAcc]}
@@ -116,6 +119,9 @@ compile_dir(Dir, Module, Options0) ->
         [Error|_] -> Error %% just the first error?
     end.
 
+parse(Data) ->
+    parse(Data, #dtl_context{}).
+
 
 %%====================================================================
 %% Internal functions
@@ -149,7 +155,8 @@ update_defaults(Options) ->
     Options1 = maybe_add_env_default_opts(Options),
     case proplists:get_value(compiler_options, Options1) of
         undefined ->
-            [{compiler_options, [verbose, report_errors]}|Options1];
+            [{compiler_options, (#dtl_context{})#dtl_context.compiler_options}
+             |Options1];
         _ -> Options1
     end.
 
@@ -321,7 +328,7 @@ init_context(IsCompilingDir, ParseTrail, DefDir, Module, Options) ->
                  reader = proplists:get_value(reader, Options, Ctx#dtl_context.reader),
                  compiler_options = proplists:append_values(compiler_options, Options),
                  binary_strings = proplists:get_value(binary_strings, Options, Ctx#dtl_context.binary_strings),
-                 force_recompile = proplists:get_value(force_recompile, Options, Ctx#dtl_context.force_recompile),
+                 force_recompile = proplists:get_bool(force_recompile, Options),
                  locale = proplists:get_value(locale, Options, Ctx#dtl_context.locale),
                  verbose = proplists:get_value(verbose, Options, Ctx#dtl_context.verbose),
                  is_compiling_dir = IsCompilingDir,
@@ -386,18 +393,22 @@ is_up_to_date(CheckSum, Context) ->
             false
     end.
 
-parse(Data) ->
-    parse(Data, #dtl_context{}).
-
-parse(Data, #dtl_context{ scanner_module=Scanner }=Context)
+parse(Data, Context)
   when is_binary(Data) ->
-    check_scan(apply(Scanner, scan, [binary_to_list(Data)]), Context);
+    CheckSum = binary_to_list(erlang:md5(Data)),
+    case is_up_to_date(CheckSum, Context) of
+        true -> up_to_date;
+        false ->
+            case do_parse(Data, Context) of
+                {ok, Val} -> {ok, Val, CheckSum};
+                Err -> Err
+            end
+    end;
 parse(File, Context) ->
     {M, F} = Context#dtl_context.reader,
     case catch M:F(File) of
-        {ok, Data} ->
-            CheckSum = binary_to_list(erlang:md5(Data)),
-            case parse(CheckSum, Data, Context) of
+        {ok, Data} when is_binary(Data) ->
+            case parse(Data, Context) of
                 {error, Msg} when is_list(Msg) ->
                     {error, File ++ ": " ++ Msg};
                 {error, Msg} ->
@@ -409,17 +420,10 @@ parse(File, Context) ->
             {error, {File, [{0, Context#dtl_context.module, "Failed to read file"}]}}
     end.
 
-parse(CheckSum, Data, Context) ->
-    case is_up_to_date(CheckSum, Context) of
-        true -> ok;
-        _ ->
-            case parse(Data, Context) of
-                {ok, Val} ->
-                    {ok, Val, CheckSum};
-                Err ->
-                    Err
-            end
-    end.
+do_parse(Data, #dtl_context{ scanner_module=Scanner }=Context) ->
+    check_scan(
+      apply(Scanner, scan, [binary_to_list(Data)]),
+      Context).
 
 call_extension(#dtl_context{ extension_module=undefined }, _Fun, _Args) ->
     undefined;
@@ -1066,7 +1070,7 @@ blocktrans_ast(ArgList, Contents, Context, TreeWalker) ->
                                                                                default ->
                                                                                    {AstInfoAcc, ThisTreeWalker, ClauseAcc};
                                                                                Body ->
-                                                                                   {ok, DjangoParseTree} = parse(Body, Context),
+                                                                                   {ok, DjangoParseTree} = do_parse(Body, Context),
                                                                                    {{ThisAst, ThisAstInfo}, TreeWalker3} = body_ast(DjangoParseTree, NewContext, ThisTreeWalker),
                                                                                    {merge_info(ThisAstInfo, AstInfoAcc), TreeWalker3,
                                                                                     [erl_syntax:clause([erl_syntax:string(Locale)], none, [ThisAst])|ClauseAcc]}
@@ -1145,21 +1149,24 @@ include_ast(File, ArgList, Scopes, Context, TreeWalker) ->
     FilePath = full_path(File, Context#dtl_context.doc_root),
     case parse(FilePath, Context) of
         {ok, InclusionParseTree, CheckSum} ->
-            {NewScope, {ArgInfo, TreeWalker1}} = lists:mapfoldl(fun
-                                                                    ({{identifier, _, LocalVarName}, Value}, {AstInfo1, TreeWalker1}) ->
-                                                                       {{Ast, Info}, TreeWalker2} = value_ast(Value, false, false, Context, TreeWalker1),
-                                                                       {{LocalVarName, Ast}, {merge_info(AstInfo1, Info), TreeWalker2}}
-                                                               end, {#ast_info{}, TreeWalker}, ArgList),
-
-            {{BodyAst, BodyInfo}, TreeWalker2} = with_dependency({FilePath, CheckSum},
-                                                                 body_ast(InclusionParseTree, Context#dtl_context{
-                                                                                                parse_trail = [FilePath | Context#dtl_context.parse_trail],
-                                                                                                local_scopes = [NewScope|Scopes]
-                                                                                               }, TreeWalker1)),
+            {NewScope, {ArgInfo, TreeWalker1}}
+                = lists:mapfoldl(
+                    fun ({{identifier, _, LocalVarName}, Value}, {AstInfo1, TreeWalker1}) ->
+                            {{Ast, Info}, TreeWalker2} = value_ast(Value, false, false, Context, TreeWalker1),
+                            {{LocalVarName, Ast}, {merge_info(AstInfo1, Info), TreeWalker2}}
+                    end, {#ast_info{}, TreeWalker}, ArgList),
+
+            {{BodyAst, BodyInfo}, TreeWalker2} = with_dependency(
+                                                   {FilePath, CheckSum},
+                                                   body_ast(
+                                                     InclusionParseTree,
+                                                     Context#dtl_context{
+                                                       parse_trail = [FilePath | Context#dtl_context.parse_trail],
+                                                       local_scopes = [NewScope|Scopes]
+                                                      }, TreeWalker1)),
 
             {{BodyAst, merge_info(BodyInfo, ArgInfo)}, TreeWalker2};
-        Err ->
-            throw(Err)
+        Err -> throw(Err)
     end.
 
                                                 % include at run-time

src/i18n/sources_parser.erl

@@ -16,42 +16,42 @@
 %% API Functions
 %%
 parse() ->
-	Parsed_Files = parse(["./views/*/*.html"]),
-	io:format("Parsed files are ~p~n",[Parsed_Files]).
+    Parsed_Files = parse(["./views/*/*.html"]),
+    io:format("Parsed files are ~p~n",[Parsed_Files]).
 parse(Pattern) ->
-	%%We assume a basedir
-	GetFiles = fun(Path,Acc) -> Acc ++ filelib:wildcard(Path) end,
-	Files = lists:foldl(GetFiles,[],Pattern),
-	io:format("Parsing files ~p~n",[Files]),
-	ParsedFiles = lists:map(fun(File)-> parse_file(File) end, Files),
-	lists:flatten(ParsedFiles).
+    %%We assume a basedir
+    GetFiles = fun(Path,Acc) -> Acc ++ filelib:wildcard(Path) end,
+    Files = lists:foldl(GetFiles,[],Pattern),
+    io:format("Parsing files ~p~n",[Files]),
+    ParsedFiles = lists:map(fun(File)-> parse_file(File) end, Files),
+    lists:flatten(ParsedFiles).
 
 %%
 %% Local Functions
 %%
 parse_file(Path) ->
-	case file:read_file((Path)) of
-		{ok,Content} ->
-			process_content(Path,Content);	
-		Error ->
-			throw(io_lib:format("Cannot read file ~s problem ~p~n", [Path,Error]))
-	end.
+    case file:read_file((Path)) of
+        {ok, Content} ->
+            process_content(Path, Content);
+        Error ->
+            bail("Cannot read file ~s problem ~p~n", [Path, Error])
+    end.
 
 process_content(Path,Content)->
-	case erlydtl_compiler:parse(Content) of
-		{ok,Data} -> 
-			{ok,Result} = process_ast(Path, Data),
-			Result;
-		_Error ->
-			throw(io_lib:format("Template parsing failed for template ~s, cause ~p~n",[Path,_Error]))
-	end.
+    case erlydtl_compiler:parse(Content) of
+        {ok, Data, _} ->
+            {ok, Result} = process_ast(Path, Data),
+            Result;
+        Error ->
+            bail("Template parsing failed for template ~s, cause ~p~n", [Path, Error])
+    end.
 
 
 process_ast(Fname, Tokens) -> {ok, process_ast(Fname, Tokens ,[]) }.
 process_ast(_Fname, [],Acc) -> Acc;
 process_ast(Fname,[Head|Tail], Acc) ->
-	NewAcc = process_token(Fname,Head,Acc),
-	process_ast(Fname, Tail, NewAcc).
+    NewAcc = process_token(Fname,Head,Acc),
+    process_ast(Fname, Tail, NewAcc).
 
 %%Block are recursivelly processed, trans are accumulated and other tags are ignored
 process_token(Fname, {block,{identifier,{_Line,_Col},_Identifier},Children}, Acc ) -> process_ast(Fname, Children, Acc);
@@ -59,10 +59,12 @@ process_token(Fname, {trans,{string_literal,{Line,Col},String}}, Acc ) -> [{unes
 process_token(_Fname, {apply_filter, _Value, _Filter}, Acc) -> Acc;
 process_token(_Fname, {date, now, _Filter}, Acc) -> Acc;
 process_token(Fname, {_Instr, _Cond, Children}, Acc) -> process_ast(Fname, Children, Acc);
-process_token(Fname, {_Instr, _Cond, Children, Children2}, Acc) -> 
-	AccModified = process_ast(Fname, Children, Acc),
-	process_ast(Fname, Children2, AccModified);
+process_token(Fname, {_Instr, _Cond, Children, Children2}, Acc) ->
+    AccModified = process_ast(Fname, Children, Acc),
+    process_ast(Fname, Children2, AccModified);
 process_token(_,_AST,Acc) -> Acc.
 
 unescape(String) ->string:sub_string(String, 2, string:len(String) -1).
-	
+
+bail(Fmt, Args) ->
+    throw(lists:flatten(io_lib:format(Fmt, Args))).

tests/src/erlydtl_functional_tests.erl

@@ -248,7 +248,7 @@ test_compile_render(Name) ->
         {CompileStatus, CompileVars} ->
             Options = [
                        {vars, CompileVars},
-                       {force_recompile, true},
+                       force_recompile,
                        %% debug_compiler,
                        {custom_tags_modules, [erlydtl_custom_tags]}],
             io:format("compiling ... "),

tests/src/erlydtl_unittests.erl

@@ -1303,7 +1303,9 @@ format_error(Name, Class, Error) ->
     {Name, io_lib:format("~s:~p~n  ~p", [Class, Error, erlang:get_stacktrace()])}.
 
 compile_test(DTL, Opts) ->
-    Options = [{custom_filters_modules, [erlydtl_contrib_humanize]}|Opts],
+    Options = [force_recompile,
+               {custom_filters_modules, [erlydtl_contrib_humanize]}
+               |Opts],
     timer:tc(erlydtl, compile, [DTL, erlydtl_running_test, Options]).
 
 render_test(Vars, RenderOpts) ->

tests/src/sources_parser_unittests.erl

@@ -3,44 +3,53 @@
 -export([run_tests/0]).
 
 tests() ->
-    [
-	{"trans", [
-                    {"block with no trans", <<"<html>{% block main %} {% endblock %}</html>">>, []},
-
-                    {"block with trans", <<"<html>{% block main %} {% trans \"Hello\" %} {% endblock %}</html>">>, [{"Hello",{"dummy_path",1,33}}]},
-
-                    {"for with trans", <<"<html>{% block main %} {%for thing in things %}{% trans \"Hello inside a for\" %}  {% endfor %} {% endblock %}</html>">>, [{"Hello inside a for",{"dummy_path",1,57}}]},
-
-		    {"if with trans", <<"<html>{% block content %}{% if thing %} {% trans \"Hello inside an if\" %} {% endif %} {% endblock %}</html>">>, [{"Hello inside an if",{"dummy_path",1,50}}]},
-
-		    {"if with trans inside a for", <<"<html>{% block content %}{%for thin in things %}{% if thing %} {% trans \"Hello inside an if inside a for\" %} {% endif %} {% endfor %}{% endblock %}</html>">>, [{"Hello inside an if inside a for",{"dummy_path",1,73}}]},
-
-		    {"if and else both with trans", <<"<html>{% block content %}{% if thing %} {% trans \"Hello inside an if\" %} {% else %} {% trans \"Hello inside an else\" %} {% endif %} {% endblock %}</html>">>, [ {"Hello inside an else",{"dummy_path",1,94}}, {"Hello inside an if",{"dummy_path",1,50}}]}  
-	]}
+    [{"trans",
+      [{"block with no trans",
+        <<"<html>{% block main %} {% endblock %}</html>">>,
+        []},
+       {"block with trans",
+        <<"<html>{% block main %} {% trans \"Hello\" %} {% endblock %}</html>">>,
+        [{"Hello",{"dummy_path",1,33}}]},
+       {"for with trans",
+        <<"<html>{% block main %} {%for thing in things %}{% trans \"Hello inside a for\" %}  {% endfor %} {% endblock %}</html>">>,
+        [{"Hello inside a for",{"dummy_path",1,57}}]},
+       {"if with trans",
+        <<"<html>{% block content %}{% if thing %} {% trans \"Hello inside an if\" %} {% endif %} {% endblock %}</html>">>,
+        [{"Hello inside an if",{"dummy_path",1,50}}]},
+       {"if with trans inside a for",
+        <<"<html>{% block content %}{%for thin in things %}{% if thing %} {% trans \"Hello inside an if inside a for\" %} {% endif %} {% endfor %}{% endblock %}</html>">>,
+        [{"Hello inside an if inside a for",{"dummy_path",1,73}}]},
+       {"if and else both with trans",
+        <<"<html>{% block content %}{% if thing %} {% trans \"Hello inside an if\" %} {% else %} {% trans \"Hello inside an else\" %} {% endif %} {% endblock %}</html>">>,
+        [ {"Hello inside an else",{"dummy_path",1,94}}, {"Hello inside an if",{"dummy_path",1,50}}]}
+      ]}
     ].
 
 run_tests() ->
     io:format("Running source parser unit tests...~n"),
     Failures = lists:foldl(
-        fun({Group, Assertions}, GroupAcc) ->
-                io:format(" Test group ~p...~n", [Group]),
-                lists:foldl(fun({Name, Content, Output}, Acc) -> 
-				process_unit_test(Content, Output, Acc, Group, Name)                              
-                            end, GroupAcc, Assertions)
-        end, [], tests()),
+                 fun({Group, Assertions}, GroupAcc) ->
+                         io:format(" Test group ~p...~n", [Group]),
+                         lists:foldl(fun({Name, Content, Output}, Acc) ->
+                                             process_unit_test(Content, Output, Acc, Group, Name)
+                                     end, GroupAcc, Assertions)
+                 end, [], tests()),
 
     case Failures of
         [] ->
-            io:format("All tests PASS~n~n");
+            io:format("All source parser tests PASS~n~n");
         _ ->
-            io:format("Unit test failures: ~p~n", [Failures]),
+            io:format("Source parser unit test failures~n"),
+            [io:format("  Test: ~s.~s~n    Expected: ~p~n    Actual: ~s~n",
+                       [Group, Name, Expected, Actual])
+             || {Group, Name, _, {expected, Expected}, {found, Actual}} <- Failures],
             throw(failed)
     end.
 
-process_unit_test(Content, Output,Acc, Group, Name) ->
-	Tokens = sources_parser:process_content("dummy_path", Content),
-	%%io:format("Tokens are: ~p~n", [Tokens]),
-	case Tokens of
-		Output -> Acc;
-		_ -> 	[{Group, Name, 'binary', {expected, Output}, {found, Tokens} } | Acc]
-	end.
+process_unit_test(Content, Output, Acc, Group, Name) ->
+    Tokens = (catch sources_parser:process_content("dummy_path", Content)),
+    %%io:format("Tokens are: ~p~n", [Tokens]),
+    case Tokens of
+        Output -> Acc;
+        _ ->    [{Group, Name, 'binary', {expected, Output}, {found, Tokens} } | Acc]
+    end.