Merge branch 'master' of github.com:evanmiller/erlydtl

.gitignore

@@ -4,3 +4,6 @@ erl_crash.dump
 examples/rendered_output
 src/erlydtl_parser.erl
 *~
+ebintest
+tests/src/erlydtl_extension_testparser.erl
+tests/output

include/erlydtl_ext.hrl

@@ -1,8 +1,44 @@
 
--record(scanner_state,
-	{
-	  template=[],
-	  scanned=[],
-	  pos={1,1},
-	  state=in_text
-	}).
+-record(dtl_context, {
+          local_scopes = [], 
+          block_dict = dict:new(), 
+          blocktrans_fun = none,
+          blocktrans_locales = [],
+          auto_escape = off, 
+          doc_root = "", 
+          parse_trail = [],
+          vars = [],
+          filter_modules = [],
+          custom_tags_dir = [],
+          custom_tags_modules = [],
+          reader = {file, read_file},
+          module = [],
+          compiler_options = [verbose, report_errors],
+          binary_strings = true,
+          force_recompile = false,
+          locale = none,
+          verbose = false,
+          is_compiling_dir = false,
+          extension_module = undefined
+         }).
+
+-record(ast_info, {
+          dependencies = [],
+          translatable_strings = [],
+          translated_blocks= [],
+          custom_tags = [],
+          var_names = [],
+          pre_render_asts = []}).
+
+-record(treewalker, {
+          counter = 0,
+          safe = false,
+          extension = undefined
+         }).    
+
+-record(scanner_state, {
+          template=[],
+          scanned=[],
+          pos={1,1},
+          state=in_text
+        }).

src/erlydtl_compiler.erl

@@ -43,43 +43,14 @@
 -export([compile/2, compile/3, compile_dir/2, compile_dir/3, parse/1]).
 
 %% exported for use by extension modules
--export([merge_info/2, value_ast/5]).
-
--record(dtl_context, {
-	  local_scopes = [], 
-	  block_dict = dict:new(), 
-	  blocktrans_fun = none,
-	  blocktrans_locales = [],
-	  auto_escape = off, 
-	  doc_root = "", 
-	  parse_trail = [],
-	  vars = [],
-	  filter_modules = [],
-	  custom_tags_dir = [],
-	  custom_tags_modules = [],
-	  reader = {file, read_file},
-	  module = [],
-	  compiler_options = [verbose, report_errors],
-	  binary_strings = true,
-	  force_recompile = false,
-	  locale = none,
-	  verbose = false,
-	  is_compiling_dir = false,
-	  extension_module = undefined
-	 }).
-
--record(ast_info, {
-	  dependencies = [],
-	  translatable_strings = [],
-	  translated_blocks= [],
-	  custom_tags = [],
-	  var_names = [],
-	  pre_render_asts = []}).
-
--record(treewalker, {
-	  counter = 0,
-	  safe = false
-	 }).    
+-export([
+         merge_info/2, 
+         format/3, 
+         value_ast/5,
+         resolve_scoped_variable_ast/2
+        ]).
+
+-include("erlydtl_ext.hrl").
 
 compile(Binary, Module) when is_binary(Binary) ->
     compile(Binary, Module, []);
@@ -191,7 +162,7 @@ write_binary(Module1, Bin, Options, Warnings) ->
     end.
 
 compile_multiple_to_binary(Dir, ParserResults, Context) ->
-    MatchAst = options_match_ast(), 
+    MatchAst = options_match_ast(Context), 
     {Functions, {AstInfo, _}} = lists:mapfoldl(fun({File, DjangoParseTree, CheckSum}, {AstInfo, TreeWalker}) ->
 						       FilePath = full_path(File, Context#dtl_context.doc_root),
 						       {{BodyAst, BodyInfo}, TreeWalker1} = with_dependency({FilePath, CheckSum}, body_ast(DjangoParseTree, Context, TreeWalker)),
@@ -204,18 +175,18 @@ compile_multiple_to_binary(Dir, ParserResults, Context) ->
 										       [erl_syntax:clause([erl_syntax:variable("_Variables"), erl_syntax:variable("RenderOptions")], none,
 													  MatchAst ++ [BodyAst])]),
 						       {{FunctionName, Function1, Function2}, {merge_info(AstInfo, BodyInfo), TreeWalker1}}
-					       end, {#ast_info{}, #treewalker{}}, ParserResults),
+					       end, {#ast_info{}, init_treewalker(Context)}, ParserResults),
     Forms = custom_forms(Dir, Context#dtl_context.module, Functions, AstInfo),
-    compile_forms_and_reload(Dir, Forms, Context#dtl_context.compiler_options).
+    compile_forms_and_reload(Dir, Forms, Context).
 
 compile_to_binary(File, DjangoParseTree, Context, CheckSum) ->
-    try body_ast(DjangoParseTree, Context, #treewalker{}) of
+    try body_ast(DjangoParseTree, Context, init_treewalker(Context)) of
         {{BodyAst, BodyInfo}, BodyTreeWalker} ->
             try custom_tags_ast(BodyInfo#ast_info.custom_tags, Context, BodyTreeWalker) of
                 {{CustomTagsAst, CustomTagsInfo}, _} ->
                     Forms = forms(File, Context#dtl_context.module, {BodyAst, BodyInfo}, 
-				  {CustomTagsAst, CustomTagsInfo}, Context#dtl_context.binary_strings, CheckSum), 
-                    compile_forms_and_reload(File, Forms, Context#dtl_context.compiler_options)
+				  {CustomTagsAst, CustomTagsInfo}, CheckSum, Context, BodyTreeWalker), 
+                    compile_forms_and_reload(File, Forms, Context)
             catch 
                 throw:Error -> Error
             end
@@ -223,14 +194,21 @@ compile_to_binary(File, DjangoParseTree, Context, CheckSum) ->
         throw:Error -> Error
     end.
 
-compile_forms_and_reload(File, Forms, CompilerOptions) ->
-    case proplists:get_value(debug_compiler, CompilerOptions) of
+compile_forms_and_reload(File, Forms, Context) ->
+    case proplists:get_value(debug_compiler, Context#dtl_context.compiler_options) of
 	true ->
-	    io:format("Template ~p compiled with options: ~p~n", [File, CompilerOptions]),
-	    [io:format("~s~n", [erl_pp:form(Form)]) || Form <- Forms];
+	    io:format("template source:~n~s~n",
+                      [try 
+                           erl_prettypr:format(erl_syntax:form_list(Forms))
+                       catch
+                           error:Err ->
+                               io_lib:format("Pretty printing failed: ~p~nContext: ~n~p~nForms: ~n~p~n",
+                                             [Err, Context, Forms])
+                       end
+                      ]);
 	_ -> nop
     end,
-    case compile:forms(Forms, CompilerOptions) of
+    case compile:forms(Forms, Context#dtl_context.compiler_options) of
         {ok, Module1, Bin} -> 
             load_code(Module1, Bin, []);
         {ok, Module1, Bin, Warnings} ->
@@ -250,7 +228,7 @@ load_code(Module, Bin, Warnings) ->
 
 init_context(IsCompilingDir, ParseTrail, DefDir, Module, Options) ->
     Ctx = #dtl_context{},
-    #dtl_context{
+    Context = #dtl_context{
 		  parse_trail = ParseTrail,
 		  module = Module,
 		  doc_root = proplists:get_value(doc_root, Options, DefDir),
@@ -268,7 +246,11 @@ init_context(IsCompilingDir, ParseTrail, DefDir, Module, Options) ->
 		  verbose = proplists:get_value(verbose, Options, Ctx#dtl_context.verbose),
 		  is_compiling_dir = IsCompilingDir,
 		  extension_module = proplists:get_value(extension_module, Options, Ctx#dtl_context.extension_module)
-		}.
+		},
+    case call_extension(Context, init_context, [Context]) of
+        {ok, C} when is_record(C, dtl_context) -> C;
+        undefined -> Context
+    end.
 
 init_dtl_context(File, Module, Options) when is_list(Module) ->
     init_dtl_context(File, list_to_atom(Module), Options);
@@ -280,6 +262,13 @@ init_dtl_context_dir(Dir, Module, Options) when is_list(Module) ->
 init_dtl_context_dir(Dir, Module, Options) ->
     init_context(true, [], Dir, Module, Options).
 
+init_treewalker(Context) ->
+    TreeWalker = #treewalker{},
+    case call_extension(Context, init_treewalker, [TreeWalker]) of
+        {ok, TW} when is_record(TW, treewalker) -> TW;
+        undefined -> TreeWalker
+    end.
+
 is_up_to_date(_, #dtl_context{force_recompile = true}) ->
     false;
 is_up_to_date(CheckSum, Context) ->
@@ -349,8 +338,9 @@ parse(CheckSum, Data, Context) ->
             end
     end.
 
-recover(undefined, _Fun, _Args) -> undefined;
-recover(Mod, Fun, Args) 
+call_extension(#dtl_context{ extension_module=undefined }, _Fun, _Args) ->
+    undefined;
+call_extension(#dtl_context{ extension_module=Mod }, Fun, Args) 
   when is_atom(Mod), is_atom(Fun), is_list(Args) ->
     M = case code:is_loaded(Mod) of
 	    false ->
@@ -374,13 +364,17 @@ recover(Mod, Fun, Args)
     end.
 
 check_scan({ok, Tokens}, Context) ->
-    check_parse(erlydtl_parser:parse(Tokens), [], Context);
+    Tokens1 = case call_extension(Context, post_scan, [Tokens]) of
+                  undefined -> Tokens;
+                  {ok, T} -> T
+              end,
+    check_parse(erlydtl_parser:parse(Tokens1), [], Context);
 check_scan({error, Err, State}, Context) ->
-    case recover(Context#dtl_context.extension_module, scan, [State]) of
+    case call_extension(Context, scan, [State]) of
         undefined ->
             {error, Err};
         {ok, NewState} ->
-            %% io:format("recover from:~p~nto: ~p~n", [State, NewState]),
+            %% io:format("call_extension from:~p~nto: ~p~n", [State, NewState]),
             check_scan(erlydtl_scanner:resume(NewState), Context);
         ExtRes ->
             ExtRes
@@ -394,7 +388,7 @@ check_parse({error, _}=Err, _, _Context) -> Err;
 check_parse({error, Err, State}, Acc, Context) ->
     %% io:format("parse error: ~p~nstate: ~p~n",[Err, State]),
     {State1, Parsed} = reset_parse_state(State),
-    case recover(Context#dtl_context.extension_module, parse, [State1]) of
+    case call_extension(Context, parse, [State1]) of
         undefined ->
             {error, Err};
         {ok, ExtParsed} ->
@@ -416,19 +410,39 @@ check_parse({error, Err, State}, Acc, Context) ->
 reset_parse_state([Ts, Tzr, _, [0 | []], [Parsed | []]]) ->
     {[Ts, Tzr, 0, [], []], Parsed};
 reset_parse_state([Ts, Tzr, _, [S | Ss], [T | Stack]]) -> 
-    reset_parse_state([[T | Ts], Tzr, S, Ss, Stack]).
+    reset_parse_state([[T | Ts], Tzr, S, Ss, Stack]);
+reset_parse_state([_, _, 0, [], []]=State) -> 
+    {State, []}.
 
 custom_tags_ast(CustomTags, Context, TreeWalker) ->
     {{CustomTagsClauses, CustomTagsInfo}, TreeWalker1} = custom_tags_clauses_ast(CustomTags, Context, TreeWalker),
-    {{erl_syntax:function(erl_syntax:atom(render_tag), CustomTagsClauses), CustomTagsInfo}, TreeWalker1}.
+    %% This doesn't work since erl_syntax:revert/1 chokes on the airity_qualifier in the -compile attribute..
+    %% bug report sent to the erlang-bugs mailing list.
+    %% {{erl_syntax:form_list(
+    %%     [erl_syntax:attribute(
+    %%        erl_syntax:atom(compile),
+    %%        [erl_syntax:tuple(
+    %%           [erl_syntax:atom(nowarn_unused_function),
+    %%            erl_syntax:arity_qualifier(
+    %%              erl_syntax:atom(render_tag),
+    %%              erl_syntax:integer(3))])]),
+    %%      erl_syntax:function(erl_syntax:atom(render_tag), CustomTagsClauses)]),
+    {{erl_syntax:function(erl_syntax:atom(render_tag), CustomTagsClauses),
+      CustomTagsInfo},
+     TreeWalker1}.
 
 custom_tags_clauses_ast(CustomTags, Context, TreeWalker) ->
     custom_tags_clauses_ast1(CustomTags, [], [], #ast_info{}, Context, TreeWalker).
 
 custom_tags_clauses_ast1([], _ExcludeTags, ClauseAcc, InfoAcc, _Context, TreeWalker) ->
-    {{lists:reverse([erl_syntax:clause([erl_syntax:variable("TagName"), erl_syntax:underscore(), erl_syntax:underscore()], none, 
-				       [erl_syntax:list([])])|ClauseAcc
-		    ]), InfoAcc}, TreeWalker};
+    {{lists:reverse(
+        [erl_syntax:clause(
+           [erl_syntax:variable("_TagName"), erl_syntax:underscore(), erl_syntax:underscore()], 
+           none, 
+           [erl_syntax:list([])])
+         |ClauseAcc]),
+      InfoAcc}, 
+     TreeWalker};
 custom_tags_clauses_ast1([Tag|CustomTags], ExcludeTags, ClauseAcc, InfoAcc, Context, TreeWalker) ->
     case lists:member(Tag, ExcludeTags) of
         true ->
@@ -441,7 +455,7 @@ custom_tags_clauses_ast1([Tag|CustomTags], ExcludeTags, ClauseAcc, InfoAcc, Cont
                         {ok, DjangoParseTree, CheckSum} ->
                             {{BodyAst, BodyAstInfo}, TreeWalker1} = with_dependency(
 								      {CustomTagFile, CheckSum}, body_ast(DjangoParseTree, Context, TreeWalker)),
-                            MatchAst = options_match_ast(), 
+                            MatchAst = options_match_ast(Context, TreeWalker), 
                             Clause = erl_syntax:clause(
 				       [key_to_string(Tag), erl_syntax:variable("_Variables"), erl_syntax:variable("RenderOptions")],
 				       none, MatchAst ++ [BodyAst]),
@@ -508,7 +522,7 @@ custom_forms(Dir, Module, Functions, AstInfo) ->
     [erl_syntax:revert(X) || X <- [ModuleAst, ExportAst, SourceFunctionAst, DependenciesFunctionAst, TranslatableStringsFunctionAst
 				   | FunctionAsts] ++ AstInfo#ast_info.pre_render_asts].
 
-forms(File, Module, {BodyAst, BodyInfo}, {CustomTagsFunctionAst, CustomTagsInfo}, BinaryStrings, CheckSum) ->
+forms(File, Module, {BodyAst, BodyInfo}, {CustomTagsFunctionAst, CustomTagsInfo}, CheckSum, Context, TreeWalker) ->
     MergedInfo = merge_info(BodyInfo, CustomTagsInfo),
     Render0FunctionAst = erl_syntax:function(erl_syntax:atom(render),
 					     [erl_syntax:clause([], none, [erl_syntax:application(none, 
@@ -543,13 +557,13 @@ forms(File, Module, {BodyAst, BodyInfo}, {CustomTagsFunctionAst, CustomTagsInfo}
 
     VariablesAst = variables_function(MergedInfo#ast_info.var_names),
 
-    MatchAst = options_match_ast(), 
+    MatchAst = options_match_ast(Context, TreeWalker), 
 
     BodyAstTmp = MatchAst ++ [
 			      erl_syntax:application(
 				erl_syntax:atom(erlydtl_runtime),
 				erl_syntax:atom(stringify_final),
-				[BodyAst, erl_syntax:atom(BinaryStrings)])
+				[BodyAst, erl_syntax:atom(Context#dtl_context.binary_strings)])
 			     ],
 
     RenderInternalFunctionAst = erl_syntax:function(
@@ -573,12 +587,16 @@ forms(File, Module, {BodyAst, BodyInfo}, {CustomTagsFunctionAst, CustomTagsInfo}
 						       erl_syntax:arity_qualifier(erl_syntax:atom(variables), erl_syntax:integer(0))
 						      ])]),
 
-    [erl_syntax:revert(X) || X <- [ModuleAst, ExportAst, Render0FunctionAst, Render1FunctionAst, Render2FunctionAst,
-				   SourceFunctionAst, DependenciesFunctionAst, TranslatableStringsAst,
-				   TranslatedBlocksAst, VariablesAst, RenderInternalFunctionAst, 
-				   CustomTagsFunctionAst | BodyInfo#ast_info.pre_render_asts]].    
+    erl_syntax:revert_forms(
+      erl_syntax:form_list(
+        [ModuleAst, ExportAst, Render0FunctionAst, Render1FunctionAst, Render2FunctionAst,
+         SourceFunctionAst, DependenciesFunctionAst, TranslatableStringsAst,
+         TranslatedBlocksAst, VariablesAst, RenderInternalFunctionAst, 
+         CustomTagsFunctionAst
+         |BodyInfo#ast_info.pre_render_asts])).
 
-options_match_ast() -> 
+options_match_ast(Context) -> options_match_ast(Context, undefined).
+options_match_ast(Context, TreeWalker) -> 
     [
      erl_syntax:match_expr(
        erl_syntax:variable("_TranslationFun"),
@@ -592,6 +610,10 @@ options_match_ast() ->
 	 erl_syntax:atom(proplists),
 	 erl_syntax:atom(get_value),
 	 [erl_syntax:atom(locale), erl_syntax:variable("RenderOptions"), erl_syntax:atom(none)]))
+     | case call_extension(Context, setup_render_ast, [Context, TreeWalker]) of
+           undefined -> [];
+           Ast when is_list(Ast) -> Ast
+       end
     ].
 
 						% child templates should only consist of blocks at the top level
@@ -788,11 +810,13 @@ value_ast(ValueToken, AsString, EmptyIfUndefined, Context, TreeWalker) ->
             {{Ast, #ast_info{var_names = [VarName]}}, TreeWalker};
         {'variable', _} = Variable ->
             {Ast, VarName} = resolve_variable_ast(Variable, Context, EmptyIfUndefined),
-            {{Ast, #ast_info{var_names = [VarName]}}, TreeWalker}
+            {{Ast, #ast_info{var_names = [VarName]}}, TreeWalker};
+        {extension, Tag} ->
+            extension_ast(Tag, Context, TreeWalker)
     end.
 
 extension_ast(Tag, Context, TreeWalker) ->
-    case recover(Context#dtl_context.extension_module, compile_ast, [Tag, Context, TreeWalker]) of
+    case call_extension(Context, compile_ast, [Tag, Context, TreeWalker]) of
         undefined ->
             throw({error, {unknown_extension, Tag}});
         Result ->