Add support for the extension module in the parser.

The parser can be resumed after resolving parse errors
(it is up to the extension module to do that).

The extension module test still fails since the compiler part has not yet been implemented,
and the tests run at a functional level instead of unit level.

Makefile

@@ -1,14 +1,15 @@
 ERL=erl
+ERLC=erlc
 REBAR=./rebar
 
-
 all: compile
 
-compile: 
+compile:
 	@$(REBAR) compile
 
 compile_test:
 	-mkdir -p ebintest
+	$(ERLC) -o tests/src -I include/erlydtl_preparser.hrl tests/src/erlydtl_extension_testparser.yrl
 	$(ERL) -make
 
 test: compile compile_test

include/erlydtl_preparser.hrl

@@ -29,7 +29,7 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 % The parser generator will insert appropriate declarations before this line.%
 
--export([parse/1, parse_and_scan/1, format_error/1, recover/1]).
+-export([parse/1, parse_and_scan/1, format_error/1, resume/1]).
 
 -type yecc_ret() :: {'error', _} | {'ok', _}.
 
@@ -44,7 +44,7 @@ parse_and_scan({F, A}) -> % Fun or {M, F}
 parse_and_scan({M, F, A}) ->
     yeccpars0([], {{{M, F}, A}, no_line}, 0, [], []).
 
-recover([Tokens, Tzr, State, States, Vstack]) ->
+resume([Tokens, Tzr, State, States, Vstack]) ->
     yeccpars0(Tokens, Tzr, State, States, Vstack).
 
 -spec format_error(any()) -> [char() | list()].
@@ -140,7 +140,7 @@ yeccpars1(State1, State, States, Vstack, Token0, [Token | Tokens], Tzr) ->
     ?checkparse(
        yeccpars2(State, element(1, Token), [State1 | States],
 		 [Token0 | Vstack], Token, Tokens, Tzr),
-       [[Token0, Token|Tokens], Tzr, [State, State1|States], Vstack]
+       [[Token0, Token | Tokens], Tzr, State1, States, Vstack]
       );
 yeccpars1(State1, State, States, Vstack, Token0, [], {{_F,_A}, _Line}=Tzr) ->
     yeccpars1([], Tzr, State, [State1 | States], [Token0 | Vstack]);

rebar.config

@@ -1 +1,2 @@
 {erl_opts, [debug_info]}.
+{yrl_opts, [{includefile, "include/erlydtl_preparser.hrl"}]}.

src/erlydtl_compiler.erl

@@ -2,6 +2,7 @@
 %%% File:      erlydtl_compiler.erl
 %%% @author    Roberto Saccon <rsaccon@gmail.com> [http://rsaccon.com]
 %%% @author    Evan Miller <emmiller@gmail.com>
+%%% @author    Andreas Stenius <kaos@astekk.se>
 %%% @copyright 2008 Roberto Saccon, Evan Miller
 %%% @doc  
 %%% ErlyDTL template compiler
@@ -34,6 +35,7 @@
 -module(erlydtl_compiler).
 -author('rsaccon@gmail.com').
 -author('emmiller@gmail.com').
+-author('Andreas Stenius <kaos@astekk.se>').
 
 %% --------------------------------------------------------------------
 %% Definitions
@@ -314,8 +316,6 @@ parse(Data) ->
 
 parse(Data, Context) when is_binary(Data) ->
     check_scan(erlydtl_scanner:scan(binary_to_list(Data)), Context);
-parse(State, Context) when is_tuple(State) ->
-    check_scan(erlydtl_scanner:scan(State), Context);
 parse(File, Context) ->  
     {M, F} = Context#dtl_context.reader,
     case catch M:F(File) of
@@ -346,24 +346,85 @@ parse(CheckSum, Data, Context) ->
             end
     end.
 
-check_scan({ok, Tokens}, _Context) ->
-    erlydtl_parser:parse(Tokens);
+recover(Mod, Fun, Args) 
+  when is_atom(Mod), is_atom(Fun), is_list(Args) ->
+    M = case code:is_loaded(Mod) of
+	    false ->
+		case code:load_file(Mod) of
+		    {module, Mod} ->
+			Mod;
+		    _ ->
+			undefined
+		end;
+	    _ -> Mod
+	end,
+    if M /= undefined ->
+	    case erlang:function_exported(M, Fun, length(Args)) of
+		true ->
+		    apply(M, Fun, Args);
+		false ->
+		    undefined
+	    end;
+       true ->
+	    undefined
+    end.
+
+check_scan({ok, Tokens}, Context) ->
+    check_parse(erlydtl_parser:parse(Tokens), [], Context);
 check_scan({error, Err, State}, Context) ->
     case Context#dtl_context.extension_module of
 	undefined ->
 	    {error, Err};
 	M ->
-	    case erlydtl_scanner:recover(M, State) of
+	    case recover(M, scan, [State]) of
 		{ok, NewState} ->
 		    %% io:format("recover from:~p~nto: ~p~n", [State, NewState]),
-		    parse(NewState, Context);
+		    check_scan(erlydtl_scanner:resume(NewState), Context);
 		undefined ->
 		    {error, Err};
-		ExtErr ->
-		    ExtErr
+		ExtRes ->
+		    ExtRes
 	    end
     end.
 
+check_parse({ok, _}=Ok, [], _Context) -> Ok;
+check_parse({ok, _, _}=Ok, [], _Context) -> Ok;
+check_parse({ok, Parsed}, Acc, _Context) -> {ok, Acc ++ Parsed};
+check_parse({ok, Parsed, C}, Acc, _Context) -> {ok, Acc ++ Parsed, C};
+check_parse({error, _}=Err, _, _Context) -> Err;
+check_parse({error, Err, State}, Acc, Context) ->
+    io:format("parse error: ~p~nstate: ~p~n",[Err, State]),
+    case Context#dtl_context.extension_module of
+        undefined ->
+            {error, Err};
+        M ->
+            {State1, Parsed} = reset_parse_state(State),
+            case recover(M, parse, [State1]) of
+                {ok, ExtParsed} ->
+                    io:format("parse recovered: ~p~n", [ExtParsed]),
+                    {ok, Acc ++ Parsed ++ ExtParsed};
+                {error, ExtErr, ExtState} ->
+                    case reset_parse_state(ExtState) of
+                        {_, []} ->
+                            %% todo: see if this is indeed a sensible ext error,
+                            %% or if we should rather present the original Err message
+                            {error, ExtErr};
+                        {State2, ExtParsed} ->
+                            check_parse(erlydtl_parser:resume(State2), Acc ++ Parsed ++ ExtParsed, Context)
+                    end;
+                undefined ->
+                    {error, Err};
+                ExtRes ->
+                    ExtRes
+            end
+    end.
+
+%% backtrack up to the Rootsymbol, and keep the current top-level value stack
+reset_parse_state([Ts, Tzr, _, [0 | []], [Parsed | []]]) ->
+    {[Ts, Tzr, 0, [], []], Parsed};
+reset_parse_state([Ts, Tzr, _, [S | Ss], [T | Stack]]) -> 
+    reset_parse_state([[T | Ts], Tzr, S, Ss, Stack]).
+
 custom_tags_ast(CustomTags, Context, TreeWalker) ->
     {{CustomTagsClauses, CustomTagsInfo}, TreeWalker1} = custom_tags_clauses_ast(CustomTags, Context, TreeWalker),
     {{erl_syntax:function(erl_syntax:atom(render_tag), CustomTagsClauses), CustomTagsInfo}, TreeWalker1}.

src/erlydtl_scanner.erl

@@ -2,6 +2,7 @@
 %%% File:      erlydtl_scanner.erl
 %%% @author    Roberto Saccon <rsaccon@gmail.com> [http://rsaccon.com]
 %%% @author    Evan Miller <emmiller@gmail.com>
+%%% @author    Andreas Stenius <kaos@astekk.se>
 %%% @copyright 2008 Roberto Saccon, Evan Miller
 %%% @doc 
 %%% Template language scanner
@@ -34,8 +35,9 @@
 -module(erlydtl_scanner).
 -author('rsaccon@gmail.com').
 -author('emmiller@gmail.com').
+-author('Andreas Stenius <kaos@astekk.se>').
 
--export([scan/1, recover/2]). 
+-export([scan/1, resume/1]).
 -include("erlydtl_ext.hrl").
 
 
@@ -50,33 +52,12 @@
 %% an error.
 %% @end
 %%--------------------------------------------------------------------
-scan(#scanner_state{ template=Template, scanned=Scanned, 
-		     pos=Pos, state=State}) ->
-    scan(Template, Scanned, Pos, State);
 scan(Template) ->
     scan(Template, [], {1, 1}, in_text).    
 
-recover(Mod, State) ->
-    M = case code:is_loaded(Mod) of
-	    false ->
-		case code:load_file(Mod) of
-		    {module, Mod} ->
-			Mod;
-		    _ ->
-			undefined
-		end;
-	    _ -> Mod
-	end,
-    if M /= undefined ->
-	    case erlang:function_exported(M, scan, 1) of
-		true ->
-		    M:scan(State);
-		false ->
-		    undefined
-	    end;
-       true ->
-	    undefined
-    end.
+resume(#scanner_state{ template=Template, scanned=Scanned, 
+		     pos=Pos, state=State}) ->
+    scan(Template, Scanned, Pos, State).
 
 scan([], Scanned, _, in_text) ->
     Tokens = lists:reverse(Scanned),

tests/src/erlydtl_extension_test.erl

@@ -1,6 +1,6 @@
 -module(erlydtl_extension_test).
 
--export([scan/1]).
+-export([scan/1, parse/1]).
 -include("erlydtl_ext.hrl").
 
 %% look for a foo identifer followed by a #
@@ -11,9 +11,16 @@ scan(#scanner_state{ template="#" ++ T,
     {ok, S#scanner_state{ template=T,
 			  scanned=[{identifier, Loc, "rab"}|Scanned],
 			  pos={L, C+1} }};
-scan(#scanner_state{ template="#" ++ T, pos={L, C} }) ->
+scan(#scanner_state{ template="#" ++ _T, pos={L, C} }) ->
     %% give error when # not follows foo
     {error, {L,?MODULE,lists:concat(["Unexpected '#' in code at column ", C])}};
 scan(_) -> 
     %% for anything else, fallback to the error message from erlydtl_scanner..
     undefined.
+
+parse(State) ->
+    io:format("extension:parse got: ~p~n", [State]),
+    %code:load_file(erlydtl_extension_testparser),
+    Res=erlydtl_extension_testparser:resume(State),
+    io:format("extension result: ~p~n", [Res]),
+    Res.

tests/src/erlydtl_unittests.erl

@@ -1104,10 +1104,12 @@ tests() ->
 			  ]},
      %% custom syntax stuff
      {"extension_module", [
-			   %% the erlydtl_extension_test module replaces a foo identifier with bar when hitting a #.
+			   %% the erlydtl_extension_test module replaces a foo identifier with bar when hitting a # following foo.
 			   {"replace parsed token", <<"{{ foo # }}">>, [{bar, "ok"}], [], [{extension_module, erlydtl_extension_test}], <<"ok">>},
 			   {"proper error message", <<"{{ bar # }}">>, [{bar, "ok"}], [], [{extension_module, erlydtl_extension_test}],
-			    {error, {1,erlydtl_extension_test,"Unexpected '#' in code at column 8"}}}
+			    {error, {1,erlydtl_extension_test,"Unexpected '#' in code at column 8"}}},
+                           %% accept identifiers as expressions (this is a dummy functionality to test the parser extensibility)
+			   {"identifiers as expressions", <<"{{ test }} data {{ foo.bar or baz }}">>, [{baz, "ok"}], [], [{extension_module, erlydtl_extension_test}], <<"ok">>}
 			  ]}
     ].