support recovering from a scanner error using the extension module.

.gitignore

@@ -3,3 +3,4 @@ ebin
 erl_crash.dump
 examples/rendered_output
 src/erlydtl_parser.erl
+*~

Emakefile

@@ -1 +1 @@
-{"tests/src/*", [debug_info, {outdir, "ebintest"}]}.
+{"tests/src/*", [debug_info, {outdir, "ebintest"}, {i, "include"}]}.

include/erlydtl_ext.hrl

@@ -0,0 +1,8 @@
+
+-record(scanner_state,
+	{
+	  template=[],
+	  scanned=[],
+	  pos={1,1},
+	  state=in_text
+	}).

src/erlydtl_compiler.erl

@@ -306,13 +306,10 @@ is_up_to_date(CheckSum, Context) ->
 parse(Data) ->
     parse(Data, #dtl_context{}).
 
-parse(Data, _Context) when is_binary(Data) ->
-    case erlydtl_scanner:scan(binary_to_list(Data)) of
-        {ok, Tokens} ->
-            erlydtl_parser:parse(Tokens);
-        Err ->
-            Err
-    end;
+parse(Data, Context) when is_binary(Data) ->
+    check_scan(erlydtl_scanner:scan(binary_to_list(Data)), Context);
+parse(State, Context) when is_tuple(State) ->
+    check_scan(erlydtl_scanner:scan(State), Context);
 parse(File, Context) ->  
     {M, F} = Context#dtl_context.reader,
     case catch M:F(File) of
@@ -343,6 +340,24 @@ parse(CheckSum, Data, Context) ->
             end
     end.
 
+check_scan({ok, Tokens}, _Context) ->
+    erlydtl_parser:parse(Tokens);
+check_scan({error, Err, State}, Context) ->
+    case Context#dtl_context.extension_module of
+	undefined ->
+	    {error, Err};
+	M ->
+	    case erlydtl_scanner:recover(M, State) of
+		{ok, NewState} ->
+		    %% io:format("recover from:~p~nto: ~p~n", [State, NewState]),
+		    parse(NewState, Context);
+		undefined ->
+		    {error, Err};
+		ExtErr ->
+		    ExtErr
+	    end
+    end.
+
 custom_tags_ast(CustomTags, Context, TreeWalker) ->
     {{CustomTagsClauses, CustomTagsInfo}, TreeWalker1} = custom_tags_clauses_ast(CustomTags, Context, TreeWalker),
     {{erl_syntax:function(erl_syntax:atom(render_tag), CustomTagsClauses), CustomTagsInfo}, TreeWalker1}.

src/erlydtl_scanner.erl

@@ -35,7 +35,8 @@
 -author('rsaccon@gmail.com').
 -author('emmiller@gmail.com').
 
--export([scan/1]). 
+-export([scan/1, recover/2]). 
+-include("erlydtl_ext.hrl").
 
 
 %%====================================================================
@@ -49,8 +50,33 @@
 %% an error.
 %% @end
 %%--------------------------------------------------------------------
+scan(#scanner_state{ template=Template, scanned=Scanned, 
+		     pos=Pos, state=State}) ->
+    scan(Template, Scanned, Pos, State);
 scan(Template) ->
-    scan(Template, [], {1, 1}, in_text).
+    scan(Template, [], {1, 1}, in_text).    
+
+recover(Mod, State) ->
+    M = case code:is_loaded(Mod) of
+	    false ->
+		case code:load_file(Mod) of
+		    {module, Mod} ->
+			Mod;
+		    _ ->
+			undefined
+		end;
+	    _ -> Mod
+	end,
+    if M /= undefined ->
+	    case erlang:function_exported(M, scan, 1) of
+		true ->
+		    M:scan(State);
+		false ->
+		    undefined
+	    end;
+       true ->
+	    undefined
+    end.
 
 scan([], Scanned, _, in_text) ->
     Tokens = lists:reverse(Scanned),
@@ -256,7 +282,8 @@ scan([H | T], Scanned, {Row, Column}, {in_code, Closer}) ->
         digit ->
             scan(T, [{number_literal, {Row, Column}, [H]} | Scanned], {Row, Column + 1}, {in_number, Closer});
         _ ->
-            {error, {Row, ?MODULE, lists:concat(["Illegal character in column ", Column])}}
+            {error, {Row, ?MODULE, lists:concat(["Illegal character in column ", Column])},
+	     #scanner_state{ template=[H|T], scanned=Scanned, pos={Row, Column}, state={in_code, Closer}}}
     end;
 
 scan([H | T], Scanned, {Row, Column}, {in_number, Closer}) ->
@@ -264,7 +291,8 @@ scan([H | T], Scanned, {Row, Column}, {in_number, Closer}) ->
         digit ->
             scan(T, append_char(Scanned, H), {Row, Column + 1}, {in_number, Closer});
         _ ->
-            {error, {Row, ?MODULE, lists:concat(["Illegal character in column ", Column])}}
+            {error, {Row, ?MODULE, lists:concat(["Illegal character in column ", Column])},
+	     #scanner_state{ template=[H|T], scanned=Scanned, pos={Row, Column}, state={in_code, Closer}}}
     end;
 
 scan([H | T], Scanned, {Row, Column}, {in_identifier, Closer}) ->
@@ -274,7 +302,8 @@ scan([H | T], Scanned, {Row, Column}, {in_identifier, Closer}) ->
         digit ->
             scan(T, append_char(Scanned, H), {Row, Column + 1}, {in_identifier, Closer});
         _ ->
-            {error, {Row, ?MODULE, lists:concat(["Illegal character in column ", Column])}}
+            {error, {Row, ?MODULE, lists:concat(["Illegal character in column ", Column])},
+	     #scanner_state{ template=[H|T], scanned=Scanned, pos={Row, Column}, state={in_code, Closer}}}
     end.
 
 						% internal functions

tests/src/erlydtl_extension_test.erl

@@ -0,0 +1,19 @@
+-module(erlydtl_extension_test).
+
+-export([scan/1]).
+-include("erlydtl_ext.hrl").
+
+%% look for a foo identifer followed by a #
+scan(#scanner_state{ template="#" ++ T, 
+		     scanned=[{identifier, Loc, "oof"}|Scanned], 
+		     pos={L,C} }=S) ->
+    %% return new state with the hash dropped, and the foo identifer replaced with bar
+    {ok, S#scanner_state{ template=T,
+			  scanned=[{identifier, Loc, "rab"}|Scanned],
+			  pos={L, C+1} }};
+scan(#scanner_state{ template="#" ++ T, pos={L, C} }) ->
+    %% give error when # not follows foo
+    {error, {L,?MODULE,lists:concat(["Unexpected '#' in code at column ", C])}};
+scan(_) -> 
+    %% for anything else, fallback to the error message from erlydtl_scanner..
+    undefined.

tests/src/erlydtl_unittests.erl

@@ -1101,6 +1101,13 @@ tests() ->
 			    <<"{{ a|intcomma }} {{ b|intcomma }} {{ c|intcomma }} {{ d|intcomma }}">>,
 			    [{a, 999}, {b, 123456789}, {c, 12345}, {d, 1234567890}],
 			    <<"999 123,456,789 12,345 1,234,567,890">>}
+			  ]},
+     %% custom syntax stuff
+     {"extension_module", [
+			   %% the erlydtl_extension_test module replaces a foo identifier with bar when hitting a #.
+			   {"replace parsed token", <<"{{ foo # }}">>, [{bar, "ok"}], [], [{extension_module, erlydtl_extension_test}], <<"ok">>},
+			   {"proper error message", <<"{{ bar # }}">>, [{bar, "ok"}], [], [{extension_module, erlydtl_extension_test}],
+			    {error, {1,erlydtl_extension_test,"Unexpected '#' in code at column 8"}}}
 			  ]}
     ].
 
@@ -1141,6 +1148,7 @@ process_unit_test(CompiledTemplate, Vars, RenderOpts, Output,Acc, Group, Name) -
 		    [{Group, Name, 'list', Unexpected1, Output},
 		     {Group, Name, 'binary', Unexpected2, Output} | Acc]
 	    end;
+	Output -> Acc;
 	Err ->
 	    [{Group, Name, Err} | Acc]
     end.