Slightly nicer error messages.

Use more atoms in the scanner so that Yecc produces nicer errors.

src/erlydtl/erlydtl_compiler.erl

@@ -197,12 +197,12 @@ parse(File, Context) ->
                 {error, Msg} when is_list(Msg) ->
                     {error, File ++ ": " ++ Msg};
                 {error, Msg} ->
-                    {error, {File, Msg}};
+                    {error, {File, [Msg]}};
                 Result ->
                     Result
             end;
         _ ->
-            {error, "reading " ++ File ++ " failed "}
+            {error, {File, ["Failed to read file"]}}
     end.
         
 parse(CheckSum, Data, Context) ->
@@ -319,9 +319,9 @@ body_ast(DjangoParseTree, Context, TreeWalker) ->
             ({'date', 'now', {string_literal, _Pos, FormatString}}, TreeWalkerAcc) ->
                 now_ast(FormatString, Context, TreeWalkerAcc);
             ({'autoescape', {identifier, _, OnOrOff}, Contents}, TreeWalkerAcc) ->
-                body_ast(Contents, Context#dtl_context{auto_escape = list_to_atom(OnOrOff)}, 
+                body_ast(Contents, Context#dtl_context{auto_escape = OnOrOff}, 
                     TreeWalkerAcc);
-            ({'text', _Pos, String}, TreeWalkerAcc) -> 
+            ({'string', _Pos, String}, TreeWalkerAcc) -> 
                 string_ast(String, TreeWalkerAcc);
 	    ({'trans', {string_literal, _Pos, FormatString}}, TreeWalkerAcc) ->
                 translated_ast(FormatString, Context, TreeWalkerAcc);
@@ -359,7 +359,7 @@ body_ast(DjangoParseTree, Context, TreeWalker) ->
                 for_loop_ast(IteratorList, Variable, Contents, EmptyAstInfo, Context, TreeWalker1);
             ({'load', Names}, TreeWalkerAcc) ->
                 load_ast(Names, Context, TreeWalkerAcc);
-            ({'tag', {identifier, _, Name}, Args}, TreeWalkerAcc) ->
+            ({'tag', {'identifier', _, Name}, Args}, TreeWalkerAcc) ->
                 tag_ast(Name, Args, Context, TreeWalkerAcc);            
             ({'call', {'identifier', _, Name}}, TreeWalkerAcc) ->
             	call_ast(Name, TreeWalkerAcc);
@@ -379,7 +379,7 @@ body_ast(DjangoParseTree, Context, TreeWalker) ->
         fun({Ast, Info}, {InfoAcc, TreeWalkerAcc}) -> 
                 PresetVars = lists:foldl(fun
                         (X, Acc) ->
-                            case proplists:lookup(list_to_atom(X), Context#dtl_context.vars) of
+                            case proplists:lookup(X, Context#dtl_context.vars) of
                                 none ->
                                     Acc;
                                 Val ->
@@ -505,7 +505,7 @@ filter_ast(Variable, Filter, Context, TreeWalker) ->
             {{UnescapedAst, Info}, TreeWalker2}
     end.
 
-filter_ast_noescape(Variable, [{identifier, _, "escape"}], Context, TreeWalker) ->
+filter_ast_noescape(Variable, [{identifier, _, 'escape'}], Context, TreeWalker) ->
     value_ast(Variable, true, Context, TreeWalker);
 filter_ast_noescape(Variable, Filter, Context, TreeWalker) ->
     {{VariableAst, Info}, TreeWalker2} = value_ast(Variable, true, Context, TreeWalker),
@@ -530,7 +530,7 @@ search_for_escape_filter(_, _, #dtl_context{auto_escape = did}) ->
 search_for_escape_filter(Variable, Filter, _) ->
     search_for_escape_filter(Variable, Filter).
 
-search_for_escape_filter(_, [{identifier, _, "escape"}]) ->
+search_for_escape_filter(_, [{identifier, _, 'escape'}]) ->
     on;
 search_for_escape_filter({apply_filter, Variable, Filter}, _) ->
     search_for_escape_filter(Variable, Filter);
@@ -568,7 +568,7 @@ resolve_variable_ast(What, _Context, _FinderFunction) ->
 resolve_scoped_variable_ast(VarName, Context) ->
     lists:foldl(fun(Scope, Value) ->
                 case Value of
-                    undefined -> proplists:get_value(list_to_atom(VarName), Scope);
+                    undefined -> proplists:get_value(VarName, Scope);
                     _ -> Value
                 end
         end, undefined, Context#dtl_context.local_scopes).
@@ -616,20 +616,20 @@ ifelse_ast(Expression, {IfContentsAst, IfContentsInfo}, {ElseContentsAst, ElseCo
 
 for_loop_ast(IteratorList, LoopValue, Contents, {EmptyContentsAst, EmptyContentsInfo}, Context, TreeWalker) ->
     Vars = lists:map(fun({identifier, _, Iterator}) -> 
-                    erl_syntax:variable("Var_" ++ Iterator) 
+                erl_syntax:variable(lists:concat(["Var_", Iterator])) 
             end, IteratorList),
     {{InnerAst, Info}, TreeWalker1} = body_ast(Contents,
         Context#dtl_context{local_scopes = [
                 [{'forloop', erl_syntax:variable("Counters")} | lists:map(
                     fun({identifier, _, Iterator}) ->
-                            {list_to_atom(Iterator), erl_syntax:variable("Var_" ++ Iterator)} 
+                            {Iterator, erl_syntax:variable(lists:concat(["Var_", Iterator]))} 
                     end, IteratorList)] | Context#dtl_context.local_scopes]}, TreeWalker),
     CounterAst = erl_syntax:application(erl_syntax:atom(erlydtl_runtime), 
         erl_syntax:atom(increment_counter_stats), [erl_syntax:variable("Counters")]),
 
     {{LoopValueAst, LoopValueInfo}, TreeWalker2} = value_ast(LoopValue, false, Context, TreeWalker1),
 
-    CounterVars0 = case resolve_scoped_variable_ast("forloop", Context) of
+    CounterVars0 = case resolve_scoped_variable_ast('forloop', Context) of
         undefined ->
             erl_syntax:application(erl_syntax:atom(erlydtl_runtime), erl_syntax:atom(init_counter_stats), [LoopValueAst]);
         Value ->
@@ -727,9 +727,9 @@ tag_ast(Name, Args, Context, TreeWalker) ->
         true ->
             InterpretedArgs = lists:map(fun
                     ({{identifier, _, Key}, {string_literal, _, Value}}) ->
-                        {list_to_atom(Key), erl_syntax:string(unescape_string_literal(Value))};
+                        {Key, erl_syntax:string(unescape_string_literal(Value))};
                     ({{identifier, _, Key}, Value}) ->
-                        {list_to_atom(Key), format(resolve_variable_ast(Value, Context), Context)}
+                        {Key, format(resolve_variable_ast(Value, Context), Context)}
                 end, Args),
             DefaultFilePath = filename:join([erlydtl_deps:get_base_dir(), "priv", "custom_tags", Name]),
             case Context#dtl_context.custom_tags_dir of
@@ -762,7 +762,7 @@ tag_ast(Name, Args, Context, TreeWalker) ->
             throw({error, lists:concat(["Custom tag '", Name, "' not loaded"])})
     end.
  
- tag_ast2({Source, _} = Dep, TagParseTree, InterpretedArgs, Context, TreeWalker) ->
+tag_ast2({Source, _} = Dep, TagParseTree, InterpretedArgs, Context, TreeWalker) ->
     with_dependency(Dep, body_ast(TagParseTree, Context#dtl_context{
         local_scopes = [ InterpretedArgs | Context#dtl_context.local_scopes ],
         parse_trail = [ Source | Context#dtl_context.parse_trail ]}, TreeWalker)).
@@ -795,5 +795,4 @@ call_ast(Module, Variable, AstInfo, TreeWalker) ->
 		 none,
 		 [ErrStrAst]),
     CallAst = erl_syntax:case_expr(AppAst, [OkAst, ErrorAst]),   
-    Module2 = list_to_atom(Module),
-    with_dependencies(Module2:dependencies(), {{CallAst, AstInfo}, TreeWalker}).
+    with_dependencies(Module:dependencies(), {{CallAst, AstInfo}, TreeWalker}).

src/erlydtl/erlydtl_parser.yrl

@@ -110,10 +110,7 @@ Terminals
     close_tag
     close_var
     comment_keyword
-    colon
-    comma
     cycle_keyword
-    dot
     else_keyword
     empty_keyword
     endautoescape_keyword
@@ -123,7 +120,6 @@ Terminals
     endif_keyword
     endifequal_keyword
     endifnotequal_keyword
-    equal
     extends_keyword
     firstof_keyword
     for_keyword
@@ -140,11 +136,11 @@ Terminals
     or_keyword
     open_tag
     open_var
-    pipe
     string_literal
-    text
+    string
     trans_keyword
     with_keyword
+    ',' '|' '=' ':' '.'
     '==' '!='
     '>=' '<='
     '>' '<'
@@ -160,7 +156,7 @@ Nonassoc 300 '==' '!=' '>=' '<=' '>' '<'.
 Unary 600 Unot.
 
 Elements -> '$empty' : [].
-Elements -> Elements text : '$1' ++ ['$2'].
+Elements -> Elements string : '$1' ++ ['$2'].
 Elements -> Elements TransTag : '$1' ++ ['$2'].
 Elements -> Elements ValueBraced : '$1' ++ ['$2'].
 Elements -> Elements ExtendsTag : '$1' ++ ['$2'].
@@ -182,12 +178,12 @@ Elements -> Elements CallWithTag : '$1' ++ ['$2'].
 
 ValueBraced -> open_var Value close_var : '$2'.
 
-Value -> Value pipe Filter : {apply_filter, '$1', '$3'}.
+Value -> Value '|' Filter : {apply_filter, '$1', '$3'}.
 Value -> Variable : '$1'.
 Value -> Literal : '$1'.
 
 Variable -> identifier : {variable, '$1'}.
-Variable -> Value dot identifier : {attribute, {'$3', '$1'}}.
+Variable -> Value '.' identifier : {attribute, {'$3', '$1'}}.
 
 TransTag -> open_tag trans_keyword string_literal close_tag : {trans, '$3'}.
 
@@ -213,8 +209,8 @@ CycleTag -> open_tag cycle_keyword CycleNames close_tag : {cycle, '$3'}.
 CycleNames -> Value : ['$1'].
 CycleNames -> CycleNames Value : '$1' ++ ['$2'].
 
-CycleNamesCompat -> identifier comma : ['$1'].
-CycleNamesCompat -> CycleNamesCompat identifier comma : '$1' ++ ['$2'].
+CycleNamesCompat -> identifier ',' : ['$1'].
+CycleNamesCompat -> CycleNamesCompat identifier ',' : '$1' ++ ['$2'].
 CycleNamesCompat -> CycleNamesCompat identifier : '$1' ++ ['$2'].
 
 FirstofTag -> open_tag firstof_keyword FirstofList close_tag : '$3'.
@@ -229,7 +225,7 @@ ForBraced -> open_tag for_keyword ForExpression close_tag : '$3'.
 EndForBraced -> open_tag endfor_keyword close_tag.
 ForExpression -> ForGroup in_keyword Variable : {'in', '$1', '$3'}.
 ForGroup -> identifier : ['$1'].
-ForGroup -> ForGroup comma identifier : '$1' ++ ['$3'].
+ForGroup -> ForGroup ',' identifier : '$1' ++ ['$3'].
 
 IfBlock -> IfBraced Elements ElseBraced Elements EndIfBraced : {ifelse, '$1', '$2', '$4'}.
 IfBlock -> IfBraced Elements EndIfBraced : {'if', '$1', '$2'}.
@@ -270,7 +266,7 @@ AutoEscapeBraced -> open_tag autoescape_keyword identifier close_tag : '$3'.
 EndAutoEscapeBraced -> open_tag endautoescape_keyword close_tag.
 
 Filter -> identifier : ['$1'].
-Filter -> identifier colon Literal : ['$1', '$3'].
+Filter -> identifier ':' Literal : ['$1', '$3'].
 
 Literal -> string_literal : '$1'.
 Literal -> number_literal : '$1'.
@@ -278,7 +274,7 @@ Literal -> number_literal : '$1'.
 CustomTag -> open_tag identifier Args close_tag : {tag, '$2', '$3'}.
 
 Args -> '$empty' : [].
-Args -> Args identifier equal Value : '$1' ++ [{'$2', '$4'}].
+Args -> Args identifier '=' Value : '$1' ++ [{'$2', '$4'}].
 
 CallTag -> open_tag call_keyword identifier close_tag : {call, '$3'}.
 CallWithTag -> open_tag call_keyword identifier with_keyword Value close_tag : {call, '$3', '$5'}.

src/erlydtl/erlydtl_runtime.erl

@@ -151,6 +151,8 @@ stringify_final([El | Rest], Out) when is_atom(El) ->
    stringify_final(Rest, [atom_to_list(El) | Out]);
 stringify_final([El | Rest], Out) when is_list(El) ->
    stringify_final(Rest, [stringify_final(El) | Out]);
+stringify_final([El | Rest], Out) when is_tuple(El) ->
+   stringify_final(Rest, [io_lib:print(El) | Out]);
 stringify_final([El | Rest], Out) ->
    stringify_final(Rest, [El | Out]).
 

src/erlydtl/erlydtl_scanner.erl

@@ -69,9 +69,12 @@ scan([], Scanned, _, in_text) ->
                             _ ->
                                 identifier
                         end,
-                        {Type, Pos, RevString};
-                    ({Type, Pos, String}) ->
-                        {Type, Pos, lists:reverse(String)} 
+                        {Type, Pos, list_to_atom(RevString)};
+                    ({Category, Pos, String}) when  Category =:= string; 
+                                                    Category =:= string_literal; 
+                                                    Category =:= number_literal ->
+                        {Category, Pos, lists:reverse(String)};
+                    (Other) -> Other
                 end, Scanned))};
 
 scan([], _Scanned, _, {in_comment, _}) ->
@@ -81,10 +84,10 @@ scan([], _Scanned, _, _) ->
     {error, "Reached end of file inside a code block."};
 
 scan("<!--{{" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, [{open_var, {Row, Column}, "<!--{{"} | Scanned], {Row, Column + length("<!--{{")}, {in_code, "}}-->"});
+    scan(T, [{open_var, {Row, Column}, '<!--{{'} | Scanned], {Row, Column + length("<!--{{")}, {in_code, "}}-->"});
 
 scan("{{" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, [{open_var, {Row, Column}, "{{"} | Scanned], {Row, Column + 2}, {in_code, "}}"});
+    scan(T, [{open_var, {Row, Column}, '{{'} | Scanned], {Row, Column + 2}, {in_code, "}}"});
 
 scan("<!--{#" ++ T, Scanned, {Row, Column}, in_text) ->
     scan(T, Scanned, {Row, Column + length("<!--{#")}, {in_comment, "#}-->"});
@@ -99,11 +102,11 @@ scan("#}" ++ T, Scanned, {Row, Column}, {in_comment, "#}"}) ->
     scan(T, Scanned, {Row, Column + 2}, in_text);
 
 scan("<!--{%" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, [{open_tag, {Row, Column}, lists:reverse("<!--{%")} | Scanned], 
+    scan(T, [{open_tag, {Row, Column}, '<!--{%'} | Scanned], 
         {Row, Column + length("<!--{%")}, {in_code, "%}-->"});
 
 scan("{%" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, [{open_tag, {Row, Column}, lists:reverse("{%")} | Scanned], 
+    scan(T, [{open_tag, {Row, Column}, '{%'} | Scanned], 
         {Row, Column + 2}, {in_code, "%}"});
 
 scan([_ | T], Scanned, {Row, Column}, {in_comment, Closer}) ->
@@ -155,58 +158,58 @@ scan([H | T], Scanned, {Row, Column}, {in_single_quote, Closer}) ->
 
 
 scan("}}-->" ++ T, Scanned, {Row, Column}, {_, "}}-->"}) ->
-    scan(T, [{close_var, {Row, Column}, lists:reverse("}}-->")} | Scanned], 
+    scan(T, [{close_var, {Row, Column}, '}}-->'} | Scanned], 
         {Row, Column + 2}, in_text);
 
 scan("}}" ++ T, Scanned, {Row, Column}, {_, "}}"}) ->
-    scan(T, [{close_var, {Row, Column}, "}}"} | Scanned], {Row, Column + 2}, in_text);
+    scan(T, [{close_var, {Row, Column}, '}}'} | Scanned], {Row, Column + 2}, in_text);
 
 scan("%}-->" ++ T, Scanned, {Row, Column}, {_, "%}-->"}) ->
-    scan(T, [{close_tag, {Row, Column}, lists:reverse("%}-->")} | Scanned], 
+    scan(T, [{close_tag, {Row, Column}, '%}-->'} | Scanned], 
         {Row, Column + 2}, in_text);
 
 scan("%}" ++ T, Scanned, {Row, Column}, {_, "%}"}) ->
-    scan(T, [{close_tag, {Row, Column}, lists:reverse("%}")} | Scanned], 
+    scan(T, [{close_tag, {Row, Column}, '%}'} | Scanned], 
         {Row, Column + 2}, in_text);
 
 scan("==" ++ T, Scanned, {Row, Column}, {_, Closer}) ->
-    scan(T, [{'==', {Row, Column}, "=="} | Scanned], {Row, Column + 2}, {in_code, Closer});
+    scan(T, [{'==', {Row, Column}} | Scanned], {Row, Column + 2}, {in_code, Closer});
 
 scan("!=" ++ T, Scanned, {Row, Column}, {_, Closer}) ->
-    scan(T, [{'!=', {Row, Column}, "!="} | Scanned], {Row, Column + 2}, {in_code, Closer});
+    scan(T, [{'!=', {Row, Column}} | Scanned], {Row, Column + 2}, {in_code, Closer});
 
 scan(">=" ++ T, Scanned, {Row, Column}, {_, Closer}) ->
-    scan(T, [{'>=', {Row, Column}, ">="} | Scanned], {Row, Column + 2}, {in_code, Closer});
+    scan(T, [{'>=', {Row, Column}} | Scanned], {Row, Column + 2}, {in_code, Closer});
 
 scan("<=" ++ T, Scanned, {Row, Column}, {_, Closer}) ->
-    scan(T, [{'<=', {Row, Column}, "<="} | Scanned], {Row, Column + 2}, {in_code, Closer});
+    scan(T, [{'<=', {Row, Column}} | Scanned], {Row, Column + 2}, {in_code, Closer});
 
 scan("<" ++ T, Scanned, {Row, Column}, {_, Closer}) ->
-    scan(T, [{'<', {Row, Column}, "<"} | Scanned], {Row, Column + 1}, {in_code, Closer});
+    scan(T, [{'<', {Row, Column}} | Scanned], {Row, Column + 1}, {in_code, Closer});
 
 scan(">" ++ T, Scanned, {Row, Column}, {_, Closer}) ->
-    scan(T, [{'>', {Row, Column}, ">"} | Scanned], {Row, Column + 1}, {in_code, Closer});
+    scan(T, [{'>', {Row, Column}} | Scanned], {Row, Column + 1}, {in_code, Closer});
 
 scan("("++ T, Scanned, {Row, Column}, {_, Closer}) ->
-    scan(T, [{'(', {Row, Column}, "("} | Scanned], {Row, Column + 1}, {in_code, Closer});
+    scan(T, [{'(', {Row, Column}} | Scanned], {Row, Column + 1}, {in_code, Closer});
 
 scan(")" ++ T, Scanned, {Row, Column}, {_, Closer}) ->
-    scan(T, [{')', {Row, Column}, ")"} | Scanned], {Row, Column + 1}, {in_code, Closer});
+    scan(T, [{')', {Row, Column}} | Scanned], {Row, Column + 1}, {in_code, Closer});
 
 scan("," ++ T, Scanned, {Row, Column}, {_, Closer}) ->
-    scan(T, [{comma, {Row, Column}, ","} | Scanned], {Row, Column + 1}, {in_code, Closer});
+    scan(T, [{',', {Row, Column}} | Scanned], {Row, Column + 1}, {in_code, Closer});
 
 scan("|" ++ T, Scanned, {Row, Column}, {_, Closer}) ->
-    scan(T, [{pipe, {Row, Column}, "|"} | Scanned], {Row, Column + 1}, {in_code, Closer});
+    scan(T, [{'|', {Row, Column}} | Scanned], {Row, Column + 1}, {in_code, Closer});
 
 scan("=" ++ T, Scanned, {Row, Column}, {_, Closer}) ->
-    scan(T, [{equal, {Row, Column}, "="} | Scanned], {Row, Column + 1}, {in_code, Closer});
+    scan(T, [{'=', {Row, Column}} | Scanned], {Row, Column + 1}, {in_code, Closer});
 
 scan(":" ++ T, Scanned, {Row, Column}, {_, Closer}) ->
-    scan(T, [{colon, {Row, Column}, ":"} | Scanned], {Row, Column + 1}, {in_code, Closer});
+    scan(T, [{':', {Row, Column}} | Scanned], {Row, Column + 1}, {in_code, Closer});
 
 scan("." ++ T, Scanned, {Row, Column}, {_, Closer}) ->
-    scan(T, [{dot, {Row, Column}, "."} | Scanned], {Row, Column + 1}, {in_code, Closer});
+    scan(T, [{'.', {Row, Column}} | Scanned], {Row, Column + 1}, {in_code, Closer});
 
 scan(" " ++ T, Scanned, {Row, Column}, {_, Closer}) ->
     scan(T, Scanned, {Row, Column + 1}, {in_code, Closer});
@@ -219,7 +222,7 @@ scan([H | T], Scanned, {Row, Column}, {in_code, Closer}) ->
         digit ->
             scan(T, [{number_literal, {Row, Column}, [H]} | Scanned], {Row, Column + 1}, {in_number, Closer});
         _ ->
-            {error, lists:concat(["Illegal character line ", Row, " column ", Column])}
+            {error, {Row, ?MODULE, lists:concat(["Illegal character in column ", Column])}}
     end;
 
 scan([H | T], Scanned, {Row, Column}, {in_number, Closer}) ->
@@ -227,7 +230,7 @@ scan([H | T], Scanned, {Row, Column}, {in_number, Closer}) ->
         digit ->
             scan(T, append_char(Scanned, H), {Row, Column + 1}, {in_number, Closer});
         _ ->
-            {error, lists:concat(["Illegal character line ", Row, " column ", Column])}
+            {error, {Row, ?MODULE, lists:concat(["Illegal character in column ", Column])}}
     end;
 
 scan([H | T], Scanned, {Row, Column}, {in_identifier, Closer}) ->
@@ -237,7 +240,7 @@ scan([H | T], Scanned, {Row, Column}, {in_identifier, Closer}) ->
         digit ->
             scan(T, append_char(Scanned, H), {Row, Column + 1}, {in_identifier, Closer});
         _ ->
-            {error, lists:concat(["Illegal character line ", Row, " column ", Column])}
+            {error, {Row, ?MODULE, lists:concat(["Illegal character in column ", Column])}}
     end.
 
 % internal functions
@@ -249,14 +252,14 @@ append_char(Scanned, Char) ->
 append_text_char(Scanned, {Row, Column}, Char) ->
     case length(Scanned) of
         0 ->
-            [{text, {Row, Column}, [Char]}];
+            [{string, {Row, Column}, [Char]}];
         _ ->
             [Token | Scanned1] = Scanned,
             case element(1, Token) of
-                text ->
-                    [{text, element(2, Token), [Char | element(3, Token)]} | Scanned1];
+                string ->
+                    [{string, element(2, Token), [Char | element(3, Token)]} | Scanned1];
                 _ ->
-                    [{text, element(2, Token), [Char]} | Scanned]
+                    [{string, element(2, Token), [Char]} | Scanned]
             end
     end.