Release 0.10.0

Merge branch 'master' into stable

Conflicts:
	NEWS.md (note to self to resolve: git co -- NEWS.md ; git show master:NEWS.md)

Update:
        src/erlydtl.app.src

.travis.yml

@@ -1,7 +1,7 @@
 language: erlang
 otp_release:
 # Test on all supported releases accepted by the `require_otp_vsn` in rebar.config
-  - 17.0
+  - 17.3
   - R16B03-1
 #  - R16B03 this version is broken!
   - R16B02

Makefile

@@ -2,7 +2,7 @@ ERL=erl
 ERLC=erlc
 REBAR=./rebar $(REBAR_ARGS)
 
-all: compile tests
+all: compile
 
 compile: check-slex get-deps
 	@$(REBAR) compile

NEWS.md

@@ -5,6 +5,25 @@ suggested by the [GNU Coding
 Standards](http://www.gnu.org/prep/standards/html_node/NEWS-File.html#NEWS-File).
 
 
+## 0.10.0 (2014-12-20)
+
+* Fix issue with generated code for `for` loops (#167).
+
+* Fix issue with using keywords as attributes (#177), or as variables (#194).
+
+* Fix issue when including multiple templates extending a common base template (#176).
+
+* New `w` option for toggling compile time warnings. Currently, there is only one, `non_block_tag`,
+  which is triggered on any non-block data in an extends-template.
+
+* Add missing features to the `cycle` tag (#195) (still missing
+  independtly stepping the cycle value).
+
+* Support records in regroup tag (#191).
+
+* Support for maps (#196).
+
+
 ## 0.9.4 (2014-04-15)
 
 * Fix compile time variables and constants (#61)

README.markdown

@@ -57,6 +57,21 @@ in this directory.
 The erl syntax tools is broken in Erlang R16B03, use R16B03-1 or any
 other supported version instead.
 
+#### Do use a recent version of rebar
+
+In case of compilation issues, make sure your version of rebar is up-to-date.
+
+Older versions of rebar does not support the `raw` option for dependencies, and may produce an error
+message like this:
+
+```
+ERROR: Invalid dependency specification {merl,".*",
+                                         {git,
+                                          "git://github.com/erlydtl/merl.git",
+                                          "28e5b3829168199e8475fa91b997e0c03b90d280"},
+                                         [raw]} 
+```
+
 
 Template compilation
 --------------------
@@ -274,6 +289,12 @@ Options is a proplist possibly containing:
 * `verbose` - Enable verbose printing of compilation progress. Add
   several for even more verbose (e.g. debug) output.
 
+* `w` - Enable/Disable compile time checks.
+
+  Available checks:
+  - `non_block_tag` indicated that there is other data than `block` tags in an extends-template
+    (e.g. a template that begins with the `extends` tag).
+
 * `warnings_as_errors` - Treat warnings as errors.
 
 
@@ -507,6 +528,8 @@ Differences from standard Django Template Language
 * Erlang specifics: Template variables may be prefixed with underscore
   (`_`) to avoid "unused variable" warnings (see
   [#164](https://github.com/erlydtl/erlydtl/issues/164)).
+* `cycle` tags do not support independently moving the cycle value
+  from the original loop.
 
 
 Tests

include/erlydtl_ext.hrl

@@ -35,7 +35,8 @@
           warnings = #error_info{},
           bin = undefined,
           lists_0_based = false,
-          tuples_0_based = false
+          tuples_0_based = false,
+          checks = [non_block_tag]
          }).
 
 %% ALL fields of ast_info{} must be lists (see erlydtl_compiler_utils:merge_info/2)

rebar.config

@@ -26,11 +26,13 @@
  ]}.
 
 {pre_hooks,
- [{compile, "make -C $REBAR_DEPS_DIR/merl all -W test"},
-  {"freebsd", compile, "gmake -C $REBAR_DEPS_DIR/merl all -W test"},
+ [{"(linux|darwin|solaris)", compile, "make -C \"$REBAR_DEPS_DIR/merl\" all -W test"},
+  {"(freebsd|netbsd|openbsd)", compile, "gmake -C \"$REBAR_DEPS_DIR/merl\" all"},
+  {"win32", compile, "make -C \"%REBAR_DEPS_DIR%/merl\" all -W test"},
   {eunit,
    "erlc -I include/erlydtl_preparser.hrl -o test"
    " test/erlydtl_extension_testparser.yrl"},
-  {eunit, "make -C $REBAR_DEPS_DIR/merl test"},
-  {"freebsd", eunit, "gmake -C $REBAR_DEPS_DIR/merl test"}
+  {"(linux|darwin|solaris)", eunit, "make -C \"$REBAR_DEPS_DIR/merl\" test"},
+  {"(freebsd|netbsd|openbsd)", eunit, "gmake -C \"$REBAR_DEPS_DIR/merl\" test"},
+  {"win32", eunit, "make -C \"%REBAR_DEPS_DIR%/merl\" test"}
  ]}.

src/erlydtl.app.src

@@ -1,8 +1,8 @@
 %% -*- mode: erlang -*-
 {application, erlydtl,
  [{description, "Django Template Language for Erlang"},
-  {vsn, "0.9.4"},
+  {vsn, "0.10.0"},
   {modules, []},
-  {applications, [kernel, stdlib, compiler, syntax_tools]},
+  {applications, [kernel, stdlib, compiler, syntax_tools, merl]},
   {registered, []}
  ]}.

src/erlydtl_beam_compiler.erl

@@ -63,8 +63,9 @@
          empty_scope/0, get_current_file/1, add_errors/2,
          add_warnings/2, merge_info/2, call_extension/3,
          init_treewalker/1, resolve_variable/2, resolve_variable/3,
-         reset_parse_trail/2, load_library/3, load_library/4,
-         shorten_filename/2, push_auto_escape/2, pop_auto_escape/1]).
+         reset_block_dict/2, reset_parse_trail/2, load_library/3,
+         load_library/4, shorten_filename/2, push_auto_escape/2,
+         pop_auto_escape/1, token_pos/1, is_stripped_token_empty/1]).
 
 -include_lib("merl/include/merl.hrl").
 -include("erlydtl_ext.hrl").
@@ -111,6 +112,8 @@ format_error({translation_fun, Fun}) ->
                            io_lib:format("~s:~s/~p", [proplists:get_value(K, Info) || K <- [module, name, arity]]);
                       true -> io_lib:format("~p", [Fun])
                    end]);
+format_error(non_block_tag) ->
+    "Non-block tag in extends-template.";
 format_error(Error) ->
     erlydtl_compiler:format_error(Error).
 
@@ -275,7 +278,9 @@ maybe_debug_template(Forms, Context) ->
             Options = Context#dtl_context.compiler_options,
             ?LOG_DEBUG("Compiler options: ~p~n", [Options], Context),
             try
-                Source = erl_prettypr:format(erl_syntax:form_list(Forms)),
+                Source = erl_prettypr:format(
+                           erl_syntax:form_list(Forms),
+                           [{ribbon, 100}, {paper, 200}]),
                 SourceFile = lists:concat(
                                [proplists:get_value(source, Options),".erl"]),
                 File = case proplists:get_value(
@@ -289,17 +294,13 @@ maybe_debug_template(Forms, Context) ->
                                    SourceFile,
                                    Context#dtl_context.doc_root),
                                        Dir),
-                               case filelib:is_dir(Dir) of
-                                   true -> Abs;
-                                   false ->
-                                       case filelib:ensure_dir(Abs) of
-                                           ok -> Abs;
-                                           {error, Reason} ->
-                                               io:format(
-                                                 "Failed to ensure directories for file '~s': ~p~n",
-                                                 [Abs, Reason]),
-                                               undefined
-                                       end
+                               case filelib:ensure_dir(Abs) of
+                                   ok -> Abs;
+                                   {error, Reason} ->
+                                       io:format(
+                                         "Failed to ensure directories for file '~s': ~p~n",
+                                         [Abs, Reason]),
+                                       undefined
                                end
                        end,
                 if File =/= undefined ->
@@ -517,6 +518,7 @@ options_match_ast(Context, TreeWalker) ->
 
 %% child templates should only consist of blocks at the top level
 body_ast([{'extends', {string_literal, _Pos, String}} | ThisParseTree], #treewalker{ context=Context }=TreeWalker) ->
+    ThisFile = get_current_file(Context),
     File = full_path(unescape_string_literal(String), Context#dtl_context.doc_root),
     case lists:member(File, Context#dtl_context.parse_trail) of
         true ->
@@ -524,32 +526,48 @@ body_ast([{'extends', {string_literal, _Pos, String}} | ThisParseTree], #treewal
         _ ->
             case parse_file(File, Context) of
                 {ok, ParentParseTree, CheckSum} ->
-                    BlockDict = lists:foldl(
-                                  fun ({block, {identifier, _, Name}, Contents}, Dict) ->
-                                          dict:store(Name, Contents, Dict);
-                                      (_, Dict) -> Dict
-                                  end,
-                                  dict:new(),
-                                  ThisParseTree),
+                    {BlockDict, Context1} = lists:foldl(
+                                              fun ({block, {identifier, Pos, Name}, Contents}, {Dict, Ctx}) ->
+                                                      {dict:store(Name, [{ThisFile, Pos, Contents}], Dict), Ctx};
+                                                  (Token, {Dict, Ctx}) ->
+                                                      case proplists:get_bool(non_block_tag, Ctx#dtl_context.checks) of
+                                                          true ->
+                                                              case is_stripped_token_empty(Token) of
+                                                                  false ->
+                                                                      {Dict, ?WARN({token_pos(Token), non_block_tag}, Ctx)};
+                                                                  true ->
+                                                                      {Dict, Ctx}
+                                                              end;
+                                                          false ->
+                                                              {Dict, Ctx}
+                                                      end
+                                              end,
+                                              {dict:new(), Context},
+                                              ThisParseTree),
                     {Info, TreeWalker1} = with_dependency(
                                             {File, CheckSum},
                                             body_ast(
                                               ParentParseTree,
                                               TreeWalker#treewalker{
-                                                context=Context#dtl_context{
+                                                context=Context1#dtl_context{
                                                           block_dict = dict:merge(
-                                                                         fun(_Key, _ParentVal, ChildVal) -> ChildVal end,
+                                                                         fun(_Key, ParentVal, ChildVal) ->
+                                                                                 ChildVal ++ ParentVal
+                                                                         end,
                                                                          BlockDict, Context#dtl_context.block_dict),
-                                                          parse_trail = [File | Context#dtl_context.parse_trail]
+                                                          parse_trail = [File | Context1#dtl_context.parse_trail]
                                                          }
                                                })),
-                    {Info, reset_parse_trail(Context#dtl_context.parse_trail, TreeWalker1)};
+                    {Info, reset_parse_trail(
+                             Context1#dtl_context.parse_trail,
+                             reset_block_dict(
+                               Context1#dtl_context.block_dict,
+                               TreeWalker1))};
                 {error, Reason} ->
                     empty_ast(?ERR(Reason, TreeWalker))
             end
     end;
 
-
 body_ast(DjangoParseTree, TreeWalker) ->
     body_ast(DjangoParseTree, empty_scope(), TreeWalker).
 
@@ -559,20 +577,22 @@ body_ast(DjangoParseTree, BodyScope, TreeWalker) ->
         fun ({'autoescape', {identifier, _, OnOrOff}, Contents}, TW) ->
                 {Info, BodyTW} = body_ast(Contents, push_auto_escape(OnOrOff, TW)),
                 {Info, pop_auto_escape(BodyTW)};
-            ({'block', {identifier, Pos, Name}, Contents}, #treewalker{ context=Context }=TW) ->
-                {Block, BlockScope} =
-                    case dict:find(Name, Context#dtl_context.block_dict) of
-                        {ok, ChildBlock} ->
-                            {{ContentsAst, _ContentsInfo}, _ContentsTW} = body_ast(Contents, TW),
-                            {ChildBlock,
-                             create_scope(
-                               [{block, ?Q("[{super, _@ContentsAst}]")}],
-                               Pos, TW)
-                            };
-                        _ ->
-                            {Contents, empty_scope()}
-                    end,
-                body_ast(Block, BlockScope, TW);
+            ({'block', {identifier, _Pos, Name}, Contents}, #treewalker{ context=Context }=TW) ->
+                ContentsAst = body_ast(Contents, TW),
+                case dict:find(Name, Context#dtl_context.block_dict) of
+                    {ok, ChildBlocks} ->
+                        lists:foldr(
+                          fun ({ChildFile, ChildPos, ChildBlock}, {{SuperAst, SuperInfo}, AccTW}) ->
+                                  BlockScope = create_scope(
+                                                 [{block, ?Q("[{super, _@SuperAst}]"), safe}],
+                                                 ChildPos, ChildFile, AccTW),
+                                  {{BlockAst, BlockInfo}, BlockTW} = body_ast(ChildBlock, BlockScope, AccTW),
+                                  {{BlockAst, merge_info(SuperInfo, BlockInfo)}, BlockTW}
+                          end,
+                          ContentsAst, ChildBlocks);
+                    _ ->
+                        ContentsAst
+                end;
             ({'blocktrans', Args, Contents, PluralContents}, TW) ->
                 blocktrans_ast(Args, Contents, PluralContents, TW);
             ({'call', {identifier, _, Name}}, TW) ->
@@ -583,8 +603,8 @@ body_ast(DjangoParseTree, BodyScope, TreeWalker) ->
                 empty_ast(TW);
             ({'comment_tag', _, _}, TW) ->
                 empty_ast(TW);
-            ({'cycle', Names}, TW) ->
-                cycle_ast(Names, TW);
+            ({'cycle', Names, AsVar}, TW) ->
+                cycle_ast(Names, AsVar, TW);
             ({'cycle_compat', Names}, TW) ->
                 cycle_compat_ast(Names, TW);
             ({'date', 'now', {string_literal, _Pos, FormatString}}, TW) ->
@@ -739,13 +759,13 @@ extension_ast(Tag, TreeWalker) ->
     end.
 
 
-with_dependencies([], Args) ->
-    Args;
-with_dependencies([Dependency | Rest], Args) ->
-    with_dependencies(Rest, with_dependency(Dependency, Args)).
+with_dependencies([], Ast) -> Ast;
+with_dependencies([Dependency | Rest], Ast) ->
+    with_dependencies(Rest, with_dependency(Dependency, Ast)).
 
 with_dependency(FilePath, {{Ast, Info}, TreeWalker}) ->
-    {{Ast, Info#ast_info{dependencies = [FilePath | Info#ast_info.dependencies]}}, TreeWalker}.
+    Dependencies = [FilePath | Info#ast_info.dependencies],
+    {{Ast, Info#ast_info{ dependencies = Dependencies }}, TreeWalker}.
 
 
 empty_ast(TreeWalker) ->
@@ -974,6 +994,7 @@ string_ast(Arg, Context) ->
 
 include_ast(File, ArgList, Scopes, #treewalker{ context=Context }=TreeWalker) ->
     FilePath = full_path(File, Context#dtl_context.doc_root),
+    ?LOG_TRACE("include file: ~s~n", [FilePath], Context),
     case parse_file(FilePath, Context) of
         {ok, InclusionParseTree, CheckSum} ->
             {NewScope, {ArgInfo, TreeWalker1}}
@@ -1162,7 +1183,8 @@ resolve_variable_ast1({attribute, {{_, Pos, Attr}, Variable}}, {Runtime, Finder}
      TreeWalker1};
 
 resolve_variable_ast1({variable, {identifier, Pos, VarName}}, {Runtime, Finder}, TreeWalker) ->
-    Ast = case resolve_variable(VarName, TreeWalker) of
+    {Source, Value, Filters} = resolve_variable(VarName, TreeWalker),
+    Ast = case {Source, Value} of
               {_, undefined} ->
                   FileName = get_current_file(TreeWalker),
                   {?Q(["'@Runtime@':'@Finder@'(",
@@ -1191,14 +1213,30 @@ resolve_variable_ast1({variable, {identifier, Pos, VarName}}, {Runtime, Finder},
               {scope, Val} ->
                   {Val, #ast_info{}}
           end,
-    {Ast, TreeWalker}.
+    lists:foldr(
+      fun ({escape, []}, {{AccAst, AccInfo}, TW}) ->
+              {{?Q("erlydtl_filters:force_escape(_@AccAst)"), AccInfo}, TW#treewalker{ safe = true }};
+          ({Safe, []}, {Acc, TW}) when Safe == safe; Safe == safeseq ->
+              {Acc, TW#treewalker{ safe = true }};
+          ({Filter, Args}, {{AccAst, AccInfo}, TW})
+            when is_atom(Filter), is_list(Args) ->
+              case filter_ast2(Filter, [AccAst|Args], TW#treewalker.context) of
+                  {ok, FilteredAst} ->
+                      {{FilteredAst, AccInfo}, TW};
+                  Error ->
+                      empty_ast(?WARN({Pos, Error}, TW))
+              end
+      end,
+      {Ast, TreeWalker},
+      Filters
+     ).
 
 resolve_reserved_variable(ReservedName, TreeWalker) ->
     resolve_reserved_variable(ReservedName, merl:term(undefined), TreeWalker).
 
 resolve_reserved_variable(ReservedName, Default, TreeWalker) ->
     case resolve_variable(ReservedName, Default, TreeWalker) of
-        {Src, Value} when Src =:= scope; Value =:= Default ->
+        {Src, Value, []} when Src =:= scope; Value =:= Default ->
             {Value, TreeWalker};
         _ ->
             {Default, ?ERR({reserved_variable, ReservedName}, TreeWalker)}
@@ -1286,7 +1324,10 @@ regroup_ast(ListVariable, GrouperVariable, LocalVarName, TreeWalker) ->
 
     {Id, TreeWalker2} = begin_scope(
                           {[{LocalVarName, LocalVarAst}],
-                           [?Q("_@LocalVarAst = erlydtl_runtime:regroup(_@ListAst, _@regroup)",
+                           [?Q(["_@LocalVarAst = erlydtl_runtime:regroup(",
+                                "  _@ListAst, _@regroup,",
+                                "  [{record_info, _RecordInfo}]",
+                                ")"],
                                [{regroup, regroup_filter(GrouperVariable, [])}])
                            ]},
                           TreeWalker1),
@@ -1340,19 +1381,14 @@ for_loop_ast(IteratorList, LoopValue, IsReversed, Contents,
 
     {ParentLoop, TreeWalker3} = resolve_reserved_variable('forloop', TreeWalker2),
 
-    %% call for loop
-    {{?Q(["case erlydtl_runtime:forloop(",
+    {{?Q(["erlydtl_runtime:forloop(",
           "  fun (_@Vars, _@Counters) ->",
           "    {_@IteratorVars} = if is_tuple(_@Vars), size(_@Vars) == _@IteratorCount@ -> _@Vars;",
           "                          _@___ifclauses -> _",
           "                       end,",
           "    {_@LoopBodyAst, erlydtl_runtime:increment_counter_stats(_@Counters)}",
           "  end,",
-          "  _@LoopValueAst0, _@ParentLoop)",
-          "of",
-          "  empty -> _@EmptyContentsAst;",
-          "  {L, _} -> L",
-          "end"],
+          "  _@LoopValueAst0, _@ParentLoop, _@EmptyContentsAst)"],
          [{ifclauses, if IteratorCount > 1 ->
                               ?Q(["() when is_list(_@Vars), length(_@Vars) == _@IteratorCount@ ->",
                                   "  list_to_tuple(_@Vars);",
@@ -1389,7 +1425,7 @@ ifchanged_contents_ast(Contents, {IfContentsAst, IfContentsInfo}, {ElseContentsA
       merge_info(IfContentsInfo, ElseContentsInfo)},
      TreeWalker}.
 
-cycle_ast(Names, #treewalker{ context=Context }=TreeWalker) ->
+cycle_ast(Names, undefined, #treewalker{ context=Context }=TreeWalker) ->
     {NamesTuple, VarNames}
         = lists:mapfoldl(
             fun ({string_literal, _, Str}, VarNamesAcc) ->
@@ -1406,7 +1442,20 @@ cycle_ast(Names, #treewalker{ context=Context }=TreeWalker) ->
     {ForLoop, TreeWalker1} = resolve_reserved_variable('forloop', TreeWalker),
     {{?Q("erlydtl_runtime:cycle({_@NamesTuple}, _@ForLoop)"),
       #ast_info{ var_names = VarNames }},
-     TreeWalker1}.
+     TreeWalker1};
+cycle_ast(Names, [{identifier, _, VarName}|Opts], TreeWalker) ->
+    {{VarAst, AstInfo}, TW1} = cycle_ast(Names, undefined, TreeWalker),
+    VarNameAst = varname_ast(VarName),
+    {Scope, TW2} = begin_scope(
+                     {[{VarName, VarNameAst}],
+                      [?Q("_@VarNameAst = _@VarAst")
+                       | case Opts of
+                             [silent] -> [];
+                             [] -> [VarAst]
+                         end
+                      ]},
+                     TW1),
+    {{Scope, AstInfo}, TW2}.
 
 %% Older Django templates treat cycle with comma-delimited elements as strings
 cycle_compat_ast(Names, #treewalker{ context=Context }=TreeWalker) ->
@@ -1520,9 +1569,14 @@ call_ast(Module, Variable, AstInfo, TreeWalker) ->
 create_scope(Vars, VarScope) ->
     {Scope, Values} =
         lists:foldl(
-          fun ({Name, Value}, {VarAcc, ValueAcc}) ->
+          fun (Var, {VarAcc, ValueAcc}) ->
+                  {Name, Value, Filters} =
+                      case Var of
+                          {N, V} -> {N, V, []};
+                          {_, _, _} -> Var
+                      end,
                   NameAst = varname_ast(lists:concat(["_", Name, VarScope])),
-                  {[{Name, NameAst}|VarAcc],
+                  {[{Name, NameAst, Filters}|VarAcc],
                    [?Q("_@NameAst = _@Value")|ValueAcc]
                   }
           end,
@@ -1530,9 +1584,9 @@ create_scope(Vars, VarScope) ->
           Vars),
     {Scope, [Values]}.
 
-create_scope(Vars, {Row, Col}, #treewalker{ context=Context }) ->
+create_scope(Vars, {Row, Col}, FileName, #treewalker{ context=Context }) ->
     Level = length(Context#dtl_context.local_scopes),
-    create_scope(Vars, lists:concat(["/", Level, "_", Row, ":", Col])).
+    create_scope(Vars, lists:concat(["::", FileName, "[", Level, ",", Row, ":", Col, "]"])).
 
 varname_ast([$_|VarName]) ->
     merl:var(lists:concat(["_Var__", VarName]));

src/erlydtl_compiler.erl

@@ -278,14 +278,27 @@ init_context(ParseTrail, DefDir, Module, Options) ->
            errors = init_error_info(errors, Ctx#dtl_context.errors, Options),
            warnings = init_error_info(warnings, Ctx#dtl_context.warnings, Options),
            lists_0_based = proplists:get_value(lists_0_based, Options, Ctx#dtl_context.lists_0_based),
-           tuples_0_based = proplists:get_value(tuples_0_based, Options, Ctx#dtl_context.tuples_0_based)
+           tuples_0_based = proplists:get_value(tuples_0_based, Options, Ctx#dtl_context.tuples_0_based),
+           checks = proplists:substitute_negations(
+                      [{no_non_block_tag, non_block_tag}],
+                      proplists:get_all_values(w, Options))
           },
-    Context = load_libraries(proplists:get_value(default_libraries, Options, []), Context0),
+    Context1 = load_libraries(proplists:get_value(default_libraries, Options, []), Context0),
+    Context = default_checks(Ctx#dtl_context.checks, Context1),
     case call_extension(Context, init_context, [Context]) of
         {ok, C} when is_record(C, dtl_context) -> C;
         undefined -> Context
     end.
 
+default_checks([], Context) -> Context;
+default_checks([C|Cs], #dtl_context{ checks = Checks0 } = Context) ->
+    Checks =
+        case proplists:get_value(C, Checks0) of
+            undefined -> [C|Checks0];
+            _ -> Checks0
+        end,
+    default_checks(Cs, Context#dtl_context{ checks = Checks }).
+
 init_error_info(warnings, Ei, Options) ->
     case proplists:get_bool(warnings_as_errors, Options) of
         true -> warnings_as_errors;

src/erlydtl_compiler_utils.erl

@@ -59,10 +59,12 @@
          full_path/2,
          get_current_file/1,
          init_treewalker/1,
+         is_stripped_token_empty/1,
          load_library/2, load_library/3, load_library/4,
          merge_info/2,
          print/3, print/4,
          push_scope/2,
+         reset_block_dict/2,
          reset_parse_trail/2,
          resolve_variable/2, resolve_variable/3,
          restore_scope/2,
@@ -70,7 +72,8 @@
          to_string/2,
          unescape_string_literal/1,
          push_auto_escape/2,
-         pop_auto_escape/1
+         pop_auto_escape/1,
+         token_pos/1
         ]).
 
 -include("erlydtl_ext.hrl").
@@ -199,12 +202,12 @@ resolve_variable(VarName, Default, #treewalker{ context=Context }) ->
             case proplists:get_value(VarName, Context#dtl_context.const) of
                 undefined ->
                     case proplists:get_value(VarName, Context#dtl_context.vars) of
-                        undefined -> {default, Default};
-                        Value -> {default_vars, Value}
+                        undefined -> {default, Default, []};
+                        Value -> {default_vars, Value, []}
                     end;
-                Value -> {constant, Value}
+                Value -> {constant, Value, []}
             end;
-        Value -> {scope, Value}
+        {Value, Filters} -> {scope, Value, Filters}
     end.
 
 push_scope(Scope, #treewalker{ context=Context }=TreeWalker) ->
@@ -230,6 +233,11 @@ end_scope(Fun, Id, AstList, TreeWalker) ->
 
 empty_scope() -> {[], []}.
 
+reset_block_dict(BlockDict, #treewalker{ context=Context }=TreeWalker) ->
+    TreeWalker#treewalker{ context=reset_block_dict(BlockDict, Context) };
+reset_block_dict(BlockDict, Context) ->
+    Context#dtl_context{ block_dict=BlockDict }.
+
 reset_parse_trail(ParseTrail, #treewalker{ context=Context }=TreeWalker) ->
     TreeWalker#treewalker{ context=reset_parse_trail(ParseTrail, Context) };
 reset_parse_trail(ParseTrail, Context) ->
@@ -297,6 +305,25 @@ pop_auto_escape(#dtl_context{ auto_escape=[_|AutoEscape] }=Context)
     Context#dtl_context{ auto_escape=AutoEscape };
 pop_auto_escape(Context) -> Context.
 
+
+token_pos(Token) when is_tuple(Token) ->
+    token_pos(tuple_to_list(Token));
+token_pos([T|Ts]) when is_tuple(T) ->
+    case T of
+        {R, C}=P when is_integer(R), is_integer(C) -> P;
+        _ -> token_pos(tuple_to_list(T) ++ Ts)
+    end;
+token_pos([T|Ts]) when is_list(T) -> token_pos(T ++ Ts);
+token_pos([_|Ts]) -> token_pos(Ts);
+token_pos([]) -> none.
+
+is_stripped_token_empty({string, _, S}) ->
+    [] == [C || C <- S, C /= 32, C /= $\r, C /= $\n, C /= $\t];
+is_stripped_token_empty({comment, _}) -> true;
+is_stripped_token_empty({comment_tag, _, _}) -> true;
+is_stripped_token_empty(_) -> false.
+
+
 format_error({load_library, Name, Mod, Reason}) ->
     io_lib:format("Failed to load library '~p' (~p): ~p", [Name, Mod, Reason]);
 format_error({load_from, Name, Mod, Tag}) ->
@@ -386,22 +413,25 @@ pos_info({Line, Col}) when is_integer(Line), is_integer(Col) ->
 
 resolve_variable1([], _VarName) -> undefined;
 resolve_variable1([Scope|Scopes], VarName) ->
-    case proplists:get_value(VarName, get_scope(Scope)) of
-        undefined ->
+    case lists:keyfind(VarName, 1, get_scope(Scope)) of
+        false ->
             resolve_variable1(Scopes, VarName);
-        Value -> Value
+        {_, Value} -> {Value, []};
+        {_, Value, Filters} when is_list(Filters) -> {Value, Filters};
+        {_, Value, Filter} when is_atom(Filter) -> {Value, [{Filter, []}]};
+        {_, Value, Filter} -> {Value, [Filter]}
     end.
 
+get_scope({_Id, Scope, _Values}) -> Scope;
+get_scope(Scope) -> Scope.
+
 merge_info1(1, _, _, Info) -> Info;
 merge_info1(FieldIdx, Info1, Info2, Info) ->
-    Value = lists:merge(
-              lists:sort(element(FieldIdx, Info1)),
-              lists:sort(element(FieldIdx, Info2))),
+    Value = lists:umerge(
+              lists:usort(element(FieldIdx, Info1)),
+              lists:usort(element(FieldIdx, Info2))),
     merge_info1(FieldIdx - 1, Info1, Info2, setelement(FieldIdx, Info, Value)).
 
-get_scope({_Id, Scope, _Values}) -> Scope;
-get_scope(Scope) -> Scope.
-
 close_scope(Fun, Id, AstList, TreeWalker) ->
     case merge_scopes(Id, TreeWalker) of
         {[], TreeWalker1} -> {AstList, TreeWalker1};

src/erlydtl_filters.erl

@@ -504,20 +504,13 @@ pluralize(Number) ->
     pluralize(Number, "s").
 
 pluralize_io(Number, Suffix) ->
-    case lists:member($, , Suffix) of
-        true ->
-            [Singular, Plural] = string:tokens(Suffix,","),
-            case Number of
-                0 -> Plural;
-                1 -> Singular;
-                _ -> Plural
-            end;
-        false ->
-            case Number of
-                0 -> Suffix;
-                1 -> [];
-                _ -> Suffix
-            end
+    [Singular, Plural] =
+        case string:tokens(Suffix,",") of
+            [P] -> ["", P];
+            [S, P|_] -> [S, P]
+        end,
+    if Number == 1; Number == "1"; Number == <<"1">> -> Singular;
+       true -> Plural
     end.
 
 %% @doc "pretty print" arbitrary data structures.  Used for debugging.
@@ -531,7 +524,7 @@ random(_) ->
     "".
 
 random_num(Value) ->
-    random:seed(now()),
+    _ = random:seed(now()),
     random:uniform(Value).
 
 %% random tags to be used when using erlydtl in testing

src/erlydtl_i18n.erl

@@ -1,17 +0,0 @@
-%% Author: dave
-%% Created: Feb 25, 2010
-%% Description: Bridge between erlydtl compiler and gettext server 
--module(erlydtl_i18n).
-
-%%
-%% Include files
-%%
-%% Exported Functions
-%%
--export([translate/2]).
-
-%%
-%% API Functions
-%%
-%% Makes i18n conversion using gettext
-translate(String, Locale) -> gettext:key2str(String, Locale).

src/erlydtl_parser.yrl

@@ -57,6 +57,7 @@ Nonterminals
     CommentBraced
     EndCommentBraced
 
+    CycleAs
     CycleTag
     CycleNames
     CycleNamesCompat
@@ -198,6 +199,7 @@ Terminals
     plural_keyword
     regroup_keyword
     reversed_keyword
+    silent_keyword
     spaceless_keyword
     ssi_keyword
     string_literal
@@ -314,11 +316,15 @@ EndCommentBraced -> open_tag endcomment_keyword close_tag.
 CommentTag -> comment_tag : '$1'.
 
 CycleTag -> open_tag cycle_keyword CycleNamesCompat close_tag : {cycle_compat, '$3'}.
-CycleTag -> open_tag cycle_keyword CycleNames close_tag : {cycle, '$3'}.
+CycleTag -> open_tag cycle_keyword CycleNames CycleAs close_tag : {cycle, '$3', '$4'}.
 
 CycleNames -> Value : ['$1'].
 CycleNames -> CycleNames Value : '$1' ++ ['$2'].
 
+CycleAs -> '$empty' : undefined.
+CycleAs -> as_keyword identifier : ['$2'].
+CycleAs -> as_keyword identifier silent_keyword : ['$2', silent].
+
 CycleNamesCompat -> identifier ',' : ['$1'].
 CycleNamesCompat -> CycleNamesCompat identifier ',' : '$1' ++ ['$2'].
 CycleNamesCompat -> CycleNamesCompat identifier : '$1' ++ ['$2'].

src/erlydtl_runtime.erl

@@ -20,7 +20,7 @@
 
 find_value(Key, Data, Options) when is_atom(Key), is_tuple(Data) ->
     Rec = element(1, Data),
-    Info = proplists:get_value(record_info, Options),
+    Info = proplists:get_value(record_info, Options, []),
     case proplists:get_value(Rec, Info) of
         Fields when is_list(Fields), length(Fields) == size(Data) - 1 ->
             case proplists:get_value(Key, Fields) of
@@ -107,6 +107,32 @@ find_value(Key, Tuple) when is_tuple(Tuple) ->
                 _ ->
                     undefined
             end
+    end;
+find_value(Key, Map) ->
+    case erlang:is_builtin(erlang, is_map, 1) andalso erlang:is_map(Map) of
+        true  -> find_map_value(Key, Map);
+        false -> undefined
+    end.
+
+find_map_value(Key, Map) when is_atom(Key) ->
+    case maps:find(Key, Map) of
+        error           -> find_map_value(atom_to_list(Key), Map);
+        {ok, Value}     -> Value
+    end;
+find_map_value(Key, Map) when is_list(Key) ->
+    case maps:find(Key, Map) of
+        error           -> find_map_value(list_to_binary(Key), Map);
+        {ok, Value}     -> Value
+    end;
+find_map_value(Key, Map) when is_binary(Key) ->
+    case maps:find(Key, Map) of
+        error           -> undefined;
+        {ok, Value}     -> Value
+    end;
+find_map_value(Key, Map) ->
+    case maps:find(Key, Map) of
+        error           -> undefined;
+        {ok, Value}     -> Value
     end.
 
 fetch_value(Key, Data, Options) ->
@@ -118,28 +144,33 @@ fetch_value(Key, Data, Options, Default) ->
         Val -> Val
     end.
 
-find_deep_value([Key|Rest],Item) ->
-    case find_value(Key,Item) of
+find_deep_value(Key, Data) ->
+    find_deep_value(Key, Data, []).
+
+find_deep_value([Key|Rest], Item, Opts) ->
+    case find_value(Key, Item, Opts) of
         undefined -> undefined;
-        NewItem -> find_deep_value(Rest,NewItem)
+        NewItem -> find_deep_value(Rest, NewItem, Opts)
     end;
-find_deep_value([],Item) -> Item.
+find_deep_value([], Item, _Opts) -> Item.
 
 regroup(List, Attribute) ->
-    regroup(List, Attribute, []).
+    do_regroup(List, Attribute, [], []).
 
-regroup([], _, []) ->
-    [];
-regroup([], _, [[{grouper, LastGrouper}, {list, LastList}]|Acc]) ->
+regroup(List, Attribute, Options) ->
+    do_regroup(List, Attribute, Options, []).
+
+do_regroup([], _, _, []) -> [];
+do_regroup([], _, _, [[{grouper, LastGrouper}, {list, LastList}]|Acc]) ->
     lists:reverse([[{grouper, LastGrouper}, {list, lists:reverse(LastList)}]|Acc]);
-regroup([Item|Rest], Attribute, []) ->
-    regroup(Rest, Attribute, [[{grouper, find_deep_value(Attribute, Item)}, {list, [Item]}]]);
-regroup([Item|Rest], Attribute, [[{grouper, PrevGrouper}, {list, PrevList}]|Acc]) ->
-    case find_deep_value(Attribute, Item) of
+do_regroup([Item|Rest], Attribute, Options, []) ->
+    do_regroup(Rest, Attribute, Options, [[{grouper, find_deep_value(Attribute, Item, Options)}, {list, [Item]}]]);
+do_regroup([Item|Rest], Attribute, Options, [[{grouper, PrevGrouper}, {list, PrevList}]|Acc]) ->
+    case find_deep_value(Attribute, Item, Options) of
         Value when Value =:= PrevGrouper ->
-            regroup(Rest, Attribute, [[{grouper, PrevGrouper}, {list, [Item|PrevList]}]|Acc]);
+            do_regroup(Rest, Attribute, Options, [[{grouper, PrevGrouper}, {list, [Item|PrevList]}]|Acc]);
         Value ->
-            regroup(Rest, Attribute, [[{grouper, Value}, {list, [Item]}], [{grouper, PrevGrouper}, {list, lists:reverse(PrevList)}]|Acc])
+            do_regroup(Rest, Attribute, Options, [[{grouper, Value}, {list, [Item]}], [{grouper, PrevGrouper}, {list, lists:reverse(PrevList)}]|Acc])
     end.
 
 -spec init_translation(init_translation()) -> none | translate_fun().
@@ -243,6 +274,7 @@ are_equal(_, _) ->
 is_false("") -> true;
 is_false(false) -> true;
 is_false(undefined) -> true;
+is_false(null) -> true;
 is_false(0) -> true;
 is_false("0") -> true;
 is_false(<<"0">>) -> true;
@@ -361,13 +393,18 @@ increment_counter_stats([{counter, Counter}, {counter0, Counter0}, {revcounter,
      {first, false}, {last, RevCounter0 =:= 1},
      {parentloop, Parent}].
 
-forloop(_Fun, [], _Parent) -> empty;
-forloop(Fun, Values, Parent) ->
+forloop(_Fun, [], _Parent, Default) -> Default;
+forloop(Fun, Values, Parent, _Default) ->
     push_ifchanged_context(),
-    Result = lists:mapfoldl(Fun, init_counter_stats(Values, Parent), Values),
+    {Result, _Acc} = lists:mapfoldl(Fun, init_counter_stats(Values, Parent), Values),
     pop_ifchanged_context(),
     Result.
 
+%% keep old version for backwards compatibility..
+forloop(_Fun, [], _Parent) -> empty;
+forloop(Fun, Values, Parent) ->
+    {forloop(Fun, Values, Parent, undefined), undefined}.
+
 push_ifchanged_context() ->
     IfChangedContextStack = case get(?IFCHANGED_CONTEXT_VARIABLE) of
                                 undefined -> [];

src/erlydtl_scanner.erl

@@ -36,7 +36,7 @@
 %%%-------------------------------------------------------------------
 -module(erlydtl_scanner).
 
-%% This file was generated 2014-04-15 19:15:09 UTC by slex 0.2.1.
+%% This file was generated 2014-12-16 18:46:16 UTC by slex 0.2.1-2-g7814678.
 %% http://github.com/erlydtl/slex
 -slex_source(["src/erlydtl_scanner.slex"]).
 
@@ -92,6 +92,7 @@ is_keyword(any, "context") -> true;
 is_keyword(any, "noop") -> true;
 is_keyword(close, "only") -> true;
 is_keyword(close, "parsed") -> true;
+is_keyword(close, "silent") -> true;
 is_keyword(close, "reversed") -> true;
 is_keyword(close, "openblock") -> true;
 is_keyword(close, "closeblock") -> true;
@@ -350,7 +351,7 @@ scan(" " ++ T, S, {R, C} = P,
      {in_verbatim_code, E} = St) ->
     {Tag, Backtrack} = E,
     scan(T, S, {R, C + 1},
-	 {in_verbatim_code, {Tag, [$  | Backtrack]}});
+	 {in_verbatim_code, {Tag, [$\s | Backtrack]}});
 scan("endverbatim%}" ++ T, S, {R, C} = P,
      {in_verbatim_code, E} = St)
     when element(1, E) =:= undefined ->
@@ -366,7 +367,8 @@ scan(" " ++ T, S, {R, C} = P,
     when element(3, E) =:= "" ->
     {Tag, Backtrack, EndTag} = E,
     scan(T, S, {R, C + 1},
-	 {in_endverbatim_code, {Tag, [$  | Backtrack], EndTag}});
+	 {in_endverbatim_code,
+	  {Tag, [$\s | Backtrack], EndTag}});
 scan([H | T], S, {R, C} = P,
      {in_endverbatim_code, E} = St)
     when H >= $a andalso H =< $z orelse
@@ -380,7 +382,7 @@ scan(" " ++ T, S, {R, C} = P,
     when element(1, E) =:= element(3, E) ->
     {Tag, Backtrack, Tag} = E,
     scan(T, S, {R, C + 1},
-	 {in_endverbatim_code, {Tag, [$  | Backtrack], Tag}});
+	 {in_endverbatim_code, {Tag, [$\s | Backtrack], Tag}});
 scan("%}" ++ T, S, {R, C} = P,
      {in_endverbatim_code, E} = St)
     when element(1, E) =:= element(3, E) ->
@@ -563,6 +565,14 @@ post_process([{open_tag, _, _} | _],
 post_process([{open_tag, _, _} | _],
 	     {identifier, _, L} = T, _) ->
     is_keyword(open_tag, T);
+post_process([{open_var, _, _} | _],
+	     {identifier, _, L} = T, _) ->
+    setelement(3, T,
+	       begin L1 = lists:reverse(L), L2 = to_atom(L1), L2 end);
+post_process([{'.', _} | _], {identifier, _, L} = T,
+	     _) ->
+    setelement(3, T,
+	       begin L1 = lists:reverse(L), L2 = to_atom(L1), L2 end);
 post_process(_, {identifier, _, L} = T, close_tag) ->
     is_keyword(close_tag, T);
 post_process(_, {identifier, _, L} = T, _) ->

src/erlydtl_scanner.slex

@@ -266,6 +266,8 @@ close_tag: to_atom.
 
 open_tag identifier, close_tag: expr is_keyword(all, T) end.
 open_tag identifier: expr is_keyword(open_tag, T) end.
+open_var identifier: lists reverse, to_atom.
+\. - identifier: lists reverse, to_atom.
 identifier, close_tag: expr is_keyword(close_tag, T) end.
 identifier: expr is_keyword(any, T) end.
 
@@ -315,6 +317,7 @@ form \
   \
   is_keyword(close, "only") -> true; \
   is_keyword(close, "parsed") -> true; \
+  is_keyword(close, "silent") -> true; \
   is_keyword(close, "reversed") -> true; \
   is_keyword(close, "openblock") -> true; \
   is_keyword(close, "closeblock") -> true; \

src/i18n/Makefile

@@ -1,4 +0,0 @@
-include ../../../../support/include.mk
-EBIN_DIR := ../../ebin
-
-all: $(EBIN_FILES_NO_DOCS) 

src/i18n/blocktrans_extractor.erl

@@ -1,50 +0,0 @@
--module(blocktrans_extractor).
-
--export([extract/1]).
-
--include("include/erlydtl_ext.hrl").
-
-extract(Path) when is_list(Path) ->
-    {ok, Contents} = file:read_file(Path),
-    extract(Contents);
-
-extract(Contents) when is_binary(Contents) ->
-    case erlydtl_compiler:do_parse_template(Contents, #dtl_context{}) of
-        {ok, ParseTree} ->
-            Blocks = process_tree(ParseTree),
-            {ok, Blocks};
-        Error ->
-            Error
-    end.
-
-process_tree(ParseTree) ->
-    process_tree(ParseTree, []).
-
-process_tree([], Acc) ->
-    lists:reverse(Acc);
-process_tree([{'autoescape', _, Contents}|Rest], Acc) ->
-    process_tree(Rest, lists:reverse(process_tree(Contents), Acc));
-process_tree([{'block', _, Contents}|Rest], Acc) ->
-    process_tree(Rest, lists:reverse(process_tree(Contents), Acc));
-process_tree([{'blocktrans', _, Contents}|Rest], Acc) ->
-    process_tree(Rest, [lists:flatten(erlydtl_unparser:unparse(Contents))|Acc]); % <-- where all the action happens
-process_tree([{'filter', _, Contents}|Rest], Acc) ->
-    process_tree(Rest, lists:reverse(process_tree(Contents), Acc));
-process_tree([{'for', _, Contents}|Rest], Acc) ->
-    process_tree(Rest, lists:reverse(process_tree(Contents), Acc));
-process_tree([{'for', _, Contents, EmptyPartContents}|Rest], Acc) ->
-    process_tree(Rest, lists:reverse(process_tree(Contents) ++ process_tree(EmptyPartContents), Acc));
-process_tree([{Instruction, _, Contents}|Rest], Acc) when Instruction =:= 'if'; 
-                                                          Instruction =:= 'ifequal'; 
-                                                          Instruction =:= 'ifnotequal' ->
-    process_tree(Rest, lists:reverse(process_tree(Contents), Acc));
-process_tree([{Instruction, _, IfContents, ElseContents}|Rest], Acc) when Instruction =:= 'ifelese'; 
-                                                                          Instruction =:= 'ifequalelse'; 
-                                                                          Instruction =:= 'ifnotequalelse' ->
-    process_tree(Rest, lists:reverse(process_tree(IfContents) ++ process_tree(ElseContents), Acc));
-process_tree([{'spaceless', Contents}|Rest], Acc) ->
-    process_tree(Rest, lists:reverse(process_tree(Contents), Acc));
-process_tree([{'with', _, Contents}|Rest], Acc) ->
-    process_tree(Rest, lists:reverse(process_tree(Contents), Acc));
-process_tree([_|Rest], Acc) ->
-    process_tree(Rest, Acc).

src/i18n/blocktrans_parser.erl

@@ -1,13 +0,0 @@
--module(blocktrans_parser).
-
--export([parse/1]).
-
-parse(Tokens) ->
-    parse(Tokens, []).
-
-parse([], Acc) ->
-    lists:reverse(Acc);
-parse([{open_blocktrans, _, _}, {text, _, Text}, {close_blocktrans, _}|Rest], Acc) ->
-    parse(Rest, [Text|Acc]);
-parse([{text, _, _}|Rest], Acc) ->
-    parse(Rest, Acc).

src/i18n/blocktrans_scanner.erl

@@ -1,146 +0,0 @@
-% Module for extracting blocktrans blocks with original source formatting preserved.
-
--module(blocktrans_scanner).
-
--export([scan/1]).
-
-scan(Template) ->
-    scan(Template, [], {1, 1}, in_text).
-
-scan([], Scanned, _, in_text) ->
-    {ok, lists:reverse(lists:map(
-                fun
-                    ({text, Pos, Text}) ->
-                        {text, Pos, lists:reverse(Text)};
-                    (Other) ->
-                        Other
-                end, Scanned))};
-
-scan([], _Scanned, _, {in_comment, _}) ->
-    {error, "Reached end of file inside a comment."};
-
-scan([], _Scanned, _, _) ->
-    {error, "Reached end of file inside a code block."};
-
-scan("<!--{{" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, append_text("<!--{{", {Row, Column}, Scanned), {Row, Column + length("<!--{{")}, {in_code, "}}-->"});
-
-scan("{{" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, append_text("{{", {Row, Column}, Scanned), {Row, Column + 2}, {in_code, "}}"});
-
-scan("<!--{#" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, append_text("<!--{#", {Row, Column}, Scanned), {Row, Column + length("<!--{#")}, {in_comment, "#}-->"});
-
-scan("{#" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, append_text("{#", {Row, Column}, Scanned), {Row, Column + length("{#")}, {in_comment, "#}"});
-
-scan("#}-->" ++ T, Scanned, {Row, Column}, {in_comment, "#}-->"}) ->
-    scan(T, append_text("#}-->", {Row, Column}, Scanned), {Row, Column + length("#}-->")}, in_text);
-
-scan("#}" ++ T, Scanned, {Row, Column}, {in_comment, "#}"}) ->
-    scan(T, append_text("#}", {Row, Column}, Scanned), {Row, Column + length("#}")}, in_text);
-
-scan("<!--{% blocktrans " ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, [{open_blocktrans, {Row, Column}, ""} | Scanned], 
-        {Row, Column + length("<!--{% blocktrans ")}, {in_code, "%}-->"});
-
-scan("{% blocktrans " ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, [{open_blocktrans, {Row, Column}, ""} | Scanned], 
-        {Row, Column + length("{% blocktrans ")}, {in_code, "%}"});
-
-scan("{% endblocktrans %}" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, [{close_blocktrans, {Row, Column}} | Scanned],
-        {Row, Column + length("{% endblocktrans %}")}, in_text);
-
-scan("<!--{% endblocktrans %}-->" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, [{close_blocktrans, {Row, Column}} | Scanned],
-        {Row, Column + length("<!--{% endblocktrans %}-->")}, {in_text});
-
-scan("<!--{%" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, append_text("<!--{%", {Row, Column}, Scanned),
-        {Row, Column + length("<!--{%")}, {in_code, "%}-->"});
-
-scan("{%" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, append_text("{%", {Row, Column}, Scanned),
-        {Row, Column + length("{%")}, {in_code, "%}"});
-
-scan([H | T], Scanned, {Row, Column}, {in_comment, Closer}) ->
-    scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_comment, Closer});
-
-scan("\n" ++ T, Scanned, {Row, Column}, in_text) ->
-    scan(T, append_text("\n", {Row, Column}, Scanned), {Row + 1, 1}, in_text);
-
-scan([H | T], Scanned, {Row, Column}, in_text) ->
-    scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, in_text);
-
-scan("\"" ++ T, Scanned, {Row, Column}, {in_code, Closer}) ->
-    scan(T, append_text("\"", {Row, Column}, Scanned), {Row, Column + 1} , {in_double_quote, Closer});
-
-scan("\'" ++ T, Scanned, {Row, Column}, {in_code, Closer}) ->
-    scan(T, append_text("\'", {Row, Column}, Scanned), {Row, Column + 1}, {in_single_quote, Closer});
-
-scan("\\" ++ T, Scanned, {Row, Column}, {in_double_quote, Closer}) ->
-    scan(T, append_text("\\", {Row, Column}, Scanned), {Row, Column + 1}, {in_double_quote_slash, Closer});
-
-scan([H | T], Scanned, {Row, Column}, {in_double_quote_slash, Closer}) ->
-    scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_double_quote, Closer});
-
-scan("\\" ++ T, Scanned, {Row, Column}, {in_single_quote, Closer}) ->
-    scan(T, append_text("\\", {Row, Column}, Scanned), {Row, Column + 1}, {in_single_quote_slash, Closer});
-
-scan([H | T], Scanned, {Row, Column}, {in_single_quote_slash, Closer}) ->
-    scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_single_quote, Closer});
-
-% end quote
-scan("\"" ++ T, Scanned, {Row, Column}, {in_double_quote, Closer}) ->
-    scan(T, append_text("\"", {Row, Column}, Scanned), {Row, Column + 1}, {in_code, Closer});
-
-scan("\'" ++ T, Scanned, {Row, Column}, {in_single_quote, Closer}) ->
-    scan(T, append_text("\'", {Row, Column}, Scanned), {Row, Column + 1}, {in_code, Closer});
-
-scan([H | T], Scanned, {Row, Column}, {in_double_quote, Closer}) ->
-    scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_double_quote, Closer});
-
-scan([H | T], Scanned, {Row, Column}, {in_single_quote, Closer}) ->
-    scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_single_quote, Closer});
-
-
-scan("}}-->" ++ T, Scanned, {Row, Column}, {_, "}}-->"}) ->
-    scan(T, append_text("}}-->", {Row, Column}, Scanned),
-        {Row, Column + length("}}-->")}, in_text);
-
-scan("}}" ++ T, Scanned, {Row, Column}, {_, "}}"}) ->
-    scan(T, append_text("}}", {Row, Column}, Scanned), {Row, Column + length("}}")}, in_text);
-
-scan("%}-->" ++ T, Scanned, {Row, Column}, {_, "%}-->"}) ->
-    scan(T, append_text("%}-->", {Row, Column}, Scanned),
-        {Row, Column + length("%}-->")}, in_text);
-
-scan("%}" ++ T, Scanned, {Row, Column}, {_, "%}"}) ->
-    scan(T, append_text("%}", {Row, Column}, Scanned),
-        {Row, Column + length("%}")}, in_text);
-
-
-scan([H | T], Scanned, {Row, Column}, {in_code, Closer}) ->
-    scan(T, append_text([H], {Row, Column}, Scanned), {Row, Column + 1}, {in_code, Closer}).
-
-% internal functions
-
-append_text(Text, Pos, []) ->
-    [{text, Pos, Text}];
-append_text(Text, Pos, [{close_blocktrans, _}|_] = Scanned) ->
-    [{text, Pos, Text}|Scanned];
-append_text([C], _Pos, [{open_blocktrans, BPos, ""}|Rest]) when ((C >= $a) and (C =< $z)) or ((C >= $A) and (C =< $Z)) or (C =:= $_) ->
-    [{open_blocktrans, BPos, [C]}|Rest];
-append_text(" ", _Pos, [{open_blocktrans, BPos, ""}|Rest]) ->
-    [{open_blocktrans, BPos, ""}|Rest];
-append_text([C], _Pos, [{open_blocktrans, BPos, Name}|Rest]) when ((C >= $a) and (C =< $z)) or ((C >= $A) and (C =< $Z)) or (C =:= $_) orelse (C >= $0 andalso C =< $9) ->
-    [{open_blocktrans, BPos, [C|Name]}|Rest];
-append_text(" ", _Pos, [{open_blocktrans, BPos, Name}|Rest]) when is_list(Name) ->
-    [{open_blocktrans, BPos, lists:reverse(Name)}|Rest];
-append_text("%}", {Row, Column}, [{open_blocktrans, _BPos, _Name}|_] = Scanned) ->
-    [{text, {Row, Column + 2}, ""}|Scanned];
-append_text(_Chars, _Pos, [{open_blocktrans, _BPos, _Name}|_] = Scanned) ->
-    Scanned;
-append_text(Chars, _Pos, [{text, TPos, TChars}|Rest]) ->
-    [{text, TPos, lists:reverse(Chars, TChars)}|Rest].

test/erlydtl_test_defs.erl

@@ -3,6 +3,7 @@
 -export([tests/0]).
 -include("testrunner.hrl").
 -record(testrec, {foo, bar, baz}).
+-record(person, {first_name, gender}).
 
 %% {Name, DTL, Vars, Output}
 %% {Name, DTL, Vars, RenderOpts, Output}
@@ -28,8 +29,32 @@ all_test_defs() ->
         [{var1, "foo"}], <<"foo">>},
        {"Variable name is a tag name",
         <<"{{ comment }}">>,
-        [{comment, "Nice work!"}], <<"Nice work!">>}
+        [{comment, "Nice work!"}], <<"Nice work!">>},
+       #test{
+          title = "reserved name ok as variable name",
+          source = <<"{{ from }}">>,
+          render_vars = [{from, "test"}],
+          output = <<"test">>
+         }
       ]},
+     {"maps",
+      case erlang:is_builtin(erlang, is_map, 1) of
+          false -> [];
+          true ->
+              [#test{
+                  title = "simple test",
+                  source = <<"{{ msg.hello }}">>,
+                  render_vars = [{msg, maps:put(hello, "world", maps:new())}],
+                  output = <<"world">>
+                 },
+               #test{
+                  title = "various key types",
+                  source = <<"{{ msg.key1 }},{{ msg.key2 }},{{ msg.key3 }},{{ msg.4 }}">>,
+                  render_vars = [{msg, maps:from_list([{key1, 1}, {"key2", 2}, {<<"key3">>, 3}, {4, "value4"}])}],
+                  output = <<"1,2,3,value4">>
+                 }
+              ]
+      end},
      {"comment",
       [{"comment block is excised",
         <<"bob {% comment %}(moron){% endcomment %} loblaw">>,
@@ -62,13 +87,37 @@ all_test_defs() ->
         <<"{{ \"foo\"|add:\"\\\"\" }}">>, [], <<"foo\"">>}
       ]},
      {"cycle",
-      [{"Cycling through quoted strings",
+      [#test{
+          title = "deprecated cycle syntax",
+          source = <<"{% for i in test %}{% cycle a,b %}{{ i }},{% endfor %}">>,
+          render_vars = [{test, [0,1,2,3,4]}],
+          output = <<"a0,b1,a2,b3,a4,">>
+         },
+       {"Cycling through quoted strings",
         <<"{% for i in test %}{% cycle 'a' 'b' %}{{ i }},{% endfor %}">>,
         [{test, ["0", "1", "2", "3", "4"]}], <<"a0,b1,a2,b3,a4,">>},
        {"Cycling through normal variables",
         <<"{% for i in test %}{% cycle aye bee %}{{ i }},{% endfor %}">>,
         [{test, ["0", "1", "2", "3", "4"]}, {aye, "a"}, {bee, "b"}],
-        <<"a0,b1,a2,b3,a4,">>}
+        <<"a0,b1,a2,b3,a4,">>},
+       #test{
+          title = "mix strings and variables",
+          source = <<"{% for i in test %}{% cycle 'a' b 'c' %}{{ i }},{% endfor %}">>,
+          render_vars = [{test, [0,1,2,3,4]}, {b, 'B'}],
+          output = <<"a0,B1,c2,a3,B4,">>
+         },
+       #test{
+          title = "keep current value in local variable",
+          source = <<"{% for i in test %}{% cycle 'a' 'b' as c %}{{ i }}{{ c }},{% endfor %}">>,
+          render_vars = [{test, [0,1,2,3,4]}],
+          output = <<"a0a,b1b,a2a,b3b,a4a,">>
+         },
+       #test{
+          title = "keep current value silently in local variable",
+          source = <<"{% for i in test %}{% cycle 'a' 'b' as c silent %}{{ i }}{{ c }},{% endfor %}">>,
+          render_vars = [{test, [0,1,2,3,4]}],
+          output = <<"0a,1b,2a,3b,4a,">>
+         }
       ]},
      {"number literal",
       [{"Render integer",
@@ -139,7 +188,17 @@ all_test_defs() ->
        {"Index all tuple elements 1-based (selected at render time)",
         <<"{{ var1.1 }},{{ var1.2 }},{{ var1.3 }}.">>,
         [{var1, {a, b, c}}], [], [{tuples_0_based, defer}],
-        <<"a,b,c.">>}
+        <<"a,b,c.">>},
+       {"Index tuple using a \"reserved\" keyword",
+        <<"{{ list.count }}">>,
+        [{list, [{count, 123}]}],
+        <<"123">>},
+       {"Index list value",
+        <<"{{ content.description }}">>,
+        [{content, "test"}], <<"">>},
+       {"Index binary value",
+        <<"{{ content.description }}">>,
+        [{content, <<"test">>}], <<"">>}
       ]},
      {"now",
       [{"now functional",
@@ -683,6 +742,12 @@ all_test_defs() ->
        {"|pluralize:\"y,es\" (list)",
         <<"{{ num|pluralize:\"y,es\" }}">>, [{num, 2}],
         <<"es">>},
+       {"|length|pluralize",
+        <<"{{ list|length|pluralize:\"plural\" }}">>, [{list, [foo, bar]}],
+        <<"plural">>},
+       {"|length|pluralize",
+        <<"{{ list|length|pluralize:\"plural\" }}">>, [{list, [foo]}],
+        <<"">>},
        {"|random",
         <<"{{ var1|random }}">>, [{var1, ["foo", "foo", "foo"]}],
         <<"foo">>},
@@ -1256,7 +1321,19 @@ all_test_defs() ->
         <<"People: {% regroup people by gender as gender_list %}{% for gender in gender_list %}{{ gender.grouper }}\n{% for item in gender.list %}{{ item.first_name }}\n{% endfor %}{% endfor %}Done.">>,
         [{people, [[{first_name, "George"}, {gender, "Male"}], [{first_name, "Bill"}, {gender, "Male"}],
                    [{first_name, "Margaret"}, {gender, "Female"}], [{first_name, "Condi"}, {gender, "Female"}]]}],
-        <<"People: Male\nGeorge\nBill\nFemale\nMargaret\nCondi\nDone.">>}
+        <<"People: Male\nGeorge\nBill\nFemale\nMargaret\nCondi\nDone.">>},
+       #test{
+          title = "regroup record",
+          source = <<"{% regroup people by gender as gender_list %}{% for gender in gender_list %}{{ gender.grouper }}:\n{% for person in gender.list %} - {{ person.first_name }}\n{% endfor %}{% endfor %}">>,
+          compile_opts = [{record_info, [{person, record_info(fields, person)}]} | (#test{})#test.compile_opts],
+          render_vars = [{people, [#person{ first_name = "George", gender = "Male" },
+                                   #person{ first_name = "Bill", gender = "Male" },
+                                   #person{ first_name = "Margaret", gender = "Female" },
+                                   #person{ first_name = "Condi", gender = "Female" }
+                                  ]}
+                        ],
+          output = <<"Male:\n - George\n - Bill\nFemale:\n - Margaret\n - Condi\n">>
+         }
       ]},
      {"spaceless",
       [{"Beginning", <<"{% spaceless %}    <b>foo</b>{% endspaceless %}">>, [], <<"<b>foo</b>">>},
@@ -1648,17 +1725,17 @@ all_test_defs() ->
       end},
      {"functional",
       [functional_test(F)
-       %% order is important.
-       || F <- ["autoescape", "comment", "extends", "filters", "for", "for_list",
-                "for_tuple", "for_list_preset", "for_preset", "for_records",
-                "for_records_preset", "include", "if", "if_preset", "ifequal",
-                "ifequal_preset", "ifnotequal", "ifnotequal_preset", "now",
-                "var", "var_preset", "cycle", "custom_tag", "custom_tag1",
-                "custom_tag2", "custom_tag3", "custom_tag4", "custom_call",
-                "include_template", "include_path", "ssi", "extends_path",
-                "extends_path2", "trans", "extends2", "extends3",
-                "recursive_block", "extend_recursive_block", "missing",
-                "block_super"]
+       %% order is important for a few of these tests, unfortunately.
+
+       || F <- ["autoescape", "comment", "extends", "filters", "for", "for_list", "for_tuple",
+                "for_list_preset", "for_preset", "for_records", "for_records_preset", "include",
+                "if", "if_preset", "ifequal", "ifequal_preset", "ifnotequal", "ifnotequal_preset",
+                "now", "var", "var_preset", "cycle", "custom_tag", "custom_tag1", "custom_tag2",
+                "custom_tag3", "custom_tag4", "custom_call", "include_template", "include_path",
+                "ssi", "extends_path", "extends_path2", "trans", "extends_for", "extends2",
+                "extends3", "recursive_block", "extend_recursive_block", "missing", "block_super",
+                "wrapper", "extends4", "super_escaped", "extends_chain"]
+
       ]},
      {"compile_dir",
       [setup_compile(T)
@@ -1765,10 +1842,13 @@ functional_test(F) ->
 setup_compile(#test{ title=F, compile_opts=Opts }=T) ->
     CompileOpts = [{doc_root, "../test/files/input"}|Opts],
     case setup_compile(F) of
-        {ok, [CV|CO]} ->
+        {ok, [CV|Other]} ->
+            CO = proplists:get_value(compile_opts, Other, []),
+            Ws = proplists:get_value(warnings, Other, []),
             setup(T#test{
                     compile_vars = CV,
-                    compile_opts = CO ++ CompileOpts
+                    compile_opts = CO ++ CompileOpts,
+                    warnings = Ws
                    });
         {error, Es, Ws} ->
             T#test{
@@ -1801,6 +1881,9 @@ setup_compile("ifnotequal_preset") ->
 setup_compile("var_preset") ->
     CompileVars = [{preset_var1, "preset-var1"}, {preset_var2, "preset-var2"}],
     {ok, [CompileVars]};
+setup_compile("extends_for") ->
+	CompileVars = [{veggie_list, ["broccoli", "beans", "peas", "carrots"]}],
+	{ok, [CompileVars]};
 setup_compile("extends2") ->
     File = template_file(input, "extends2"),
     Error = {none, erlydtl_beam_compiler, unexpected_extends_tag},
@@ -1810,16 +1893,22 @@ setup_compile("extends3") ->
     Include = template_file(input, "imaginary"),
     Error = {none, erlydtl_beam_compiler, {read_file, Include, enoent}},
     {error, [{File, [Error]}], []};
+setup_compile("extends4") ->
+    File = template_file(input, "extends4"),
+    Warning = {{1,21}, erlydtl_beam_compiler, non_block_tag},
+    {ok, [[]|[{warnings, [{File, [Warning]}]}]]};
 setup_compile("missing") ->
     File = template_file(input, "missing"),
     Error = {none, erlydtl_compiler, {read_file, File, enoent}},
     {error, [{File, [Error]}], []};
 setup_compile("custom_tag") ->
-    {ok, [[]|[{custom_tags_modules, [erlydtl_custom_tags]}]]};
+    {ok, [[]|[{compile_opts, [{custom_tags_modules, [erlydtl_custom_tags]}]}]]};
 setup_compile("custom_tag1") -> setup_compile("custom_tag");
 setup_compile("custom_tag2") -> setup_compile("custom_tag");
 setup_compile("custom_tag3") -> setup_compile("custom_tag");
 setup_compile("custom_tag4") -> setup_compile("custom_tag");
+setup_compile("super_escaped") ->
+    {ok, [[]|[{compile_opts, [auto_escape]}]]};
 setup_compile(_) ->
     {ok, [[]]}.
 
@@ -1936,6 +2025,10 @@ setup("custom_tag4") ->
 setup("ssi") ->
     RenderVars = [{path, "ssi_include.html"}],
     {ok, RenderVars};
+setup("wrapper") ->
+    RenderVars = [{types, ["b", "a", "c"]}],
+    {ok, RenderVars};
+
 %%--------------------------------------------------------------------
 %% Custom tags
 %%--------------------------------------------------------------------

test/files/expect/extends4

@@ -0,0 +1,11 @@
+
+
+base template
+
+base title
+
+more of base template
+
+base content
+
+end of base template

test/files/expect/extends_chain

@@ -0,0 +1,15 @@
+<html>
+<head>
+</head>
+<body>
+
+    
+    
+    <p>A</p>
+
+    <p>B</p>
+
+    <p>C</p>
+
+</body>
+</html>

test/files/expect/extends_for

@@ -0,0 +1,17 @@
+before
+
+
+<ul>
+
+<li>broccoli</li>
+
+<li>beans</li>
+
+<li>peas</li>
+
+<li>carrots</li>
+
+</ul>
+
+
+after

test/files/expect/super_escaped

@@ -0,0 +1,12 @@
+<html>
+<head>
+</head>
+<body>
+
+    
+    <p>A</p>
+
+    <p>B</p>
+
+</body>
+</html>

test/files/expect/wrapper

@@ -0,0 +1,49 @@
+
+    
+        including base b now
+        
+
+base template
+
+base b
+
+more of base template
+
+base content
+
+end of base template
+
+    
+
+    
+        including base a now
+        
+
+base template
+
+base a
+
+more of base template
+
+base content
+
+end of base template
+
+    
+
+    
+        including base c now
+        
+
+base template
+
+base c
+
+more of base template
+
+base content
+
+end of base template
+
+    
+

test/files/input/base_a

@@ -0,0 +1,3 @@
+{% extends "base" %}
+
+{% block title %}base a{% endblock %}

test/files/input/base_b

@@ -0,0 +1,3 @@
+{% extends "base" %}
+
+{% block title %}base b{% endblock %}

test/files/input/base_c

@@ -0,0 +1,3 @@
+{% extends "base" %}
+
+{% block title %}base c{% endblock %}

test/files/input/base_escape

@@ -0,0 +1,9 @@
+<html>
+<head>
+</head>
+<body>
+{% block my_block %}
+    <p>A</p>
+{% endblock %}
+</body>
+</html>

test/files/input/base_for

@@ -0,0 +1,11 @@
+before
+
+{% block forloop %}
+<ul>
+{% for iterator in fruit_list %}
+<li>{{ forloop.counter }}. {{ iterator }}</li>
+{% endfor %}
+</ul>
+{% endblock %}
+
+after

test/files/input/extends4

@@ -0,0 +1,3 @@
+{% extends "base" %}
+
+bad, only block level tags should be here..

test/files/input/extends_chain

@@ -0,0 +1,6 @@
+{% extends "super_escaped" %}
+
+{% block my_block %}
+    {{ block.super }}
+    <p>C</p>
+{% endblock %}

test/files/input/extends_for

@@ -0,0 +1,8 @@
+{% extends "base_for" %}
+{% block forloop %}
+<ul>
+{% for iterator in veggie_list %}
+<li>{{ iterator }}</li>
+{% endfor %}
+</ul>
+{% endblock %}

test/files/input/super_escaped

@@ -0,0 +1,6 @@
+{% extends "base_escape" %}
+
+{% block my_block %}
+    {{ block.super }}
+    <p>B</p>
+{% endblock %}

test/files/input/wrapper

@@ -0,0 +1,12 @@
+{% for type in types %}
+    {% if type=="a" %}
+        including base a now
+        {% include "base_a" %}
+    {% elif type=="b" %}
+        including base b now
+        {% include "base_b" %}
+    {% else %}
+        including base c now
+        {% include "base_c" %}
+    {% endif %}
+{% endfor %}