support translation context in `trans` tags (#131)

README.markdown

@@ -100,9 +100,10 @@ Options is a proplist possibly containing:
   (rather than lists). Defaults to `true`.
 
 * `blocktrans_fun` - A two-argument fun to use for translating
-  `blocktrans` blocks, `trans` tags and `_(..)` expressions. This will
-  be called once for each pair of translated element and locale
-  specified in `blocktrans_locales`. The fun should take the form:
+  `blocktrans` blocks, `trans` tags and `_(..)` expressions at compile
+  time. This will be called once for each pair of translated element
+  and locale specified in `blocktrans_locales`. The fun should take
+  the form:
 
   ```erlang
   Fun(Block::string(), Locale::string()) -> <<"ErlyDTL code">>::binary() | default
@@ -307,11 +308,30 @@ my_compiled_template:render(Variables, Options) -> {ok, IOList} | {error, Err}
 
 Same as `render/1`, but with the following options:
 
-* `translation_fun` - A fun/1 that will be used to translate strings
-  appearing inside `{% trans %}` and `{% blocktrans %}` tags. The
-  simplest TranslationFun would be `fun(Val) -> Val end`. Placeholders
-  for blocktrans variable interpolation should be wrapped to `{{` and
-  `}}`.
+* `translation_fun` - A `fun/1` or `fun/2` that will be used to
+  translate strings appearing inside `{% trans %}` and `{% blocktrans
+  %}` tags at render-time. The simplest TranslationFun would be
+  `fun(Val) -> Val end`. Placeholders for blocktrans variable
+  interpolation should be wrapped in `{{` and `}}`. In case of
+  `fun/2`, the extra argument is the current locale, possibly together
+  with a translation context in a tuple:
+
+  ```erlang
+  fun (Val, {Locale, Context}) -> Translated_Val;
+      (Val, Locale) -> Translated_Val
+  end
+  ```
+
+  The context is present when specified in the translation
+  tag. Example:
+
+  ```django
+  {% trans "Some text to translate" context "app-specific" %}
+    or
+  {% blocktrans context "another-context" %}
+    Translate this for {{ name }}.
+  {% endblocktrans %}
+  ```
 
 * `lists_0_based` - If the compile option `lists_0_based` was set to
   `defer`, pass this option (or set it to true, `{lists_0_based,

src/erlydtl_beam_compiler.erl

@@ -662,6 +662,8 @@ body_ast(DjangoParseTree, BodyScope, TreeWalker) ->
                 templatetag_ast(TagName, TW);
             ({'trans', Value}, TW) ->
                 translated_ast(Value, TW);
+            ({'trans', Value, Context}, TW) ->
+                translated_ast(Value, Context, TW);
             ({'widthratio', Numerator, Denominator, Scale}, TW) ->
                 widthratio_ast(Numerator, Denominator, Scale, TW);
             ({'with', Args, Contents}, TW) ->
@@ -741,6 +743,9 @@ with_dependency(FilePath, {{Ast, Info}, TreeWalker}) ->
 empty_ast(TreeWalker) ->
     {{erl_syntax:list([]), #ast_info{}}, TreeWalker}.
 
+
+%%% Note: Context here refers to the translation context, not the #dtl_context{} record
+
 blocktrans_ast(ArgList, Contents, TreeWalker) ->
     %% add new scope using 'with' values
     {NewScope, {ArgInfo, TreeWalker1}} =
@@ -769,7 +774,7 @@ blocktrans_ast(ArgList, Contents, TreeWalker) ->
             {FinalAst, restore_scope(TreeWalker1, FinalTW)};
         BlockTransFun when is_function(BlockTransFun) ->
             %% translate in compile-time
-            {FinalAstInfo, FinalTreeWalker, Clauses} = 
+            {FinalAstInfo, FinalTreeWalker, Clauses} =
                 lists:foldr(
                   fun (Locale, {AstInfoAcc, TreeWalkerAcc, ClauseAcc}) ->
                           case BlockTransFun(SourceText, Locale) of
@@ -808,26 +813,39 @@ blocktrans_runtime_ast({DefaultAst, Info}, SourceText, Contents, TreeWalker) ->
                         "end"]),
     {{BlockTransAst, Info}, TreeWalker1}.
 
-translated_ast({string_literal, _, String}, TreeWalker) ->
-    UnescapedStr = unescape_string_literal(String),
-    case call_extension(TreeWalker, translate_ast, [UnescapedStr, TreeWalker]) of
-        undefined ->
-            AstInfo = #ast_info{translatable_strings = [UnescapedStr]},
-            case TreeWalker#treewalker.context#dtl_context.trans_fun of
-                none -> runtime_trans_ast({{erl_syntax:string(UnescapedStr), AstInfo}, TreeWalker});
-                _ -> compiletime_trans_ast(UnescapedStr, AstInfo, TreeWalker)
+translated_ast(Phrase, TreeWalker) ->
+    translated_ast(Phrase, undefined, TreeWalker).
+
+translated_ast(Phrase, {string_literal, _, Context}, TreeWalker) ->
+    translated_ast(Phrase, unescape_string_literal(Context), TreeWalker);
+translated_ast(Phrase, Context, TreeWalker) ->
+    case extract_phrase(Phrase, TreeWalker) of
+        {compiletime, Phrase1, TreeWalker1} ->
+            case call_extension(TreeWalker, translate_ast, [Phrase1, Context, TreeWalker1]) of
+                undefined ->
+                    case TreeWalker#treewalker.context#dtl_context.trans_fun of
+                        none -> runtime_trans_ast(Phrase1, Context, TreeWalker1);
+                        _ -> compiletime_trans_ast(Phrase1, Context, TreeWalker1)
+                    end;
+                TranslatedAst ->
+                    TranslatedAst
             end;
-        Translated ->
-            Translated
-    end;
-translated_ast(ValueToken, TreeWalker) ->
-    runtime_trans_ast(value_ast(ValueToken, true, false, TreeWalker)).
+        {runtime, PhraseAst} ->
+            runtime_trans_ast(PhraseAst, Context)
+    end.
+
+runtime_trans_ast(Phrase, Context, TreeWalker) ->
+    Info = #ast_info{ translatable_strings = [Phrase] },
+    runtime_trans_ast({{merl:term(Phrase), Info}, TreeWalker}, Context).
 
-runtime_trans_ast({{ValueAst, AstInfo}, TreeWalker}) ->
-    {{?Q("erlydtl_runtime:translate(_@ValueAst, _TranslationFun)"),
+runtime_trans_ast({{ValueAst, AstInfo}, TreeWalker}, undefined) ->
+    {{?Q("erlydtl_runtime:translate(_@ValueAst, _CurrentLocale, _TranslationFun)"),
+      AstInfo}, TreeWalker};
+runtime_trans_ast({{ValueAst, AstInfo}, TreeWalker}, Context) ->
+    {{?Q("erlydtl_runtime:translate(_@ValueAst, {_CurrentLocale, _@Context@}, _TranslationFun)"),
       AstInfo}, TreeWalker}.
 
-compiletime_trans_ast(String, AstInfo,
+compiletime_trans_ast(Phrase, LContext,
                       #treewalker{
                          context=#dtl_context{
                                     trans_fun=TFun,
@@ -837,19 +855,40 @@ compiletime_trans_ast(String, AstInfo,
     ClAst = lists:foldl(
               fun(Locale, ClausesAcc) ->
                       [?Q("_@Locale@ -> _@translated",
-                          [{translated, case TFun(String, Locale) of
-                                            default -> string_ast(String, Context);
+                          [{translated, case TFun(Phrase, phrase_context(Locale, LContext)) of
+                                            default -> phrase_ast(Phrase, Context);
                                             Translated -> string_ast(Translated, Context)
                                         end}])
                        |ClausesAcc]
               end,
               [], TLocales),
-    CaseAst = ?Q(["case _CurrentLocale of",
-                  "  _@_ClAst -> _;",
-                  " _ -> _@string",
-                  "end"],
-                 [{string, string_ast(String, Context)}]),
-    {{CaseAst, AstInfo}, TreeWalker}.
+    {{?Q(["case _CurrentLocale of",
+          "  _@_ClAst -> _;",
+          " _ -> _@string",
+          "end"],
+         [{string, phrase_ast(Phrase, Context)}]),
+      #ast_info{translatable_strings = [Phrase]}},
+     TreeWalker}.
+
+phrase_context(Locale, undefined) -> Locale;
+phrase_context(Locale, Context) -> {Locale, Context}.
+
+extract_phrase({{string_literal, _, String}, {string_literal, _, Plural}}, TreeWalker) ->
+    {compiletime, {unescape_string_literal(String), unescape_string_literal(Plural)}, TreeWalker};
+extract_phrase({string_literal, _, String}, TreeWalker) ->
+    {compiletime, unescape_string_literal(String), TreeWalker};
+extract_phrase({Value, {string_literal, _, Plural}}, TreeWalker) ->
+    {{ValueAst, ValueInfo}, TreeWalker1} = value_ast(Value, true, false, TreeWalker),
+    ContextAst = string_ast(unescape_string_literal(Plural), TreeWalker1),
+    {runtime, {{erl_syntax:tuple([ValueAst, ContextAst]), ValueInfo}, TreeWalker1}};
+extract_phrase(Value, TreeWalker) ->
+    {runtime, value_ast(Value, true, false, TreeWalker)}.
+
+phrase_ast({Text, _}, TreeWalker) -> string_ast(Text, TreeWalker);
+phrase_ast(Text, TreeWalker) -> string_ast(Text, TreeWalker).
+
+%%% end of context being translation context
+
 
 %% Completely unnecessary in ErlyDTL (use {{ "{%" }} etc), but implemented for compatibility.
 templatetag_ast("openblock", TreeWalker) ->

src/erlydtl_parser.yrl

@@ -113,7 +113,8 @@ Nonterminals
 
     BlockTransBlock
     BlockTransContent
-    TransTag    
+    TransTag
+    TransArgs
     TransText
 
     TemplatetagTag
@@ -145,6 +146,7 @@ Terminals
     close_var
     comment_tag
     comment_keyword
+    context_keyword
     cycle_keyword
     elif_keyword
     else_keyword
@@ -401,9 +403,12 @@ Templatetag -> closebrace_keyword : '$1'.
 Templatetag -> opencomment_keyword : '$1'.
 Templatetag -> closecomment_keyword : '$1'.
 
-TransTag -> open_tag trans_keyword TransText close_tag : {trans, '$3'}.
-TransTag -> open_tag trans_keyword TransText as_keyword identifier close_tag : {scope_as, '$5', [{trans, '$3'}]}.
-TransTag -> open_tag trans_keyword TransText noop_keyword close_tag : '$3'.
+TransTag -> open_tag trans_keyword TransArgs close_tag : '$3'.
+TransTag -> open_tag trans_keyword TransArgs as_keyword identifier close_tag : {scope_as, '$5', ['$3']}.
+TransTag -> open_tag trans_keyword TransArgs noop_keyword close_tag : element(2, '$3').
+
+TransArgs -> TransText : {trans, '$1'}.
+TransArgs -> TransText context_keyword string_literal: {trans, '$1', '$3'}.
 
 TransText -> string_literal : '$1'.
 TransText -> Variable : '$1'.

src/erlydtl_runtime.erl

@@ -2,7 +2,13 @@
 
 -compile(export_all).
 
--type translate_fun() :: fun((string() | binary()) -> string() | binary() | undefined).
+-type text() :: string() | binary().
+-type phrase() :: text() | {text(), {PluralPhrase::text(), non_neg_integer()}}.
+-type locale() :: term() | {Locale::term(), Context::binary()}.
+
+-type old_translate_fun() :: fun((text()) -> text() | undefined).
+-type new_translate_fun() :: fun((phrase(), locale()) -> text() | undefined).
+-type translate_fun() :: new_translate_fun() | old_translate_fun().
 
 -define(IFCHANGED_CONTEXT_VARIABLE, erlydtl_ifchanged_context).
 
@@ -130,18 +136,30 @@ regroup([Item|Rest], Attribute, [[{grouper, PrevGrouper}, {list, PrevList}]|Acc]
             regroup(Rest, Attribute, [[{grouper, Value}, {list, [Item]}], [{grouper, PrevGrouper}, {list, lists:reverse(PrevList)}]|Acc])
     end.
 
--spec translate(Str, none | translate_fun()) -> Str when
-      Str :: string() | binary().
-translate(String, none) -> String;
-translate(String, TranslationFun)
-  when is_function(TranslationFun) ->
-    case TranslationFun(String) of
-        undefined -> String;
-        <<"">> -> String;
-        "" -> String;
-        Str -> Str
+-spec translate(Phrase, Locale, Fun) -> text() | undefined when
+      Phrase :: phrase(),
+      Locale :: locale(),
+      Fun :: none | translate_fun().
+translate(Phrase, _Locale, none) -> trans_text(Phrase);
+translate(Phrase, Locale, TranslationFun) ->
+    case do_translate(Phrase, Locale, TranslationFun) of
+        undefined -> trans_text(Phrase);
+        <<"">> -> trans_text(Phrase);
+        "" -> trans_text(Phrase);
+        Translated -> Translated
     end.
 
+trans_text({Text, _}) -> Text;
+trans_text(Text) -> Text.
+
+do_translate(Phrase, _Locale, TranslationFun)
+  when is_function(TranslationFun, 1) ->
+    TranslationFun(trans_text(Phrase));
+do_translate(Phrase, Locale, TranslationFun)
+  when is_function(TranslationFun, 2) ->
+    TranslationFun(Phrase, Locale).
+
+
 %% @doc Translate and interpolate 'blocktrans' content.
 %% Pre-requisites:
 %%  * `Variables' should be sorted

src/erlydtl_scanner.erl

@@ -36,7 +36,7 @@
 %%%-------------------------------------------------------------------
 -module(erlydtl_scanner).
 
-%% This file was generated 2014-04-09 13:00:24 UTC by slex 0.2.1.
+%% This file was generated 2014-04-10 04:34:52 UTC by slex 0.2.1.
 %% http://github.com/erlydtl/slex
 -slex_source(["src/erlydtl_scanner.slex"]).
 
@@ -87,6 +87,7 @@ is_keyword(any, "as") -> true;
 is_keyword(any, "by") -> true;
 is_keyword(any, "with") -> true;
 is_keyword(any, "from") -> true;
+is_keyword(any, "context") -> true;
 is_keyword(close, "only") -> true;
 is_keyword(close, "parsed") -> true;
 is_keyword(close, "noop") -> true;

src/erlydtl_scanner.slex

@@ -309,6 +309,7 @@ form \
   is_keyword(any, "by") -> true; \
   is_keyword(any, "with") -> true; \
   is_keyword(any, "from") -> true; \
+  is_keyword(any, "context") -> true; \
   \
   is_keyword(close, "only") -> true; \
   is_keyword(close, "parsed") -> true; \

test/erlydtl_test_defs.erl

@@ -1289,8 +1289,8 @@ all_test_defs() ->
         <<"Hello {% trans var1 noop %}">>, [{var1, <<"Hi">>}], [{translation_fun, fun(<<"Hi">>) -> <<"Konichiwa">> end}], [],
         <<"Hello Hi">>},
        {"trans as",
-        <<"{% trans 'Hans' as name %}Hello {{ name }}">>, [], <<"Hello Hans">>
-       }
+        <<"{% trans 'Hans' as name %}Hello {{ name }}">>, [],
+        <<"Hello Hans">>}
       ]},
      {"blocktrans",
       [{"blocktrans default locale",
@@ -1316,6 +1316,31 @@ all_test_defs() ->
         [{translation_fun, fun("Hello, {{ name }}! See {{ v1 }}.") -> <<"Guten tag, {{name}}! Sehen {{    v1   }}.">> end}],
         [], <<"Guten tag, Mr. President! Sehen rubber-duck.">>}
       ]},
+     {"extended translation features (#131)",
+      [{"trans default locale",
+        <<"test {% trans 'message' %}">>,
+        [], [{translation_fun, fun ("message", none) -> "ok" end}],
+        <<"test ok">>},
+       {"trans foo locale",
+        <<"test {% trans 'message' %}">>,
+        [], [{locale, "foo"}, {translation_fun, fun ("message", "foo") -> "ok" end}],
+        <<"test ok">>},
+       {"trans context (run-time)",
+        <<"test {% trans 'message' context 'foo' %}">>,
+        [], [{translation_fun, fun ("message", {none, "foo"}) -> "ok" end}],
+        <<"test ok">>},
+       {"trans context (compile-time)",
+        <<"test {% trans 'message' context 'foo' %}">>,
+        [], [{locale, "baz"}],
+        [{blocktrans_locales, ["bar", "baz"]},
+         {blocktrans_fun, fun ("message", {L, "foo"}) ->
+                                  case L of
+                                      "bar" -> "rab";
+                                      "baz" -> "ok"
+                                  end
+                          end}],
+        <<"test ok">>}
+      ]},
      {"verbatim",
       [{"Plain verbatim",
         <<"{% verbatim %}{{ oh no{% foobar %}{% endverbatim %}">>, [],
@@ -1523,7 +1548,7 @@ all_test_defs() ->
                      {"default_variables/0",
                       default_variables, [], []},
                      {"default_variables/0 w. defaults",
-                      default_variables, [var1], [debug_compiler, {default_vars, [{var1, aaa}]}]},
+                      default_variables, [var1], [{default_vars, [{var1, aaa}]}]},
                      {"default_variables/0 w. constants",
                       default_variables, [], [{constants, [{var1, bbb}]}]},
                      {"constants/0",

test/sources_parser_tests.erl

@@ -119,7 +119,9 @@ compare_token({'firstof', Vars1}, {'firstof', Vars2}) ->
 %% TODO...
 %% compare_token({'for', {'in', IteratorList, Identifier}, Contents}, {'for', {'in', IteratorList, Identifier}, Contents}) -> ok;
 %% compare_token({'for', {'in', IteratorList, Identifier}, Contents, EmptyPartsContents}, {'for', {'in', IteratorList, Identifier}, Contents, EmptyPartsContents}) -> ok;
-%% compare_token({'if', Expression, Contents}, {'if', Expression, Contents}) -> ok;
+compare_token({'if', Expression1, Contents1}, {'if', Expression2, Contents2}) ->
+    compare_expression(Expression1, Expression2),
+    compare_tree(Contents1, Contents2);
 %% compare_token({'ifchanged', Expression, IfContents}, {'ifchanged', Expression, IfContents}) -> ok;
 %% compare_token({'ifchangedelse', Expression, IfContents, ElseContents}, {'ifchangedelse', Expression, IfContents, ElseContents}) -> ok;
 %% compare_token({'ifelse', Expression, IfContents, ElseContents}, {'ifelse', Expression, IfContents, ElseContents}) -> ok;