#488: nested i18n expressions for makemessages

misago/core/management/commands/makemessages.py

@@ -9,19 +9,91 @@ from django.core.management.commands.makemessages import Command as BaseCommand
 from django.utils.text import smart_split
 
 
+I18N_HELPERS = {
+    # helper: min valid expression len
+    'gettext': 2,
+    'ngettext': 4,
+    'gettext_noop': 2,
+    'pgettext': 3,
+    'npgettext': 5
+}
+
+HBS_HELPERS = ('unbound', 'if')
 HBS_EXPRESSION = re.compile(r'({{{(.*?)}}})|({{(.*?)}})')
 
-HELPERS = {
-    'gettext': 1,
-    'ngettext': 3,
-    'gettext_noop': 1,
-    'pgettext': 2,
-    'npgettext': 4
-}
+
+class HandlebarsExpression(object):
+    def __init__(self, unparsed_expression):
+        cleaned_expression = self.clean_expression(unparsed_expression)
+        all_helpers = self.parse_expression(
+            unparsed_expression, cleaned_expression)
+
+        self.i18n_helpers = self.clean_helpers(all_helpers)
+
+    def get_i18n_helpers(self):
+        return self.i18n_helpers
+
+    def clean_expression(self, unparsed):
+        cleaned = u''
+
+        for piece in smart_split(unparsed):
+            if not cleaned and piece in HBS_HELPERS:
+                continue
+            if not piece.startswith('=') and not cleaned.endswith('='):
+                cleaned += ' '
+            cleaned += piece
+
+        return cleaned.strip()
+
+    def parse_expression(self, unparsed, cleaned):
+        helper = []
+        helpers = [helper]
+        stack = [helper]
+
+        for piece in smart_split(cleaned):
+            if piece.endswith(')'):
+                stack[-1].append(piece.rstrip(')').strip())
+                while piece.endswith(')'):
+                    piece = piece[:-1].strip()
+                    stack.pop()
+                continue
+
+            if not piece.startswith(('\'', '"')):
+                if piece.startswith('('):
+                    piece = piece[1:].strip()
+                    if piece.startswith('('):
+                        continue
+                    else:
+                        helper = [piece]
+                        helpers.append(helper)
+                        stack.append(helper)
+                else:
+                    is_kwarg = re.match(r'^[_a-zA-Z]+([_a-zA-Z0-9]+?)=', piece)
+                    if is_kwarg and not piece.endswith('='):
+                        piece = piece[len(is_kwarg.group(0)):]
+                        if piece.startswith('('):
+                            helper = [piece[1:].strip()]
+                            helpers.append(helper)
+                            stack.append(helper)
+                    else:
+                        stack[-1].append(piece)
+            else:
+                stack[-1].append(piece)
+
+        return helpers
+
+    def clean_helpers(self, all_helpers):
+        i18n_helpers = []
+        for helper in all_helpers:
+            i18n_helper_len = I18N_HELPERS.get(helper[0])
+            if i18n_helper_len and len(helper) >= i18n_helper_len:
+                i18n_helpers.append(helper[:i18n_helper_len])
+        return i18n_helpers
 
 
 class HandlebarsTemplate(object):
     def __init__(self, content):
+        self.expressions = {}
         self.content = content
 
     def get_converted_content(self):
@@ -33,13 +105,15 @@ class HandlebarsTemplate(object):
     def strip_expressions(self, content):
         def replace_expression(matchobj):
             trimmed_expression = matchobj.group(0).lstrip('{').rstrip('}')
-            trimmed_expression = trimmed_expression.strip()
+            parsed_expression = HandlebarsExpression(trimmed_expression)
 
-            expression_words = trimmed_expression.split()
-            if expression_words[0] in HELPERS:
+            expression_i18n_helpers = parsed_expression.get_i18n_helpers()
+
+            if expression_i18n_helpers:
+                self.expressions[matchobj.group(0)] = expression_i18n_helpers
                 return matchobj.group(0)
             else:
-                return ' ' * len(matchobj.group(0))
+                return ''
 
         return HBS_EXPRESSION.sub(replace_expression, self.content)
 
@@ -63,18 +137,11 @@ class HandlebarsTemplate(object):
 
     def replace_expressions(self, content):
         def replace_expression(matchobj):
-            trimmed_expression = matchobj.group(0).lstrip('{').rstrip('}')
-            trimmed_expression = trimmed_expression.strip()
-
-            expression_bits = [b for b in smart_split(trimmed_expression)]
-            function = expression_bits[0]
-
-            args_count = HELPERS[function] + 1
-            if len(expression_bits) >= args_count:
-                args = expression_bits[1:HELPERS[function] + 1]
-                return '%s(%s);' % (function, ', '.join(args))
-            else:
-                return ''
+            js_functions = []
+            for helper in self.expressions.get(matchobj.group(0)):
+                function, args = helper[0], helper[1:]
+                js_functions.append('%s(%s);' % (function, ', '.join(args)))
+            return ' '.join(js_functions)
 
         return HBS_EXPRESSION.sub(replace_expression, content)
 
@@ -89,7 +156,7 @@ class HandlebarsFile(object):
             self.make_js_file(self.hbs_path, self.js_path)
 
     def make_js_path_suffix(self, hbs_path):
-        return '%s.tmp.js' % md5(hbs_path).hexdigest()[:8]
+        return '%s.makemessages.js' % md5(hbs_path).hexdigest()[:8]
 
     def make_js_path(self, hbs_path, path_suffix):
         return Path('%s.%s' % (unicode(hbs_path), path_suffix))

misago/core/tests/test_makemessages.py

@@ -1,21 +1,34 @@
 from django.test import TestCase
 
-from misago.core.management.commands.makemessages import (HandlebarsTemplate,
-                                                          HandlebarsFile)
+from misago.core.management.commands.makemessages import (
+    HandlebarsExpression, HandlebarsTemplate, HandlebarsFile)
 
 
-class HandlebarsFileTests(TestCase):
-    def test_make_js_path(self):
-        """Object correctly translates hbs path to temp js path"""
-        hbs_path = "templates/application.hbs"
-        test_file = HandlebarsFile(hbs_path, False)
+class HandlebarsExpressionTests(TestCase):
+    def test_get_i18n_helpers(self):
+        """expression parser finds i18n helpers"""
+        expression = HandlebarsExpression("some.expression")
+        self.assertFalse(expression.get_i18n_helpers())
 
-        suffix = test_file.make_js_path_suffix(hbs_path)
-        self.assertTrue(suffix.endswith(".tmp.js"))
+        expression = HandlebarsExpression("bind-attr src=user.avatar")
+        self.assertFalse(expression.get_i18n_helpers())
 
-        js_path = test_file.make_js_path(hbs_path, suffix)
-        self.assertTrue(js_path.startswith(hbs_path))
-        self.assertTrue(js_path.endswith(suffix))
+        expression = HandlebarsExpression("gettext 'misiek'")
+        self.assertTrue(expression.get_i18n_helpers())
+
+        expression = HandlebarsExpression("gettext '%(user)s has %(trait)s' user=user.username trait=(gettext user.trait)")
+        helpers = expression.get_i18n_helpers()
+        self.assertEqual(len(helpers), 2)
+        self.assertEqual(helpers[0], ['gettext', "'%(user)s has %(trait)s'"])
+        self.assertEqual(helpers[1], ['gettext', "user.trait"])
+
+        expression = HandlebarsExpression('gettext "%(param)s!" param = (gettext "nested once" param = (gettext "nested twice")) otherparam= (gettext "nested once again")')
+        helpers = expression.get_i18n_helpers()
+        self.assertEqual(len(helpers), 4)
+        self.assertEqual(helpers[0], ['gettext', '"%(param)s!"'])
+        self.assertEqual(helpers[1], ['gettext', '"nested once"'])
+        self.assertEqual(helpers[2], ['gettext', '"nested twice"'])
+        self.assertEqual(helpers[3], ['gettext', '"nested once again"'])
 
 
 class HandlebarsTemplateTests(TestCase):
@@ -57,113 +70,154 @@ class HandlebarsTemplateTests(TestCase):
 
     def test_valid_expression_replace(self):
         """valid i18n expressions are replaced"""
-        template = HandlebarsTemplate("{{gettext 'Lorem ipsum'}}")
-        self.assertEqual(template.get_converted_content(),
-                         "gettext('Lorem ipsum');")
-
-        template = HandlebarsTemplate("{{gettext 'Lorem %(vis)s' vis=name}}")
-        self.assertEqual(template.get_converted_content(),
-                         "gettext('Lorem %(vis)s');")
-
-        template = HandlebarsTemplate("{{gettext some_variable}}")
-        self.assertEqual(template.get_converted_content(),
-                         "gettext(some_variable);")
-
-        template = HandlebarsTemplate("{{gettext 'Lorem ipsum'}}")
-        self.assertEqual(template.get_converted_content(),
-                         "gettext('Lorem ipsum');")
-
-        template = HandlebarsTemplate("{{gettext 'Lorem %(vis)s' vis=name}}")
-        self.assertEqual(template.get_converted_content(),
-                         "gettext('Lorem %(vis)s');")
-
-        template = HandlebarsTemplate("{{gettext some_variable}}")
-        self.assertEqual(template.get_converted_content(),
-                         "gettext(some_variable);")
-
-        template = HandlebarsTemplate("{{gettext some_variable user=user.username}}")
-        self.assertEqual(template.get_converted_content(),
-                         "gettext(some_variable);")
-
-        template = HandlebarsTemplate("{{ngettext '%(count)s apple' '%(count)s apples' apples_count}}")
-        self.assertEqual(template.get_converted_content(),
-                         "ngettext('%(count)s apple', '%(count)s apples', apples_count);")
-
-        template = HandlebarsTemplate("{{ngettext '%(user)s has %(count)s apple' '%(user)s has %(count)s apples' apples_count user=user.username}}")
-        self.assertEqual(template.get_converted_content(),
-                         "ngettext('%(user)s has %(count)s apple', '%(user)s has %(count)s apples', apples_count);")
-
-        template = HandlebarsTemplate("{{ngettext apple apples apples_count}}")
-        self.assertEqual(template.get_converted_content(),
-                         "ngettext(apple, apples, apples_count);")
-
-        template = HandlebarsTemplate("{{ngettext '%(count)s apple' apples apples_count}}")
-        self.assertEqual(template.get_converted_content(),
-                         "ngettext('%(count)s apple', apples, apples_count);")
-
-        template = HandlebarsTemplate("{{ngettext '%(user)s has %(count)s apple' apples apples_count user=user.username}}")
-        self.assertEqual(template.get_converted_content(),
-                         "ngettext('%(user)s has %(count)s apple', apples, apples_count);")
-
-        template = HandlebarsTemplate("{{gettext_noop 'Lorem ipsum'}}")
-        self.assertEqual(template.get_converted_content(),
-                         "gettext_noop('Lorem ipsum');")
-
-        template = HandlebarsTemplate("{{gettext_noop 'Lorem %(vis)s' vis=name}}")
-        self.assertEqual(template.get_converted_content(),
-                         "gettext_noop('Lorem %(vis)s');")
-
-        template = HandlebarsTemplate("{{gettext_noop some_variable}}")
-        self.assertEqual(template.get_converted_content(),
-                         "gettext_noop(some_variable);")
-
-        template = HandlebarsTemplate("{{gettext_noop 'Lorem ipsum'}}")
-        self.assertEqual(template.get_converted_content(),
-                         "gettext_noop('Lorem ipsum');")
-
-        template = HandlebarsTemplate("{{gettext_noop 'Lorem %(vis)s' vis=name}}")
-        self.assertEqual(template.get_converted_content(),
-                         "gettext_noop('Lorem %(vis)s');")
-
-        template = HandlebarsTemplate("{{gettext_noop some_variable}}")
-        self.assertEqual(template.get_converted_content(),
-                         "gettext_noop(some_variable);")
-
-        template = HandlebarsTemplate("{{pgettext 'month' 'may'}}")
-        self.assertEqual(template.get_converted_content(),
-                         "pgettext('month', 'may');")
-
-        template = HandlebarsTemplate("{{pgettext 'month' month_name}}")
-        self.assertEqual(template.get_converted_content(),
-                         "pgettext('month', month_name);")
-
-        template = HandlebarsTemplate("{{pgettext 'day of month' 'May, %(day)s' day=calendar.day}}")
-        self.assertEqual(template.get_converted_content(),
-                         "pgettext('day of month', 'May, %(day)s');")
-
-        template = HandlebarsTemplate("{{pgettext context value day=calendar.day}}")
-        self.assertEqual(template.get_converted_content(),
-                         "pgettext(context, value);")
-
-        template = HandlebarsTemplate("{{npgettext 'fruits' '%(count)s apple' '%(count)s apples' apples_count}}")
-        self.assertEqual(template.get_converted_content(),
-                         "npgettext('fruits', '%(count)s apple', '%(count)s apples', apples_count);")
-
-        template = HandlebarsTemplate("{{npgettext 'fruits' '%(user)s has %(count)s apple' '%(user)s has %(count)s apples' apples_count user=user.username}}")
-        self.assertEqual(template.get_converted_content(),
-                         "npgettext('fruits', '%(user)s has %(count)s apple', '%(user)s has %(count)s apples', apples_count);")
-
-        template = HandlebarsTemplate("{{npgettext context apple apples apples_count}}")
-        self.assertEqual(template.get_converted_content(),
-                         "npgettext(context, apple, apples, apples_count);")
-
-        template = HandlebarsTemplate("{{npgettext context '%(count)s apple' apples apples_count}}")
-        self.assertEqual(template.get_converted_content(),
-                         "npgettext(context, '%(count)s apple', apples, apples_count);")
-
-        template = HandlebarsTemplate("{{npgettext 'fruits' '%(user)s has %(count)s apple' apples apples_count user=user.username}}")
-        self.assertEqual(template.get_converted_content(),
-                         "npgettext('fruits', '%(user)s has %(count)s apple', apples, apples_count);")
+        VALID_CASES = (
+            '%s',
+            'unbound %s',
+            'something (%s)',
+            'unbound something (%s)',
+            'if condition (%s)',
+            'something "lorem ipsum" some.var kwarg=(%s) otherkwarg=(helper something)'
+        )
+
+        for case in VALID_CASES:
+            self.subtest("{{%s}}" % case)
+            self.subtest("{{ %s }}" % case)
+            self.subtest("{{{%s}}}" % case)
+            self.subtest("{{{ %s }}}" % case)
+
+    def subtest(self, case_template):
+        CASES = (
+            (
+                "gettext 'Lorem ipsum'",
+                "gettext('Lorem ipsum');"
+            ),
+            (
+                "gettext 'Lorem %(vis)s' vis=name",
+                "gettext('Lorem %(vis)s');"
+            ),
+            (
+                "gettext 'Lorem %(vis)s' vis=(gettext user.vis)",
+                "gettext('Lorem %(vis)s'); gettext(user.vis);"
+            ),
+            (
+                "gettext some_variable",
+                "gettext(some_variable);"
+            ),
+            (
+                "gettext 'Lorem ipsum'",
+                "gettext('Lorem ipsum');"
+            ),
+            (
+                "gettext 'Lorem %(vis)s' vis=name",
+                "gettext('Lorem %(vis)s');"
+            ),
+            (
+                "gettext some_variable",
+                "gettext(some_variable);"
+            ),
+            (
+                "gettext some_variable user=user.username",
+                "gettext(some_variable);"
+            ),
+            (
+                "ngettext '%(count)s apple' '%(count)s apples' apples_count",
+                "ngettext('%(count)s apple', '%(count)s apples', apples_count);"
+            ),
+            (
+                "ngettext '%(user)s has %(count)s apple' '%(user)s has %(count)s apples' apples_count user=user.username",
+                "ngettext('%(user)s has %(count)s apple', '%(user)s has %(count)s apples', apples_count);"
+            ),
+            (
+                "ngettext apple apples apples_count",
+                "ngettext(apple, apples, apples_count);"
+            ),
+            (
+                "ngettext '%(count)s apple' apples apples_count",
+                "ngettext('%(count)s apple', apples, apples_count);"
+            ),
+            (
+                "ngettext '%(user)s has %(count)s apple' apples apples_count user=user.username",
+                "ngettext('%(user)s has %(count)s apple', apples, apples_count);"
+            ),
+            (
+                "gettext_noop 'Lorem ipsum'",
+                "gettext_noop('Lorem ipsum');"
+            ),
+            (
+                "gettext_noop 'Lorem %(vis)s' vis=name",
+                "gettext_noop('Lorem %(vis)s');"
+            ),
+            (
+                "gettext_noop some_variable",
+                "gettext_noop(some_variable);"
+            ),
+            (
+                "gettext_noop 'Lorem ipsum'",
+                "gettext_noop('Lorem ipsum');"
+            ),
+            (
+                "gettext_noop 'Lorem %(vis)s' vis=name",
+                "gettext_noop('Lorem %(vis)s');"
+            ),
+            (
+                "gettext_noop some_variable",
+                "gettext_noop(some_variable);"
+            ),
+            (
+                "pgettext 'month' 'may'",
+                "pgettext('month', 'may');"
+            ),
+            (
+                "pgettext 'month' month_name",
+                "pgettext('month', month_name);"
+            ),
+            (
+                "pgettext 'day of month' 'May, %(day)s' day=calendar.day",
+                "pgettext('day of month', 'May, %(day)s');"
+            ),
+            (
+                "pgettext context value day=calendar.day",
+                "pgettext(context, value);"
+            ),
+            (
+                "npgettext 'fruits' '%(count)s apple' '%(count)s apples' apples_count",
+                "npgettext('fruits', '%(count)s apple', '%(count)s apples', apples_count);"
+            ),
+            (
+                "npgettext 'fruits' '%(user)s has %(count)s apple' '%(user)s has %(count)s apples' apples_count user=user.username",
+                "npgettext('fruits', '%(user)s has %(count)s apple', '%(user)s has %(count)s apples', apples_count);"
+            ),
+            (
+                "npgettext context apple apples apples_count",
+                "npgettext(context, apple, apples, apples_count);"
+            ),
+            (
+                "npgettext context '%(count)s apple' apples apples_count",
+                "npgettext(context, '%(count)s apple', apples, apples_count);"
+            ),
+            (
+                "npgettext 'fruits' '%(user)s has %(count)s apple' apples apples_count user=user.username",
+                "npgettext('fruits', '%(user)s has %(count)s apple', apples, apples_count);"
+            ),
+        )
+
+        assertion_msg = """
+HBS template was parsed incorrectly:
+
+input:      %s
+output:     %s
+expected:   %s
+"""
+
+        for test, expected_output in CASES:
+            test_input = case_template % test
+
+            template = HandlebarsTemplate(case_template % test)
+            test_output = template.get_converted_content()
+
+            self.assertEqual(
+                test_output, expected_output,
+                assertion_msg % (test_input, test_output, expected_output))
 
     def test_multiple_expressions(self):
         """multiple expressions are handled"""
@@ -182,3 +236,17 @@ class HandlebarsTemplateTests(TestCase):
             {{gettext user.rank.title}}""")
         self.assertEqual(template.get_converted_content(),
                          "\ngettext('Posted by:');\n\ngettext(user.rank.title);")
+
+
+class HandlebarsFileTests(TestCase):
+    def test_make_js_path(self):
+        """Object correctly translates hbs path to temp js path"""
+        hbs_path = "templates/application.hbs"
+        test_file = HandlebarsFile(hbs_path, False)
+
+        suffix = test_file.make_js_path_suffix(hbs_path)
+        self.assertTrue(suffix.endswith(".makemessages.js"))
+
+        js_path = test_file.make_js_path(hbs_path, suffix)
+        self.assertTrue(js_path.startswith(hbs_path))
+        self.assertTrue(js_path.endswith(suffix))