Merge pull request #615 from sheybey/mistune-2

Upgrade mistune to version 2

docs/development/hooks/startup.rst

@@ -24,5 +24,7 @@ The hooks below are listed in the order they are called.
 .. autofunction:: flaskbb_load_translations
 .. autofunction:: flaskbb_load_post_markdown_class
 .. autofunction:: flaskbb_load_nonpost_markdown_class
+.. autofunction:: flaskbb_load_post_markdown_plugins
+.. autofunction:: flaskbb_load_nonpost_markdown_plugins
 .. autofunction:: flaskbb_additional_setup
 

flaskbb/markup.py

@@ -9,9 +9,9 @@
     :license: BSD, see LICENSE for more details.
 """
 import logging
-import re
 
 import mistune
+from mistune.plugins import plugin_strikethrough, plugin_url
 from flask import url_for
 from markupsafe import Markup
 from pluggy import HookimplMarker
@@ -24,33 +24,30 @@ impl = HookimplMarker('flaskbb')
 
 logger = logging.getLogger(__name__)
 
-_re_user = re.compile(r'@(\w+)', re.I)
+_re_user = r'(?i)@(\w+)'
 
 
-def userify(match):
-    value = match.group(1)
-    user = "<a href='{url}'>@{user}</a>".format(
-        url=url_for("user.profile", username=value, _external=False),
-        user=value
-    )
-    return user
+def parse_user_link(inline, match, state):
+    url = url_for("user.profile", username=match.group(1), _external=False)
+    return 'link', url, match.group(0)
 
 
-class FlaskBBRenderer(mistune.Renderer):
-    """Markdown with some syntactic sugar, such as @user gettting linked
-    to the user's profile.
-    """
+def plugin_userify(md):
+    """Mistune plugin that transforms @username references to links."""
+    md.inline.register_rule('flaskbb_user_link', _re_user, parse_user_link)
+    md.inline.rules.append('flaskbb_user_link')
 
-    def __init__(self, **kwargs):
-        super(FlaskBBRenderer, self).__init__(**kwargs)
 
-    def paragraph(self, text):
-        """Render paragraph tags, autolinking user handles."""
+DEFAULT_PLUGINS = [plugin_url, plugin_strikethrough, plugin_userify]
+
 
-        text = _re_user.sub(userify, text)
-        return super(FlaskBBRenderer, self).paragraph(text)
+class FlaskBBRenderer(mistune.HTMLRenderer):
+    """Mistune renderer that uses pygments to apply code highlighting."""
+
+    def __init__(self, **kwargs):
+        super(FlaskBBRenderer, self).__init__(**kwargs)
 
-    def block_code(self, code, lang):
+    def block_code(self, code, lang=None):
         if lang:
             try:
                 lexer = get_lexer_by_name(lang, stripall=True)
@@ -78,16 +75,34 @@ def flaskbb_load_nonpost_markdown_class():
 @impl
 def flaskbb_jinja_directives(app):
     render_classes = app.pluggy.hook.flaskbb_load_post_markdown_class(app=app)
-    app.jinja_env.filters['markup'] = make_renderer(render_classes)
+    plugins = DEFAULT_PLUGINS[:]
+    app.pluggy.hook.flaskbb_load_post_markdown_plugins(
+        plugins=plugins,
+        app=app
+    )
+    app.jinja_env.filters['markup'] = make_renderer(render_classes, plugins)
 
     render_classes = app.pluggy.hook.flaskbb_load_nonpost_markdown_class(
         app=app
     )
-    app.jinja_env.filters['nonpost_markup'] = make_renderer(render_classes)
+    plugins = DEFAULT_PLUGINS[:]
+    plugins = app.pluggy.hook.flaskbb_load_nonpost_markdown_plugins(
+        plugins=plugins,
+        app=app
+    )
+    app.jinja_env.filters['nonpost_markup'] = make_renderer(
+        render_classes,
+        plugins
+    )
 
 
-def make_renderer(classes):
+def make_renderer(classes, plugins):
     RenderCls = type('FlaskBBRenderer', tuple(classes), {})
 
-    markup = mistune.Markdown(renderer=RenderCls(escape=True, hard_wrap=True))
-    return lambda text: Markup(markup.render(text))
+    markup = mistune.create_markdown(
+        renderer=RenderCls(),
+        plugins=plugins,
+        escape=True,
+        hard_wrap=True
+    )
+    return lambda text: Markup(markup(text))

flaskbb/plugins/spec.py

@@ -134,6 +134,71 @@ def flaskbb_load_nonpost_markdown_class(app):
 
 
 @spec
+def flaskbb_load_post_markdown_plugins(plugins, app):
+    """
+    Hook for loading mistune renderer plugins used when rendering markdown on
+    posts and user signatures. Implementations should modify the `plugins`
+    list directly.
+
+    Example of adding plugins::
+
+        from mistune.plugins import plugin_abbr, plugin_table
+
+        @impl
+        def flaskbb_load_post_markdown_plugins(plugins):
+            # add the built-in mistune table and abbr plugins
+            plugins.extend([plugin_abbr, plugin_table])
+
+    Example of removing plugins::
+
+        from flaskbb.markup import plugin_userify
+
+        @impl
+        def flaskbb_load_post_markdown_plugins(plugins):
+            try:
+                # remove the FlaskBB user mention link plugin
+                plugins.remove(plugin_userify)
+            except ValueError:
+                # other FlaskBB plugins might beat you to removing a plugin,
+                # which is not an error. You should not raise an exception in
+                # this case.
+                pass
+
+    :param plugins: List of mistune plugins to load.
+    :type plugins: list
+    :param app: The application object.
+    :type app: Flask
+
+    .. seealso::
+        https://mistune.readthedocs.io/en/v2.0.2/advanced.html#create-plugins
+            Mistune plugin documentation.
+        :data:`~flaskbb.markup.plugin_userify`
+            FlaskBB-provided plugin that links user mentions to their profiles.
+        :data:`~flaskbb.markup.DEFAULT_PLUGINS`
+            List of plugins loaded by default.
+        :func:`flaskbb_load_nonpost_markdown_plugins`
+            Hook to modify the list of plugins for markdown rendering in
+            non-post areas.
+    """
+
+
+@spec
+def flaskbb_load_nonpost_markdown_plugins(plugins, app):
+    """
+    Hook for loading mistune renderer plugins used when rendering markdown in
+    locations other than posts, for example in category or forum
+    descriptions. Implementations should modify the `plugins` list directly.
+
+    See :func:`flaskbb_load_post_markdown_plugins` for more details.
+
+    :param plugins: List of mistune plugins to load.
+    :type plugins: list
+    :param app: The application object.
+    :type app: Flask
+    """
+
+
+@spec
 def flaskbb_cli(cli, app):
     """Hook for registering CLI commands.
 

requirements.txt

@@ -39,7 +39,7 @@ kombu==5.1.0
 limits==1.5.1
 Mako==1.1.5
 MarkupSafe==2.0.1
-mistune==0.8.4
+mistune==2.0.2
 Pillow==8.3.2
 pluggy==1.0.0
 prompt-toolkit==3.0.20

tests/unit/utils/test_markup.py

@@ -1,13 +1,19 @@
-from flaskbb.markup import FlaskBBRenderer, make_renderer
+from flaskbb.markup import FlaskBBRenderer, make_renderer, DEFAULT_PLUGINS
 
-markdown = make_renderer([FlaskBBRenderer])
 
+markdown = make_renderer([FlaskBBRenderer], DEFAULT_PLUGINS)
 
-def test_custom_renderer():
-    # custom paragraph
-    p_plain = "@sh4nks is developing flaskbb."
-    assert "/user/sh4nks" in markdown(p_plain)
 
+def test_userify():
+    # user link rendering plugin
+    result = markdown("@sh4nks is developing flaskbb.")
+    assert all(
+        substring in result
+        for substring in ("/user/sh4nks", "<a href=")
+    )
+
+
+def test_highlighting():
     # custom block code with pygments highlighting (jus)
     b_plain = """
 ```