Misago/misago/markup/parser.py

import bleach
import markdown
from bs4 import BeautifulSoup
from django.http import Http404
from django.urls import resolve
from htmlmin.minify import html_minify
from markdown.extensions.fenced_code import FencedCodeExtension

from ..conf import settings
from .bbcode import blocks, inline
from .md.shortimgs import ShortImagesExtension
from .md.striketrough import StriketroughExtension
from .mentions import add_mentions
from .pipeline import pipeline

MISAGO_ATTACHMENT_VIEWS = ("misago:attachment", "misago:attachment-thumbnail")


def parse(
    text,
    request,
    poster,
    allow_mentions=True,
    allow_links=True,
    allow_images=True,
    allow_blocks=True,
    force_shva=False,
    minify=True,
):
    """
    Message parser

    Utility for flavours to call

    Breaks text into paragraphs, supports code, spoiler and quote blocks,
    headers, lists, images, spoilers, text styles

    Returns dict object
    """
    md = md_factory(
        allow_links=allow_links, allow_images=allow_images, allow_blocks=allow_blocks
    )

    parsing_result = {
        "original_text": text,
        "parsed_text": "",
        "markdown": md,
        "mentions": [],
        "images": [],
        "internal_links": [],
        "outgoing_links": [],
    }

    # Parse text
    parsed_text = md.convert(text)

    # Clean and store parsed text
    parsing_result["parsed_text"] = parsed_text.strip()

    if allow_links:
        linkify_paragraphs(parsing_result)

    parsing_result = pipeline.process_result(parsing_result)

    if allow_mentions:
        add_mentions(request, parsing_result)

    if allow_links or allow_images:
        clean_links(request, parsing_result, force_shva)

    if minify:
        minify_result(parsing_result)
    return parsing_result


def md_factory(allow_links=True, allow_images=True, allow_blocks=True):
    """creates and configures markdown object"""
    md = markdown.Markdown(extensions=["markdown.extensions.nl2br"])

    # Remove HTML allowances
    del md.preprocessors["html_block"]
    del md.inlinePatterns["html"]

    # Remove references
    del md.preprocessors["reference"]
    del md.inlinePatterns["reference"]
    del md.inlinePatterns["image_reference"]
    del md.inlinePatterns["short_reference"]

    # Add [b], [i], [u]
    md.inlinePatterns.add("bb_b", inline.bold, "<strong")
    md.inlinePatterns.add("bb_i", inline.italics, "<emphasis")
    md.inlinePatterns.add("bb_u", inline.underline, "<emphasis2")

    # Add ~~deleted~~
    striketrough_md = StriketroughExtension()
    striketrough_md.extendMarkdown(md)

    if allow_links:
        # Add [url]
        md.inlinePatterns.add("bb_url", inline.url(md), "<link")
    else:
        # Remove links
        del md.inlinePatterns["link"]
        del md.inlinePatterns["autolink"]
        del md.inlinePatterns["automail"]

    if allow_images:
        # Add [img]
        md.inlinePatterns.add("bb_img", inline.image(md), "<image_link")
        short_images_md = ShortImagesExtension()
        short_images_md.extendMarkdown(md)
    else:
        # Remove images
        del md.inlinePatterns["image_link"]

    if allow_blocks:
        # Add [hr] and [quote] blocks
        md.parser.blockprocessors.add(
            "bb_hr", blocks.BBCodeHRProcessor(md.parser), ">hr"
        )

        fenced_code = FencedCodeExtension()
        fenced_code.extendMarkdown(md, None)

        code_bbcode = blocks.CodeBlockExtension()
        code_bbcode.extendMarkdown(md)

        quote_bbcode = blocks.QuoteExtension()
        quote_bbcode.extendMarkdown(md)
    else:
        # Remove blocks
        del md.parser.blockprocessors["hashheader"]
        del md.parser.blockprocessors["setextheader"]
        del md.parser.blockprocessors["code"]
        del md.parser.blockprocessors["quote"]
        del md.parser.blockprocessors["hr"]
        del md.parser.blockprocessors["olist"]
        del md.parser.blockprocessors["ulist"]

    return pipeline.extend_markdown(md)


def linkify_paragraphs(result):
    result["parsed_text"] = bleach.linkify(
        result["parsed_text"],
        callbacks=settings.MISAGO_BLEACH_CALLBACKS,
        skip_tags=["a", "code", "pre"],
        parse_email=True,
    )


def clean_links(request, result, force_shva=False):
    host = request.get_host()

    soup = BeautifulSoup(result["parsed_text"], "html5lib")
    for link in soup.find_all("a"):
        if is_internal_link(link["href"], host):
            link["href"] = clean_internal_link(link["href"], host)
            result["internal_links"].append(link["href"])
            link["href"] = clean_attachment_link(link["href"], force_shva)
        else:
            result["outgoing_links"].append(clean_link_prefix(link["href"]))
            link["href"] = assert_link_prefix(link["href"])
            link["rel"] = "nofollow noopener"

        if link.string:
            link.string = clean_link_prefix(link.string)

    for img in soup.find_all("img"):
        img["alt"] = clean_link_prefix(img["alt"])
        if is_internal_link(img["src"], host):
            img["src"] = clean_internal_link(img["src"], host)
            result["images"].append(img["src"])
            img["src"] = clean_attachment_link(img["src"], force_shva)
        else:
            result["images"].append(clean_link_prefix(img["src"]))
            img["src"] = assert_link_prefix(img["src"])

    # [6:-7] trims <body></body> wrap
    result["parsed_text"] = str(soup.body)[6:-7]


def is_internal_link(link, host):
    if link.startswith("/") and not link.startswith("//"):
        return True

    link = clean_link_prefix(link).lstrip("www.").lower()
    return link.lower().startswith(host.lstrip("www."))


def clean_link_prefix(link):
    if link.lower().startswith("https:"):
        link = link[6:]
    if link.lower().startswith("http:"):
        link = link[5:]
    if link.startswith("//"):
        link = link[2:]
    return link


def assert_link_prefix(link):
    if link.lower().startswith("https:"):
        return link
    if link.lower().startswith("http:"):
        return link
    if link.startswith("//"):
        return "http:%s" % link

    return "http://%s" % link


def clean_internal_link(link, host):
    link = clean_link_prefix(link)

    if link.lower().startswith("www."):
        link = link[4:]
    if host.lower().startswith("www."):
        host = host[4:]

    if link.lower().startswith(host):
        link = link[len(host) :]

    return link or "/"


def clean_attachment_link(link, force_shva=False):
    try:
        resolution = resolve(link)
        url_name = ":".join(resolution.namespaces + [resolution.url_name])
    except (Http404, ValueError):
        return link

    if url_name in MISAGO_ATTACHMENT_VIEWS:
        if force_shva:
            link = "%s?shva=1" % link
        elif link.endswith("?shva=1"):
            link = link[:-7]
    return link


def minify_result(result):
    result["parsed_text"] = html_minify(result["parsed_text"])
    result["parsed_text"] = strip_html_head_body(result["parsed_text"])


def strip_html_head_body(parsed_text):
    # [25:-14] trims <html><head></head><body> and </body></html>
    return parsed_text[25:-14]