#600: update posts contents

misago/datamover/management/commands/buildmovesindex.py

@@ -1,13 +1,14 @@
+from misago.core.pgutils import batch_update
+
 from ...models import MovedId, OldIdRedirect
 from ..base import BaseCommand
 
 
 MAPPINGS = {
-    'attachment': 0,
-    'category': 1,
-    'post': 2,
-    'thread': 3,
-    'user': 4,
+    'category': OldIdRedirect.CATEGORY,
+    'post': OldIdRedirect.POST,
+    'thread': OldIdRedirect.THREAD,
+    'user': OldIdRedirect.USER,
 }
 
 
@@ -22,8 +23,12 @@ class Command(BaseCommand):
         counter = 1
         self.start_timer()
 
-        for moved_id in MovedId.objects.exclude(model='label').iterator():
+        for moved_id in batch_update(MovedId.objects):
             counter += 1
+
+            if moved_id.model not in MAPPINGS:
+                continue
+
             OldIdRedirect.objects.create(
                 model=MAPPINGS[moved_id.model],
                 old_id=moved_id.old_id,

misago/datamover/markup.py

@@ -1,111 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from misago.core.pgutils import batch_update
-from misago.threads.checksums import update_post_checksum
-from misago.threads.models import Post
-
-from . import fetch_assoc, movedids
-
-
-def clean_posts():
-    for post in batch_update(Post.objects):
-        print '----' * 8
-        post.original = clean_original(post.original)
-        post.parsed = clean_parsed(post.parsed)
-
-        # update_post_checksum(post)
-        # post.save()
-
-
-def clean_original(post):
-    post = convert_quotes_to_bbcode(post)
-
-    return post
-
-
-def clean_parsed(post):
-    post = update_quotes_markup(post)
-
-    return post
-
-
-def convert_quotes_to_bbcode(post):
-    if not (post[0] == '>' or '\n>' in post):
-        return post
-
-    clean_lines = []
-
-    in_quote = False
-    quote_author = None
-    quote = []
-
-    for i, line in enumerate(post.splitlines() + ['']):
-        if in_quote:
-            if line.startswith('>'):
-                quote.append(line[1:].lstrip())
-            else:
-                clean_lines.append('')
-                if quote_author:
-                    clean_lines.append('[quote="%s"]' % quote_author)
-                clean_lines += quote
-                clean_lines.append('[/quote]')
-                clean_lines.append('')
-
-                in_quote = False
-                quote_author = None
-                quote = []
-
-                clean_lines.append(line)
-        elif line.startswith('>'):
-            in_quote = True
-
-            if clean_lines and clean_lines[-1].startswith('@'):
-                quote_author = clean_lines.pop(-1)
-
-            quote.append(line[1:].lstrip())
-        else:
-            clean_lines.append(line)
-
-    return convert_quotes_to_bbcode('\n'.join(clean_lines))
-
-
-MD_QUOTE_RE = re.compile(r'''
-<blockquote>(?P<content>.*?)</blockquote>
-'''.strip(), re.IGNORECASE | re.MULTILINE | re.DOTALL);
-MD_QUOTE_TITLE_RE = re.compile(r'''
-<quotetitle>(?P<content>.*?)</quotetitle>
-'''.strip(), re.IGNORECASE | re.MULTILINE | re.DOTALL);
-MD_QUOTE_CONTENT_RE = re.compile(r'''
-<article>(?P<content>.*?)</article>
-'''.strip(), re.IGNORECASE | re.MULTILINE | re.DOTALL);
-
-
-def update_quotes_markup(post):
-    if not ('<blockquote>' in post and '<article>' in post):
-        return post
-
-    post = MD_QUOTE_RE.sub(replace_quote, post)
-
-    return update_quotes_markup(post)
-
-
-def replace_quote(matchobj):
-    title_match = MD_QUOTE_TITLE_RE.search(matchobj.group('content'))
-    content_match = MD_QUOTE_CONTENT_RE.search(matchobj.group('content'))
-
-    if not (title_match and content_match):
-        return matchobj.group('content')
-
-    title = title_match.group('content')
-    content = content_match.group('content')
-
-    return '\n'.join([
-        '<aside class="quote-block">',
-        '<div class="quote-heading">%s</div>' % title,
-        '<blockquote class="quote-body">',
-        content.strip(),
-        '</blockquote>',
-        '</aside>',
-    ])

misago/datamover/markup/__init__.py

@@ -0,0 +1,44 @@
+from __future__ import unicode_literals
+
+from django.utils.crypto import get_random_string
+
+from misago.core.pgutils import batch_update
+from misago.markup import common_flavour
+from misago.threads.checksums import update_post_checksum
+from misago.threads.models import Post
+
+from .attachments import update_attachments_urls
+from .quotes import convert_quotes_to_bbcode
+
+
+def clean_posts():
+    for post in batch_update(Post.objects):
+        post.original = clean_original(post.original)
+
+        parsed_post = common_flavour(FakeRequest(), FakeUser(), post.original)
+        post.parsed = parsed_post['parsed_text']
+
+        update_post_checksum(post)
+        post.save()
+
+
+def clean_original(post):
+    post = convert_quotes_to_bbcode(post)
+    post = update_attachments_urls(post)
+
+    return post
+
+
+"""
+Fake request and user for parser
+"""
+class FakeUser(object):
+    slug = get_random_string(40)
+
+
+class FakeRequest(object):
+    scheme = 'http'
+    user = FakeUser()
+
+    def get_host(self):
+        return '{}.com'.format(get_random_string(40))

misago/datamover/markup/attachments.py

@@ -0,0 +1,48 @@
+from __future__ import unicode_literals
+
+import re
+
+from misago.threads.models import Attachment
+
+from .. import fetch_assoc, movedids
+
+
+ATTACHMENT_RE = re.compile(r'/attachment/(?P<hash>[a-z0-9]+)/')
+ATTACHMENT_THUMB_RE = re.compile(r'/attachment/thumb/(?P<hash>[a-z0-9]+)/')
+
+
+def update_attachments_urls(post):
+    if '/attachment/' not in post:
+        return post
+
+    post = ATTACHMENT_THUMB_RE.sub(update_thumb_url, post)
+    post = ATTACHMENT_RE.sub(update_full_url, post)
+
+    return post
+
+
+def update_thumb_url(matchobj):
+    hash = matchobj.group('hash')
+    attachment = get_attachment_for_hash(hash)
+    if attachment:
+        return attachment.get_thumbnail_url()
+    return matchobj.group(0)
+
+
+def update_full_url(matchobj):
+    hash = matchobj.group('hash')
+    attachment = get_attachment_for_hash(hash)
+    if attachment:
+        return attachment.get_absolute_url()
+    return matchobj.group(0)
+
+
+def get_attachment_for_hash(hash):
+    query = 'SELECT * FROM misago_attachment WHERE hash_id = %s'
+    for attachment in fetch_assoc(query, [hash]):
+        attachment_pk = movedids.get('attachment', attachment['id'])
+        try:
+            return Attachment.objects.get(pk=attachment_pk)
+        except Attachment.DoesNotExist:
+            return None
+    return None

misago/datamover/markup/quotes.py

@@ -0,0 +1,41 @@
+from __future__ import unicode_literals
+
+
+def convert_quotes_to_bbcode(post):
+    if not (post[0] == '>' or '\n>' in post):
+        return post
+
+    clean_lines = []
+
+    in_quote = False
+    quote_author = None
+    quote = []
+
+    for i, line in enumerate(post.splitlines() + ['']):
+        if in_quote:
+            if line.startswith('>'):
+                quote.append(line[1:].lstrip())
+            else:
+                clean_lines.append('')
+                if quote_author:
+                    clean_lines.append('[quote="%s"]' % quote_author)
+                clean_lines += quote
+                clean_lines.append('[/quote]')
+                clean_lines.append('')
+
+                in_quote = False
+                quote_author = None
+                quote = []
+
+                clean_lines.append(line)
+        elif line.startswith('>'):
+            in_quote = True
+
+            if clean_lines and clean_lines[-1].startswith('@'):
+                quote_author = clean_lines.pop(-1)
+
+            quote.append(line[1:].lstrip())
+        else:
+            clean_lines.append(line)
+
+    return convert_quotes_to_bbcode('\n'.join(clean_lines))

misago/datamover/models.py

@@ -8,11 +8,10 @@ class MovedId(models.Model):
 
 
 class OldIdRedirect(models.Model):
-    ATTACHMENT = 0
-    CATEGORY = 1
-    POST = 2
-    THREAD = 3
-    USER = 4
+    CATEGORY = 0
+    POST = 1
+    THREAD = 2
+    USER = 3
 
     model = models.PositiveIntegerField()
     old_id = models.PositiveIntegerField()