#600: move posts attachments

misago/datamover/attachments.py

@@ -0,0 +1,109 @@
+from __future__ import unicode_literals
+
+import os
+
+from django.contrib.auth import get_user_model
+from django.core.files import File
+
+from misago.threads.models import Attachment, AttachmentType, Thread, Post
+from misago.threads.serializers import AttachmentSerializer
+
+from . import OLD_FORUM, fetch_assoc, movedids, localise_datetime
+
+
+UserModel = get_user_model()
+
+IMAGE_TYPES = (
+    'image/gif',
+    'image/jpeg',
+    'image/png',
+)
+
+
+def move_attachments(stdout, style):
+    query = '''
+        SELECT *
+        FROM
+            misago_attachment
+        ORDER BY
+            id
+    '''
+
+    posts = []
+
+    attachment_types = {}
+    for attachment_type in AttachmentType.objects.all():
+        for mimetype in attachment_type.mimetypes_list:
+            attachment_types[mimetype] = attachment_type
+
+    for attachment in fetch_assoc(query):
+        if attachment['content_type'] not in attachment_types:
+            stdout.write(style.WARNING(
+                "Skipping attachment: %s (invalid type)" % attachment['name']))
+            continue
+
+        if not attachment['post_id']:
+            stdout.write(style.WARNING(
+                "Skipping attachment: %s (orphaned)" % attachment['name']))
+            continue
+
+        filetype = attachment_types[attachment['content_type']]
+
+        post_pk = movedids.get('post', attachment['post_id'])
+        post = Post.objects.get(pk=post_pk)
+
+        if post_pk not in posts:
+            posts.append(post_pk)
+
+        uploader = None
+        if attachment['user_id']:
+            uploader_pk = movedids.get('user', attachment['user_id'])
+            uploader = UserModel.objects.get(pk=uploader_pk)
+
+        file_path = os.path.join(OLD_FORUM['ATTACHMENTS'], attachment['path'])
+        upload = OldAttachmentFile(open(file_path, 'rb'), attachment)
+
+        new_attachment = Attachment(
+            secret=Attachment.generate_new_secret(),
+            filetype=filetype,
+            post=post,
+            uploaded_on=localise_datetime(attachment['date']),
+            uploader=uploader,
+            uploader_name=attachment['user_name'],
+            uploader_slug=attachment['user_name_slug'],
+            uploader_ip=attachment['ip'],
+            filename=attachment['name'],
+            size=attachment['size'],
+        )
+
+        if attachment['content_type'] in IMAGE_TYPES:
+            new_attachment.set_image(upload)
+        else:
+            new_attachment.set_file(upload)
+
+        new_attachment.save()
+
+        movedids.set('attachment', attachment['id'], new_attachment.pk)
+
+    update_attachments_caches(posts)
+
+
+def update_attachments_caches(posts):
+    for post in Post.objects.filter(pk__in=posts).iterator():
+        attachments = post.attachment_set.order_by('id')
+        post.attachments_cache = AttachmentSerializer(attachments, many=True).data
+        for attachment in post.attachments_cache:
+            del attachment['acl']
+            del attachment['post']
+            del attachment['uploader_ip']
+        post.save()
+
+
+class OldAttachmentFile(File):
+    def __init__(self, file, attachment):
+        self._attachment = attachment
+        self.name = attachment['name']
+
+        self.file = file
+        if hasattr(file, 'mode'):
+            self.mode = file.mode

misago/datamover/management/commands/movethreads.py

@@ -1,4 +1,4 @@
-from ... import threads
+from ... import attachments, threads
 from ..base import BaseCommand
 
 
@@ -32,3 +32,8 @@ class Command(BaseCommand):
         threads.move_likes()
         self.stdout.write(
             self.style.SUCCESS("Moved likes in %s" % self.stop_timer()))
+
+        self.start_timer()
+        attachments.move_attachments(self.stdout, self.style)
+        self.stdout.write(
+            self.style.SUCCESS("Moved attachments in %s" % self.stop_timer()))

misago/datamover/threads.py

@@ -55,6 +55,7 @@ def move_threads(stdout, style):
 
 def move_posts():
     special_categories = get_special_categories_dict()
+
     for post in fetch_assoc('SELECT * FROM misago_post ORDER BY id'):
         if special_categories.get(post['forum_id']) == 'reports':
             continue
@@ -69,7 +70,6 @@ def move_posts():
             poster_id = movedids.get('user', post['user_id'])
             poster = UserModel.objects.get(pk=poster_id)
 
-
         if post['edit_user_id']:
             last_editor_id = movedids.get('user', post['edit_user_id'])
             last_editor = UserModel.objects.get(pk=last_editor_id)

misago/threads/api/postingendpoint/attachments.py

@@ -113,7 +113,6 @@ class AttachmentsSerializer(serializers.Serializer):
         post.update_fields.append('attachments_cache')
 
 
-
 def validate_attachments_count(data):
     total_attachments = len(data)
     if total_attachments > settings.MISAGO_POST_ATTACHMENTS_LIMIT:

misago/threads/migrations/0003_attachment_types.py

@@ -37,6 +37,24 @@ ATTACHMENTS = (
         'size_limit': 4 * 1024
     },
     {
+        'name': 'Text',
+        'extensions': ('txt',),
+        'mimetypes': ('text/plain',),
+        'size_limit': 4 * 1024
+    },
+    {
+        'name': 'Markdown',
+        'extensions': ('md',),
+        'mimetypes': ('text/markdown',),
+        'size_limit': 4 * 1024
+    },
+    {
+        'name': 'reStructuredText',
+        'extensions': ('rst',),
+        'mimetypes': ('text/x-rst',),
+        'size_limit': 4 * 1024
+    },
+    {
         'name': '7Z',
         'extensions': ('7z',),
         'mimetypes': ('application/x-7z-compressed',),

misago/threads/models/thread.py

@@ -130,7 +130,11 @@ class Thread(models.Model):
         except ObjectDoesNotExist:
             self.has_poll = False
 
-        self.replies = self.post_set.filter(is_event=False, is_unapproved=False).count()
+        self.replies = self.post_set.filter(
+            is_event=False,
+            is_unapproved=False
+        ).count()
+
         if self.replies > 0:
             self.replies -= 1