small cleanup, task for cleaning expired orphaned attachments

docs/developers/settings.rst

@@ -212,6 +212,12 @@ Max dimensions (width and height) of user-uploaded images embedded in posts. If
    Because user-uploaded GIF's may be smaller than dimensions specified, but still be considerably heavy due to animation, Misago always generates thumbnails for user-uploaded GIFS, stripping the animations from them.
 
 
+MISAGO_ATTACHMENT_ORPHANED_EXPIRE
+---------------------------------
+
+How old (in minutes) should attachments unassociated with any be before they'll automatically deleted by ``clearattachments`` task.
+
+
 MISAGO_ATTACHMENT_SECRET_LENGTH
 -------------------------------
 

misago/conf/defaults.py

@@ -286,6 +286,11 @@ MISAGO_ATTACHMENT_IMAGE_SIZE_LIMIT = (500, 500)
 # Length of secret used for attachments url tokens and filenames
 MISAGO_ATTACHMENT_SECRET_LENGTH = 64
 
+# How old (in minutes) should attachments unassociated with any be before they'll
+# automatically deleted by "clearattachments" task
+MISAGO_ATTACHMENT_ORPHANED_EXPIRE = 24 * 60
+
+
 # Names of files served when user requests file that doesn't exist or is unavailable
 # Those files will be sought within STATIC_ROOT directory
 MISAGO_404_IMAGE = 'misago/img/error-404.png'

misago/threads/management/commands/clearattachments.py

@@ -0,0 +1,46 @@
+import time
+from datetime import timedelta
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from misago.core.management.progressbar import show_progress
+from misago.core.pgutils import batch_update
+
+from ...models import Attachment
+
+
+class Command(BaseCommand):
+    help = "Deletes attachments unassociated with any posts"
+
+    def handle(self, *args, **options):
+        cutoff = timezone.now() - timedelta(minutes=settings.MISAGO_ATTACHMENT_ORPHANED_EXPIRE)
+        queryset = Attachment.objects.filter(
+            post__isnull=True,
+            uploaded_on__lt=cutoff
+        )
+
+        attachments_to_sync = queryset.count()
+
+        if not attachments_to_sync:
+            self.stdout.write("\n\nNo attachments were found")
+        else:
+            self.sync_attachments(queryset, attachments_to_sync)
+
+    def sync_attachments(self, queryset, attachments_to_sync):
+        message = "Clearing %s attachments...\n"
+        self.stdout.write(message % attachments_to_sync)
+
+        message = "\n\nCleared %s attachments"
+
+        synchronized_count = 0
+        show_progress(self, synchronized_count, attachments_to_sync)
+        start_time = time.time()
+        for attachment in batch_update(queryset):
+            attachment.delete()
+
+            synchronized_count += 1
+            show_progress(self, synchronized_count, attachments_to_sync, start_time)
+
+        self.stdout.write(message % synchronized_count)

misago/threads/management/commands/synchronizethreads.py

@@ -9,21 +9,21 @@ from ...models import Thread
 
 
 class Command(BaseCommand):
-    help = 'Synchronizes threads'
+    help = "Synchronizes threads"
 
     def handle(self, *args, **options):
         threads_to_sync = Thread.objects.count()
 
         if not threads_to_sync:
-            self.stdout.write('\n\nNo threads were found')
+            self.stdout.write("\n\nNo threads were found")
         else:
             self.sync_threads(threads_to_sync)
 
     def sync_threads(self, threads_to_sync):
-        message = 'Synchronizing %s threads...\n'
+        message = "Synchronizing %s threads...\n"
         self.stdout.write(message % threads_to_sync)
 
-        message = '\n\nSynchronized %s threads'
+        message = "\n\nSynchronized %s threads"
 
         synchronized_count = 0
         show_progress(self, synchronized_count, threads_to_sync)
@@ -33,7 +33,6 @@ class Command(BaseCommand):
             thread.save()
 
             synchronized_count += 1
-            show_progress(
-                self, synchronized_count, threads_to_sync, start_time)
+            show_progress(self, synchronized_count, threads_to_sync, start_time)
 
         self.stdout.write(message % synchronized_count)

misago/threads/migrations/0001_initial.py

@@ -225,7 +225,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('secret', models.CharField(max_length=64)),
-                ('uploaded_on', models.DateTimeField(default=django.utils.timezone.now)),
+                ('uploaded_on', models.DateTimeField(default=django.utils.timezone.now, db_index=True)),
                 ('uploader_name', models.CharField(max_length=255)),
                 ('uploader_slug', models.CharField(max_length=255)),
                 ('uploader_ip', models.GenericIPAddressField()),

misago/threads/models/attachment.py

@@ -41,7 +41,7 @@ class Attachment(models.Model):
         on_delete=models.SET_NULL
     )
 
-    uploaded_on = models.DateTimeField(default=timezone.now)
+    uploaded_on = models.DateTimeField(default=timezone.now, db_index=True)
 
     uploader = models.ForeignKey(
         settings.AUTH_USER_MODEL,

misago/threads/tests/test_clearattachments.py

@@ -0,0 +1,83 @@
+from datetime import timedelta
+
+from django.conf import settings
+from django.test import TestCase
+from django.utils import timezone
+from django.utils.six import StringIO
+
+from misago.categories.models import Category
+
+from .. import testutils
+from ..management.commands import clearattachments
+from ..models import Attachment, AttachmentType
+
+
+class ClearAttachmentsTests(TestCase):
+    def test_no_attachments_sync(self):
+        """command works when there are no attachments"""
+        command = clearattachments.Command()
+
+        out = StringIO()
+        command.execute(stdout=out)
+        command_output = out.getvalue().strip()
+
+        self.assertEqual(command_output, "No attachments were found")
+
+    def test_attachments_sync(self):
+        """command synchronizes attachments"""
+        filetype = AttachmentType.objects.order_by('id').last()
+
+        # create 5 expired orphaned attachments
+        cutoff = timezone.now() - timedelta(minutes=settings.MISAGO_ATTACHMENT_ORPHANED_EXPIRE)
+        cutoff -= timedelta(minutes=5)
+
+        for i in range(5):
+            Attachment.objects.create(
+                secret=Attachment.generate_new_secret(),
+                filetype=filetype,
+                size=1000,
+                uploaded_on=cutoff,
+                uploader_name='bob',
+                uploader_slug='bob',
+                uploader_ip='127.0.0.1',
+                filename='testfile_{}.zip'.format(Attachment.objects.count() + 1),
+            )
+
+        # create 5 expired non-orphaned attachments
+        category = Category.objects.get(slug='first-category')
+        post = testutils.post_thread(category).first_post
+
+        for i in range(5):
+            Attachment.objects.create(
+                secret=Attachment.generate_new_secret(),
+                filetype=filetype,
+                size=1000,
+                uploaded_on=cutoff,
+                post=post,
+                uploader_name='bob',
+                uploader_slug='bob',
+                uploader_ip='127.0.0.1',
+                filename='testfile_{}.zip'.format(Attachment.objects.count() + 1),
+            )
+
+        # create 5 fresh orphaned attachments
+        for i in range(5):
+            Attachment.objects.create(
+                secret=Attachment.generate_new_secret(),
+                filetype=filetype,
+                size=1000,
+                uploader_name='bob',
+                uploader_slug='bob',
+                uploader_ip='127.0.0.1',
+                filename='testfile_{}.zip'.format(Attachment.objects.count() + 1),
+            )
+
+        command = clearattachments.Command()
+
+        out = StringIO()
+        command.execute(stdout=out)
+
+        command_output = out.getvalue().splitlines()[-1].strip()
+        self.assertEqual(command_output, "Cleared 5 attachments")
+
+        self.assertEqual(Attachment.objects.count(), 10)

misago/threads/tests/test_synchronizethreads.py

@@ -17,7 +17,7 @@ class SynchronizeThreadsTests(TestCase):
         command.execute(stdout=out)
         command_output = out.getvalue().strip()
 
-        self.assertEqual(command_output, 'No threads were found')
+        self.assertEqual(command_output, "No threads were found")
 
     def test_threads_sync(self):
         """command synchronizes threads"""
@@ -39,4 +39,4 @@ class SynchronizeThreadsTests(TestCase):
             self.assertEqual(db_thread.replies, i)
 
         command_output = out.getvalue().splitlines()[-1].strip()
-        self.assertEqual(command_output, 'Synchronized 10 threads')
+        self.assertEqual(command_output, "Synchronized 10 threads")