Super basic data collector

misago/users/datacollector.py

@@ -0,0 +1,91 @@
+import json
+import os
+import shutil
+
+from django.utils import timezone
+from django.utils.crypto import get_random_string
+
+
+class DataWriter(object):
+    def write_json_file(self, name, data):
+        file_path = os.path.join(self.data_dir_path, '{}.json'.format(name))
+        with open(file_path, 'w+') as fp:
+            json.dump(data, fp, ensure_ascii=False, indent=2)
+
+    def write_file(self, model_file):
+        if not model_file:
+            return None
+
+        target_filename = model_file.name.split('/')[-1]
+        target_path = os.path.join(self.data_dir_path, target_filename)
+
+        with open(target_path, 'wb') as fp:
+            for chunk in model_file.chunks():
+                fp.write(chunk)
+
+        return target_path
+        
+
+class DataCollection(DataWriter):
+    def __init__(self, user, data_dir_path):
+        self.user = user
+        self.data_dir_path = data_dir_path
+        os.mkdir(data_dir_path)
+
+
+class DataCollector(DataWriter):
+    def __init__(self, user, working_dir_path):
+        self.user = user
+        self.working_dir_path = working_dir_path
+
+        self.archive_path = None
+        self.tmp_dir_path = self.create_tmp_dir()
+        self.data_dir_path = self.create_data_dir()
+
+    def get_tmp_dir_name(self):
+        dir_name_bits = [
+            self.user.slug,
+            timezone.now().strftime('%Y%m%d-%H%M%S'),
+            get_random_string(6),
+        ]
+
+        return '-'.join(dir_name_bits)
+
+    def create_tmp_dir(self):
+        tmp_dir_name = self.get_tmp_dir_name()
+        tmp_dir_path = os.path.join(self.working_dir_path, tmp_dir_name)
+
+        os.mkdir(tmp_dir_path)
+
+        return tmp_dir_path
+
+    def get_data_dir_name(self):
+        dir_name_bits = [
+            self.user.slug,
+            timezone.now().strftime('%Y%m%d-%H%M%S'),
+        ]
+
+        return '-'.join(dir_name_bits)
+
+    def create_data_dir(self):
+        data_dir_name = self.get_data_dir_name()
+        data_dir_path = os.path.join(self.tmp_dir_path, data_dir_name)
+
+        os.mkdir(data_dir_path)
+
+        return data_dir_path
+
+    def create_collection(self, name):
+        collection_dir_path = os.path.join(self.data_dir_path, name)
+        return DataCollection(self.user, collection_dir_path)
+
+    def create_archive(self):
+        archive_name = self.get_tmp_dir_name()
+        archive_path = os.path.join(self.working_dir_path, archive_name)
+        shutil.make_archive(archive_path, 'zip', self.tmp_dir_path)
+
+        self.archive_path = archive_path
+        return archive_path
+
+    def delete_tmp_dir(self):
+        shutil.rmtree(self.tmp_dir_path)

misago/users/management/commands/prepareuserdatadownloads.py

@@ -0,0 +1,57 @@
+import logging
+
+from django.contrib.auth import get_user_model
+from django.core.management.base import BaseCommand
+
+from misago.conf import settings
+from misago.core.pgutils import chunk_queryset
+from misago.users.datacollector import DataCollector
+from misago.users.models import DataDownload
+
+
+logger = logging.getLogger('misago.users.datadownloads')
+
+UserModel = get_user_model()
+
+
+class Command(BaseCommand):
+    help = "Prepares user data downloads."
+
+    def handle(self, *args, **options):
+        working_dir = settings.MISAGO_USER_DATA_DOWNLOADS_WORKING_DIR
+        if not working_dir:
+            self.stdout.write(
+                "MISAGO_USER_DATA_DOWNLOADS_WORKING_DIR has to be set in order for "
+                "this feature to work.")
+            return
+        
+        downloads_prepared = 0
+        queryset = DataDownload.objects.select_related('user')
+        queryset = queryset.filter(status=DataDownload.STATUS_PENDING)
+        for data_download in chunk_queryset(queryset):
+            data_collector = DataCollector(data_download.user, working_dir)
+            try:
+                collect_user_data(data_download.user, data_collector)
+                data_collector.create_archive()
+                data_download.save()
+            except Exception as e:
+                print(e)
+                logger.exception(e)
+            data_collector.delete_tmp_dir()
+
+            downloads_prepared += 1
+
+        self.stdout.write("Data downloads prepared: {}".format(downloads_prepared))
+
+
+def collect_user_data(user, data_collector):
+    data_collector.write_json_file('details', {
+        'username': user.username,
+        'email': user.email,
+    })
+
+    avatars = data_collector.create_collection('avatars')
+    avatars.write_file(user.avatar_tmp)
+    avatars.write_file(user.avatar_src)
+    for avatar in user.avatar_set.iterator():
+        avatars.write_file(avatar.image)

misago/users/management/commands/processuserdatadownloads.py

@@ -1,19 +0,0 @@
-from django.contrib.auth import get_user_model
-from django.core.management.base import BaseCommand, CommandError
-
-from misago.conf import settings
-from misago.core.pgutils import chunk_queryset
-from misago.users.models import DataDownload
-
-UserModel = get_user_model()
-
-
-class Command(BaseCommand):
-    help = "Processes user data downloads."
-
-    def handle(self, *args, **options):
-        if not settings.MISAGO_USER_DATA_DOWNLOADS_WORKING_DIR:
-            self.stdout.write(
-                "Data downloads working directory has to be set for this feature to work.")
-            return
-