Generate user data archives

misago/threads/signals.py

@@ -1,3 +1,5 @@
+from collections import OrderedDict
+
 from django.contrib.auth import get_user_model
 from django.db import transaction
 from django.db.models.signals import pre_delete
@@ -125,37 +127,40 @@ def delete_user_threads(sender, **kwargs):
 
 @receiver(archive_user_data)
 def archive_user_attachments(sender, archive=None, **kwargs):
-    collection = archive.create_collection('attachment')
     queryset = sender.attachment_set.order_by('id')
     for attachment in chunk_queryset(queryset):
-        collection.write_model_file(attachment.image or attachment.file)
+        item_path = [attachment.uploaded_on.year, attachment.uploaded_on.month, attachment.uploaded_on.day]
+        archive.add_model_file(attachment.image or attachment.file, item_path)
 
 
 @receiver(archive_user_data)
 def archive_user_posts(sender, archive=None, **kwargs):
-    collection = archive.create_collection('post')
     queryset = sender.post_set.order_by('id')
     for post in chunk_queryset(queryset):
-        collection.write_data_file(post.posted_on, post.parsed)
+        item_name = post.posted_on.strftime('post_%H%M%S')
+        item_path = [post.posted_on.year, post.posted_on.month, post.posted_on.day]
+        archive.add_text(item_name, post.parsed, item_path)
 
 
 @receiver(archive_user_data)
 def archive_user_posts_edits(sender, archive=None, **kwargs):
-    collection = archive.create_collection('post_edit')
     queryset = sender.postedit_set.order_by('id')
     for post_edit in chunk_queryset(queryset):
-        collection.write_data_file(post_edit.edited_on, post_edit.edited_from)
+        item_name = post_edit.edited_on.strftime('post_edit_%H%M%S')
+        item_path = [post_edit.edited_on.year, post_edit.edited_on.month, post_edit.edited_on.day]
+        archive.add_text(item_name, post_edit.edited_from, item_path)
 
 
 @receiver(archive_user_data)
 def archive_user_polls(sender, archive=None, **kwargs):
-    collection = archive.create_collection('poll')
     queryset = sender.poll_set.order_by('id')
     for poll in chunk_queryset(queryset):
-        collection.write_data_file(poll.posted_on, {
-            _("Question"): poll.question,
-            _("Choices"): [c['label'] for c in poll.choices],
-        })
+        item_name = poll.posted_on.strftime('poll_%H%M%S')
+        item_path = [poll.posted_on.year, poll.posted_on.month, poll.posted_on.day]
+        archive.add_dict(item_name, OrderedDict([
+            (_("Question"), poll.question),
+            (_("Choices"), u', '.join([c['label'] for c in poll.choices])),
+        ]), item_path)
 
 
 @receiver(anonymize_user_content)

misago/users/dataarchive.py

@@ -107,7 +107,7 @@ class DataArchive(object):
             return final_path
 
         for path_item in path:
-            final_path = os.path.join(final_path, path_item)
+            final_path = os.path.join(final_path, six.text_type(path_item))
             if not os.path.isdir(final_path):
                 os.mkdir(final_path)
         return final_path

misago/users/management/commands/prepareuserdatadownloads.py

@@ -7,7 +7,7 @@ from misago.conf import settings
 from misago.core.pgutils import chunk_queryset
 from misago.users.dataarchive import DataArchive
 from misago.users.models import DataDownload
-from misago.users.signals import archive_user_personal_data
+from misago.users.signals import archive_user_data
 
 
 logger = logging.getLogger('misago.users.datadownloads')
@@ -33,7 +33,7 @@ class Command(BaseCommand):
             user = data_download.user
             with DataArchive(user, working_dir) as archive:
                 try:
-                    archive_user_personal_data.send(user, archive=archive)
+                    archive_user_data.send(user, archive=archive)
                     data_download.file = archive.get_file()
                     #data_download.save()
                 except Exception as e:

misago/users/signals.py

@@ -1,3 +1,4 @@
+from collections import OrderedDict
 from datetime import timedelta
 
 from django.contrib.auth import get_user_model
@@ -24,17 +25,17 @@ username_changed = Signal()
 
 @receiver(archive_user_data)
 def archive_user_details(sender, archive=None, **kwargs):
-    archive.write_data_file('details', {
-        _('Username'): sender.username,
-        _('E-mail'): sender.email,
-        _('Joined on'): sender.joined_on,
-        _('Joined from ip'): sender.joined_from_ip or 'unavailable',
-    })
+    archive.add_dict('details', OrderedDict([
+        (_('Username'), sender.username),
+        (_('E-mail'), sender.email),
+        (_('Joined on'), sender.joined_on),
+        (_('Joined from ip'), sender.joined_from_ip or 'unavailable'),
+    ]))
 
 
 @receiver(archive_user_data)
 def archive_user_profile_fields(sender, archive=None, **kwargs):
-    clean_profile_fields = {}
+    clean_profile_fields = OrderedDict()
     for profile_fields_group in profilefields.get_fields_groups():
         for profile_field in profile_fields_group['fields']:
             if sender.profile_fields.get(profile_field.fieldname):
@@ -42,34 +43,37 @@ def archive_user_profile_fields(sender, archive=None, **kwargs):
                 clean_profile_fields[str(profile_field.label)] = field_value
                 
     if clean_profile_fields:
-        archive.write_data_file('profile_fields', clean_profile_fields)
+        archive.add_dict('profile_fields', clean_profile_fields)
 
 
 @receiver(archive_user_data)
 def archive_user_avatar(sender, archive=None, **kwargs):
-    collection = archive.create_collection('avatar')
-    collection.write_model_file(sender.avatar_tmp)
-    collection.write_model_file(sender.avatar_src)
+    avatars_path = ['avatar']
+
+    archive.add_model_file(sender.avatar_tmp, avatars_path)
+    archive.add_model_file(sender.avatar_src, avatars_path)
     for avatar in sender.avatar_set.iterator():
-        collection.write_model_file(avatar.image)
+        archive.add_model_file(avatar.image, avatars_path)
 
 
 @receiver(archive_user_data)
 def archive_user_audit_trail(sender, archive=None, **kwargs):
-    collection = archive.create_collection('audit_trail')
     queryset = sender.audittrail_set.order_by('id')
     for audit_trail in chunk_queryset(queryset):
-        collection.write_data_file(audit_trail.created_at, audit_trail.ip_address)
+        item_name = audit_trail.created_at.strftime('audit_trail_%H%M%S')
+        item_path = [audit_trail.created_at.year, audit_trail.created_at.month, audit_trail.created_at.day]
+        archive.add_text(item_name, audit_trail.ip_address, item_path)
 
 
 @receiver(archive_user_data)
 def archive_user_name_history(sender, archive=None, **kwargs):
-    collection = archive.create_collection('name_history')
     for name_change in sender.namechanges.order_by('id').iterator():
-        collection.write_data_file(name_change.changed_on, {
-            _("New username"): name_change.new_username,
-            _("Old username"): name_change.old_username,
-        })
+        item_name = name_change.changed_on.strftime('name_change_%H%M%S')
+        item_path = [name_change.changed_on.year, name_change.changed_on.month, name_change.changed_on.day]
+        archive.add_dict(item_name, OrderedDict([
+            (_("New username"), name_change.new_username),
+            (_("Old username"), name_change.old_username),
+        ]), item_path)
 
 
 @receiver(username_changed)