#600: first go at moving threads and posts

docs/upgrading_from_05.rst

@@ -78,7 +78,7 @@ Todo
 Wrapping up migration
 =====================
 
-Not everything is moved over. Thread labels will be turned into subcategories of their categories. With exception of pre-made superuser accounts, all users will be assigned to "Members" rank and will have only default roles of forum members. Likewise no permissions will be moved over to users, categories or ranks, and you will have to reset those manually.
+Not everything is moved over. Thread labels will be turned into subcategories of their categories. With exception of pre-made superuser accounts, all users will be assigned to "Members" rank and will have only default roles of forum members. Likewise no permissions will be moved over to users, categories or ranks, and you will have to reset those manually. Reports and events will also be omitted by migration.
 
 
 Recounting data
@@ -86,7 +86,7 @@ Recounting data
 
 After moving users data over to new site, you'll need to rebuild their stats, trackers and bans. To do this use ``invalidatebans``, ``populateonlinetracker`` and ``synchronizeusers`` commands that you can run via ``manage.py``.
 
-Likewise you'll need to rebuild threads and categories via ``synchronizethreads`` and then ``synchronizecategories``.
+Likewise you'll need to rebuild threads and categories via ``synchronizethreads``, then ``synchronizecategories`` and ``rebuildpostssearch``.
 
 
 Changed links

misago/datamover/avatars.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import os
 
 from django.contrib.auth import get_user_model

misago/datamover/bans.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from misago.users.models import Ban
 
 from . import fetch_assoc, localise_datetime

misago/datamover/categories.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from misago.categories.models import Category
 
 from . import fetch_assoc, movedids

misago/datamover/management/commands/movecategories.py

@@ -3,10 +3,10 @@ from ..base import BaseCommand
 
 
 class Command(BaseCommand):
-    help = "Moves categories and labels from Misago 0.5 installation"
+    help = "Moves categories and labels from Misago 0.5"
 
     def handle(self, *args, **options):
-        self.stdout.write("Moving users from Misago 0.5:")
+        self.stdout.write("Moving categories from Misago 0.5:")
 
         self.start_timer()
         categories.move_categories(self.stdout, self.style)

misago/datamover/management/commands/movesettings.py

@@ -3,7 +3,7 @@ from ..base import BaseCommand
 
 
 class Command(BaseCommand):
-    help = "Moves settings from Misago 0.5 installation"
+    help = "Moves settings from Misago 0.5"
 
     def handle(self, *args, **options):
         self.stdout.write("Moving settings from Misago 0.5:")

misago/datamover/management/commands/movethreads.py

@@ -0,0 +1,19 @@
+from ... import threads
+from ..base import BaseCommand
+
+
+class Command(BaseCommand):
+    help = "Moves threads and posts from Misago 0.5"
+
+    def handle(self, *args, **options):
+        self.stdout.write("Moving threads from Misago 0.5:")
+
+        self.start_timer()
+        threads.move_threads(self.stdout, self.style)
+        self.stdout.write(
+            self.style.SUCCESS("Moved threads in %s" % self.stop_timer()))
+
+        self.start_timer()
+        threads.move_posts()
+        self.stdout.write(
+            self.style.SUCCESS("Moved posts in %s" % self.stop_timer()))

misago/datamover/management/commands/moveusers.py

@@ -5,7 +5,7 @@ from ..base import BaseCommand
 class Command(BaseCommand):
     help = (
         "Moves users, avatars, followers, blocks "
-        "and bans from Misago 0.5 installation"
+        "and bans from Misago 0.5"
     )
 
     def handle(self, *args, **options):

misago/datamover/markup.py

@@ -0,0 +1,6 @@
+def convert_original(content):
+    return content
+
+
+def convert_parsed(content):
+    return content

misago/datamover/settings.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from misago.conf import db_settings
 from misago.conf.models import Setting
 

misago/datamover/threads.py

@@ -0,0 +1,136 @@
+from __future__ import unicode_literals
+
+from django.contrib.auth import get_user_model
+from django.utils import timezone
+
+from misago.categories.models import Category
+from misago.threads.checksums import update_post_checksum
+from misago.threads.models import Thread, Post
+
+from . import fetch_assoc, markup, movedids, localise_datetime
+
+
+UserModel = get_user_model()
+
+
+def move_threads(stdout, style):
+    special_categories = get_special_categories_dict()
+
+    for thread in fetch_assoc('SELECT * FROM misago_thread ORDER BY id'):
+        if special_categories.get(thread['forum_id']) == 'reports':
+            stdout.write(style.WARNING(
+                "Skipping report: %s" % thread['name']))
+            continue
+
+        if not thread['start_post_id']:
+            stdout.write(style.ERROR(
+                "Corrupted thread: %s" % thread['name']))
+            continue
+
+        if special_categories.get(thread['forum_id']) == 'private_threads':
+            category = Category.objects.private_threads()
+        else:
+            if thread['prefix_id']:
+                label_id = '%s-%s' % (thread['prefix_id'], thread['forum_id'])
+                category_pk = movedids.get('label', label_id)
+            else:
+                category_pk = movedids.get('category', thread['forum_id'])
+            category = Category.objects.get(pk=category_pk)
+
+        new_thread = Thread.objects.create(
+            category=category,
+            title=thread['name'],
+            slug=thread['slug'],
+            started_on=timezone.now(),
+            last_post_on=timezone.now(),
+            starter_name='none',
+            starter_slug='none',
+            weight=thread['weight'],
+            is_hidden=thread['deleted'],
+            is_closed=thread['closed'],
+        )
+
+        movedids.set('thread', thread['id'], new_thread.pk)
+
+
+def move_posts():
+    special_categories = get_special_categories_dict()
+    for post in fetch_assoc('SELECT * FROM misago_post ORDER BY id'):
+        if special_categories.get(post['forum_id']) == 'reports':
+            continue
+
+        thread_pk = movedids.get('thread', post['thread_id'])
+        thread = Thread.objects.select_related('category').get(pk=thread_pk)
+
+        poster = None
+        last_editor = None
+
+        if post['user_id']:
+            poster_id = movedids.get('user', post['user_id'])
+            poster = UserModel.objects.get(pk=poster_id)
+
+
+        if post['edit_user_id']:
+            last_editor_id = movedids.get('user', post['edit_user_id'])
+            last_editor = UserModel.objects.get(pk=last_editor_id)
+
+        deleter = None
+        deleter_name = None
+        deleter_slug = None
+        if post['deleted']:
+            deleter = UserModel.objects.filter(
+                is_staff=True
+            ).order_by('id').last()
+
+            if deleter:
+                deleter_name = deleter.username
+                deleter_slug = deleter.slug
+            else:
+                deleter = poster
+                deleter_name = poster.username
+                deleter_slug = poster.slug
+
+        new_post = Post.objects.create(
+            category=thread.category,
+            thread=thread,
+            poster=poster,
+            poster_name=post['user_name'],
+            poster_ip=post['ip'],
+            original=markup.convert_original(post['post']),
+            parsed=markup.convert_parsed(post['post_preparsed']),
+            posted_on=localise_datetime(post['date']),
+            updated_on=localise_datetime(post['current_date'] or post['date']),
+            hidden_on=localise_datetime(post['delete_date'] or post['date']),
+            edits=post['edits'],
+            last_editor=last_editor,
+            last_editor_name=post['edit_user_name'],
+            last_editor_slug=post['edit_user_slug'],
+            hidden_by=deleter,
+            hidden_by_name=deleter_name,
+            hidden_by_slug=deleter_slug,
+            is_hidden=post['deleted'],
+            is_protected=post['protected'],
+            likes=post['upvotes']
+        )
+
+        update_post_checksum(new_post)
+        new_post.save()
+
+        movedids.set('post', post['id'], new_post.pk)
+
+
+def get_special_categories_dict():
+    special_categories = {}
+
+    query = '''
+        SELECT
+            id, special
+        FROM
+            misago_forum
+        WHERE
+            special IS NOT NULL
+    '''
+
+    for special in fetch_assoc(query):
+        special_categories[special['id']] = special['special']
+    return special_categories

misago/datamover/users.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from django.contrib.auth import get_user_model
 from django.core.exceptions import ValidationError
 from django.utils.crypto import get_random_string