#600: move over data, redirect from old urls

docs/upgrading_from_05.rst

@@ -72,7 +72,12 @@ In case of username collision, Misago will append digits to new user's username
 Moving threads
 ==============
 
-Todo
+To move threads and categories over, run following commands::
+
+    python manage.py movecategories
+    python manage.py movethreads
+
+This will move first the categories and then threads, posts, polls, attachments and finally private threads. This step will also take care of updating your posts markup.
 
 
 Wrapping up migration
@@ -92,4 +97,20 @@ Likewise you'll need to rebuild threads and categories via ``synchronizethreads`
 Changed links
 -------------
 
-Todo
+Links in Misago have changed with 0.6 release, but Misago will not update posted url's for you. Instead it comes with small utility that will catch old urls and return 301 (permament) redirect to new url, keeping old urls alive.
+
+To enable this feature you'll need to insert new url in your forum's ``urls.py``, so it looks like this::
+
+    urlpatterns = [
+        # insert below line above url with namespace='misago'
+        url(r'^', include('misago.datamover.urls')),
+        url(r'^', include('misago.urls', namespace='misago')),
+
+This will make Misago redirect users from old urls to new ones, altrough it'll wont preserve the meaning:
+
+- All links to forum will redirect to category's start page
+- All links to different profile pages of user profile will redirect to user's profile start page
+- All links to thread will lead to thread's first page
+- All links to post will lead to redirect to post in thread view
+
+This script also comes with one limitation: Because it comes before Misago's urls, it will catch all requests to ranks whose names end with number and try to map them to old user profiles. This means that naming the rank "Squadron 42" will produce url ``/users/squadron-42/`` that will be interpreted as link to old user. To avoid this make sure your ranks names end with non-alphametical characters, eg. "Squadron 42th" will produce ``/users/squardon-42th/`` as link that will successfully resolve to rank.

misago/core/pgutils.py

@@ -52,8 +52,7 @@ def batch_update(queryset, step=50):
     """
     Util because psycopg2 iterators aren't really memory effective
     """
-    queryset = queryset.order_by('pk')
-    paginator = Paginator(queryset, step)
+    paginator = Paginator(queryset.order_by('pk'), step)
     for page_number in paginator.page_range:
         for obj in paginator.page(page_number).object_list:
             yield obj

misago/datamover/management/commands/buildmovesindex.py

@@ -0,0 +1,34 @@
+from ...models import MovedId, OldIdRedirect
+from ..base import BaseCommand
+
+
+MAPPINGS = {
+    'attachment': 0,
+    'category': 1,
+    'post': 2,
+    'thread': 3,
+    'user': 4,
+}
+
+
+class Command(BaseCommand):
+    help = (
+        "Builds moves index for redirects from old urls to new ones."
+    )
+
+    def handle(self, *args, **options):
+        self.stdout.write("Building moves index...")
+
+        counter = 1
+        self.start_timer()
+
+        for moved_id in MovedId.objects.exclude(model='label').iterator():
+            counter += 1
+            OldIdRedirect.objects.create(
+                model=MAPPINGS[moved_id.model],
+                old_id=moved_id.old_id,
+                new_id=moved_id.new_id,
+            )
+
+        summary = "Indexed %s items in %s" % (counter, self.stop_timer())
+        self.stdout.write(self.style.SUCCESS(summary))

misago/datamover/migrations/0001_initial.py

@@ -22,4 +22,17 @@ class Migration(migrations.Migration):
                 ('new_id', models.CharField(max_length=255)),
             ],
         ),
+        migrations.CreateModel(
+            name='OldIdRedirect',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('model', models.PositiveIntegerField()),
+                ('old_id', models.PositiveIntegerField()),
+                ('new_id', models.PositiveIntegerField()),
+            ],
+        ),
+        migrations.AlterIndexTogether(
+            name='oldidredirect',
+            index_together=set([('model', 'old_id')]),
+        ),
     ]

misago/datamover/models.py

@@ -5,3 +5,20 @@ class MovedId(models.Model):
     model = models.CharField(max_length=255)
     old_id = models.CharField(max_length=255)
     new_id = models.CharField(max_length=255)
+
+
+class OldIdRedirect(models.Model):
+    ATTACHMENT = 0
+    CATEGORY = 1
+    POST = 2
+    THREAD = 3
+    USER = 4
+
+    model = models.PositiveIntegerField()
+    old_id = models.PositiveIntegerField()
+    new_id = models.PositiveIntegerField()
+
+    class Meta:
+        index_together = [
+            ['model', 'old_id'],
+        ]

misago/datamover/urls.py

@@ -0,0 +1,98 @@
+from django.conf.urls import url
+
+from . import views
+
+
+urlpatterns = [
+    url(r'^category/(?P<slug>(\w|-)+)-(?P<forum>\d+)/$', views.category_redirect),
+    url(r'^forum/(?P<slug>(\w|-)+)-(?P<forum>\d+)/$', views.category_redirect),
+    url(r'^forum/(?P<slug>(\w|-)+)-(?P<forum>\d+)/(?P<page>[1-9]([0-9]+)?)/$', views.category_redirect),
+    url(r'^forum/(?P<slug>(\w|-)+)-(?P<forum>\d+)/prefix/(?P<prefix>(\w|-)+)/$', views.category_redirect),
+    url(r'^forum/(?P<slug>(\w|-)+)-(?P<forum>\d+)/prefix/(?P<prefix>(\w|-)+)/(?P<page>[1-9]([0-9]+)?)/$', views.category_redirect),
+    url(r'^forum/(?P<slug>(\w|-)+)-(?P<forum>\d+)/start/$', views.category_redirect),
+]
+
+urlpatterns += [
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/edit/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/reply/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/vote/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/poll/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/reply/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/edit/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<page>[1-9]([0-9]+)?)/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/last/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/find-(?P<post>\d+)/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/new/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/moderated/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/reported/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/show-hidden/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/watch/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/watch/email/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/unwatch/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/unwatch/email/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/upvote/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/downvote/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/report/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/show-report/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/delete/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/hide/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/show/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/delete/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/hide/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/show/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/checkpoint/(?P<checkpoint>\d+)/delete/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/checkpoint/(?P<checkpoint>\d+)/hide/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/checkpoint/(?P<checkpoint>\d+)/show/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/info/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/votes/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/changelog/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/changelog/(?P<change>\d+)/$', views.thread_redirect),
+    url(r'^thread/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/changelog/(?P<change>\d+)/revert/$', views.thread_redirect),
+]
+
+urlpatterns += [
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/edit/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/reply/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/vote/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/poll/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/reply/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/edit/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<page>[1-9]([0-9]+)?)/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/last/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/find-(?P<post>\d+)/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/new/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/moderated/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/reported/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/show-hidden/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/watch/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/watch/email/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/unwatch/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/unwatch/email/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/upvote/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/downvote/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/report/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/show-report/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/delete/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/hide/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/show/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/delete/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/hide/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/show/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/checkpoint/(?P<checkpoint>\d+)/delete/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/checkpoint/(?P<checkpoint>\d+)/hide/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/checkpoint/(?P<checkpoint>\d+)/show/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/info/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/votes/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/changelog/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/changelog/(?P<change>\d+)/$', views.private_thread_redirect),
+    url(r'^private-threads/(?P<slug>(\w|-)+)-(?P<thread>\d+)/(?P<post>\d+)/changelog/(?P<change>\d+)/revert/$', views.private_thread_redirect),
+]
+
+urlpatterns += [
+    url(r'^users/(?P<username>\w+)-(?P<user>\d+)/', views.user_redirect),
+    url(r'^users/(?P<username>\w+)-(?P<user>\d+)/(?P<page>\d+)/', views.user_redirect),
+    url(r'^users/(?P<username>\w+)-(?P<user>\d+)/(?P<subpage>(\w|-)+)/', views.user_redirect),
+    url(r'^users/(?P<username>\w+)-(?P<user>\d+)/(?P<subpage>(\w|-)+)/(?P<page>\d+)/', views.user_redirect),
+]

misago/datamover/views.py

@@ -0,0 +1,50 @@
+from django.contrib.auth import get_user_model
+from django.shortcuts import get_object_or_404, redirect
+
+from misago.threads.viewmodels import (
+    ForumThread, PrivateThread, ThreadPost, ThreadsCategory)
+
+from .models import OldIdRedirect
+
+
+def category_redirect(request, **kwargs):
+    category_pk = get_new_id_or_404(OldIdRedirect.CATEGORY, kwargs['forum'])
+    category = ThreadsCategory(request, pk=category_pk)
+    return redirect(category.get_absolute_url(), permanent=True)
+
+
+def thread_redirect(request, **kwargs):
+    thread_pk = get_new_id_or_404(OldIdRedirect.THREAD, kwargs['thread'])
+    thread = ForumThread(request, pk=thread_pk)
+
+    if 'post' in kwargs:
+        post_pk = get_new_id_or_404(OldIdRedirect.POST, kwargs['post'])
+        post = ThreadPost(request, thread, pk=post_pk)
+        return redirect(post.get_absolute_url(), permanent=True)
+
+    return redirect(thread.get_absolute_url(), permanent=True)
+
+
+def private_thread_redirect(request, **kwargs):
+    thread_pk = get_new_id_or_404(OldIdRedirect.THREAD, kwargs['thread'])
+    thread = PrivateThread(request, pk=thread_pk)
+
+    if 'post' in kwargs:
+        post_pk = get_new_id_or_404(OldIdRedirect.POST, kwargs['post'])
+        post = ThreadPost(request, thread, pk=post_pk)
+        return redirect(post.get_absolute_url(), permanent=True)
+
+    return redirect(thread.get_absolute_url(), permanent=True)
+
+
+def user_redirect(request, **kwargs):
+    user_pk = get_new_id_or_404(OldIdRedirect.USER, kwargs['user'])
+
+    UserModel = get_user_model()
+    user = get_object_or_404(UserModel, pk=user_pk)
+
+    return redirect(user.get_absolute_url(), permanent=True)
+
+
+def get_new_id_or_404(model, old_id):
+    return get_object_or_404(OldIdRedirect, model=model, old_id=old_id).new_id

misago/users/management/commands/synchronizeusers.py

@@ -28,8 +28,10 @@ class Command(BaseCommand):
         show_progress(self, synchronized_count, users_to_sync)
         start_time = time.time()
         for user in batch_update(get_user_model().objects.all()):
-            user.threads = user.thread_set.count()
-            user.posts = user.post_set.count()
+            if user.pk == 1:
+                print user.username
+            user.threads = user.thread_set.filter(is_unapproved=False).count()
+            user.posts = user.post_set.filter(is_unapproved=False).count()
             user.followers = user.followed_by.count()
             user.following = user.follows.count()
             user.save()