#600: users, avatars, follows, blocks

docs/upgrading_from_05.rst

@@ -42,10 +42,46 @@ You'll actually won't need all of your old forum's files for move, only attachme
     }
 
 
+Creating superuser
+------------------
+
+Its good idea to create superuser accounts for all site administrators. Don't worry about their e-mails being same as ones on old forum. If this happens Misago will simply reuse those accounts, instead of creating new ones. 
+
+
 Moving forum configuration
---------------------------
+==========================
 
 To move configuration over to new forum, run ``python manage.py movesettings`` command.
 
 .. note::
-   Some settings have been moved from admin to configuration file, or removed. Those will not be migrated. Please consult :doc:`configuration reference </developers/settings>`.
+   Some settings have been moved from admin to configuration file or removed. Those will not be migrated. Please consult :doc:`configuration reference </developers/settings>` for available settings that you will need to add yourself.
+
+
+Moving users
+============
+
+To move users over to new forum, run ``python manage.py moveusers`` command.
+
+Moved users will be assigned default rank and permissions as those aren't moved by the datamover. If user with such e-mail address already exists in database (because you've used this e-mail ealier to create superuser account), his or her permissions and rank will be left as they are in new database.
+
+If user avatar could not be moved (for .eg because uploaded picture is smaller than allowed by ``MISAGO_AVATARS_SIZES``), default avatar will be set for this user.
+
+In case of username collision, Misago will append digits to new user's username and print a warning in console to let you know about this.
+
+
+Moving threads
+==============
+
+Todo
+
+
+Wrapping up migration
+=====================
+
+Not everything is moved over. Thread labels will be turned into subcategories of their categories. With exception of pre-made superuser accounts, all users will be assigned to "Members" rank and will have only default roles of forum members. Likewise no permissions will be moved over to users, categories or ranks, and you will have to reset those manually.
+
+
+Changed links
+-------------
+
+Todo

misago/datamover/__init__.py

@@ -1,8 +1,17 @@
+from django.utils.timezone import make_aware, utc
+
 from .conf import OLD_FORUM
 from .db import fetch_assoc
+import movedids
 
 
 class DevNull(object):
     def write(self, *args, **kwargs):
         pass
 defstdout = DevNull()
+
+
+def localise_datetime(datetime):
+    if datetime:
+        return make_aware(datetime, utc)
+    return None

misago/datamover/avatars.py

@@ -0,0 +1,71 @@
+import os
+
+from django.contrib.auth import get_user_model
+from django.core.exceptions import ValidationError
+
+from misago.conf import settings
+from misago.users.avatars import store, gravatar, dynamic, gallery, uploaded
+
+from . import OLD_FORUM, defstdout, fetch_assoc, movedids
+
+
+UserModel = get_user_model()
+
+
+def move_avatars(stdout=None, style=None):
+    stdout = stdout or defstdout
+
+    for old_user in fetch_assoc('SELECT * FROM misago_user ORDER BY id'):
+        user = UserModel.objects.get(pk=movedids.get('user', old_user['id']))
+
+        if old_user['avatar_ban'] or old_user['avatar_type'] == 'gallery':
+            dynamic.set_avatar(user)
+        else:
+            if old_user['avatar_type'] == 'gravatar':
+                try:
+                    gravatar.set_avatar(user)
+                except gravatar.GravatarError:
+                    dynamic.set_avatar(user)
+                    print_warning(
+                        '%s: failed to download Gravatar' % user, stdout, style)
+            else:
+                try:
+                    if not old_user['avatar_original'] or not old_user['avatar_crop']:
+                        raise ValidationError("Invalid avatar upload data.")
+
+                    image_path = os.path.join(
+                        OLD_FORUM['MEDIA'], 'avatars', old_user['avatar_original'])
+                    image = uploaded.validate_dimensions(image_path)
+
+                    cleaned_crop = convert_crop(image, old_user)
+                    uploaded.clean_crop(image, cleaned_crop)
+
+                    store.store_temporary_avatar(user, image)
+                    uploaded.crop_source_image(user, 'tmp', cleaned_crop)
+                except ValidationError as e:
+                    dynamic.set_avatar(user)
+                    print_warning('%s: %s' % (user, e.args[0]), stdout, style)
+
+        user.save()
+
+
+def print_warning(warning, stdout, style):
+    if style:
+        stdout.write(style.ERROR(warning))
+    else:
+        stdout.write(warning)
+
+
+def convert_crop(image, user):
+    min_size = max(settings.MISAGO_AVATARS_SIZES)
+    x, y, s = [float(v) for v in user['avatar_crop'].split(',')]
+
+    zoom = min_size / s
+
+    return {
+        'offset': {
+            'x': x * zoom * -1,
+            'y': y * zoom * -1,
+        },
+        'zoom': zoom
+    }

misago/datamover/bans.py

@@ -0,0 +1,2 @@
+def move_bans(stdout=None):
+    pass

misago/datamover/db.py

@@ -5,11 +5,11 @@ def fetch_assoc(query, *args):
     """
     Return all rows from a cursor as a dict
     """
-    cursor = connections['misago05'].cursor()
-    cursor.execute(query, *args)
+    with connections['misago05'].cursor() as cursor:
+        cursor.execute(query, *args)
 
-    columns = [col[0] for col in cursor.description]
-    row = cursor.fetchone()
-    while row:
-        yield dict(zip(columns, row))
+        columns = [col[0] for col in cursor.description]
         row = cursor.fetchone()
+        while row:
+            yield dict(zip(columns, row))
+            row = cursor.fetchone()

misago/datamover/management/commands/moveusers.py

@@ -0,0 +1,42 @@
+from ... import avatars, bans, users
+from ..base import BaseCommand, CommandError
+
+
+class Command(BaseCommand):
+    help = (
+        "Moves users, avatars, followers, blocks "
+        "and bans from Misago 0.5 installation"
+    )
+
+    def handle(self, *args, **options):
+        self.stdout.write("Moving users from Misago 0.5:")
+
+        self.start_timer()
+        users.move_users(self.stdout, self.style)
+        self.stdout.write(
+            self.style.SUCCESS("Moved users in %s" % self.stop_timer()))
+
+        self.start_timer()
+        avatars.move_avatars(self.stdout, self.style)
+        self.stdout.write(
+            self.style.SUCCESS("Moved avatars in %s" % self.stop_timer()))
+
+        self.start_timer()
+        users.move_followers(self.stdout)
+        self.stdout.write(
+            self.style.SUCCESS("Moved followers in %s" % self.stop_timer()))
+
+        self.start_timer()
+        users.move_blocks(self.stdout)
+        self.stdout.write(
+            self.style.SUCCESS("Moved blocks in %s" % self.stop_timer()))
+
+        self.start_timer()
+        users.move_namehistory(self.stdout)
+        self.stdout.write(
+            self.style.SUCCESS("Moved name history in %s" % self.stop_timer()))
+
+        self.start_timer()
+        bans.move_bans(self.stdout)
+        self.stdout.write(
+            self.style.SUCCESS("Moved bans in %s" % self.stop_timer()))

misago/datamover/migrations/0001_initial.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.4 on 2016-12-30 22:47
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='MovedId',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('model', models.CharField(max_length=255)),
+                ('old_id', models.CharField(max_length=255)),
+                ('new_id', models.CharField(max_length=255)),
+            ],
+        ),
+    ]

misago/datamover/models.py

@@ -0,0 +1,7 @@
+from django.db import models
+
+
+class MovedId(models.Model):
+    model = models.CharField(max_length=255)
+    old_id = models.CharField(max_length=255)
+    new_id = models.CharField(max_length=255)

misago/datamover/movedids.py

@@ -0,0 +1,14 @@
+def get(model, id):
+    from .models import MovedId
+    try:
+        return MovedId.objects.get(model=model, old_id=id).new_id
+    except MovedId.DoesNotExist:
+        return None
+
+def set(model, old_id, new_id):
+    from .models import MovedId
+    MovedId.objects.create(
+        model=model,
+        old_id=old_id,
+        new_id=new_id,
+    )

misago/datamover/settings.py

@@ -2,7 +2,6 @@ from misago.conf import db_settings
 from misago.conf.models import Setting
 
 from . import defstdout, fetch_assoc
-import pprint
 
 
 def copy_value(setting):
@@ -55,8 +54,16 @@ SETTING_CONVERTER = {
     'avatars_types': convert_allow_custom_avatars,
     'default_avatar': copy_value('default_avatar'),
     'upload_limit': copy_value('avatar_upload_limit'),
-    'subscribe_start': copy_value('subscribe_start'),
-    'subscribe_reply': copy_value('subscribe_reply'),
+    'subscribe_start': map_value('subscribe_start', {
+        '0': 'no',
+        '1': 'watch',
+        '2': 'watch_email',
+    }),
+    'subscribe_reply': map_value('subscribe_reply', {
+        '0': 'no',
+        '1': 'watch',
+        '2': 'watch_email',
+    }),
     'bots_registration': map_value('captcha_type', {
         'no': 'no',
         'recaptcha': 're',
@@ -70,7 +77,7 @@ SETTING_CONVERTER = {
 }
 
 
-def move_settings(stdout):
+def move_settings(stdout=None):
     stdout = stdout or defstdout
 
     for row in fetch_assoc('SELECT * FROM misago_setting'):

misago/datamover/users.py

@@ -0,0 +1,122 @@
+from django.contrib.auth import get_user_model
+from django.core.exceptions import ValidationError
+from django.utils.crypto import get_random_string
+
+from misago.users.signatures import make_signature_checksum
+
+from . import defstdout, fetch_assoc, movedids, localise_datetime
+
+
+UserModel = get_user_model()
+
+
+PRIVATE_THREAD_INVITES = {
+    0: 0,
+    1: 0,
+    2: 1,
+    3: 2
+}
+
+
+def move_users(stdout=None, style=None):
+    stdout = stdout or defstdout
+
+    from .models import MovedId
+    MovedId.objects.all().delete()
+    [u.delete() for u in UserModel.objects.filter(pk__gt=1).iterator()]
+
+    existing_users = get_existing_users()
+
+    for user in fetch_assoc('SELECT * FROM misago_user ORDER BY id'):
+        if user['email'].lower() in existing_users:
+            user_new_pk = existing_users[user['email'].lower()]
+            new_user = UserModel.objects.get(pk=user_new_pk)
+        else:
+            try:
+                new_user = UserModel.objects.create_user(
+                    user['username'], user['email'], 'Pass.123')
+            except ValidationError:
+                new_name = ''.join([user['username'], get_random_string(4)])
+                new_user = UserModel.objects.create_user(
+                    new_name, user['email'], 'Pass.123')
+
+                if style:
+                    formats = (user['username'], new_name)
+                    stdout.write(style.ERROR(
+                        '"%s" has been registered as "%s"' % formats))
+                else:
+                    stdout.write(
+                        '"%s" has been registered as "%s"' % formats)
+
+            new_user.password = user['password']
+
+        new_user.joined_on = localise_datetime(user['join_date'])
+        new_user.joined_from_ip = user['join_ip']
+
+        new_user.last_login = localise_datetime(user['last_date'])
+        new_user.last_ip = user['last_ip']
+
+        new_user.is_hiding_presence = bool(user['hide_activity'])
+        new_user.title = user['title'] or None
+
+        new_user.requires_activation = user['activation']
+        if new_user.requires_activation > 2:
+            new_user.requires_activation = 1
+
+        new_user.is_avatar_locked = user['avatar_ban']
+        new_user.avatar_lock_user_message = user['avatar_ban_reason_user'] or None
+        new_user.avatar_lock_staff_message = user['avatar_ban_reason_admin'] or None
+
+        if user['signature'] and user['signature_preparsed']:
+            new_user.signature = user['signature']
+            new_user.signature_parsed = user['signature_preparsed']
+            new_user.signature_checksum = make_signature_checksum(
+                user['signature_preparsed'], new_user)
+
+        new_user.is_signature_locked = user['signature_ban']
+        new_user.signature_lock_user_message = user['signature_ban_reason_user'] or None
+        new_user.signature_lock_staff_message = user['signature_ban_reason_admin'] or None
+
+        new_user.limits_private_thread_invites_to = PRIVATE_THREAD_INVITES[user['allow_pds']]
+
+        new_user.subscribe_to_started_threads = int(user['subscribe_start'])
+        new_user.subscribe_to_replied_threads = int(user['subscribe_reply'])
+
+        new_user.save()
+        movedids.set('user', user['id'], new_user.pk)
+
+
+def get_existing_users():
+    existing_users = {}
+
+    queryset = UserModel.objects.values_list('id', 'email')
+    for pk, email in queryset.iterator():
+        existing_users[email.lower()] = pk
+    return existing_users
+
+
+def move_followers(stdout=None):
+    for follow in fetch_assoc('SELECT * FROM misago_user_follows ORDER BY id'):
+        from_user_id = movedids.get('user', follow['from_user_id'])
+        to_user_id = movedids.get('user', follow['to_user_id'])
+
+        from_user = UserModel.objects.get(pk=from_user_id)
+        to_user = UserModel.objects.get(pk=to_user_id)
+
+        from_user.follows.add(to_user)
+
+
+def move_blocks(stdout=None):
+    for follow in fetch_assoc('SELECT * FROM misago_user_ignores ORDER BY id'):
+        from_user_id = movedids.get('user', follow['from_user_id'])
+        to_user_id = movedids.get('user', follow['to_user_id'])
+
+        from_user = UserModel.objects.get(pk=from_user_id)
+        to_user = UserModel.objects.get(pk=to_user_id)
+
+        from_user.blocks.add(to_user)
+
+
+def move_namehistory(stdout):
+    pass
+

misago/users/avatars/gallery.py

@@ -42,7 +42,8 @@ def get_available_galleries(include_default=False):
 
 
 def galleries_exist():
-    return bool(get_available_galleries())
+    from ..models import AvatarGallery
+    return AvatarGallery.objects.exists()
 
 
 def load_avatar_galleries():

misago/users/avatars/gravatar.py

@@ -15,7 +15,7 @@ class GravatarError(RuntimeError):
     pass
 
 
-class NoGravatarAvailable(RuntimeError):
+class NoGravatarAvailable(GravatarError):
     pass
 
 

misago/users/migrations/0001_initial.py

@@ -71,7 +71,6 @@ class Migration(migrations.Migration):
                 ('threads', models.PositiveIntegerField(default=0)),
                 ('posts', models.PositiveIntegerField(default=0, db_index=True)),
                 ('last_posted_on', models.DateTimeField(null=True, blank=True)),
-                ('last_searched_on', models.DateTimeField(null=True, blank=True)),
             ],
             options={
                 'abstract': False,
@@ -180,7 +179,7 @@ class Migration(migrations.Migration):
                 ('image', models.ImageField(max_length=255, upload_to=misago.users.avatars.store.upload_to)),
             ],
             options={
-                'ordering': ['gallery', 'name'],
+                'ordering': ['gallery', 'pk'],
             },
         ),
         migrations.CreateModel(

misago/users/models/user.py

@@ -282,7 +282,6 @@ class User(AbstractBaseUser, PermissionsMixin):
     posts = models.PositiveIntegerField(default=0, db_index=True)
 
     last_posted_on = models.DateTimeField(null=True, blank=True)
-    last_searched_on = models.DateTimeField(null=True, blank=True)
 
     USERNAME_FIELD = 'slug'
     REQUIRED_FIELDS = ['email']

misago/users/serializers/user.py

@@ -152,7 +152,7 @@ class UserSerializer(BaseSerializer):
 
     def get_signature(self, obj):
         if obj.has_valid_signature:
-            return obj.signature.signature_parsed
+            return obj.signature_parsed
         else:
             return None
 

misago/users/signatures.py

@@ -6,8 +6,7 @@ def set_user_signature(request, user, signature):
 
     if signature:
         user.signature_parsed = signature_flavour(request, user, signature)
-        user.signature_checksum = make_signature_checksum(
-            user.signature_parsed, user)
+        user.signature_checksum = make_signature_checksum( user.signature_parsed, user)
     else:
         user.signature_parsed = ''
         user.signature_checksum = ''