Merge pull request #1176 from ali-alsabbah/add-django-celery

added redis to docker-compose.yml

devproject/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ["celery_app"]

devproject/celery.py

@@ -0,0 +1,22 @@
+import os
+
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "devproject.settings")
+
+app = Celery("devproject")
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+
+
+@app.task(bind=True)
+def debug_task(self):
+    print("Request: {0!r}".format(self.request))

devproject/settings.py

@@ -169,6 +169,7 @@ INSTALLED_APPS = [
     "django.contrib.messages",
     "django.contrib.staticfiles",
     # 3rd party apps used by Misago
+    "celery",
     "debug_toolbar",
     "mptt",
     "rest_framework",
@@ -415,6 +416,14 @@ MISAGO_PROFILE_FIELDS = [
 
 EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
 
+CELERY_BROKER_URL = "redis://redis:6379/0"
+
+# Celery workers may leak the memory, eventually depriving the instance of free
+# resources
+# This setting forces celery to stop worker, clean after it and create new one
+# after worker has processed 10 tasks.
+
+CELERY_WORKER_MAX_TASKS_PER_CHILD = 10
 
 # Display debug toolbar if IN_MISAGO_DOCKER enviroment var is set to "1"
 

docker-compose.yaml

@@ -4,11 +4,15 @@ version: "3.0"
 services:
   postgres:
     image: postgres:10
+    restart: unless-stopped
     environment:
       - POSTGRES_USER=misago
       - POSTGRES_PASSWORD=misago
     ports:
       - '127.0.0.1:5432:5432'
+  redis:
+    image: redis:5
+    restart: unless-stopped
   misago:
     build: .
     command: python manage.py runserver 0.0.0.0:8000
@@ -29,8 +33,28 @@ services:
       - "${MISAGO_DEVSERVER_PORT:-8000}:8000"
     depends_on:
       - postgres
+      - redis
     tty: true
     volumes:
       # Map in the entire project into the container
       # This makes sure files in the container updates on the fly as we were working locally
       - .:/srv/misago:Z
+  celery:
+    build: .
+    command: celery -A devproject worker --loglevel=info
+    environment:
+      # Postgres
+      - POSTGRES_USER=misago
+      - POSTGRES_PASSWORD=misago
+      - POSTGRES_DB=misago
+      - POSTGRES_HOST=postgres
+      - POSTGRES_TEST_DB=misago_test
+    depends_on:
+      - postgres
+      - redis
+    tty: true
+    volumes:
+      # Map in the entire project into the container
+      # This makes sure files in the container updates on the fly as we were working locally
+      - .:/srv/misago:Z
+  

misago/threads/migrations/0001_initial.py

@@ -1,12 +1,11 @@
 import django.db.models.deletion
 import django.utils.timezone
+import misago.threads.models.attachment
 from django.conf import settings
 from django.contrib.postgres.fields import JSONField
 from django.contrib.postgres.search import SearchVectorField
 from django.db import migrations, models
 
-import misago.threads.models.attachment
-
 
 class Migration(migrations.Migration):
 

misago/threads/migrations/0006_redo_partial_indexes.py

@@ -1,10 +1,9 @@
 # Generated by Django 1.11.1 on 2017-05-21 17:52
 import django.contrib.postgres.indexes
+import misago.core.pgutils
 from django.contrib.postgres.operations import BtreeGinExtension
 from django.db import migrations
 
-import misago.core.pgutils
-
 
 class Migration(migrations.Migration):
 

misago/threads/migrations/0009_auto_20180326_0010.py

@@ -1,7 +1,6 @@
 # Generated by Django 1.11.9 on 2018-03-26 00:10
-from django.db import migrations
-
 import misago.core.pgutils
+from django.db import migrations
 
 
 class Migration(migrations.Migration):

misago/users/migrations/0001_initial.py

@@ -1,11 +1,10 @@
 import django.db.models.deletion
 import django.utils.timezone
+import misago.users.avatars.store
 from django.conf import settings
 from django.contrib.postgres.fields import JSONField
 from django.db import migrations, models
 
-import misago.users.avatars.store
-
 
 class Migration(migrations.Migration):
 

misago/users/migrations/0005_dj_19_update.py

@@ -1,7 +1,6 @@
 # Generated by Django 1.9.7 on 2016-07-17 02:05
-from django.db import migrations, models
-
 import misago.users.models.user
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

misago/users/migrations/0009_redo_partial_indexes.py

@@ -1,7 +1,6 @@
 # Generated by Django 1.11.1 on 2017-05-26 21:56
-from django.db import migrations
-
 import misago.core.pgutils
+from django.db import migrations
 
 
 class Migration(migrations.Migration):

misago/users/migrations/0011_auto_20180331_2208.py

@@ -1,7 +1,6 @@
 # Generated by Django 1.11.11 on 2018-03-31 22:08
-from django.db import migrations, models
-
 import misago.core.pgutils
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

misago/users/migrations/0014_datadownload.py

@@ -1,11 +1,10 @@
 # Generated by Django 1.11.13 on 2018-06-24 00:13
 import django.db.models.deletion
 import django.utils.timezone
+import misago.users.models.datadownload
 from django.conf import settings
 from django.db import migrations, models
 
-import misago.users.models.datadownload
-
 
 class Migration(migrations.Migration):
 

requirements.in

@@ -1,5 +1,6 @@
 beautifulsoup4<4.7
 bleach<2.2
+celery[redis]
 django<2
 djangorestframework<3.7
 django-debug-toolbar<1.9

requirements.txt

@@ -4,10 +4,13 @@
 #
 #    pip-compile --output-file requirements.txt requirements.in
 #
+amqp==2.3.2               # via kombu
 atomicwrites==1.2.1       # via pytest
 attrs==18.2.0             # via pytest
 beautifulsoup4==4.6.3
+billiard==3.5.0.5         # via celery
 bleach==2.1.4
+celery[redis]==4.2.1
 certifi==2018.11.29       # via requests
 chardet==3.0.4            # via requests
 defusedxml==0.5.0         # via python3-openid, social-auth-core
@@ -20,8 +23,9 @@ djangorestframework==3.6.4
 faker==0.8.18
 html5lib==0.999999999
 idna==2.8                 # via requests
+kombu==4.2.2.post1        # via celery
 markdown==2.6.11
-more-itertools==4.3.0     # via pytest
+more-itertools==5.0.0     # via pytest
 oauthlib==2.1.0           # via requests-oauthlib, social-auth-core
 olefile==0.46             # via pillow
 pillow==4.1.1
@@ -35,6 +39,7 @@ pytest==4.0.2
 python-dateutil==2.7.5    # via faker
 python3-openid==3.1.0     # via social-auth-core
 pytz==2018.7
+redis==3.0.1              # via celery
 requests-oauthlib==1.0.0  # via social-auth-core
 requests==2.21.0
 six==1.12.0               # via bleach, faker, html5lib, more-itertools, pytest, python-dateutil, social-auth-app-django, social-auth-core
@@ -44,4 +49,5 @@ sqlparse==0.2.4           # via django-debug-toolbar
 text-unidecode==1.2       # via faker
 unidecode==0.4.21
 urllib3==1.24.1           # via requests
+vine==1.1.4               # via amqp
 webencodings==0.5.1       # via html5lib