fix #688: added default robots.txt

misago/core/tests/test_views.py

@@ -22,6 +22,12 @@ class PreloadJSDataViewTests(TestCase):
         self.assertEqual(response.status_code, 200)
 
 
+class RobotsTxtViewTests(TestCase):
+    def test_robots_txt_returns_200(self):
+        """robots.txt has no showstoppers"""
+        response = self.client.get('/robots.txt')
+        self.assertContains(response, '/api/')
+
 
 @override_settings(ROOT_URLCONF='misago.core.testproject.urls')
 class RedirectViewTests(TestCase):

misago/project_template/project_name/urls.py

@@ -1,11 +1,9 @@
 from django.conf import settings
 from django.conf.urls import include, url
-# Serve static and media files in development
 from django.conf.urls.static import static
-# Setup Django admin to work with Misago auth
 from django.contrib import admin
+from django.views.generic import TemplateView
 
-# Register default views
 from misago.core.views import javascript_catalog, momentjs_catalog
 from misago.users.forms.auth import AdminAuthenticationForm
 

misago/project_template/theme/templates/README.txt

@@ -1 +1 @@
-You can use this directory to replace default templates with custom ones as well as add new ones to your theme.
+You can use this directory to replace default templates with custom ones as well as add new ones to your theme.

misago/templates/misago/robots.txt

@@ -0,0 +1,11 @@
+# Forbid crawlers from indexing API
+User-agent: *
+Disallow: {% url 'misago:index' %}api/
+Disallow: {% url 'misago:index' %}activation/
+Disallow: {% url 'misago:banned' %}
+Disallow: {% url 'misago:blank-avatar' %}
+Disallow: {% url 'misago:forgotten-password' %}
+Disallow: {% url 'misago:login' %}
+Disallow: {% url 'misago:request-activation' %}
+Disallow: {% url 'misago:search' %}
+Disallow: {% url 'misago:index' %}u/

misago/urls.py

@@ -1,5 +1,6 @@
 from django.conf import settings
 from django.conf.urls import include, url
+from django.views.generic import TemplateView
 
 from misago.core.views import forum_index
 
@@ -12,6 +13,9 @@ urlpatterns = [
     url(r'^', include('misago.threads.urls')),
     url(r'^', include('misago.search.urls')),
 
+    # default robots.txt
+    url(r'^robots.txt$', TemplateView.as_view(template_name='misago/robots.txt')),
+
     # "misago:index" link symbolises "root" of Misago links space
     # any request with path that falls below this one is assumed to be directed
     # at Misago and will be handled by misago.views.exceptionhandler if it