Merge branch 'djsilcock-migrate-plugins' into 'master'

flaskbb/cli/plugins.py

@@ -45,8 +45,8 @@ def new_plugin(plugin_identifier, template):
     It will either accept a valid path on the filesystem
     or a URL to a Git repository which contains the cookiecutter template.
     """
-    out_dir = os.path.join(current_app.root_path, "plugins", plugin_identifier)
-    click.secho("[+] Creating new plugin {}".format(plugin_identifier),
+    out_dir = current_app.extensions['plugin_manager'].plugin_folder  #monkeypatched by test routine
+    click.secho("[+] Creating new plugin...",
                 fg="cyan")
     cookiecutter(template, output_dir=out_dir)
     click.secho("[+] Done. Created in {}".format(out_dir),
@@ -128,3 +128,52 @@ def list_plugins():
             click.secho("    - {} (version {})".format(
                 plugin.name, plugin.version), bold=True
             )
+
+
+@plugins.command("migrate")
+@click.argument("plugin_identifier")
+@click.option("--message", "-m", help="The name of the migration.")
+def migrate_plugin(plugin_identifier, message=None):
+    """Generates migration files for a plugin.
+    Migration version files are stored in
+    ``flaskbb/plugins/<plugin_dir>/migration_versions``.
+    """
+    validate_plugin(plugin_identifier)
+    plugin = get_plugin_from_all(plugin_identifier)
+    click.secho("[+] Updating plugin migrations {}...".format(plugin.name),
+                fg="cyan")
+    try:
+        plugin.migrate(message=message)
+    except Exception as e:
+        click.secho("[-] Couldn't generate migrations for plugin because of "
+                    "following exception: \n{}".format(e), fg="red")
+
+
+@plugins.command("upgrade")
+@click.argument("plugin_identifier")
+def upgrade_plugin(plugin_identifier):
+    """Upgrades database to the latest version of a plugin's models"""
+    validate_plugin(plugin_identifier)
+    plugin = get_plugin_from_all(plugin_identifier)
+    click.secho("[+] Upgrading plugin {}...".format(plugin.name), fg="cyan")
+    try:
+        plugin.upgrade_database()
+    except AttributeError:
+        pass
+
+
+@plugins.command("downgrade")
+@click.argument("plugin_identifier")
+def downgrade_plugin(plugin_identifier):
+    """Downgrades database to remove a plugin's models"""
+    validate_plugin(plugin_identifier)
+    plugin = get_plugin_from_all(plugin_identifier)
+
+    if click.confirm("Please confirm if you want to remove this plugins data "
+                     "from the database."):
+        click.secho("[+] Downgrading plugin {}...".format(plugin.name),
+                    fg="cyan")
+        try:
+            plugin.downgrade_database()
+        except AttributeError:
+            pass

flaskbb/plugins/__init__.py

@@ -9,10 +9,63 @@
     :license: BSD, see LICENSE for more details.
 """
 import warnings
+import contextlib
+import copy
+import os
+
 from flask import current_app
+from flask import g
+from flask_migrate import upgrade, downgrade, migrate
 from flask_plugins import Plugin
+from flaskbb.extensions import db, migrate as migrate_ext
+from flaskbb.management.models import SettingsGroup, Setting
+
+
+@contextlib.contextmanager  # TODO: Add tests
+def plugin_name_migrate(name):
+    """Migrations in this with block will only apply to models
+    from the named plugin"""
+    g.plugin_name = name
+    yield
+    del g.plugin_name
+
+
+def db_for_plugin(plugin_name, sqla_instance=None):
+    """Labels models as belonging to this plugin.
+    sqla_instance is a valid Flask-SQLAlchemy instance, if None,
+    then the default db is used
+
+    Usage:
+        from flaskbb.plugins import db_for_plugin
+
+        db = db_for_plugin(__name__)
+
+        mytable = db.Table(...)
+
+        class MyModel(db.Model):
+            ...
+    """
+    sqla_instance = sqla_instance or db
+    new_db = copy.copy(sqla_instance)
+
+    def Table(*args, **kwargs):
+        new_table = sqla_instance.Table(*args, **kwargs)
+        new_table._plugin = plugin_name
+        return new_table
+
+    new_db.Table = Table
+    return new_db
 
-from flaskbb.management.models import SettingsGroup
+
+@migrate_ext.configure
+def config_migrate(config):
+    """Configuration callback for plugins environment."""
+    plugins = current_app.extensions['plugin_manager'].all_plugins.values()
+    migration_dirs = [p.get_migration_version_dir() for p in plugins]
+    if config.get_main_option('version_table') == 'plugins':
+        config.set_main_option('version_locations', ' '.join(migration_dirs))
+        # current_app.logger.debug(config.get_main_option('version_locations'))
+    return config
 
 
 class FlaskBBPluginDeprecationWarning(DeprecationWarning):
@@ -23,11 +76,64 @@ warnings.simplefilter("always", FlaskBBPluginDeprecationWarning)
 
 
 class FlaskBBPlugin(Plugin):
-
     #: This is the :class:`SettingsGroup` key - if your the plugin needs to
     #: install additional things you must set it, else it won't install
     #: anything.
     settings_key = None
+    requires_install = None
+
+    def resource_filename(self, *names):
+        """Returns an absolute filename for a plugin resource."""
+        if len(names) == 1 and '/' in names[0]:
+            names = names[0].split('/')
+        fname = os.path.join(self.path, *names)
+        if ' ' in fname and '"' not in fname and "'" not in fname:
+            fname = '"%s"' % fname
+        return fname
+
+    def get_migration_version_dir(self):
+        """Returns the path to the directory which is containing the
+        migration version files
+        """
+        return self.__module__ + ':migration_versions'
+
+    def upgrade_database(self, target='head'):
+        """Updates the database to a later version of the plugin models.
+        Default behaviour is to upgrade to the latest version.
+        """
+        plugin_dir = current_app.extensions['plugin_manager'].plugin_folder
+        plugin_env = os.path.join(plugin_dir, '_migration_environment')
+        plugin_rev = "{}@{}".format(self.settings_key, target)
+        upgrade(directory=plugin_env, revision=plugin_rev)
+
+    def downgrade_database(self, target='base'):
+        """Rolls back the database to a previous version of plugin models.
+        Default behaviour is to remove the models completely.
+        """
+        plugin_dir = current_app.extensions['plugin_manager'].plugin_folder
+        plugin_env = os.path.join(plugin_dir, '_migration_environment')
+        plugin_rev = "{}@{}".format(self.settings_key, target)
+        downgrade(directory=plugin_env, revision=plugin_rev)
+
+    def migrate(self, message=None):
+        """Generates new migration files for a plugin and stores them in
+        flaskbb/plugins/<plugin_folder>/migration_versions
+
+        :param message: The message of the revision.
+        """
+        with plugin_name_migrate(self.__module__):
+            plugin_dir = current_app.extensions['plugin_manager'].plugin_folder
+            plugin_env = os.path.join(plugin_dir, '_migration_environment')
+            plugin_ver = os.path.join(self.path, 'migration_versions')
+            try:
+                migrate(directory=plugin_env, head=self.settings_key)
+            except Exception as e:  # presumably this is the initial migration?
+                migrate(
+                    message=message,
+                    directory=plugin_env,
+                    version_path=plugin_ver,
+                    branch_label=self.settings_key
+                )
 
     @property
     def has_settings(self):
@@ -63,6 +169,17 @@ class FlaskBBPlugin(Plugin):
         )
         return self.has_settings
 
+    def this_version_installed(self):
+        installed_migration = Setting.\
+            as_dict(self.settings_key).\
+            get('version', None)
+
+        if self.uninstallable:
+            if installed_migration == self.version:
+                return True
+            return False
+        return None
+
     # Some helpers
     def register_blueprint(self, blueprint, **kwargs):
         """Registers a blueprint.

flaskbb/plugins/_migration_environment/README

@@ -0,0 +1 @@
+Generic single-database configuration.

flaskbb/plugins/_migration_environment/alembic.ini

@@ -0,0 +1,45 @@
+# A generic, single database configuration.
+
+[alembic]
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+version_table = plugins
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

flaskbb/plugins/_migration_environment/env.py

@@ -0,0 +1,111 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+import logging
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+logger = logging.getLogger('alembic.env')
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+from flask import current_app, g
+
+config.set_main_option('sqlalchemy.url',
+                       current_app.config.get('SQLALCHEMY_DATABASE_URI'))
+db = current_app.extensions['migrate'].db
+target_metadata = db.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+moduletables = {k.__table__.name: k.__module__
+                for k in db.Model._decl_class_registry.values()
+                if hasattr(k, '__table__')}
+
+
+def include_object(obj, name, type_, reflected, compare_to):
+    modname = moduletables.get(name) or getattr(obj, '_plugin', None)
+    if hasattr(g, 'plugin_tables'):
+        return name in g.plugin_tables
+    elif hasattr(g, 'plugin_name'):
+        if name == 'alembic_version':
+            return False
+        if not modname:
+            return False
+        if modname and modname.startswith(g.plugin_name):
+            return True
+        return False
+    else:
+        return False
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(url=url, include_object=include_object)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+    """
+
+    # this callback is used to prevent an auto-migration from being generated
+    # when there are no changes to the schema
+    # reference: http://alembic.readthedocs.org/en/latest/cookbook.html
+    def process_revision_directives(context, revision, directives):
+        if getattr(config.cmd_opts, 'autogenerate', False):
+            script = directives[0]
+            if script.upgrade_ops.is_empty():
+                directives[:] = []
+                logger.info('No changes in schema detected.')
+
+    engine = engine_from_config(config.get_section(config.config_ini_section),
+                                prefix='sqlalchemy.',
+                                poolclass=pool.NullPool)
+
+    connection = engine.connect()
+    context.configure(connection=connection,
+                      target_metadata=target_metadata,
+                      process_revision_directives=process_revision_directives,
+                      include_object=include_object,
+                      version_table='plugins',
+                      **current_app.extensions['migrate'].configure_args)
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations()
+    finally:
+        connection.close()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

flaskbb/plugins/_migration_environment/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

tests/cli/test_plugins.py

@@ -0,0 +1,144 @@
+import zipfile
+import urllib
+import os
+import shutil
+import json
+
+import pytest
+from click.testing import CliRunner
+from flaskbb.cli import main as cli_main
+from flaskbb import plugins
+from importlib import import_module
+
+
+def test_new_plugin(tmpdir, application, monkeypatch):
+    runner = CliRunner()
+    # download the cookiecutter file to use locally
+    # (bypasses prompt about re-cloning)
+    zipfilename = str(tmpdir.join('cookiecutter.zip'))
+    urllib.urlretrieve('https://github.com/sh4nks/cookiecutter-flaskbb-plugin/archive/master.zip', zipfilename)  # noqa
+    with zipfile.ZipFile(zipfilename) as zf:
+        zf.extractall(str(tmpdir))
+    cookiecutterpath = tmpdir.join('cookiecutter-flaskbb-plugin-master')
+
+    tmp_plugin_folder = str(tmpdir.join('plugin_folder'))
+    os.mkdir(tmp_plugin_folder)
+    monkeypatch.setattr(cli_main, 'create_app', lambda s: application)
+    monkeypatch.setattr(application.extensions['plugin_manager'],
+                        'plugin_folder', tmp_plugin_folder)
+    stdin = '\n'.join([
+        'Test Name',
+        'someone@nowhere.com',
+        'Testing Plugin',
+        '',
+        'TestingPlugin',
+        'Straightforward Test Plugin',
+        'www.example.com',
+        '1.0.0'
+    ])
+
+    result = runner.invoke(
+        cli_main.flaskbb,
+        ['plugins', 'new', 'testplugin', '--template', str(cookiecutterpath)],
+        input=stdin
+    )
+
+    assert result.exit_code == 0
+    plugin_dir = os.path.join(
+        application.extensions['plugin_manager'].plugin_folder,
+        'testing_plugin'
+    )
+    assert os.path.exists(plugin_dir)
+    assert os.path.isdir(plugin_dir)
+    # add the temporary folder to the plugins path
+    # so import flaskbb.plugins.test_plugin works as expected
+    monkeypatch.setattr(
+        plugins, '__path__', plugins.__path__ + [tmp_plugin_folder]
+    )
+    assert import_module('flaskbb.plugins.testing_plugin').__plugin__ == 'TestingPlugin'  # noqa
+
+
+def test_migrate_plugin(tmpdir, monkeypatch, application):
+    pluginmanager = application.extensions['plugin_manager']
+    orig_plugin_folder = pluginmanager.plugin_folder
+    tmp_plugin_folder = str(tmpdir.join('plugin_folder'))
+    os.mkdir(tmp_plugin_folder)
+    shutil.copytree(
+        os.path.join(orig_plugin_folder, '_migration_environment'),
+        os.path.join(tmp_plugin_folder, '_migration_environment')
+    )
+    os.mkdir(os.path.join(tmp_plugin_folder, 'testplugin'))
+
+    pyfile = os.path.join(tmp_plugin_folder, 'testplugin', '__init__.py')
+    with open(pyfile, 'w') as pyfile:
+        pyfile.write('\r\n'.join([
+            "from flaskbb.plugins import FlaskBBPlugin",
+            "from flaskbb.extensions import db",
+            "class TestPlugin(FlaskBBPlugin):",
+            "    settings_key='testplugin'",
+            "    def somequery(self):",
+            "        TestModel.query.all()",
+            "class TestModel(db.Model):",
+            "    __tablename__='testtable'",
+            "    testkey=db.Column(db.Integer,primary_key=True)",
+            "",
+            "__plugin__='TestPlugin'",
+        ]))
+
+    jsoninfo = {
+        "identifier": "testplugin",
+        "name": "TestPlugin",
+        "author": "sh4nks",
+        "website": "http://flaskbb.org",
+        "license": "BSD",
+        "description": "A Test Plugin for FlaskBB",
+        "version": "0.1"
+    }
+    jsonfile = os.path.join(tmp_plugin_folder, 'testplugin', 'info.json')
+    with open(jsonfile, 'w') as jsonfile:
+        json.dump(jsoninfo, jsonfile)
+
+    monkeypatch.setattr(cli_main, 'create_app', lambda s: application)
+
+    monkeypatch.setattr(pluginmanager, 'plugin_folder', tmp_plugin_folder)
+    # add the temporary folder to the plugins path
+    # so import flaskbb.plugins.test_plugin works as expected
+    monkeypatch.setattr(
+        plugins, '__path__', plugins.__path__ + [tmp_plugin_folder]
+    )
+    pluginmanager._plugins = None
+    pluginmanager._all_plugins = None
+    pluginmanager._available_plugins = dict()
+    pluginmanager._found_plugins = dict()
+    pluginmanager.setup_plugins()
+    assert 'testplugin' in pluginmanager.plugins
+    versionsdir = os.path.join(
+        tmp_plugin_folder, 'testplugin', 'migration_versions'
+    )
+    assert not os.path.exists(versionsdir)
+    testplugin = pluginmanager.plugins['testplugin']
+
+    with application.app_context():
+        testplugin.migrate()
+        assert os.path.exists(versionsdir)
+
+        dirlist = os.listdir(versionsdir)
+        assert dirlist
+
+        dirlist = [os.path.join(versionsdir, d)
+                   for d in dirlist if d.endswith('.py')]
+
+        for d in dirlist:
+            with open(d, 'r') as f:
+                output = '\n'.join([l for l in f])
+
+        assert 'testtable' in output
+        exception_msg = 'Should not be able to run migrations twice'
+        with pytest.raises(Exception, message=exception_msg):
+            testplugin.migrate()
+
+        exception_msg = "Operations should fail as model not yet registered"
+        with pytest.raises(Exception, message=exception_msg):
+            testplugin.somequery()
+
+        testplugin.upgrade_database()