diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 84dc3af3..d7c50399 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,7 @@ Current - **BREAKING**: New configuration file format to allow further improvements (The conversion is automatic, but LDAP settings might need some attention) - **BREAKING**: Passwords are now *salted* for the *BASIC* authentication backend (The conversion is automatic too) +- **BREAKING**: If you plan to use the SQL storage along with gunicorn, you **MUST** add the *--preload* parameter (see the *gunicorn.d/burp-ui* file) - Add: `Celery `_ support for asynchronous tasks - Add: `SQLAlchemy `_ support for persistent storage - Add: `RESTful restore `_ diff --git a/burpui/__main__.py b/burpui/__main__.py index f31202cf..dbb45f42 100644 --- a/burpui/__main__.py +++ b/burpui/__main__.py @@ -33,7 +33,7 @@ def parse_args(mode=True, name=None): parser.add_argument('-m', '--mode', dest='mode', help='application mode', metavar='') options, unknown = parser.parse_known_args() - if options.mode and options.mode not in ['celery', 'manage']: + if mode and options.mode and options.mode not in ['celery', 'manage']: options = parser.parse_args() if options.version: diff --git a/migrations/README b/migrations/README new file mode 100755 index 00000000..98e4f9c4 --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/migrations/alembic.ini b/migrations/alembic.ini new file mode 100644 index 00000000..f8ed4801 --- /dev/null +++ b/migrations/alembic.ini @@ -0,0 +1,45 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py new file mode 100755 index 00000000..45938160 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,87 @@ +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig +import logging + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger('alembic.env') + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from flask import current_app +config.set_main_option('sqlalchemy.url', + current_app.config.get('SQLALCHEMY_DATABASE_URI')) +target_metadata = current_app.extensions['migrate'].db.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.readthedocs.org/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + engine = engine_from_config(config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + connection = engine.connect() + context.configure(connection=connection, + target_metadata=target_metadata, + process_revision_directives=process_revision_directives, + **current_app.extensions['migrate'].configure_args) + + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.close() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100755 index 00000000..95702017 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,22 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/225d9b2f0fb1_initial.py b/migrations/versions/225d9b2f0fb1_initial.py new file mode 100644 index 00000000..b50d3944 --- /dev/null +++ b/migrations/versions/225d9b2f0fb1_initial.py @@ -0,0 +1,35 @@ +"""initial + +Revision ID: 225d9b2f0fb1 +Revises: None +Create Date: 2016-07-18 11:22:27.613921 + +""" + +# revision identifiers, used by Alembic. +revision = '225d9b2f0fb1' +down_revision = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('task', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', sa.String(), nullable=True), + sa.Column('task', sa.String(), nullable=True), + sa.Column('user', sa.String(), nullable=True), + sa.Column('timestamp', sa.DateTime(), nullable=True), + sa.Column('expire', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('uuid') + ) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('task') + ### end Alembic commands ### diff --git a/setup.py b/setup.py index f03f8502..a7361bff 100755 --- a/setup.py +++ b/setup.py @@ -192,6 +192,8 @@ dev_requires = ['flake8', 'pylint'] datadir = os.path.join('share', 'burpui') confdir = os.path.join(datadir, 'etc') contrib = os.path.join(datadir, 'contrib') +migrations = [(os.path.join(datadir, root), [os.path.join(root, f) for f in files if not f.endswith('.pyc')]) + for root, dirs, files in os.walk('migrations')] setup( name=name, @@ -224,8 +226,7 @@ setup( (os.path.join(contrib, 'centos'), ['contrib/centos/init.sh']), (os.path.join(contrib, 'debian'), ['contrib/debian/init.sh']), (os.path.join(contrib, 'gunicorn.d'), ['contrib/gunicorn.d/burp-ui']), - (os.path.join(datadir, 'migrations'), ['migrations']), - ], + ] + migrations, install_requires=requires, extras_require={ 'ssl': ['pyOpenSSL'],